vesper-wizard 2.3.1 → 2.3.2

package/src/python/hf_fallback.py

@@ -1,298 +0,0 @@
-"""
-HuggingFace Datasets Library Fallback Downloader.
-
-Used when the HF Hub file listing finds no suitable data files
-(e.g. script-based datasets, gated datasets, datasets that use
-the `datasets` library format).
-
-Handles:
-- Legacy script-based datasets (trust_remote_code)
-- Gated/private datasets (token auth)
-- Image datasets (PIL Image columns → stripped for tabular export)
-- Various split formats (DatasetDict, single split)
-
-Usage:
-    python hf_fallback.py '{"repo_id": "user/dataset", "output_path": "/path/to/output.parquet", "token": "optional", "max_rows": 100000, "split": "train"}'
-
-Output: JSON to stdout
-    {"ok": true, "path": "/path/to/output.parquet", "rows": 12345, "columns": ["col1", "col2"]}
-    {"ok": false, "error": "..."}
-"""
-import sys
-import json
-import os
-import warnings
-
-# Suppress noisy HF warnings about trust_remote_code etc.
-warnings.filterwarnings("ignore", message=".*trust_remote_code.*")
-warnings.filterwarnings("ignore", message=".*legacy.*")
-
-
-def _detect_image_columns(ds):
-    """Detect columns that contain HF Image features or PIL Image objects."""
-    image_cols = []
-    features = getattr(ds, "features", None)
-    if features:
-        for name, feat in features.items():
-            feat_cls = feat.__class__.__name__.lower()
-            feat_str = str(feat).lower()
-            if feat_cls == "image" or "image(" in feat_str:
-                image_cols.append(name)
-    return image_cols
-
-
-def _strip_image_columns(ds, image_cols):
-    """Remove image columns from dataset so it can be exported to Parquet/CSV.
-    
-    Image columns contain PIL Image objects that can't be serialized to tabular
-    formats. We replace them with a placeholder string indicating the column
-    was an image column.
-    """
-    if not image_cols:
-        return ds
-    
-    # Remove the image columns entirely for tabular export
-    cols_to_keep = [c for c in ds.column_names if c not in image_cols]
-    if not cols_to_keep:
-        # Dataset is ALL image columns — keep them but cast to path strings if possible
-        return ds
-    
-    return ds.select_columns(cols_to_keep)
-
-
-def _load_dataset_robust(repo_id, token, split):
-    """Load a HuggingFace dataset with multiple fallback strategies.
-    
-    Strategy order:
-    1. Normal load with trust_remote_code=True (handles legacy script datasets)
-    2. Load without trust_remote_code (newer datasets that reject it)
-    3. Load with streaming=True then materialize (handles very large datasets)
-    """
-    from datasets import load_dataset, DatasetDict
-
-    errors = []
-    splits_to_try = [split] if split else ["train", "test", "validation", None]
-    
-    # Strategy 1: Normal load with trust_remote_code
-    for s in splits_to_try:
-        try:
-            kwargs = {"path": repo_id, "trust_remote_code": True}
-            if token:
-                kwargs["token"] = token
-            if s:
-                kwargs["split"] = s
-            ds = load_dataset(**kwargs)
-            return ds, s
-        except (ValueError, KeyError):
-            continue
-        except Exception as e:
-            msg = str(e)
-            # Auth errors should be raised immediately, not retried
-            if any(x in msg for x in ["401", "403", "gated", "Unauthorized", "access"]):
-                raise
-            if "split" in msg.lower() or "key" in msg.lower():
-                continue
-            errors.append(f"trust_remote_code=True, split={s}: {msg}")
-    
-    # Strategy 2: Load WITHOUT trust_remote_code (some repos reject it)
-    for s in splits_to_try:
-        try:
-            kwargs = {"path": repo_id}
-            if token:
-                kwargs["token"] = token
-            if s:
-                kwargs["split"] = s
-            ds = load_dataset(**kwargs)
-            return ds, s
-        except (ValueError, KeyError):
-            continue
-        except Exception as e:
-            msg = str(e)
-            if any(x in msg for x in ["401", "403", "gated", "Unauthorized", "access"]):
-                raise
-            if "split" in msg.lower() or "key" in msg.lower():
-                continue
-            errors.append(f"trust_remote_code=False, split={s}: {msg}")
-    
-    # Strategy 3: Streaming fallback (for very large / oddly structured datasets)
-    for s in splits_to_try:
-        if s is None:
-            continue  # streaming requires a split
-        try:
-            kwargs = {"path": repo_id, "streaming": True, "trust_remote_code": True}
-            if token:
-                kwargs["token"] = token
-            if s:
-                kwargs["split"] = s
-            ds_stream = load_dataset(**kwargs)
-            # Materialize from streaming iterator
-            from datasets import Dataset as HFDataset
-            rows = []
-            for i, row in enumerate(ds_stream):
-                if i >= 500000:
-                    break
-                rows.append(row)
-            if rows:
-                ds = HFDataset.from_list(rows)
-                return ds, s
-        except Exception:
-            continue
-    
-    # All strategies failed
-    error_summary = "; ".join(errors[:3]) if errors else "No valid configuration found"
-    return None, error_summary
-
-
-def main():
-    if len(sys.argv) < 2:
-        print(json.dumps({"ok": False, "error": "Missing payload argument"}))
-        sys.exit(1)
-
-    try:
-        payload = json.loads(sys.argv[1])
-    except json.JSONDecodeError as e:
-        print(json.dumps({"ok": False, "error": f"Invalid JSON payload: {e}"}))
-        sys.exit(1)
-
-    repo_id = payload.get("repo_id", "").strip()
-    output_path = payload.get("output_path", "").strip()
-    token = payload.get("token") or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or None
-    max_rows = payload.get("max_rows", 500000)
-    split = payload.get("split")  # None = auto-detect
-
-    if not repo_id:
-        print(json.dumps({"ok": False, "error": "repo_id is required"}))
-        sys.exit(1)
-
-    if not output_path:
-        print(json.dumps({"ok": False, "error": "output_path is required"}))
-        sys.exit(1)
-
-    try:
-        from datasets import load_dataset
-    except ImportError:
-        print(json.dumps({"ok": False, "error": "Python 'datasets' library not installed. Install with: pip install datasets"}))
-        sys.exit(1)
-
-    try:
-        import polars as pl
-    except ImportError:
-        pl = None
-
-    try:
-        ds, used_split = _load_dataset_robust(repo_id, token, split)
-
-        if ds is None:
-            print(json.dumps({"ok": False, "error": f"Could not load dataset '{repo_id}': {used_split}"}))
-            sys.exit(1)
-
-        # Handle DatasetDict (when no split specified)
-        from datasets import DatasetDict, Dataset
-        if isinstance(ds, DatasetDict):
-            # Pick the best split
-            for preferred in ["train", "test", "validation"]:
-                if preferred in ds:
-                    ds = ds[preferred]
-                    used_split = preferred
-                    break
-            else:
-                # Just pick the first available split
-                first_key = list(ds.keys())[0]
-                ds = ds[first_key]
-                used_split = first_key
-
-        # Limit rows if needed
-        total_rows = len(ds)
-        if max_rows and total_rows > max_rows:
-            ds = ds.select(range(max_rows))
-            total_rows = max_rows
-
-        # Detect and handle image columns (PIL Image objects can't be exported to Parquet)
-        image_cols = _detect_image_columns(ds)
-        has_images = len(image_cols) > 0
-        
-        if has_images:
-            # Strip image columns for tabular export, note them in output
-            export_ds = _strip_image_columns(ds, image_cols)
-        else:
-            export_ds = ds
-
-        # Ensure output directory exists
-        os.makedirs(os.path.dirname(output_path), exist_ok=True)
-
-        # Export to parquet
-        columns = export_ds.column_names
-
-        try:
-            if output_path.endswith(".parquet"):
-                export_ds.to_parquet(output_path)
-            elif output_path.endswith(".csv"):
-                export_ds.to_csv(output_path)
-            else:
-                # Default to parquet
-                if not output_path.endswith(".parquet"):
-                    output_path = output_path + ".parquet"
-                export_ds.to_parquet(output_path)
-        except Exception as export_err:
-            # If parquet export fails (e.g. complex nested types), try CSV
-            csv_path = output_path.replace(".parquet", ".csv")
-            try:
-                export_ds.to_csv(csv_path)
-                output_path = csv_path
-            except Exception:
-                raise export_err  # Re-raise original error
-
-        result = {
-            "ok": True,
-            "path": output_path,
-            "rows": total_rows,
-            "columns": columns,
-            "split": used_split
-        }
-        
-        if has_images:
-            result["image_columns"] = image_cols
-            result["note"] = (
-                f"This dataset contains image columns ({', '.join(image_cols)}). "
-                "Image data was stripped for tabular export. "
-                "Use vesper_download_assets with source='huggingface' to download the actual images."
-            )
-
-        print(json.dumps(result))
-
-    except Exception as e:
-        error_msg = str(e)
-        # Provide helpful, actionable hints
-        if "401" in error_msg or "Unauthorized" in error_msg:
-            error_msg = (
-                f"Authentication required for dataset '{repo_id}'. "
-                "This dataset may be gated or private. "
-                "Use the configure_keys tool to set your HF_TOKEN, then retry."
-            )
-        elif "403" in error_msg or "Forbidden" in error_msg:
-            error_msg = (
-                f"Access denied for dataset '{repo_id}'. "
-                "You may need to accept the dataset's usage agreement on huggingface.co, "
-                "then set HF_TOKEN via configure_keys tool."
-            )
-        elif "gated" in error_msg.lower():
-            error_msg = (
-                f"Dataset '{repo_id}' is gated. "
-                "Visit https://huggingface.co/datasets/{repo_id} to request access, "
-                "then set HF_TOKEN via configure_keys tool."
-            ).format(repo_id=repo_id)
-        elif "FileNotFoundError" in error_msg or "does not exist" in error_msg.lower() or "doesn't exist" in error_msg.lower():
-            error_msg = f"Dataset '{repo_id}' not found on HuggingFace. Check the dataset ID."
-        elif "script" in error_msg.lower() and "no longer supported" in error_msg.lower():
-            error_msg = (
-                f"Dataset '{repo_id}' uses a legacy loading script that is no longer supported "
-                "by the current version of the datasets library. "
-                "Try: pip install datasets --upgrade, or use an older datasets version."
-            )
-
-        print(json.dumps({"ok": False, "error": error_msg}))
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/src/python/image_engine.py

@@ -1,86 +0,0 @@
-import sys
-import json
-import os
-from PIL import Image
-import cv2
-import numpy as np
-
-def analyze_image(image_path):
-    stats = {
-        "path": image_path,
-        "filename": os.path.basename(image_path),
-        "status": "ok",
-        "error": None
-    }
-
-    try:
-        # 1. Basic Metadata with Pillow
-        img = Image.open(image_path)
-        stats["width"], stats["height"] = img.size
-        stats["format"] = img.format
-        stats["mode"] = img.mode
-        
-        # 2. Advanced Analysis with OpenCV
-        cv_img = cv2.imread(image_path)
-        if cv_img is None:
-            stats["status"] = "corrupted"
-            stats["error"] = "OpenCV failed to decode image"
-            return stats
-
-        # Blur detection (Laplacian variance)
-        gray = cv2.cvtColor(cv_img, cv2.COLOR_BGR2GRAY)
-        laplacian_var = cv2.Laplacian(gray, cv2.CV_64F).var()
-        stats["blur_score"] = laplacian_var
-        stats["is_blurry"] = laplacian_var < 100 # Rule of thumb threshold
-
-        # Brightness
-        stats["brightness"] = np.mean(gray)
-        
-        # Aspect Ratio
-        stats["aspect_ratio"] = stats["width"] / stats["height"]
-
-    except Exception as e:
-        stats["status"] = "failed"
-        stats["error"] = str(e)
-
-    return stats
-
-def main():
-    if len(sys.argv) < 2:
-        print(json.dumps({"error": "No path provided"}))
-        sys.exit(1)
-
-    input_path = sys.argv[1]
-    results = []
-
-    if os.path.isfile(input_path):
-        results.append(analyze_image(input_path))
-    elif os.path.isdir(input_path):
-        # Analyze first 50 images for performance in this demo
-        valid_exts = (".jpg", ".jpeg", ".png", ".bmp", ".webp")
-        files = [os.path.join(input_path, f) for f in os.listdir(input_path) if f.lower().endswith(valid_exts)]
-        for f in files[:50]:
-            results.append(analyze_image(f))
-    else:
-        print(json.dumps({"error": "Invalid path"}))
-        sys.exit(1)
-
-    # Aggregate stats
-    if not results:
-        print(json.dumps({"error": "No images found"}))
-        sys.exit(1)
-
-    report = {
-        "total_images": len(results),
-        "corrupted_count": len([r for r in results if r["status"] == "corrupted"]),
-        "failed_count": len([r for r in results if r["status"] == "failed"]),
-        "average_width": np.mean([r["width"] for r in results if "width" in r]),
-        "average_height": np.mean([r["height"] for r in results if "height" in r]),
-        "blurry_count": len([r for r in results if r.get("is_blurry")]),
-        "individual_results": results
-    }
-
-    print(json.dumps(report))
-
-if __name__ == "__main__":
-    main()

package/src/python/kaggle_engine.py

@@ -1,295 +0,0 @@
-import sys
-import os
-import json
-import tempfile
-from typing import Dict, Any, List
-from config import get_all
-
-try:
-    from kaggle.api.kaggle_api_extended import KaggleApi
-    HAS_KAGGLE = True
-except Exception:
-    HAS_KAGGLE = False
-
-
-IMAGE_EXTENSIONS = {
-    ".jpg",
-    ".jpeg",
-    ".png",
-    ".webp",
-    ".bmp",
-    ".gif",
-    ".tiff",
-    ".tif",
-    ".svg",
-}
-
-
-def _ensure_auth() -> Dict[str, Any]:
-    if not HAS_KAGGLE:
-        return {
-            "ok": False,
-            "error": "kaggle package not installed. Install with: pip install kaggle",
-        }
-
-    # Priority:
-    # 1) secure local store (keyring or ~/.vesper/config.toml)
-    # 2) existing env vars
-    # 3) ~/.kaggle/kaggle.json handled by KaggleApi.authenticate()
-    keys = get_all()
-    if keys.get("kaggle_username") and keys.get("kaggle_key"):
-        os.environ["KAGGLE_USERNAME"] = keys["kaggle_username"]
-        os.environ["KAGGLE_KEY"] = keys["kaggle_key"]
-
-    api = KaggleApi()
-    try:
-        api.authenticate()
-    except Exception as e:
-        return {
-            "ok": False,
-            "error": "Kaggle requires API key — run 'vespermcp config keys' (30 seconds) or provide ~/.kaggle/kaggle.json",
-            "details": str(e),
-        }
-
-    return {"ok": True, "api": api}
-
-
-def _dataset_to_dict(ds) -> Dict[str, Any]:
-    # kaggle API object fields differ by version; use getattr defensively
-    ref = getattr(ds, "ref", None) or getattr(ds, "datasetRef", None) or ""
-    title = getattr(ds, "title", None) or ref
-    subtitle = getattr(ds, "subtitle", None) or ""
-    owner = getattr(ds, "creatorName", None) or getattr(ds, "ownerName", None) or ""
-    votes = int(getattr(ds, "voteCount", 0) or 0)
-    downloads = int(getattr(ds, "downloadCount", 0) or 0)
-    size = int(getattr(ds, "totalBytes", 0) or 0)
-    last_updated = str(getattr(ds, "lastUpdated", ""))
-    tags = []
-    raw_tags = getattr(ds, "tags", None)
-    if raw_tags:
-        for t in raw_tags:
-            tags.append(getattr(t, "name", str(t)))
-
-    return {
-        "id": ref,
-        "source": "kaggle",
-        "name": title,
-        "description": subtitle or title,
-        "downloads": downloads,
-        "likes": votes,
-        "stars": 0,
-        "tags": tags,
-        "last_updated": last_updated,
-        "task": "unknown",
-        "domain": "unknown",
-        "languages": [],
-        "splits": [{"name": "data", "num_examples": 0, "size_bytes": size}],
-        "license": {
-            "id": "unknown",
-            "name": "unknown",
-            "category": "unknown",
-            "usage_restrictions": [],
-            "warnings": ["Kaggle license details may vary by dataset"],
-        },
-        "quality_score": 40,
-        "quality_warnings": ["Review dataset card and competition rules before use"],
-        "download_url": f"https://www.kaggle.com/datasets/{ref}",
-        "format": None,
-        "total_examples": 0,
-        "total_size_bytes": size,
-        "total_size_mb": round(size / (1024 * 1024), 2) if size else 0,
-        "columns": [],
-        "is_structured": False,
-        "has_target_column": False,
-        "is_safe_source": True,
-        "has_personal_data": False,
-        "is_paywalled": False,
-        "is_scraped_web_data": False,
-        "uses_https": True,
-        "has_train_split": False,
-        "has_test_split": False,
-        "has_validation_split": False,
-        "description_length": len(subtitle or title),
-        "has_readme": True,
-    }
-
-
-def discover(query: str, limit: int = 20) -> Dict[str, Any]:
-    auth = _ensure_auth()
-    if not auth.get("ok"):
-        return auth
-
-    api: KaggleApi = auth["api"]
-    try:
-        desired = max(1, min(limit, 100))
-
-        try:
-            datasets = api.dataset_list(search=query, page_size=desired)
-            items = [_dataset_to_dict(ds) for ds in datasets[:limit]]
-            return {"ok": True, "results": items, "count": len(items)}
-        except TypeError:
-            pass
-
-        collected = []
-        page = 1
-        while len(collected) < limit:
-            page_items = api.dataset_list(search=query, page=page)
-            if not page_items:
-                break
-
-            collected.extend(page_items)
-            if len(page_items) < 20:
-                break
-            page += 1
-
-        items = [_dataset_to_dict(ds) for ds in collected[:limit]]
-        return {"ok": True, "results": items, "count": len(items)}
-    except Exception as e:
-        return {"ok": False, "error": f"Kaggle discover failed: {str(e)}"}
-
-
-def _find_image_files(root: str) -> List[str]:
-    image_files: List[str] = []
-    for base, _, files in os.walk(root):
-        for name in files:
-            full = os.path.join(base, name)
-            if os.path.splitext(name)[1].lower() in IMAGE_EXTENSIONS:
-                image_files.append(full)
-    image_files.sort()
-    return image_files
-
-
-def _infer_image_record(root: str, full_path: str, index: int) -> Dict[str, Any]:
-    relative_path = os.path.relpath(full_path, root).replace("\\", "/")
-    parent_dir = os.path.dirname(relative_path)
-    parts = [part for part in parent_dir.split("/") if part and part != "."]
-
-    split = None
-    label = None
-    if parts:
-        first = parts[0].lower()
-        if first in {"train", "test", "val", "valid", "validation"}:
-            split = parts[0]
-            if len(parts) > 1:
-                label = parts[-1]
-        else:
-            label = parts[-1]
-
-    record: Dict[str, Any] = {
-        "id": index,
-        "image_path": os.path.abspath(full_path),
-        "relative_path": relative_path,
-        "file_name": os.path.basename(full_path),
-        "extension": os.path.splitext(full_path)[1].lower().lstrip("."),
-    }
-    if split:
-        record["split"] = split
-    if label:
-        record["label"] = label
-    return record
-
-
-def _write_image_manifest(root: str, image_files: List[str]) -> str:
-    manifest_path = os.path.join(root, "_vesper_image_manifest.jsonl")
-    with open(manifest_path, "w", encoding="utf-8") as handle:
-        for index, full_path in enumerate(image_files):
-            handle.write(json.dumps(_infer_image_record(root, full_path, index), ensure_ascii=False) + "\n")
-    return manifest_path
-
-
-def _pick_best_file(root: str) -> Dict[str, Any]:
-    candidates: List[str] = []
-    for base, _, files in os.walk(root):
-        for name in files:
-            full = os.path.join(base, name)
-            lower = name.lower()
-            if lower.endswith((".parquet", ".csv", ".jsonl", ".json", ".feather", ".arrow")):
-                candidates.append(full)
-
-    if not candidates:
-        image_files = _find_image_files(root)
-        if image_files:
-            manifest_path = _write_image_manifest(root, image_files)
-            return {
-                "local_path": manifest_path,
-                "dataset_kind": "image-manifest",
-                "image_count": len(image_files),
-            }
-        raise RuntimeError("No suitable data file found after download")
-
-    # prioritize common tabular formats
-    priorities = [".parquet", ".csv", ".jsonl", ".json", ".feather", ".arrow"]
-    for ext in priorities:
-        for c in candidates:
-            if c.lower().endswith(ext):
-                return {"local_path": c, "dataset_kind": "tabular", "image_count": 0}
-    return {"local_path": candidates[0], "dataset_kind": "tabular", "image_count": 0}
-
-
-def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
-    auth = _ensure_auth()
-    if not auth.get("ok"):
-        return auth
-
-    api: KaggleApi = auth["api"]
-
-    if not target_dir:
-        target_dir = tempfile.mkdtemp(prefix="vesper_kaggle_")
-
-    os.makedirs(target_dir, exist_ok=True)
-
-    try:
-        if "kaggle.com/datasets/" in dataset_ref:
-            dataset_ref = dataset_ref.split("kaggle.com/datasets/")[1].lstrip("/")
-
-        # unzip in place, remove zip for convenience
-        api.dataset_download_files(dataset_ref, path=target_dir, unzip=True, quiet=True)
-        artifact = _pick_best_file(target_dir)
-        return {
-            "ok": True,
-            "dataset_id": dataset_ref,
-            "target_dir": target_dir,
-            "local_path": artifact["local_path"],
-            "dataset_kind": artifact["dataset_kind"],
-            "image_count": artifact.get("image_count", 0),
-        }
-    except Exception as e:
-        msg = str(e)
-        if "401" in msg or "Unauthorized" in msg:
-            return {"ok": False, "error": "Invalid Kaggle credentials (401). Run 'vespermcp config kaggle' again."}
-        if "429" in msg or "Too Many Requests" in msg:
-            return {"ok": False, "error": "Kaggle rate limit reached. Please retry later."}
-        return {"ok": False, "error": f"Kaggle download failed: {msg}"}
-
-
-def main():
-    if len(sys.argv) < 2:
-        print(json.dumps({"ok": False, "error": "Usage: kaggle_engine.py <discover|download> ..."}))
-        sys.exit(1)
-
-    command = sys.argv[1]
-
-    if command == "discover":
-        if len(sys.argv) < 3:
-            print(json.dumps({"ok": False, "error": "Usage: kaggle_engine.py discover <query> [limit]"}))
-            sys.exit(1)
-        query = sys.argv[2]
-        limit = int(sys.argv[3]) if len(sys.argv) > 3 else 20
-        print(json.dumps(discover(query, limit)))
-        return
-
-    if command == "download":
-        if len(sys.argv) < 3:
-            print(json.dumps({"ok": False, "error": "Usage: kaggle_engine.py download <dataset_ref> [target_dir]"}))
-            sys.exit(1)
-        dataset_ref = sys.argv[2]
-        target_dir = sys.argv[3] if len(sys.argv) > 3 else ""
-        print(json.dumps(download(dataset_ref, target_dir)))
-        return
-
-    print(json.dumps({"ok": False, "error": f"Unknown command: {command}"}))
-    sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()