vesper-wizard 2.0.4 → 2.0.6

package/src/python/vesper/core/download_recipe.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+DEFAULT_RECIPES_DIR = Path.home() / ".vesper" / "recipes"
+
+
+@dataclass
+class DownloadRecipe:
+    dataset_id: str
+    source: str
+    repo_id: str
+    image_column: Optional[str]
+    download_method: str
+    requires_auth: bool
+    estimated_asset_size_gb: float
+    total_images: int
+    fallback_strategy: list[str]
+
+
+def _safe_name(value: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9._-]+", "_", value).strip("_") or "dataset"
+
+
+def build_download_recipe(dataset_info: Dict[str, Any]) -> Dict[str, Any]:
+    dataset_id = str(dataset_info.get("dataset_id") or dataset_info.get("id") or "")
+    source = str(dataset_info.get("source") or "unknown").lower()
+    repo_id = str(dataset_info.get("repo_id") or dataset_info.get("id") or dataset_id)
+
+    image_column = dataset_info.get("image_column")
+    if not image_column:
+        features = dataset_info.get("features") or {}
+        if isinstance(features, dict):
+            for key in features.keys():
+                lower = str(key).lower()
+                if lower in {"image", "images", "img", "img_path", "image_url", "url"}:
+                    image_column = key
+                    break
+
+    download_method = "url_list"
+    if source == "huggingface":
+        download_method = "hf_dataset_image_feature"
+    elif source == "kaggle":
+        download_method = "kaggle_archive"
+    elif source in {"dataworld", "openml"}:
+        download_method = "direct_file_scan"
+
+    requires_auth = bool(dataset_info.get("requires_auth", source in {"kaggle", "dataworld"}))
+
+    total_images = int(dataset_info.get("total_images") or dataset_info.get("total_examples") or 0)
+    if total_images <= 0:
+        total_images = 1000
+
+    estimated_asset_size_gb = round(float(dataset_info.get("estimated_asset_size_gb") or (total_images * 0.0004)), 3)
+
+    fallback_strategy = dataset_info.get("fallback_strategy") or [
+        "scan_archive_for_images",
+        "extract_url_column_and_download",
+        "export_metadata_only_with_actionable_error",
+    ]
+
+    recipe = DownloadRecipe(
+        dataset_id=dataset_id or repo_id,
+        source=source,
+        repo_id=repo_id,
+        image_column=image_column,
+        download_method=download_method,
+        requires_auth=requires_auth,
+        estimated_asset_size_gb=estimated_asset_size_gb,
+        total_images=total_images,
+        fallback_strategy=list(fallback_strategy),
+    )
+
+    return asdict(recipe)
+
+
+def save_recipe(recipe: Dict[str, Any], recipes_dir: Optional[str] = None) -> str:
+    root = Path(recipes_dir) if recipes_dir else DEFAULT_RECIPES_DIR
+    root.mkdir(parents=True, exist_ok=True)
+
+    dataset_id = str(recipe.get("dataset_id") or recipe.get("repo_id") or "dataset")
+    recipe_dir = root / _safe_name(dataset_id)
+    recipe_dir.mkdir(parents=True, exist_ok=True)
+
+    out_path = recipe_dir / "download_recipe.json"
+    out_path.write_text(json.dumps(recipe, indent=2, ensure_ascii=False), encoding="utf-8")
+    return str(out_path)
+
+
+def get_download_recipe(dataset_id: str, recipes_dir: Optional[str] = None) -> Optional[Dict[str, Any]]:
+    root = Path(recipes_dir) if recipes_dir else DEFAULT_RECIPES_DIR
+    path = root / _safe_name(dataset_id) / "download_recipe.json"
+    if not path.exists():
+        return None
+
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None

package/src/python/worldbank_adapter.py

@@ -0,0 +1,99 @@
+import sys
+import json
+import argparse
+import urllib.request
+import urllib.parse
+from datetime import datetime
+
+# WB API for indicators (Series)
+# Source 2 is World Development Indicators
+WB_API_URL = "https://api.worldbank.org/v2/indicator"
+
+def search_worldbank(query: str, limit: int = 10):
+    """
+    Search World Bank indicators.
+    """
+    try:
+        # The World Bank Indicators API doesn't have a direct "search" parameter for indicators 
+        # that works exactly like a search engine. We fetch a page and filter by query terms.
+        # Alternatively, we could use the 'qterm' on the documents API, but indicators are more tabular.
+        
+        params = {
+            "format": "json",
+            "per_page": 299, # Max per page to search through more indicators
+            "source": 2
+        }
+        
+        query_string = urllib.parse.urlencode(params)
+        url = f"{WB_API_URL}?{query_string}"
+        
+        req = urllib.request.Request(url)
+        with urllib.request.urlopen(req) as response:
+            data = json.load(response)
+        
+        # WB response is [metadata, data_list]
+        if len(data) < 2:
+            return []
+            
+        indicators = data[1]
+        
+        results = []
+        count = 0
+        
+        query_terms = query.lower().split()
+        
+        for ind in indicators:
+            name = ind.get('name', '')
+            source_note = ind.get('sourceNote', '')
+            text = (name + " " + source_note).lower()
+            
+            # Simple keyword matching
+            if all(term in text for term in query_terms):
+                metadata = {
+                    "id": f"wb:{ind.get('id')}", 
+                    "source": "worldbank",
+                    "name": name,
+                    "description": source_note or "No description available.",
+                    "downloads": 1000, # Placeholder (high relevance for WB)
+                    "likes": 100,
+                    "last_updated": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
+                    "quality_score": 95, # Institutional data is high quality
+                    "license": {
+                        "id": "cc-by-4.0",
+                        "name": "Creative Commons Attribution 4.0",
+                        "category": "safe",
+                        "usage_restrictions": [],
+                        "warnings": []
+                    },
+                    "tags": [ind.get('source', {}).get('value')] if ind.get('source') else [],
+                    "total_examples": 0, # Time series length varies
+                    "is_safe_source": True,
+                    "is_structured": True,
+                    "metadata_url": f"https://data.worldbank.org/indicator/{ind.get('id')}",
+                    "domain": "economics"
+                }
+                
+                results.append(metadata)
+                count += 1
+                if count >= limit:
+                    break
+                    
+        return results
+
+    except Exception as e:
+        return {"error": str(e)}
+
+def main():
+    parser = argparse.ArgumentParser(description="World Bank Adapter")
+    parser.add_argument("--action", required=True, choices=["search"])
+    parser.add_argument("--query", required=True)
+    parser.add_argument("--limit", type=int, default=10)
+    
+    args = parser.parse_args()
+    
+    if args.action == "search":
+        results = search_worldbank(args.query, args.limit)
+        print(json.dumps(results))
+
+if __name__ == "__main__":
+    main()

package/vesper-mcp-config.json

@@ -1,6 +0,0 @@
-{
-  "project": "vesper",
-  "dataDir": "./datasets",
-  "exportFormat": "parquet",
-  "tokens": {}
-}