vesper-wizard 2.3.1 → 2.3.3

package/src/python/media_engine.py

@@ -1,133 +0,0 @@
-import sys
-import json
-import os
-import cv2
-import numpy as np
-
-# Audio analysis depends on librosa/soundfile. Fallback if not available.
-try:
-    import librosa
-    AUDIO_SUPPORT = True
-except ImportError:
-    AUDIO_SUPPORT = False
-
-def analyze_audio(path):
-    if not AUDIO_SUPPORT:
-        return {"status": "error", "error": "librosa not installed"}
-    
-    try:
-        # Load audio (mono, default sr)
-        y, sr = librosa.load(path, sr=None)
-        duration = librosa.get_duration(y=y, sr=sr)
-        
-        # Audio metrics
-        rms = librosa.feature.rms(y=y)
-        avg_rms = float(np.mean(rms))
-        
-        return {
-            "status": "ok",
-            "type": "audio",
-            "filename": os.path.basename(path),
-            "sample_rate": int(sr),
-            "duration": float(duration),
-            "avg_volume_rms": avg_rms,
-            "is_silent": avg_rms < 0.001
-        }
-    except Exception as e:
-        return {"status": "error", "error": str(e)}
-
-def analyze_video(path):
-    try:
-        cap = cv2.VideoCapture(path)
-        if not cap.isOpened():
-            return {"status": "error", "error": "Could not open video file"}
-        
-        width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
-        height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-        fps = cap.get(cv2.CAP_PROP_FPS)
-        frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
-        duration = frame_count / fps if fps > 0 else 0
-        
-        # Check integrity by reading a few frames
-        test_frame_indices = [0, frame_count // 2, frame_count - 1] if frame_count > 0 else []
-        failed_frames = 0
-        
-        for idx in test_frame_indices:
-            cap.set(cv2.CAP_PROP_POS_FRAMES, idx)
-            ret, frame = cap.read()
-            if not ret or frame is None:
-                failed_frames += 1
-        
-        cap.release()
-        
-        return {
-            "status": "ok",
-            "type": "video",
-            "filename": os.path.basename(path),
-            "width": width,
-            "height": height,
-            "fps": float(fps),
-            "duration": float(duration),
-            "frame_count": frame_count,
-            "corruption_risk": "high" if failed_frames > 0 else "low"
-        }
-    except Exception as e:
-        return {"status": "error", "error": str(e)}
-
-def main():
-    if len(sys.argv) < 2:
-        print(json.dumps({"error": "No path provided"}))
-        sys.exit(1)
-
-    input_path = sys.argv[1]
-    results = []
-
-    # Supported extensions
-    AUDIO_EXTS = (".wav", ".mp3", ".flac", ".ogg", ".m4a")
-    VIDEO_EXTS = (".mp4", ".avi", ".mkv", ".mov", ".wmv")
-
-    if os.path.isfile(input_path):
-        ext = os.path.splitext(input_path.lower())[1]
-        if ext in AUDIO_EXTS:
-            results.append(analyze_audio(input_path))
-        elif ext in VIDEO_EXTS:
-            results.append(analyze_video(input_path))
-        else:
-            results.append({"status": "error", "error": f"Unsupported file type: {ext}"})
-    elif os.path.isdir(input_path):
-        files = [os.path.join(input_path, f) for f in os.listdir(input_path)]
-        for f in files[:50]: # Limit for demo
-            ext = os.path.splitext(f.lower())[1]
-            if ext in AUDIO_EXTS:
-                results.append(analyze_audio(f))
-            elif ext in VIDEO_EXTS:
-                results.append(analyze_video(f))
-    else:
-        print(json.dumps({"error": "Invalid path"}))
-        sys.exit(1)
-
-    # Filtering failed results for report aggregation
-    ok_results = [r for r in results if r.get("status") == "ok"]
-    
-    report = {
-        "total_files": len(results),
-        "ok_files": len(ok_results),
-        "failed_files": len(results) - len(ok_results),
-        "details": results
-    }
-    
-    # Calculate some averages if files were found
-    if ok_results:
-        audio_files = [r for r in ok_results if r["type"] == "audio"]
-        video_files = [r for r in ok_results if r["type"] == "video"]
-        
-        if audio_files:
-            report["avg_audio_duration"] = float(np.mean([r["duration"] for r in audio_files]))
-        if video_files:
-            report["avg_video_duration"] = float(np.mean([r["duration"] for r in video_files]))
-            report["avg_fps"] = float(np.mean([r["fps"] for r in video_files]))
-
-    print(json.dumps(report))
-
-if __name__ == "__main__":
-    main()

package/src/python/nasa_adapter.py

@@ -1,82 +0,0 @@
-import sys
-import json
-import argparse
-import urllib.request
-import urllib.parse
-from datetime import datetime
-
-# NASA Data Portal uses Socrata
-NASA_API_URL = "https://api.us.socrata.com/api/catalog/v1"
-NASA_DOMAIN = "data.nasa.gov"
-
-def search_nasa(query: str, limit: int = 10):
-    """
-    Search NASA data portal.
-    """
-    try:
-        params = {
-            "q": query,
-            "limit": limit,
-            "domains": NASA_DOMAIN,
-            "search_context": NASA_DOMAIN
-        }
-        
-        query_string = urllib.parse.urlencode(params)
-        url = f"{NASA_API_URL}?{query_string}"
-        
-        req = urllib.request.Request(url)
-        with urllib.request.urlopen(req) as response:
-            data = json.load(response)
-        
-        results = []
-        # Socrata catalog results are in 'results'
-        items = data.get('results', [])
-        
-        for item in items:
-            ds = item.get('resource', {})
-            
-            metadata = {
-                "id": f"nasa:{ds.get('id')}", 
-                "source": "nasa",
-                "name": ds.get('name'),
-                "description": ds.get('description') or "No description available.",
-                "downloads": ds.get('download_count', 0),
-                "likes": ds.get('view_count', 0) // 10,
-                "last_updated": ds.get('updatedAt') or datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                "quality_score": 90,
-                "license": {
-                    "id": "public_domain",
-                    "name": "Public Domain",
-                    "category": "safe",
-                    "usage_restrictions": [],
-                    "warnings": []
-                },
-                "tags": ds.get('tags', []),
-                "total_examples": 0,
-                "is_safe_source": True,
-                "is_structured": True,
-                "metadata_url": f"https://data.nasa.gov/d/{ds.get('id')}",
-                "domain": "science"
-            }
-            
-            results.append(metadata)
-                    
-        return results
-
-    except Exception as e:
-        return {"error": str(e)}
-
-def main():
-    parser = argparse.ArgumentParser(description="NASA Adapter")
-    parser.add_argument("--action", required=True, choices=["search"])
-    parser.add_argument("--query", required=True)
-    parser.add_argument("--limit", type=int, default=10)
-    
-    args = parser.parse_args()
-    
-    if args.action == "search":
-        results = search_nasa(args.query, args.limit)
-        print(json.dumps(results))
-
-if __name__ == "__main__":
-    main()

package/src/python/normalize_engine.py

@@ -1,83 +0,0 @@
-"""
-Normalize any supported dataset file to parquet format.
-Usage: normalize_engine.py <input_path> <output_path>
-Outputs JSON: {"ok": true, "output_path": "...", "rows": N} or {"ok": false, "error": "..."}
-"""
-import sys
-import json
-import os
-
-try:
-    import polars as pl
-except Exception:
-    print(json.dumps({"ok": False, "error": "polars is required"}))
-    sys.exit(1)
-
-
-def _load(src: str) -> pl.DataFrame:
-    ext = os.path.splitext(src)[1].lower()
-
-    if ext == ".csv":
-        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
-    if ext in (".tsv", ".tab"):
-        return pl.read_csv(src, separator="\t", ignore_errors=True, infer_schema_length=10000)
-    if ext in (".parquet", ".pq"):
-        return pl.read_parquet(src)
-    if ext in (".feather", ".ftr", ".arrow", ".ipc"):
-        return pl.read_ipc(src)
-    if ext in (".jsonl", ".ndjson"):
-        return pl.read_ndjson(src)
-    if ext == ".json":
-        raw = open(src, "r", encoding="utf-8").read().strip()
-        if raw.startswith("["):
-            return pl.read_json(src)
-        # Try NDJSON
-        if "\n" in raw and raw.split("\n")[0].strip().startswith("{"):
-            return pl.read_ndjson(src)
-        # Try wrapper object
-        obj = json.loads(raw)
-        if isinstance(obj, dict):
-            for key in ("data", "rows", "items", "records", "results", "entries", "samples"):
-                if key in obj and isinstance(obj[key], list):
-                    return pl.DataFrame(obj[key])
-            # Last resort - take first list value
-            for v in obj.values():
-                if isinstance(v, list) and len(v) > 0 and isinstance(v[0], dict):
-                    return pl.DataFrame(v)
-        return pl.read_json(src)
-    if ext == ".txt":
-        return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
-
-    # Fallback: try csv
-    return pl.read_csv(src, ignore_errors=True, infer_schema_length=10000)
-
-
-def normalize(input_path: str, output_path: str):
-    df = _load(input_path)
-    os.makedirs(os.path.dirname(output_path), exist_ok=True)
-    df.write_parquet(output_path)
-    return df.height
-
-
-def main():
-    if len(sys.argv) < 3:
-        print(json.dumps({"ok": False, "error": "Usage: normalize_engine.py <input> <output>"}))
-        sys.exit(1)
-
-    input_path = sys.argv[1]
-    output_path = sys.argv[2]
-
-    if not os.path.exists(input_path):
-        print(json.dumps({"ok": False, "error": f"File not found: {input_path}"}))
-        sys.exit(1)
-
-    try:
-        rows = normalize(input_path, output_path)
-        print(json.dumps({"ok": True, "output_path": output_path, "rows": rows}))
-    except Exception as e:
-        print(json.dumps({"ok": False, "error": str(e)}))
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/src/python/openml_engine.py

@@ -1,146 +0,0 @@
-import sys
-import json
-import argparse
-import tempfile
-import os
-from typing import Dict, Any, List
-
-try:
-    import openml
-except ImportError:
-    openml = None
-
-def _ensure_openml() -> Dict[str, Any]:
-    if openml is None:
-        return {"ok": False, "error": "openml package is not installed. Run 'pip install openml'"}
-    return {"ok": True}
-
-def _dataset_to_dict(ds: Dict[str, Any]) -> Dict[str, Any]:
-    # OpenML dataset dict from list_datasets
-    did = ds.get("did", "")
-    name = ds.get("name", f"dataset_{did}")
-    version = ds.get("version", "1")
-    status = ds.get("status", "active")
-    format = ds.get("format", "unknown")
-    
-    # Map to Vesper DatasetMetadata format
-    return {
-        "id": f"openml:{did}",
-        "name": name,
-        "source": "openml",
-        "description": f"OpenML Dataset {name} (ID: {did}, Version: {version}, Format: {format}, Status: {status})",
-        "author": "OpenML Community",
-        "license": "Public",
-        "tags": ["openml", format.lower()],
-        "downloads": ds.get("NumberOfDownloads", 0),
-        "likes": ds.get("NumberOfLikes", 0),
-        "created_at": ds.get("upload_date", ""),
-        "updated_at": ds.get("upload_date", ""),
-        "size_bytes": 0, # Not always available in list
-        "quality_score": 0.8, # Default good score for OpenML
-        "domain": "machine_learning",
-        "is_gated": False,
-        "is_nsfw": False,
-        "description_length": 100,
-        "has_readme": False,
-    }
-
-def discover(query: str, limit: int = 20) -> Dict[str, Any]:
-    check = _ensure_openml()
-    if not check.get("ok"):
-        return check
-
-    try:
-        # OpenML list_datasets doesn't have a direct text search in the python API easily exposed without downloading all.
-        # But we can filter by tag or just get a list and filter locally if needed, or use the REST API directly.
-        # Actually, openml.datasets.list_datasets() returns a dict of datasets.
-        # We can fetch a larger batch and filter by name/keyword.
-        
-        # Fetching a batch of datasets
-        datasets = openml.datasets.list_datasets(output_format='dataframe')
-        
-        if query:
-            # Simple case-insensitive search in name
-            mask = datasets['name'].str.contains(query, case=False, na=False)
-            filtered = datasets[mask]
-        else:
-            filtered = datasets
-            
-        # Sort by NumberOfDownloads if available, else just take top
-        if 'NumberOfDownloads' in filtered.columns:
-            filtered = filtered.sort_values('NumberOfDownloads', ascending=False)
-            
-        top_k = filtered.head(limit)
-        
-        # Convert to list of dicts
-        records = top_k.to_dict(orient='records')
-        items = [_dataset_to_dict(r) for r in records]
-        
-        return {"ok": True, "results": items, "count": len(items)}
-    except Exception as e:
-        return {"ok": False, "error": f"OpenML discover failed: {str(e)}"}
-
-def download(dataset_ref: str, target_dir: str) -> Dict[str, Any]:
-    check = _ensure_openml()
-    if not check.get("ok"):
-        return check
-
-    try:
-        # dataset_ref is expected to be "openml:ID"
-        if dataset_ref.startswith("openml:"):
-            did_str = dataset_ref.split(":")[1]
-        else:
-            did_str = dataset_ref
-            
-        did = int(did_str)
-        
-        if not target_dir:
-            target_dir = tempfile.mkdtemp(prefix="vesper_openml_")
-            
-        os.makedirs(target_dir, exist_ok=True)
-        
-        # Get the dataset
-        dataset = openml.datasets.get_dataset(did, download_data=True, download_qualities=False, download_features_meta_data=False)
-        
-        # Get the pandas dataframe
-        X, y, categorical_indicator, attribute_names = dataset.get_data(
-            dataset_format="dataframe"
-        )
-        
-        # If there's a target column (y), we might want to join it back if it was separated
-        # get_data() can return X and y separately if target is specified, but usually X contains everything if target=None
-        # Let's just get everything
-        df, _, _, _ = dataset.get_data(target=None, dataset_format="dataframe")
-        
-        # Save to parquet in the target directory
-        safe_name = "".join([c if c.isalnum() else "_" for c in dataset.name])
-        file_path = os.path.join(target_dir, f"{safe_name}_{did}.parquet")
-        
-        df.to_parquet(file_path, index=False)
-        
-        return {
-            "ok": True, 
-            "local_path": file_path, 
-            "target_dir": target_dir
-        }
-    except Exception as e:
-        return {"ok": False, "error": f"OpenML download failed: {str(e)}"}
-
-def main():
-    parser = argparse.ArgumentParser(description="Vesper OpenML Engine")
-    parser.add_argument("action", choices=["discover", "download"])
-    parser.add_argument("arg1", help="Query for discover, Dataset ID for download")
-    parser.add_argument("arg2", nargs="?", help="Limit for discover, Target Dir for download")
-
-    args = parser.parse_args()
-
-    if args.action == "discover":
-        limit = int(args.arg2) if args.arg2 else 20
-        result = discover(args.arg1, limit)
-        print(json.dumps(result))
-    elif args.action == "download":
-        result = download(args.arg1, args.arg2)
-        print(json.dumps(result))
-
-if __name__ == "__main__":
-    main()