vesper-wizard 2.3.1 → 2.3.2

package/src/python/target_engine.py

@@ -1,154 +0,0 @@
-import sys
-import json
-import pandas as pd
-import numpy as np
-
-# Common names for target variables in datasets
-TARGET_CANDIDATES = [
-    'target', 'label', 'class', 'outcome', 'y', 
-    'price', 'saleprice', 'sales', 'cost', 'value', 'total',
-    'diagnosis', 'species', 'churn', 'survived', 'credit_risk'
-]
-
-def load_data(file_path):
-    if file_path.endswith('.csv'):
-        return pd.read_csv(file_path)
-    elif file_path.endswith('.parquet'):
-        return pd.read_parquet(file_path)
-    else:
-        raise ValueError("Unsupported file format")
-
-def detect_target(file_path):
-    try:
-        df = load_data(file_path)
-        columns = [c.lower() for c in df.columns]
-        candidates = []
-
-        # 1. Exact Name Match
-        for col_original in df.columns:
-            col_lower = col_original.lower()
-            confidence = 0.0
-            reasons = []
-
-            if col_lower in TARGET_CANDIDATES:
-                confidence += 0.6
-                reasons.append(f"Matches common target name '{col_lower}'")
-                
-                # Boost if exact match 'target' or 'label'
-                if col_lower in ['target', 'label', 'class']:
-                    confidence += 0.2
-
-            # 2. Position Heuristic (Last column is often target)
-            if col_original == df.columns[-1]:
-                confidence += 0.3
-                reasons.append("Is the last column")
-
-            # 3. Completeness
-            missing_rate = df[col_original].isnull().mean()
-            if missing_rate > 0.5:
-                confidence -= 0.5
-                reasons.append(f"High missing rate ({missing_rate:.1%})")
-            elif missing_rate > 0:
-                confidence -= 0.1
-                reasons.append(f"Has missing values ({missing_rate:.1%})")
-
-            # 4. Cardinality / Unique Values
-            # If regression-like (many unique numeric values) or class-like (few unique values)
-            # This is hard to score generally, but extremes are bad for targets (e.g. all unique = ID usually)
-            n_unique = df[col_original].nunique()
-            if n_unique == len(df):
-                confidence -= 0.8
-                reasons.append("All values are unique (likely ID)")
-            
-            if confidence > 0.3:
-                candidates.append({
-                    "column": col_original,
-                    "confidence": min(confidence, 1.0),
-                    "reason": reasons
-                })
-
-        # Sort by confidence
-        candidates.sort(key=lambda x: x['confidence'], reverse=True)
-
-        best_target = None
-        best_conf = 0.0
-        
-        if candidates:
-            best_target = candidates[0]['column']
-            best_conf = candidates[0]['confidence']
-
-        return {
-            "target_column": best_target,
-            "confidence": best_conf,
-            "candidates": candidates,
-            "is_unified": False # Wrapper will handle unification logic
-        }
-
-    except Exception as e:
-        return {"error": str(e)}
-
-def validate_target(file_path, target_column):
-    try:
-        df = load_data(file_path)
-        if target_column not in df.columns:
-            return {"error": f"Column '{target_column}' not found in dataset."}
-
-        series = df[target_column]
-        total_rows = len(df)
-        missing_count = series.isnull().sum()
-        
-        # Determine type
-        is_numeric = pd.api.types.is_numeric_dtype(series)
-        n_unique = series.nunique()
-        
-        problem_type = "unknown"
-        if is_numeric and n_unique > 20:
-             problem_type = "regression"
-        elif n_unique < 50: # String or few numeric values
-             problem_type = "classification"
-        else:
-             # Heuristic fallback
-             problem_type = "regression" if is_numeric else "classification"
-
-        warnings = []
-        if missing_count > 0:
-            warnings.append(f"Target has {missing_count} missing values.")
-        
-        # Imbalance check for classification
-        if problem_type == "classification":
-            counts = series.value_counts(normalize=True)
-            if counts.iloc[0] > 0.9: # Dominant class > 90%
-                warnings.append(f"Highly imbalanced target: Class '{counts.index[0]}' is {counts.iloc[0]:.1%}")
-
-        return {
-            "valid": True,
-            "problem_type": problem_type,
-            "missing_count": int(missing_count),
-            "total_rows": total_rows,
-            "warnings": warnings
-        }
-
-    except Exception as e:
-        return {"error": str(e)}
-
-if __name__ == "__main__":
-    if len(sys.argv) < 3:
-        print(json.dumps({"error": "Usage: target_engine.py <action> <file_path> [args]"}));
-        sys.exit(1)
-
-    action = sys.argv[1]
-    file_path = sys.argv[2]
-    
-    result = {}
-    if action == "detect":
-        result = detect_target(file_path)
-    elif action == "validate":
-        target_col = sys.argv[3] if len(sys.argv) > 3 else None
-        if target_col:
-            result = validate_target(file_path, target_col)
-        else:
-            result = {"error": "Target column required for validation"}
-    else:
-        result = {"error": f"Unknown action: {action}"}
-
-    print(json.dumps(result))

package/src/python/test_framework_adapters.py

@@ -1,61 +0,0 @@
-
-import sys
-import os
-import polars as pl
-import numpy as np
-
-# Mock data creation
-def create_mock_data():
-    df = pl.DataFrame({
-        "feature1": np.random.rand(100),
-        "feature2": np.random.rand(100),
-        "label": np.random.randint(0, 2, 100)
-    })
-    os.makedirs("test_adapters", exist_ok=True)
-    df.write_parquet("test_adapters/data.parquet")
-    df.write_csv("test_adapters/data.csv")
-    print("Created mock data in test_adapters/")
-
-def test_pytorch():
-    print("\n--- Testing PyTorch Adapter ---")
-    try:
-        from framework_adapters import VesperPyTorchDataset
-        import torch
-        from torch.utils.data import DataLoader
-
-        dataset = VesperPyTorchDataset("test_adapters/data.parquet", target_col="label")
-        loader = DataLoader(dataset, batch_size=10, shuffle=True)
-        
-        batch = next(iter(loader))
-        print(f"Loaded batch: {batch}")
-        print("PASS: PyTorch DataLoader works")
-        
-    except ImportError:
-        print("SKIP: PyTorch not installed")
-    except Exception as e:
-        print(f"FAIL: PyTorch test failed: {e}")
-
-def test_huggingface():
-    print("\n--- Testing HuggingFace Adapter ---")
-    try:
-        from framework_adapters import load_vesper_dataset
-        ds = load_vesper_dataset("test_adapters/data.csv")
-        print(f"Loaded dataset: {ds}")
-        print("PASS: HuggingFace Dataset works")
-        
-    except ImportError:
-        print("SKIP: HuggingFace datasets not installed")
-    except Exception as e:
-        print(f"FAIL: HuggingFace test failed: {e}")
-
-if __name__ == "__main__":
-    create_mock_data()
-    # Add src/python to path to import adapters
-    sys.path.append(os.path.join(os.getcwd(), "src", "python"))
-    
-    test_pytorch()
-    test_huggingface()
-    
-    # Cleanup
-    import shutil
-    shutil.rmtree("test_adapters")

package/src/python/test_fusion_engine.py

@@ -1,89 +0,0 @@
-import os
-import tempfile
-import polars as pl
-from fusion_engine import fuse_datasets
-
-
-def run_basic_tests():
-    tmp = tempfile.gettempdir()
-
-    # ----- Test 1: concat -----
-    p1 = os.path.join(tmp, "fuse_test_a.csv")
-    p2 = os.path.join(tmp, "fuse_test_b.csv")
-    out_concat = os.path.join(tmp, "fuse_test_concat.feather")
-
-    df1 = pl.DataFrame({
-        "id": [1, 2, 3],
-        "text": ["a", "b", "c"],
-        "price": [10.0, 20.0, 30.0],
-    })
-    df2 = pl.DataFrame({
-        "id": [4, 5, 3],
-        "text": ["d", "e", "c"],
-        "price": [40.0, 50.0, 30.0],
-        "image_path": ["img1.jpg", "img2.jpg", "img3.jpg"],
-    })
-
-    df1.write_csv(p1)
-    df2.write_csv(p2)
-
-    concat_res = fuse_datasets(
-        sources=[p1, p2],
-        strategy="concat",
-        dedup=True,
-        run_quality_after=False,
-        leakage_check=True,
-        output_path=out_concat,
-        output_format="feather",
-        compression="lz4",
-        preview=True,
-        id_column="id",
-    )
-
-    assert concat_res.get("success") is True, f"Concat failed: {concat_res}"
-    assert os.path.exists(out_concat), "Concat output file missing"
-
-    # ----- Test 2: join with conflicting column names -----
-    p3 = os.path.join(tmp, "fuse_test_c.csv")
-    p4 = os.path.join(tmp, "fuse_test_d.csv")
-    out_join = os.path.join(tmp, "fuse_test_join.parquet")
-
-    left = pl.DataFrame({
-        "id": [1, 2, 3],
-        "price": [100, 200, 300],
-        "text": ["x", "y", "z"],
-    })
-    right = pl.DataFrame({
-        "id": [2, 3, 4],
-        "price": [999, 888, 777],
-        "caption": ["two", "three", "four"],
-    })
-
-    left.write_csv(p3)
-    right.write_csv(p4)
-
-    join_res = fuse_datasets(
-        sources=[p3, p4],
-        strategy="join",
-        join_on="id",
-        how="inner",
-        dedup=True,
-        run_quality_after=False,
-        leakage_check=False,
-        output_path=out_join,
-        output_format="parquet",
-        compression="snappy",
-        preview=True,
-    )
-
-    assert join_res.get("success") is True, f"Join failed: {join_res}"
-    assert os.path.exists(out_join), "Join output file missing"
-    assert len(join_res.get("stats", {}).get("conflict_renames", [])) >= 1, "Expected conflict rename for price column"
-
-    print("✅ Fusion tests passed")
-    print("Concat:", concat_res["stats"])
-    print("Join:", join_res["stats"])
-
-
-if __name__ == "__main__":
-    run_basic_tests()

package/src/python/uci_adapter.py

@@ -1,94 +0,0 @@
-import sys
-import json
-import argparse
-import urllib.request
-import urllib.parse
-from datetime import datetime
-
-# API Endpoint found in network inspection of UCI website
-UCI_API_URL = "https://archive.ics.uci.edu/api/datasets/list"
-
-def search_uci(query: str, limit: int = 10):
-    """
-    Search UCI datasets using their internal API.
-    """
-    try:
-        # Fetch data dictionary from API
-        # Only fetching first 100 to filter locally
-        params = {
-            "skip": 0,
-            "take": 100,
-            "sort": "desc",
-            "orderBy": "NumHits",
-            "search": query 
-        }
-        
-        query_string = urllib.parse.urlencode(params)
-        url = f"{UCI_API_URL}?{query_string}"
-        
-        req = urllib.request.Request(url)
-        with urllib.request.urlopen(req) as response:
-            data = json.load(response)
-        
-        datasets = data.get('data', [])
-        if not datasets:
-            datasets = []
-
-        results = []
-        count = 0
-        
-        # We trust the API search mostly, but can do extra filtering if needed
-        # The API "search" param is supported
-        
-        for ds in datasets:
-            # Normalize to Vesper schema
-            # API fields: id, name, abstract, numHits, area, task, dateDonated
-            
-            metadata = {
-                "id": f"uci:{ds.get('id')}", 
-                "source": "uci",
-                "name": ds.get('name'),
-                "description": ds.get('abstract') or "No description available.",
-                "downloads": ds.get('numHits') or 0,
-                "likes": 0,
-                "last_updated": ds.get('dateDonated') or datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),
-                "quality_score": 80, 
-                "license": {
-                    "id": "other",
-                    "category": "open", 
-                    "usage_restrictions": [],
-                    "warnings": []
-                },
-                "tags": [t for t in [ds.get('area'), ds.get('task')] if t],
-                "total_examples": ds.get('numInstances'),
-                "is_safe_source": True,
-                "is_structured": True,
-                "metadata_url": f"https://archive.ics.uci.edu/dataset/{ds.get('id')}/{ds.get('name').replace(' ', '+')}"
-            }
-            
-            results.append(metadata)
-            count += 1
-            if count >= limit:
-                break
-                    
-        return results
-
-    except Exception as e:
-        # Fallback empty or specific error
-        return {"error": str(e)}
-
-def main():
-    parser = argparse.ArgumentParser(description="UCI Adapter")
-    parser.add_argument("--action", required=True, choices=["search"])
-    parser.add_argument("--query", required=True)
-    parser.add_argument("--limit", type=int, default=10)
-    
-    args = parser.parse_args()
-    
-    if args.action == "search":
-        results = search_uci(args.query, args.limit)
-        # JSON dump print for stdout capture
-        print(json.dumps(results))
-
-if __name__ == "__main__":
-    main()

package/src/python/vesper/__init__.py

@@ -1 +0,0 @@
-"""Vesper Python runtime package."""

package/src/python/vesper/core/__init__.py

@@ -1 +0,0 @@
-"""Core data engines for Vesper."""