isa-model 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

isa_model/eval/benchmarks.py

@@ -11,13 +11,106 @@ This module provides implementations of standard AI benchmarks:
 import os
 import json
 import logging
+import requests
+import zipfile
+import tarfile
+from pathlib import Path
 from typing import Dict, List, Any, Optional
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
+import pandas as pd
 
 logger = logging.getLogger(__name__)
 
 
+class DatasetDownloader:
+    """Utility class for downloading and caching benchmark datasets."""
+    
+    def __init__(self, cache_dir: str = "~/.isa_model/datasets"):
+        self.cache_dir = Path(cache_dir).expanduser()
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Dataset URLs and info
+        self.dataset_info = {
+            "mmlu": {
+                "url": "https://people.eecs.berkeley.edu/~hendrycks/data.tar",
+                "filename": "mmlu_data.tar",
+                "extracted_dir": "data"
+            },
+            "hellaswag": {
+                "url": "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl",
+                "filename": "hellaswag_val.jsonl"
+            },
+            "arc": {
+                "url": "https://s3-us-west-2.amazonaws.com/ai2-website/data/ARC-V1-Feb2018.zip",
+                "filename": "arc_data.zip",
+                "extracted_dir": "ARC-V1-Feb2018-2"
+            },
+            "gsm8k": {
+                "url": "https://github.com/openai/grade-school-math/raw/master/grade_school_math/data/test.jsonl",
+                "filename": "gsm8k_test.jsonl"
+            }
+        }
+    
+    def download_dataset(self, dataset_name: str, force_download: bool = False) -> Path:
+        """Download and cache a dataset."""
+        if dataset_name not in self.dataset_info:
+            raise ValueError(f"Unknown dataset: {dataset_name}")
+        
+        info = self.dataset_info[dataset_name]
+        dataset_dir = self.cache_dir / dataset_name
+        dataset_dir.mkdir(exist_ok=True)
+        
+        file_path = dataset_dir / info["filename"]
+        
+        # Check if already downloaded
+        if file_path.exists() and not force_download:
+            logger.info(f"Using cached {dataset_name} dataset at {file_path}")
+            return self._get_data_path(dataset_name, file_path)
+        
+        # Download the dataset
+        logger.info(f"Downloading {dataset_name} dataset from {info['url']}")
+        try:
+            response = requests.get(info["url"], stream=True)
+            response.raise_for_status()
+            
+            with open(file_path, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            
+            logger.info(f"Downloaded {dataset_name} dataset to {file_path}")
+            
+            # Extract if needed
+            return self._get_data_path(dataset_name, file_path)
+            
+        except Exception as e:
+            logger.error(f"Failed to download {dataset_name}: {e}")
+            # Fall back to placeholder data
+            return None
+    
+    def _get_data_path(self, dataset_name: str, file_path: Path) -> Path:
+        """Get the actual data path, extracting archives if needed."""
+        info = self.dataset_info[dataset_name]
+        
+        if "extracted_dir" in info:
+            # Need to extract
+            extract_dir = file_path.parent / info["extracted_dir"]
+            
+            if not extract_dir.exists():
+                logger.info(f"Extracting {file_path}")
+                
+                if file_path.suffix == ".zip":
+                    with zipfile.ZipFile(file_path, 'r') as zip_ref:
+                        zip_ref.extractall(file_path.parent)
+                elif file_path.suffix == ".tar" or ".tar." in file_path.name:
+                    with tarfile.open(file_path, 'r') as tar_ref:
+                        tar_ref.extractall(file_path.parent)
+            
+            return extract_dir
+        else:
+            return file_path
+
+
 @dataclass
 class BenchmarkConfig:
     """Configuration for benchmark evaluation."""
@@ -36,6 +129,8 @@ class BaseBenchmark(ABC):
         self.config = config
         self.name = config.name
         self.data = None
+        self.downloader = DatasetDownloader()
+        self.use_real_data = True  # Flag to control real vs placeholder data
     
     @abstractmethod
     def load_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
@@ -111,10 +206,62 @@ class MMLU(BaseBenchmark):
         self.subjects = subjects or self.all_subjects[:10]  # Use first 10 subjects by default
     
     def load_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
-        """Load MMLU data (simplified implementation)."""
-        # This is a simplified implementation
-        # In practice, you'd load from the actual MMLU dataset
+        """Load MMLU data with real dataset support."""
+        if self.use_real_data:
+            try:
+                return self._load_real_mmlu_data(max_samples)
+            except Exception as e:
+                logger.warning(f"Failed to load real MMLU data: {e}. Falling back to placeholder data.")
+                return self._load_placeholder_mmlu_data(max_samples)
+        else:
+            return self._load_placeholder_mmlu_data(max_samples)
+    
+    def _load_real_mmlu_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load real MMLU dataset."""
+        data_path = self.downloader.download_dataset("mmlu")
+        if not data_path or not data_path.exists():
+            raise FileNotFoundError("MMLU dataset not found")
+        
+        data = []
+        samples_per_subject = max_samples // len(self.subjects) if max_samples else None
+        
+        for subject in self.subjects:
+            subject_file = data_path / "test" / f"{subject}_test.csv"
+            if not subject_file.exists():
+                logger.warning(f"Subject file not found: {subject_file}")
+                continue
+            
+            try:
+                # Load CSV data
+                df = pd.read_csv(subject_file, header=None, 
+                               names=["question", "A", "B", "C", "D", "answer"])
+                
+                # Convert to our format
+                for idx, row in df.iterrows():
+                    if samples_per_subject and len([d for d in data if d["subject"] == subject]) >= samples_per_subject:
+                        break
+                    
+                    sample = {
+                        "subject": subject,
+                        "question": row["question"],
+                        "choices": [row["A"], row["B"], row["C"], row["D"]],
+                        "answer": str(row["answer"]).strip().upper(),
+                        "id": f"{subject}_{idx}"
+                    }
+                    data.append(sample)
+                    
+            except Exception as e:
+                logger.error(f"Error loading subject {subject}: {e}")
+                continue
         
+        if max_samples:
+            data = data[:max_samples]
+        
+        logger.info(f"Loaded {len(data)} real MMLU samples across {len(self.subjects)} subjects")
+        return data
+    
+    def _load_placeholder_mmlu_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load placeholder MMLU data."""
         data = []
         
         for subject in self.subjects:
@@ -137,7 +284,7 @@ class MMLU(BaseBenchmark):
         if max_samples:
             data = data[:max_samples]
         
-        logger.info(f"Loaded {len(data)} MMLU samples across {len(self.subjects)} subjects")
+        logger.info(f"Loaded {len(data)} placeholder MMLU samples across {len(self.subjects)} subjects")
         return data
     
     def evaluate_sample(self, sample: Dict[str, Any], prediction: str) -> bool:
@@ -194,12 +341,52 @@ class HellaSwag(BaseBenchmark):
         super().__init__(config)
     
     def load_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
-        """Load HellaSwag data (simplified implementation)."""
-        # This is a simplified implementation
-        # In practice, you'd load from the actual HellaSwag dataset
+        """Load HellaSwag data with real dataset support."""
+        if self.use_real_data:
+            try:
+                return self._load_real_hellaswag_data(max_samples)
+            except Exception as e:
+                logger.warning(f"Failed to load real HellaSwag data: {e}. Falling back to placeholder data.")
+                return self._load_placeholder_hellaswag_data(max_samples)
+        else:
+            return self._load_placeholder_hellaswag_data(max_samples)
+    
+    def _load_real_hellaswag_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load real HellaSwag dataset."""
+        data_path = self.downloader.download_dataset("hellaswag")
+        if not data_path or not data_path.exists():
+            raise FileNotFoundError("HellaSwag dataset not found")
         
         data = []
         
+        try:
+            with open(data_path, 'r', encoding='utf-8') as f:
+                for i, line in enumerate(f):
+                    if max_samples and i >= max_samples:
+                        break
+                    
+                    item = json.loads(line.strip())
+                    
+                    sample = {
+                        "context": item["ctx"],
+                        "question": "What happens next?",
+                        "choices": item["endings"],
+                        "answer": chr(65 + int(item["label"])),  # Convert 0,1,2,3 to A,B,C,D
+                        "id": f"hellaswag_{item.get('ind', i)}"
+                    }
+                    data.append(sample)
+        
+        except Exception as e:
+            logger.error(f"Error loading HellaSwag data: {e}")
+            raise
+        
+        logger.info(f"Loaded {len(data)} real HellaSwag samples")
+        return data
+    
+    def _load_placeholder_hellaswag_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load placeholder HellaSwag data."""
+        data = []
+        
         sample_contexts = [
             "A person is washing dishes in the kitchen",
             "Someone is riding a bicycle down a hill",
@@ -226,7 +413,7 @@ class HellaSwag(BaseBenchmark):
             }
             data.append(sample)
         
-        logger.info(f"Loaded {len(data)} HellaSwag samples")
+        logger.info(f"Loaded {len(data)} placeholder HellaSwag samples")
         return data
     
     def evaluate_sample(self, sample: Dict[str, Any], prediction: str) -> bool:
@@ -377,12 +564,57 @@ class GSM8K(BaseBenchmark):
         super().__init__(config)
     
     def load_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
-        """Load GSM8K data (simplified implementation)."""
-        # This is a simplified implementation
-        # In practice, you'd load from the actual GSM8K dataset
+        """Load GSM8K data with real dataset support."""
+        if self.use_real_data:
+            try:
+                return self._load_real_gsm8k_data(max_samples)
+            except Exception as e:
+                logger.warning(f"Failed to load real GSM8K data: {e}. Falling back to placeholder data.")
+                return self._load_placeholder_gsm8k_data(max_samples)
+        else:
+            return self._load_placeholder_gsm8k_data(max_samples)
+    
+    def _load_real_gsm8k_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load real GSM8K dataset."""
+        data_path = self.downloader.download_dataset("gsm8k")
+        if not data_path or not data_path.exists():
+            raise FileNotFoundError("GSM8K dataset not found")
         
         data = []
         
+        try:
+            with open(data_path, 'r', encoding='utf-8') as f:
+                for i, line in enumerate(f):
+                    if max_samples and i >= max_samples:
+                        break
+                    
+                    item = json.loads(line.strip())
+                    
+                    # Extract numerical answer from solution
+                    answer_text = item["answer"]
+                    import re
+                    numbers = re.findall(r'\d+', answer_text)
+                    answer = numbers[-1] if numbers else "0"
+                    
+                    sample = {
+                        "question": item["question"],
+                        "answer": answer,
+                        "solution": answer_text,  # Keep full solution for reference
+                        "id": f"gsm8k_{i}"
+                    }
+                    data.append(sample)
+        
+        except Exception as e:
+            logger.error(f"Error loading GSM8K data: {e}")
+            raise
+        
+        logger.info(f"Loaded {len(data)} real GSM8K samples")
+        return data
+    
+    def _load_placeholder_gsm8k_data(self, max_samples: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Load placeholder GSM8K data."""
+        data = []
+        
         sample_problems = [
             {
                 "question": "Janet has 12 apples. She gives 3 apples to her friend and eats 2 apples. How many apples does Janet have left?",
@@ -417,7 +649,7 @@ class GSM8K(BaseBenchmark):
             }
             data.append(sample)
         
-        logger.info(f"Loaded {len(data)} GSM8K samples")
+        logger.info(f"Loaded {len(data)} placeholder GSM8K samples")
         return data
     
     def evaluate_sample(self, sample: Dict[str, Any], prediction: str) -> bool:

isa_model/eval/evaluators/__init__.py

@@ -7,12 +7,18 @@ Provides specialized evaluators for different model types and evaluation tasks.
 from .base_evaluator import BaseEvaluator, EvaluationResult
 from .llm_evaluator import LLMEvaluator
 from .vision_evaluator import VisionEvaluator
-from .multimodal_evaluator import MultimodalEvaluator
+from .audio_evaluator import AudioEvaluator
+from .embedding_evaluator import EmbeddingEvaluator
+
+# MultimodalEvaluator will be implemented later
+# from .multimodal_evaluator import MultimodalEvaluator
 
 __all__ = [
     "BaseEvaluator",
     "EvaluationResult", 
     "LLMEvaluator",
     "VisionEvaluator",
-    "MultimodalEvaluator"
+    "AudioEvaluator",
+    "EmbeddingEvaluator"
+    # "MultimodalEvaluator"  # TODO: Implement later
 ]