isa-model 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

isa_model/eval/metrics.py

@@ -83,7 +83,7 @@ class LLMMetrics:
         else:
             self.ai_factory = None
     
-    def evaluate(
+    async def evaluate(
         self,
         model_path: str,
         dataset: List[Dict[str, Any]],
@@ -113,7 +113,7 @@ class LLMMetrics:
         }
         
         # Generate predictions
-        predictions, references = self._generate_predictions(
+        predictions, references = await self._generate_predictions(
             model_path, dataset, batch_size, provider, **kwargs
         )
         
@@ -149,7 +149,7 @@ class LLMMetrics:
         
         return results
     
-    def evaluate_generation(
+    async def evaluate_generation(
         self,
         model_path: str,
         prompts: List[str],
@@ -208,7 +208,7 @@ class LLMMetrics:
         
         return results
     
-    def _generate_predictions(
+    async def _generate_predictions(
         self,
         model_path: str,
         dataset: List[Dict[str, Any]],
@@ -306,7 +306,7 @@ class LLMMetrics:
         logger.info(f"Generated {len(predictions)} predictions")
         return predictions, references
     
-    def _generate_texts(
+    async def _generate_texts(
         self,
         model_path: str,
         prompts: List[str],
@@ -795,4 +795,157 @@ class BenchmarkRunner:
             
         except Exception as e:
             logger.error(f"Failed to generate prediction: {e}")
-            return "A"  # Fallback answer 
+            return "A"  # Fallback answer
+
+
+# Utility functions for evaluators
+def compute_text_metrics(predictions: Union[str, List[str]], 
+                        references: Union[str, List[str]], 
+                        aggregate: bool = False) -> Dict[str, float]:
+    """
+    Compute standard text evaluation metrics.
+    
+    Args:
+        predictions: Single prediction or list of predictions
+        references: Single reference or list of references
+        aggregate: Whether to compute aggregate metrics for lists
+        
+    Returns:
+        Dictionary of computed metrics
+    """
+    try:
+        # Handle single string inputs
+        if isinstance(predictions, str) and isinstance(references, str):
+            pred_list = [predictions]
+            ref_list = [references]
+        else:
+            pred_list = predictions if isinstance(predictions, list) else [str(predictions)]
+            ref_list = references if isinstance(references, list) else [str(references)]
+        
+        # Ensure equal lengths
+        min_len = min(len(pred_list), len(ref_list))
+        pred_list = pred_list[:min_len]
+        ref_list = ref_list[:min_len]
+        
+        metrics = {}
+        
+        # Exact match
+        exact_matches = sum(1 for p, r in zip(pred_list, ref_list) if p.strip().lower() == r.strip().lower())
+        metrics["exact_match"] = exact_matches / len(pred_list) if pred_list else 0.0
+        
+        # F1 Score (token-level)
+        f1_scores = []
+        for pred, ref in zip(pred_list, ref_list):
+            pred_tokens = set(pred.lower().split())
+            ref_tokens = set(ref.lower().split())
+            
+            if not ref_tokens and not pred_tokens:
+                f1_scores.append(1.0)
+            elif not ref_tokens or not pred_tokens:
+                f1_scores.append(0.0)
+            else:
+                intersection = len(pred_tokens & ref_tokens)
+                precision = intersection / len(pred_tokens)
+                recall = intersection / len(ref_tokens)
+                
+                if precision + recall > 0:
+                    f1 = 2 * (precision * recall) / (precision + recall)
+                    f1_scores.append(f1)
+                else:
+                    f1_scores.append(0.0)
+        
+        metrics["f1_score"] = np.mean(f1_scores) if f1_scores else 0.0
+        
+        # BLEU Score (simplified)
+        bleu_scores = []
+        for pred, ref in zip(pred_list, ref_list):
+            pred_words = pred.lower().split()
+            ref_words = ref.lower().split()
+            
+            # Simple n-gram overlap
+            overlap = len(set(pred_words) & set(ref_words))
+            total = len(set(pred_words) | set(ref_words))
+            
+            bleu_scores.append(overlap / total if total > 0 else 0.0)
+        
+        metrics["bleu_score"] = np.mean(bleu_scores) if bleu_scores else 0.0
+        
+        # ROUGE-L (simplified)
+        rouge_scores = []
+        for pred, ref in zip(pred_list, ref_list):
+            pred_words = set(pred.lower().split())
+            ref_words = set(ref.lower().split())
+            
+            if len(ref_words) > 0:
+                rouge_l = len(pred_words & ref_words) / len(ref_words)
+                rouge_scores.append(rouge_l)
+            else:
+                rouge_scores.append(0.0)
+        
+        metrics["rouge_l"] = np.mean(rouge_scores) if rouge_scores else 0.0
+        
+        # Response length metrics
+        pred_lengths = [len(p.split()) for p in pred_list]
+        ref_lengths = [len(r.split()) for r in ref_list]
+        
+        metrics["avg_prediction_length"] = np.mean(pred_lengths) if pred_lengths else 0.0
+        metrics["avg_reference_length"] = np.mean(ref_lengths) if ref_lengths else 0.0
+        metrics["length_ratio"] = (np.mean(pred_lengths) / np.mean(ref_lengths)) if np.mean(ref_lengths) > 0 else 0.0
+        
+        # Diversity metrics for predictions
+        if len(pred_list) > 1:
+            all_words = []
+            for pred in pred_list:
+                all_words.extend(pred.lower().split())
+            
+            unique_words = len(set(all_words))
+            total_words = len(all_words)
+            
+            metrics["vocabulary_diversity"] = unique_words / total_words if total_words > 0 else 0.0
+        
+        return metrics
+        
+    except Exception as e:
+        logger.error(f"Error computing text metrics: {e}")
+        return {"text_metrics_error": 1.0}
+
+
+def compute_vision_metrics(predictions: List[Any], 
+                          references: List[Any], 
+                          task_type: str = "general") -> Dict[str, float]:
+    """
+    Compute vision-specific evaluation metrics.
+    
+    Args:
+        predictions: List of vision model predictions
+        references: List of reference outputs
+        task_type: Type of vision task (ocr, detection, etc.)
+        
+    Returns:
+        Dictionary of computed metrics
+    """
+    try:
+        metrics = {}
+        
+        # Basic success rate
+        successful_predictions = sum(1 for p in predictions if p is not None)
+        metrics["prediction_success_rate"] = successful_predictions / len(predictions) if predictions else 0.0
+        
+        # Task-specific metrics would be computed by individual evaluators
+        # This is a placeholder for common vision metrics
+        
+        if task_type == "ocr":
+            # OCR-specific metrics would be computed in VisionEvaluator
+            pass
+        elif task_type == "detection":
+            # Object detection metrics (IoU, mAP, etc.)
+            pass
+        elif task_type == "classification":
+            # Image classification metrics
+            pass
+        
+        return metrics
+        
+    except Exception as e:
+        logger.error(f"Error computing vision metrics: {e}")
+        return {"vision_metrics_error": 1.0} 

isa_model/eval/tests/unit/test_basic.py

@@ -0,0 +1,396 @@
+"""
+Unit tests for basic ISA Model evaluation framework functionality.
+
+This test file focuses on core functionality without complex dependencies.
+"""
+
+import pytest
+import asyncio
+from dataclasses import dataclass, field
+from typing import Dict, List, Any
+from abc import ABC, abstractmethod
+
+
+@dataclass
+class MockEvaluationResult:
+    """Mock evaluation result for testing."""
+    metrics: Dict[str, float] = field(default_factory=dict)
+    predictions: List[Any] = field(default_factory=list)
+    references: List[Any] = field(default_factory=list)
+    
+    def to_dict(self):
+        """Convert to dictionary."""
+        return {
+            "metrics": self.metrics,
+            "predictions": self.predictions,
+            "references": self.references
+        }
+
+
+class MockBaseEvaluator(ABC):
+    """Mock base evaluator for testing."""
+    
+    def __init__(self, config: Dict = None):
+        self.config = config or {}
+    
+    @abstractmethod
+    async def evaluate(self, model_interface, dataset, **kwargs):
+        pass
+
+
+class TestEvaluationResult:
+    """Test the EvaluationResult data structure."""
+    
+    def test_evaluation_result_creation(self):
+        """Test basic EvaluationResult creation and properties."""
+        result = MockEvaluationResult(
+            metrics={"accuracy": 0.85, "f1_score": 0.78},
+            predictions=["response1", "response2"],
+            references=["expected1", "expected2"]
+        )
+        
+        assert result.metrics["accuracy"] == 0.85
+        assert result.metrics["f1_score"] == 0.78
+        assert len(result.predictions) == 2
+        assert len(result.references) == 2
+    
+    def test_evaluation_result_default_values(self):
+        """Test EvaluationResult with default values."""
+        result = MockEvaluationResult()
+        
+        assert isinstance(result.metrics, dict)
+        assert isinstance(result.predictions, list)
+        assert isinstance(result.references, list)
+        assert len(result.metrics) == 0
+        assert len(result.predictions) == 0
+        assert len(result.references) == 0
+    
+    def test_evaluation_result_to_dict(self):
+        """Test EvaluationResult serialization."""
+        result = MockEvaluationResult(
+            metrics={"accuracy": 0.9},
+            predictions=["test"],
+            references=["expected"]
+        )
+        
+        result_dict = result.to_dict()
+        assert isinstance(result_dict, dict)
+        assert "metrics" in result_dict
+        assert result_dict["metrics"]["accuracy"] == 0.9
+
+
+class MockModelInterface:
+    """Mock model interface for testing."""
+    
+    def __init__(self, responses: List[str] = None):
+        self.responses = responses or ["mock response"]
+        self.call_count = 0
+    
+    async def generate_response(self, prompt: str, **kwargs) -> str:
+        """Mock response generation."""
+        response = self.responses[self.call_count % len(self.responses)]
+        self.call_count += 1
+        await asyncio.sleep(0.01)  # Simulate async processing
+        return response
+
+
+class TestBasicMetrics:
+    """Test basic metric calculation functions."""
+    
+    def test_exact_match_metric(self):
+        """Test exact match calculation."""
+        predictions = ["Paris", "London", "Berlin"]
+        references = ["Paris", "Madrid", "Berlin"]
+        
+        def calculate_exact_match(pred_list, ref_list):
+            """Simple exact match implementation."""
+            matches = sum(1 for p, r in zip(pred_list, ref_list) 
+                         if p.strip().lower() == r.strip().lower())
+            return matches / len(pred_list)
+        
+        accuracy = calculate_exact_match(predictions, references)
+        assert accuracy == 2/3  # 2 out of 3 matches
+    
+    def test_f1_score_calculation(self):
+        """Test F1 score calculation."""
+        predictions = ["The cat sits", "A dog runs"]
+        references = ["The cat sits on mat", "The dog runs fast"]
+        
+        def calculate_f1_score(pred_list, ref_list):
+            """Simple token-based F1 calculation."""
+            total_f1 = 0
+            for pred, ref in zip(pred_list, ref_list):
+                pred_tokens = set(pred.lower().split())
+                ref_tokens = set(ref.lower().split())
+                
+                if len(pred_tokens) == 0 and len(ref_tokens) == 0:
+                    f1 = 1.0
+                elif len(pred_tokens) == 0 or len(ref_tokens) == 0:
+                    f1 = 0.0
+                else:
+                    intersection = len(pred_tokens & ref_tokens)
+                    precision = intersection / len(pred_tokens)
+                    recall = intersection / len(ref_tokens)
+                    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
+                
+                total_f1 += f1
+            
+            return total_f1 / len(pred_list)
+        
+        f1 = calculate_f1_score(predictions, references)
+        assert isinstance(f1, float)
+        assert 0 <= f1 <= 1
+
+
+class TestBasicEvaluator:
+    """Test basic evaluator functionality."""
+    
+    @pytest.fixture
+    def mock_config(self):
+        """Create a mock evaluation config."""
+        return {
+            "batch_size": 2,
+            "max_concurrent_requests": 3,
+            "timeout_seconds": 30
+        }
+    
+    @pytest.fixture
+    def sample_dataset(self):
+        """Create a sample dataset for testing."""
+        return [
+            {
+                "id": "test_1",
+                "prompt": "What is 2+2?",
+                "expected_output": "4",
+                "metadata": {"category": "math"}
+            },
+            {
+                "id": "test_2", 
+                "input": "Name the capital of France",
+                "expected_output": "Paris",
+                "metadata": {"category": "geography"}
+            },
+            {
+                "id": "test_3",
+                "prompt": "What color is the sky?",
+                "expected_output": "blue",
+                "metadata": {"category": "general"}
+            }
+        ]
+    
+    def test_evaluator_initialization(self, mock_config):
+        """Test basic evaluator initialization."""
+        class TestEvaluator(MockBaseEvaluator):
+            async def evaluate(self, model_interface, dataset, **kwargs):
+                return MockEvaluationResult()
+        
+        evaluator = TestEvaluator(config=mock_config)
+        assert evaluator.config["batch_size"] == 2
+        assert evaluator.config["max_concurrent_requests"] == 3
+    
+    @pytest.mark.asyncio
+    async def test_mock_evaluation_workflow(self, sample_dataset, mock_config):
+        """Test basic evaluation workflow with mocked components."""
+        
+        class TestEvaluator(MockBaseEvaluator):
+            async def evaluate(self, model_interface, dataset, **kwargs):
+                """Simple evaluation implementation for testing."""
+                predictions = []
+                references = []
+                
+                for item in dataset:
+                    # Mock model call
+                    response = await model_interface.generate_response(
+                        item.get("prompt", item.get("input", ""))
+                    )
+                    predictions.append(response)
+                    references.append(item["expected_output"])
+                
+                # Calculate simple accuracy
+                matches = sum(1 for p, r in zip(predictions, references) 
+                            if p.strip().lower() == r.strip().lower())
+                accuracy = matches / len(predictions) if predictions else 0
+                
+                return MockEvaluationResult(
+                    metrics={"accuracy": accuracy, "total_samples": len(dataset)},
+                    predictions=predictions,
+                    references=references
+                )
+        
+        # Create evaluator and mock model
+        evaluator = TestEvaluator(config=mock_config)
+        model_interface = MockModelInterface(responses=["4", "Paris", "blue"])
+        
+        # Run evaluation
+        result = await evaluator.evaluate(
+            model_interface=model_interface,
+            dataset=sample_dataset,
+            dataset_name="test_dataset"
+        )
+        
+        # Verify results
+        assert isinstance(result, MockEvaluationResult)
+        assert "accuracy" in result.metrics
+        assert "total_samples" in result.metrics
+        assert result.metrics["total_samples"] == 3
+        assert result.metrics["accuracy"] == 1.0  # All mock responses match expected
+        assert len(result.predictions) == 3
+        assert len(result.references) == 3
+
+
+class TestEvaluationConfig:
+    """Test evaluation configuration functionality."""
+    
+    def test_config_creation(self):
+        """Test basic config creation."""
+        config_data = {
+            "batch_size": 16,
+            "max_concurrent_requests": 5,
+            "timeout_seconds": 60,
+            "output_dir": "test_results"
+        }
+        
+        class MockConfig:
+            def __init__(self, **kwargs):
+                for k, v in kwargs.items():
+                    setattr(self, k, v)
+        
+        config = MockConfig(**config_data)
+        assert config.batch_size == 16
+        assert config.output_dir == "test_results"
+    
+    def test_config_validation(self):
+        """Test config validation logic."""
+        def validate_config(config_dict):
+            """Validate configuration values."""
+            if config_dict.get("batch_size", 1) <= 0:
+                raise ValueError("batch_size must be positive")
+            if config_dict.get("max_concurrent_requests", 1) <= 0:
+                raise ValueError("max_concurrent_requests must be positive")
+            if config_dict.get("timeout_seconds", 1) <= 0:
+                raise ValueError("timeout_seconds must be positive")
+            return True
+        
+        # Test valid config
+        valid_config = {"batch_size": 10, "max_concurrent_requests": 5, "timeout_seconds": 60}
+        assert validate_config(valid_config) is True
+        
+        # Test invalid configs
+        invalid_configs = [
+            {"batch_size": -1},
+            {"max_concurrent_requests": 0},
+            {"timeout_seconds": -5}
+        ]
+        
+        for invalid_config in invalid_configs:
+            with pytest.raises(ValueError):
+                validate_config(invalid_config)
+
+
+class TestAsyncEvaluation:
+    """Test asynchronous evaluation capabilities."""
+    
+    @pytest.mark.asyncio
+    async def test_concurrent_evaluation(self):
+        """Test that evaluations can run concurrently."""
+        async def mock_evaluation_task(task_id: int, delay: float = 0.1):
+            """Mock evaluation task with delay."""
+            await asyncio.sleep(delay)
+            return {"task_id": task_id, "result": f"completed_{task_id}"}
+        
+        # Run multiple evaluations concurrently
+        start_time = asyncio.get_event_loop().time()
+        
+        tasks = [mock_evaluation_task(i, 0.1) for i in range(3)]
+        results = await asyncio.gather(*tasks)
+        
+        end_time = asyncio.get_event_loop().time()
+        
+        # Should complete in roughly 0.1 seconds (concurrent) rather than 0.3 (sequential)
+        assert end_time - start_time < 0.2
+        assert len(results) == 3
+        assert all(r["result"].startswith("completed_") for r in results)
+    
+    @pytest.mark.asyncio 
+    async def test_batch_processing(self):
+        """Test batch processing functionality."""
+        async def process_batch(batch: List[Dict], batch_size: int = 2):
+            """Process items in batches."""
+            results = []
+            for i in range(0, len(batch), batch_size):
+                batch_items = batch[i:i + batch_size]
+                # Simulate processing time proportional to batch size
+                await asyncio.sleep(0.01 * len(batch_items))
+                batch_results = [{"processed": item["id"]} for item in batch_items]
+                results.extend(batch_results)
+            return results
+        
+        # Test data
+        test_items = [{"id": f"item_{i}"} for i in range(5)]
+        
+        # Process in batches
+        results = await process_batch(test_items, batch_size=2)
+        
+        assert len(results) == 5
+        assert all("processed" in r for r in results)
+
+
+class TestErrorHandling:
+    """Test error handling and edge cases."""
+    
+    @pytest.mark.asyncio
+    async def test_timeout_handling(self):
+        """Test timeout handling in async operations."""
+        async def slow_operation():
+            """Simulate a slow operation."""
+            await asyncio.sleep(1.0)
+            return "completed"
+        
+        # Test timeout
+        with pytest.raises(asyncio.TimeoutError):
+            await asyncio.wait_for(slow_operation(), timeout=0.1)
+    
+    def test_empty_dataset_handling(self):
+        """Test handling of empty datasets."""
+        def calculate_metrics(predictions, references):
+            """Calculate metrics with empty data handling."""
+            if not predictions or not references:
+                return {"accuracy": 0.0, "count": 0}
+            
+            matches = sum(1 for p, r in zip(predictions, references) if p == r)
+            return {
+                "accuracy": matches / len(predictions),
+                "count": len(predictions)
+            }
+        
+        # Test empty data
+        empty_metrics = calculate_metrics([], [])
+        assert empty_metrics["accuracy"] == 0.0
+        assert empty_metrics["count"] == 0
+    
+    def test_mismatched_data_lengths(self):
+        """Test handling of mismatched prediction and reference lengths."""
+        def safe_calculate_accuracy(predictions, references):
+            """Safely calculate accuracy with length mismatch handling."""
+            if len(predictions) != len(references):
+                min_len = min(len(predictions), len(references))
+                predictions = predictions[:min_len]
+                references = references[:min_len]
+            
+            if not predictions:
+                return 0.0
+            
+            matches = sum(1 for p, r in zip(predictions, references) if p == r)
+            return matches / len(predictions)
+        
+        # Test mismatched lengths
+        predictions = ["a", "b", "c"]
+        references = ["a", "b"]  # Shorter
+        
+        accuracy = safe_calculate_accuracy(predictions, references)
+        assert accuracy == 1.0  # Both "a" and "b" match
+
+
+if __name__ == "__main__":
+    # Allow running tests directly
+    pytest.main([__file__, "-v"])