isa-model 0.4.0__py3-none-any.whl → 0.4.4__py3-none-any.whl

isa_model/eval/tests/unit/test_basic.py

@@ -1,396 +0,0 @@
-"""
-Unit tests for basic ISA Model evaluation framework functionality.
-
-This test file focuses on core functionality without complex dependencies.
-"""
-
-import pytest
-import asyncio
-from dataclasses import dataclass, field
-from typing import Dict, List, Any
-from abc import ABC, abstractmethod
-
-
-@dataclass
-class MockEvaluationResult:
-    """Mock evaluation result for testing."""
-    metrics: Dict[str, float] = field(default_factory=dict)
-    predictions: List[Any] = field(default_factory=list)
-    references: List[Any] = field(default_factory=list)
-    
-    def to_dict(self):
-        """Convert to dictionary."""
-        return {
-            "metrics": self.metrics,
-            "predictions": self.predictions,
-            "references": self.references
-        }
-
-
-class MockBaseEvaluator(ABC):
-    """Mock base evaluator for testing."""
-    
-    def __init__(self, config: Dict = None):
-        self.config = config or {}
-    
-    @abstractmethod
-    async def evaluate(self, model_interface, dataset, **kwargs):
-        pass
-
-
-class TestEvaluationResult:
-    """Test the EvaluationResult data structure."""
-    
-    def test_evaluation_result_creation(self):
-        """Test basic EvaluationResult creation and properties."""
-        result = MockEvaluationResult(
-            metrics={"accuracy": 0.85, "f1_score": 0.78},
-            predictions=["response1", "response2"],
-            references=["expected1", "expected2"]
-        )
-        
-        assert result.metrics["accuracy"] == 0.85
-        assert result.metrics["f1_score"] == 0.78
-        assert len(result.predictions) == 2
-        assert len(result.references) == 2
-    
-    def test_evaluation_result_default_values(self):
-        """Test EvaluationResult with default values."""
-        result = MockEvaluationResult()
-        
-        assert isinstance(result.metrics, dict)
-        assert isinstance(result.predictions, list)
-        assert isinstance(result.references, list)
-        assert len(result.metrics) == 0
-        assert len(result.predictions) == 0
-        assert len(result.references) == 0
-    
-    def test_evaluation_result_to_dict(self):
-        """Test EvaluationResult serialization."""
-        result = MockEvaluationResult(
-            metrics={"accuracy": 0.9},
-            predictions=["test"],
-            references=["expected"]
-        )
-        
-        result_dict = result.to_dict()
-        assert isinstance(result_dict, dict)
-        assert "metrics" in result_dict
-        assert result_dict["metrics"]["accuracy"] == 0.9
-
-
-class MockModelInterface:
-    """Mock model interface for testing."""
-    
-    def __init__(self, responses: List[str] = None):
-        self.responses = responses or ["mock response"]
-        self.call_count = 0
-    
-    async def generate_response(self, prompt: str, **kwargs) -> str:
-        """Mock response generation."""
-        response = self.responses[self.call_count % len(self.responses)]
-        self.call_count += 1
-        await asyncio.sleep(0.01)  # Simulate async processing
-        return response
-
-
-class TestBasicMetrics:
-    """Test basic metric calculation functions."""
-    
-    def test_exact_match_metric(self):
-        """Test exact match calculation."""
-        predictions = ["Paris", "London", "Berlin"]
-        references = ["Paris", "Madrid", "Berlin"]
-        
-        def calculate_exact_match(pred_list, ref_list):
-            """Simple exact match implementation."""
-            matches = sum(1 for p, r in zip(pred_list, ref_list) 
-                         if p.strip().lower() == r.strip().lower())
-            return matches / len(pred_list)
-        
-        accuracy = calculate_exact_match(predictions, references)
-        assert accuracy == 2/3  # 2 out of 3 matches
-    
-    def test_f1_score_calculation(self):
-        """Test F1 score calculation."""
-        predictions = ["The cat sits", "A dog runs"]
-        references = ["The cat sits on mat", "The dog runs fast"]
-        
-        def calculate_f1_score(pred_list, ref_list):
-            """Simple token-based F1 calculation."""
-            total_f1 = 0
-            for pred, ref in zip(pred_list, ref_list):
-                pred_tokens = set(pred.lower().split())
-                ref_tokens = set(ref.lower().split())
-                
-                if len(pred_tokens) == 0 and len(ref_tokens) == 0:
-                    f1 = 1.0
-                elif len(pred_tokens) == 0 or len(ref_tokens) == 0:
-                    f1 = 0.0
-                else:
-                    intersection = len(pred_tokens & ref_tokens)
-                    precision = intersection / len(pred_tokens)
-                    recall = intersection / len(ref_tokens)
-                    f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0
-                
-                total_f1 += f1
-            
-            return total_f1 / len(pred_list)
-        
-        f1 = calculate_f1_score(predictions, references)
-        assert isinstance(f1, float)
-        assert 0 <= f1 <= 1
-
-
-class TestBasicEvaluator:
-    """Test basic evaluator functionality."""
-    
-    @pytest.fixture
-    def mock_config(self):
-        """Create a mock evaluation config."""
-        return {
-            "batch_size": 2,
-            "max_concurrent_requests": 3,
-            "timeout_seconds": 30
-        }
-    
-    @pytest.fixture
-    def sample_dataset(self):
-        """Create a sample dataset for testing."""
-        return [
-            {
-                "id": "test_1",
-                "prompt": "What is 2+2?",
-                "expected_output": "4",
-                "metadata": {"category": "math"}
-            },
-            {
-                "id": "test_2", 
-                "input": "Name the capital of France",
-                "expected_output": "Paris",
-                "metadata": {"category": "geography"}
-            },
-            {
-                "id": "test_3",
-                "prompt": "What color is the sky?",
-                "expected_output": "blue",
-                "metadata": {"category": "general"}
-            }
-        ]
-    
-    def test_evaluator_initialization(self, mock_config):
-        """Test basic evaluator initialization."""
-        class TestEvaluator(MockBaseEvaluator):
-            async def evaluate(self, model_interface, dataset, **kwargs):
-                return MockEvaluationResult()
-        
-        evaluator = TestEvaluator(config=mock_config)
-        assert evaluator.config["batch_size"] == 2
-        assert evaluator.config["max_concurrent_requests"] == 3
-    
-    @pytest.mark.asyncio
-    async def test_mock_evaluation_workflow(self, sample_dataset, mock_config):
-        """Test basic evaluation workflow with mocked components."""
-        
-        class TestEvaluator(MockBaseEvaluator):
-            async def evaluate(self, model_interface, dataset, **kwargs):
-                """Simple evaluation implementation for testing."""
-                predictions = []
-                references = []
-                
-                for item in dataset:
-                    # Mock model call
-                    response = await model_interface.generate_response(
-                        item.get("prompt", item.get("input", ""))
-                    )
-                    predictions.append(response)
-                    references.append(item["expected_output"])
-                
-                # Calculate simple accuracy
-                matches = sum(1 for p, r in zip(predictions, references) 
-                            if p.strip().lower() == r.strip().lower())
-                accuracy = matches / len(predictions) if predictions else 0
-                
-                return MockEvaluationResult(
-                    metrics={"accuracy": accuracy, "total_samples": len(dataset)},
-                    predictions=predictions,
-                    references=references
-                )
-        
-        # Create evaluator and mock model
-        evaluator = TestEvaluator(config=mock_config)
-        model_interface = MockModelInterface(responses=["4", "Paris", "blue"])
-        
-        # Run evaluation
-        result = await evaluator.evaluate(
-            model_interface=model_interface,
-            dataset=sample_dataset,
-            dataset_name="test_dataset"
-        )
-        
-        # Verify results
-        assert isinstance(result, MockEvaluationResult)
-        assert "accuracy" in result.metrics
-        assert "total_samples" in result.metrics
-        assert result.metrics["total_samples"] == 3
-        assert result.metrics["accuracy"] == 1.0  # All mock responses match expected
-        assert len(result.predictions) == 3
-        assert len(result.references) == 3
-
-
-class TestEvaluationConfig:
-    """Test evaluation configuration functionality."""
-    
-    def test_config_creation(self):
-        """Test basic config creation."""
-        config_data = {
-            "batch_size": 16,
-            "max_concurrent_requests": 5,
-            "timeout_seconds": 60,
-            "output_dir": "test_results"
-        }
-        
-        class MockConfig:
-            def __init__(self, **kwargs):
-                for k, v in kwargs.items():
-                    setattr(self, k, v)
-        
-        config = MockConfig(**config_data)
-        assert config.batch_size == 16
-        assert config.output_dir == "test_results"
-    
-    def test_config_validation(self):
-        """Test config validation logic."""
-        def validate_config(config_dict):
-            """Validate configuration values."""
-            if config_dict.get("batch_size", 1) <= 0:
-                raise ValueError("batch_size must be positive")
-            if config_dict.get("max_concurrent_requests", 1) <= 0:
-                raise ValueError("max_concurrent_requests must be positive")
-            if config_dict.get("timeout_seconds", 1) <= 0:
-                raise ValueError("timeout_seconds must be positive")
-            return True
-        
-        # Test valid config
-        valid_config = {"batch_size": 10, "max_concurrent_requests": 5, "timeout_seconds": 60}
-        assert validate_config(valid_config) is True
-        
-        # Test invalid configs
-        invalid_configs = [
-            {"batch_size": -1},
-            {"max_concurrent_requests": 0},
-            {"timeout_seconds": -5}
-        ]
-        
-        for invalid_config in invalid_configs:
-            with pytest.raises(ValueError):
-                validate_config(invalid_config)
-
-
-class TestAsyncEvaluation:
-    """Test asynchronous evaluation capabilities."""
-    
-    @pytest.mark.asyncio
-    async def test_concurrent_evaluation(self):
-        """Test that evaluations can run concurrently."""
-        async def mock_evaluation_task(task_id: int, delay: float = 0.1):
-            """Mock evaluation task with delay."""
-            await asyncio.sleep(delay)
-            return {"task_id": task_id, "result": f"completed_{task_id}"}
-        
-        # Run multiple evaluations concurrently
-        start_time = asyncio.get_event_loop().time()
-        
-        tasks = [mock_evaluation_task(i, 0.1) for i in range(3)]
-        results = await asyncio.gather(*tasks)
-        
-        end_time = asyncio.get_event_loop().time()
-        
-        # Should complete in roughly 0.1 seconds (concurrent) rather than 0.3 (sequential)
-        assert end_time - start_time < 0.2
-        assert len(results) == 3
-        assert all(r["result"].startswith("completed_") for r in results)
-    
-    @pytest.mark.asyncio 
-    async def test_batch_processing(self):
-        """Test batch processing functionality."""
-        async def process_batch(batch: List[Dict], batch_size: int = 2):
-            """Process items in batches."""
-            results = []
-            for i in range(0, len(batch), batch_size):
-                batch_items = batch[i:i + batch_size]
-                # Simulate processing time proportional to batch size
-                await asyncio.sleep(0.01 * len(batch_items))
-                batch_results = [{"processed": item["id"]} for item in batch_items]
-                results.extend(batch_results)
-            return results
-        
-        # Test data
-        test_items = [{"id": f"item_{i}"} for i in range(5)]
-        
-        # Process in batches
-        results = await process_batch(test_items, batch_size=2)
-        
-        assert len(results) == 5
-        assert all("processed" in r for r in results)
-
-
-class TestErrorHandling:
-    """Test error handling and edge cases."""
-    
-    @pytest.mark.asyncio
-    async def test_timeout_handling(self):
-        """Test timeout handling in async operations."""
-        async def slow_operation():
-            """Simulate a slow operation."""
-            await asyncio.sleep(1.0)
-            return "completed"
-        
-        # Test timeout
-        with pytest.raises(asyncio.TimeoutError):
-            await asyncio.wait_for(slow_operation(), timeout=0.1)
-    
-    def test_empty_dataset_handling(self):
-        """Test handling of empty datasets."""
-        def calculate_metrics(predictions, references):
-            """Calculate metrics with empty data handling."""
-            if not predictions or not references:
-                return {"accuracy": 0.0, "count": 0}
-            
-            matches = sum(1 for p, r in zip(predictions, references) if p == r)
-            return {
-                "accuracy": matches / len(predictions),
-                "count": len(predictions)
-            }
-        
-        # Test empty data
-        empty_metrics = calculate_metrics([], [])
-        assert empty_metrics["accuracy"] == 0.0
-        assert empty_metrics["count"] == 0
-    
-    def test_mismatched_data_lengths(self):
-        """Test handling of mismatched prediction and reference lengths."""
-        def safe_calculate_accuracy(predictions, references):
-            """Safely calculate accuracy with length mismatch handling."""
-            if len(predictions) != len(references):
-                min_len = min(len(predictions), len(references))
-                predictions = predictions[:min_len]
-                references = references[:min_len]
-            
-            if not predictions:
-                return 0.0
-            
-            matches = sum(1 for p, r in zip(predictions, references) if p == r)
-            return matches / len(predictions)
-        
-        # Test mismatched lengths
-        predictions = ["a", "b", "c"]
-        references = ["a", "b"]  # Shorter
-        
-        accuracy = safe_calculate_accuracy(predictions, references)
-        assert accuracy == 1.0  # Both "a" and "b" match
-
-
-if __name__ == "__main__":
-    # Allow running tests directly
-    pytest.main([__file__, "-v"])