isa-model 0.3.9__py3-none-any.whl → 0.4.0__py3-none-any.whl

isa_model/eval/factory.py

@@ -16,9 +16,45 @@ from typing import Optional, Dict, Any, List, Union, Callable
 from pathlib import Path
 import json
 
-from .evaluators import LLMEvaluator, VisionEvaluator, MultimodalEvaluator, EvaluationResult
-from .infrastructure import ExperimentTracker, create_experiment_tracker
-from .config import EvaluationConfig
+from .evaluators import LLMEvaluator, VisionEvaluator, AudioEvaluator, EmbeddingEvaluator, EvaluationResult
+from .isa_integration import ISAModelInterface
+try:
+    from .infrastructure import ExperimentTracker, create_experiment_tracker
+    EXPERIMENT_TRACKING_AVAILABLE = True
+except ImportError:
+    EXPERIMENT_TRACKING_AVAILABLE = False
+    logger.warning("Experiment tracking not available")
+
+try:
+    from .config import EvaluationConfig
+    CONFIG_AVAILABLE = True
+except ImportError:
+    CONFIG_AVAILABLE = False
+    # Create a simple config class
+    class EvaluationConfig:
+        def __init__(self):
+            self.batch_size = 16
+            self.output_dir = "./evaluation_results"
+            self.default_temperature = 0.7
+            self.default_max_tokens = 512
+            self.max_concurrent_evaluations = 3
+        
+        def to_dict(self):
+            return {
+                "batch_size": self.batch_size,
+                "output_dir": self.output_dir,
+                "default_temperature": self.default_temperature,
+                "default_max_tokens": self.default_max_tokens,
+                "max_concurrent_evaluations": self.max_concurrent_evaluations
+            }
+        
+        @classmethod
+        def from_dict(cls, config_dict):
+            config = cls()
+            for key, value in config_dict.items():
+                if hasattr(config, key):
+                    setattr(config, key, value)
+            return config
 
 logger = logging.getLogger(__name__)
 
@@ -103,19 +139,37 @@ class EvaluationFactory:
         
         # Initialize experiment tracker
         self.experiment_tracker = None
-        if experiment_tracking:
+        if experiment_tracking and EXPERIMENT_TRACKING_AVAILABLE:
             try:
                 self.experiment_tracker = create_experiment_tracker(**experiment_tracking)
                 logger.info(f"Initialized experiment tracking: {experiment_tracking['type']}")
             except Exception as e:
                 logger.warning(f"Failed to initialize experiment tracking: {e}")
         
+        # Initialize ISA Model interface
+        self.isa_interface = ISAModelInterface()
+        
         # Initialize evaluators
         self.llm_evaluator = LLMEvaluator(
             config=self.config.to_dict(),
             experiment_tracker=self.experiment_tracker
         )
         
+        self.vision_evaluator = VisionEvaluator(
+            config=self.config.to_dict(),
+            experiment_tracker=self.experiment_tracker
+        )
+        
+        self.audio_evaluator = AudioEvaluator(
+            config=self.config.to_dict(),
+            experiment_tracker=self.experiment_tracker
+        )
+        
+        self.embedding_evaluator = EmbeddingEvaluator(
+            config=self.config.to_dict(),
+            experiment_tracker=self.experiment_tracker
+        )
+        
         # State tracking
         self._active_evaluations: Dict[str, asyncio.Task] = {}
         
@@ -171,7 +225,7 @@ class EvaluationFactory:
         
         # Run evaluation
         result = await self.llm_evaluator.evaluate(
-            model_interface=None,  # Will use AI factory
+            model_interface=self.isa_interface,
             dataset=dataset,
             dataset_name=dataset_name,
             model_name=f"{provider}:{model_name}",
@@ -329,6 +383,219 @@ class EvaluationFactory:
         
         return results
     
+    async def evaluate_vision(self,
+                            dataset: List[Dict[str, Any]],
+                            task_type: str = "ocr",
+                            model_name: str = "gpt-4.1-mini",
+                            save_results: bool = True,
+                            experiment_name: Optional[str] = None) -> EvaluationResult:
+        """
+        Evaluate vision model on image tasks.
+        
+        Args:
+            dataset: Vision dataset with images and expected outputs
+            task_type: Vision task type (ocr, table, ui, vqa, caption)
+            model_name: Vision model name
+            save_results: Whether to save results
+            experiment_name: Custom experiment name
+            
+        Returns:
+            Vision evaluation results
+        """
+        # Configure vision evaluator
+        self.vision_evaluator.config.update({
+            "task_type": task_type,
+            "model_name": model_name
+        })
+        
+        experiment_name = experiment_name or f"vision_{task_type}_{model_name}"
+        
+        result = await self.vision_evaluator.evaluate(
+            model_interface=self.isa_interface,
+            dataset=dataset,
+            dataset_name=f"vision_{task_type}",
+            model_name=model_name
+        )
+        
+        if save_results:
+            await self._save_results(result, experiment_name)
+        
+        return result
+    
+    async def evaluate_audio(self,
+                           dataset: List[Dict[str, Any]],
+                           task_type: str = "stt",
+                           model_name: str = "isa_audio_sota_service",
+                           save_results: bool = True,
+                           experiment_name: Optional[str] = None) -> EvaluationResult:
+        """
+        Evaluate audio model on speech tasks.
+        
+        Args:
+            dataset: Audio dataset with audio files and expected outputs
+            task_type: Audio task type (stt, emotion, diarization)
+            model_name: Audio model name
+            save_results: Whether to save results
+            experiment_name: Custom experiment name
+            
+        Returns:
+            Audio evaluation results
+        """
+        # Configure audio evaluator
+        self.audio_evaluator.config.update({
+            "task_type": task_type,
+            "model_name": model_name
+        })
+        
+        experiment_name = experiment_name or f"audio_{task_type}_{model_name}"
+        
+        result = await self.audio_evaluator.evaluate(
+            model_interface=self.isa_interface,
+            dataset=dataset,
+            dataset_name=f"audio_{task_type}",
+            model_name=model_name
+        )
+        
+        if save_results:
+            await self._save_results(result, experiment_name)
+        
+        return result
+    
+    async def evaluate_embedding(self,
+                               dataset: List[Dict[str, Any]],
+                               task_type: str = "similarity",
+                               model_name: str = "text-embedding-3-small",
+                               save_results: bool = True,
+                               experiment_name: Optional[str] = None) -> EvaluationResult:
+        """
+        Evaluate embedding model on semantic tasks.
+        
+        Args:
+            dataset: Embedding dataset with text and expected outputs
+            task_type: Embedding task type (similarity, retrieval, reranking)
+            model_name: Embedding model name
+            save_results: Whether to save results
+            experiment_name: Custom experiment name
+            
+        Returns:
+            Embedding evaluation results
+        """
+        # Configure embedding evaluator
+        self.embedding_evaluator.config.update({
+            "task_type": task_type,
+            "model_name": model_name
+        })
+        
+        experiment_name = experiment_name or f"embedding_{task_type}_{model_name}"
+        
+        result = await self.embedding_evaluator.evaluate(
+            model_interface=self.isa_interface,
+            dataset=dataset,
+            dataset_name=f"embedding_{task_type}",
+            model_name=model_name
+        )
+        
+        if save_results:
+            await self._save_results(result, experiment_name)
+        
+        return result
+    
+    async def compare_models(self,
+                           models: List[Dict[str, str]],
+                           dataset_path: Optional[str] = None,
+                           dataset: Optional[List[Dict[str, Any]]] = None,
+                           evaluator_type: str = "llm",
+                           benchmark_name: Optional[str] = None,
+                           metrics: Optional[List[str]] = None,
+                           save_results: bool = True,
+                           experiment_name: Optional[str] = None) -> Dict[str, EvaluationResult]:
+        """
+        Compare multiple models on the same evaluation task.
+        
+        Args:
+            models: List of model configs [{"name": "gpt-4", "provider": "openai"}, ...]
+            dataset_path: Path to evaluation dataset
+            dataset: Direct dataset input
+            evaluator_type: Type of evaluator (llm, vision, audio, embedding)
+            benchmark_name: Benchmark name (alternative to dataset)
+            metrics: Metrics to compute
+            save_results: Whether to save comparison results
+            experiment_name: Custom experiment name
+            
+        Returns:
+            Dictionary mapping model names to evaluation results
+        """
+        results = {}
+        
+        # Load dataset if needed
+        if dataset is None and dataset_path:
+            dataset = self._load_dataset(dataset_path)
+        
+        # Run evaluations concurrently (with concurrency limits)
+        semaphore = asyncio.Semaphore(self.config.max_concurrent_evaluations)
+        
+        async def evaluate_single_model(model_config: Dict[str, str]) -> tuple:
+            async with semaphore:
+                model_name = model_config["name"]
+                provider = model_config.get("provider", "openai")
+                
+                if evaluator_type == "llm":
+                    if benchmark_name:
+                        result = await self.run_benchmark(
+                            model_name=model_name,
+                            provider=provider,
+                            benchmark_name=benchmark_name,
+                            save_results=False
+                        )
+                    else:
+                        result = await self.evaluate_llm(
+                            model_name=model_name,
+                            provider=provider,
+                            dataset=dataset,
+                            metrics=metrics,
+                            save_results=False
+                        )
+                elif evaluator_type == "vision":
+                    result = await self.evaluate_vision(
+                        dataset=dataset,
+                        model_name=model_name,
+                        save_results=False
+                    )
+                elif evaluator_type == "audio":
+                    result = await self.evaluate_audio(
+                        dataset=dataset,
+                        model_name=model_name,
+                        save_results=False
+                    )
+                elif evaluator_type == "embedding":
+                    result = await self.evaluate_embedding(
+                        dataset=dataset,
+                        model_name=model_name,
+                        save_results=False
+                    )
+                else:
+                    raise ValueError(f"Unknown evaluator type: {evaluator_type}")
+                
+                return f"{provider}:{model_name}", result
+        
+        # Execute all evaluations
+        tasks = [evaluate_single_model(model) for model in models]
+        evaluation_results = await asyncio.gather(*tasks)
+        
+        # Collect results
+        for model_id, result in evaluation_results:
+            results[model_id] = result
+        
+        # Generate comparison report
+        comparison_report = self._generate_comparison_report(results)
+        
+        # Save results if requested
+        if save_results:
+            experiment_name = experiment_name or f"model_comparison_{evaluator_type}_{len(models)}_models"
+            await self._save_comparison_results(results, comparison_report, experiment_name)
+        
+        return results
+    
     def _load_dataset(self, dataset_path: str) -> List[Dict[str, Any]]:
         """Load dataset from file."""
         with open(dataset_path, 'r', encoding='utf-8') as f: