isa-model 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

isa_model/eval/metrics.py

@@ -15,6 +15,12 @@ from typing import Dict, List, Any, Optional, Union
 from enum import Enum
 from abc import ABC, abstractmethod
 
+try:
+    from ..inference.ai_factory import AIFactory
+    AI_FACTORY_AVAILABLE = True
+except ImportError:
+    AI_FACTORY_AVAILABLE = False
+
 logger = logging.getLogger(__name__)
 
 
@@ -66,6 +72,16 @@ class LLMMetrics:
             MetricType.COHERENCE,
             MetricType.FLUENCY
         ]
+        
+        # Initialize AI factory if available
+        if AI_FACTORY_AVAILABLE:
+            try:
+                self.ai_factory = AIFactory()
+            except Exception as e:
+                logger.warning(f"Failed to initialize AIFactory: {e}")
+                self.ai_factory = None
+        else:
+            self.ai_factory = None
     
     def evaluate(
         self,
@@ -200,21 +216,92 @@ class LLMMetrics:
         provider: str,
         **kwargs
     ) -> tuple:
-        """Generate predictions from model."""
+        """Generate predictions from model using actual inference."""
         predictions = []
         references = []
         
-        # This is a simplified implementation
-        # In practice, you'd use the actual model inference
-        for item in dataset:
-            if isinstance(item, dict):
-                if "input" in item and "output" in item:
-                    # Simulate prediction (replace with actual model inference)
-                    predictions.append(f"Generated response for: {item['input']}")
-                    references.append(item["output"])
-                elif "prompt" in item and "response" in item:
-                    predictions.append(f"Generated response for: {item['prompt']}")
-                    references.append(item["response"])
+        if not self.ai_factory:
+            logger.warning("AIFactory not available, using placeholder predictions")
+            # Fallback to placeholder predictions
+            for item in dataset:
+                if isinstance(item, dict):
+                    if "input" in item and "output" in item:
+                        predictions.append(f"Generated response for: {item['input']}")
+                        references.append(item["output"])
+                    elif "prompt" in item and "response" in item:
+                        predictions.append(f"Generated response for: {item['prompt']}")
+                        references.append(item["response"])
+            return predictions, references
+        
+        try:
+            # Get LLM service
+            llm_service = self.ai_factory.get_llm(model_name=model_path, provider=provider)
+            
+            # Process dataset in batches
+            for i in range(0, len(dataset), batch_size):
+                batch = dataset[i:i + batch_size]
+                batch_predictions = []
+                batch_references = []
+                
+                for item in batch:
+                    if isinstance(item, dict):
+                        prompt = None
+                        reference = None
+                        
+                        # Extract prompt and reference based on data format
+                        if "input" in item and "output" in item:
+                            prompt = item["input"]
+                            reference = item["output"]
+                        elif "prompt" in item and "response" in item:
+                            prompt = item["prompt"]
+                            reference = item["response"]
+                        elif "question" in item and "answer" in item:
+                            prompt = item["question"]
+                            reference = item["answer"]
+                        elif "text" in item and "label" in item:
+                            prompt = item["text"]
+                            reference = str(item["label"])
+                        
+                        if prompt and reference:
+                            try:
+                                # Generate prediction using actual model
+                                response = await llm_service.ainvoke(prompt)
+                                
+                                # Extract text from response
+                                if hasattr(response, 'text'):
+                                    prediction = response.text
+                                elif isinstance(response, dict) and 'text' in response:
+                                    prediction = response['text']
+                                elif isinstance(response, str):
+                                    prediction = response
+                                else:
+                                    prediction = str(response)
+                                
+                                batch_predictions.append(prediction.strip())
+                                batch_references.append(reference)
+                                
+                            except Exception as e:
+                                logger.error(f"Failed to generate prediction for item: {e}")
+                                # Use fallback prediction
+                                batch_predictions.append(f"Error generating prediction: {str(e)}")
+                                batch_references.append(reference)
+                
+                predictions.extend(batch_predictions)
+                references.extend(batch_references)
+                
+                logger.info(f"Processed batch {i//batch_size + 1}/{(len(dataset) + batch_size - 1)//batch_size}")
+        
+        except Exception as e:
+            logger.error(f"Failed to use AIFactory for predictions: {e}")
+            # Fallback to placeholder predictions
+            for item in dataset:
+                if isinstance(item, dict):
+                    if "input" in item and "output" in item:
+                        predictions.append(f"Generated response for: {item['input']}")
+                        references.append(item["output"])
+                    elif "prompt" in item and "response" in item:
+                        predictions.append(f"Generated response for: {item['prompt']}")
+                        references.append(item["response"])
         
         logger.info(f"Generated {len(predictions)} predictions")
         return predictions, references
@@ -226,11 +313,47 @@ class LLMMetrics:
         provider: str,
         **kwargs
     ) -> List[str]:
-        """Generate texts from prompts."""
-        # Simplified implementation - replace with actual model inference
+        """Generate texts from prompts using actual model inference."""
         generated_texts = []
-        for prompt in prompts:
-            generated_texts.append(f"Generated response for: {prompt}")
+        
+        if not self.ai_factory:
+            logger.warning("AIFactory not available, using placeholder text generation")
+            # Fallback to placeholder generation
+            for prompt in prompts:
+                generated_texts.append(f"Generated response for: {prompt}")
+            return generated_texts
+        
+        try:
+            # Get LLM service
+            llm_service = self.ai_factory.get_llm(model_name=model_path, provider=provider)
+            
+            for prompt in prompts:
+                try:
+                    # Generate text using actual model
+                    response = await llm_service.ainvoke(prompt)
+                    
+                    # Extract text from response
+                    if hasattr(response, 'text'):
+                        generated_text = response.text
+                    elif isinstance(response, dict) and 'text' in response:
+                        generated_text = response['text']
+                    elif isinstance(response, str):
+                        generated_text = response
+                    else:
+                        generated_text = str(response)
+                    
+                    generated_texts.append(generated_text.strip())
+                    
+                except Exception as e:
+                    logger.error(f"Failed to generate text for prompt: {e}")
+                    # Use fallback generation
+                    generated_texts.append(f"Error generating text: {str(e)}")
+        
+        except Exception as e:
+            logger.error(f"Failed to use AIFactory for text generation: {e}")
+            # Fallback to placeholder generation
+            for prompt in prompts:
+                generated_texts.append(f"Generated response for: {prompt}")
         
         return generated_texts
     
@@ -544,6 +667,16 @@ class BenchmarkRunner:
     
     def __init__(self):
         self.supported_benchmarks = ["mmlu", "hellaswag", "arc", "gsm8k"]
+        
+        # Initialize AI factory if available
+        if AI_FACTORY_AVAILABLE:
+            try:
+                self.ai_factory = AIFactory()
+            except Exception as e:
+                logger.warning(f"Failed to initialize AIFactory: {e}")
+                self.ai_factory = None
+        else:
+            self.ai_factory = None
     
     def run(
         self,
@@ -588,9 +721,12 @@ class BenchmarkRunner:
         
         for sample in test_data:
             try:
-                # Generate prediction (simplified)
+                # Format prompt using benchmark's method
+                prompt = benchmark.format_prompt(sample)
+                
+                # Generate prediction using actual model
                 prediction = self._generate_prediction(
-                    model_path, sample, num_shots, provider, **kwargs
+                    model_path, {"prompt": prompt}, num_shots, provider, **kwargs
                 )
                 
                 # Check if correct
@@ -623,6 +759,40 @@ class BenchmarkRunner:
         provider: str,
         **kwargs
     ) -> str:
-        """Generate prediction for a sample (simplified implementation)."""
-        # This is a placeholder - replace with actual model inference
-        return "A"  # Placeholder answer 
+        """Generate prediction for a sample using actual model inference."""
+        if not self.ai_factory:
+            logger.warning("AIFactory not available, using placeholder prediction")
+            return "A"  # Placeholder answer
+        
+        try:
+            # Get LLM service
+            llm_service = self.ai_factory.get_llm(model_name=model_path, provider=provider)
+            
+            # Format the prompt (this should be done by the benchmark)
+            if hasattr(sample, 'get'):
+                prompt = sample.get('prompt', str(sample))
+            else:
+                prompt = str(sample)
+            
+            # Generate prediction using actual model
+            response = llm_service.generate(
+                prompt=prompt,
+                max_tokens=kwargs.get("max_tokens", 50),
+                temperature=kwargs.get("temperature", 0.0)  # Low temperature for consistency
+            )
+            
+            # Extract text from response
+            if hasattr(response, 'text'):
+                prediction = response.text
+            elif isinstance(response, dict) and 'text' in response:
+                prediction = response['text']
+            elif isinstance(response, str):
+                prediction = response
+            else:
+                prediction = str(response)
+            
+            return prediction.strip()
+            
+        except Exception as e:
+            logger.error(f"Failed to generate prediction: {e}")
+            return "A"  # Fallback answer