judgeval 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

judgeval/judgment_client.py

@@ -15,7 +15,8 @@ from judgeval.scorers import (
     APIJudgmentScorer, 
     JudgevalScorer, 
     ClassifierScorer, 
-    ScorerWrapper
+    ScorerWrapper,
+    score,
 )
 from judgeval.evaluation_run import EvaluationRun
 from judgeval.run_evaluation import (
@@ -26,6 +27,7 @@ from judgeval.judges import JudgevalJudge
 from judgeval.constants import JUDGMENT_EVAL_FETCH_API_URL, JUDGMENT_EVAL_DELETE_API_URL, JUDGMENT_EVAL_DELETE_PROJECT_API_URL
 from judgeval.common.exceptions import JudgmentAPIError
 from pydantic import BaseModel
+from judgeval.rules import Rule
 
 class EvalRunRequestBody(BaseModel):
     eval_name: str
@@ -34,7 +36,7 @@ class EvalRunRequestBody(BaseModel):
 
 
 class JudgmentClient:
-    def __init__(self, judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"), organization_id: str = os.getenv("ORGANIZATION_ID")):
+    def __init__(self, judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"), organization_id: str = os.getenv("JUDGMENT_ORG_ID")):
         self.judgment_api_key = judgment_api_key
         self.organization_id = organization_id
         self.eval_dataset_client = EvalDatasetClient(judgment_api_key, organization_id)
@@ -58,17 +60,69 @@ class JudgmentClient:
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         override: bool = False,
-        use_judgment: bool = True
+        use_judgment: bool = True,
+        rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
         """
         Executes an evaluation of `Example`s using one or more `Scorer`s
+        
+        Args:
+            examples (List[Example]): The examples to evaluate
+            scorers (List[Union[ScorerWrapper, JudgevalScorer]]): A list of scorers to use for evaluation
+            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
+            project_name (str): The name of the project the evaluation results belong to
+            eval_run_name (str): A name for this evaluation run
+            override (bool): Whether to override an existing evaluation run with the same name
+            use_judgment (bool): Whether to use Judgment API for evaluation
+            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
+            
+        Returns:
+            List[ScoringResult]: The results of the evaluation
         """
         try:
             # Load appropriate implementations for all scorers
-            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = [
-                scorer.load_implementation(use_judgment=use_judgment) if isinstance(scorer, ScorerWrapper) else scorer
-                for scorer in scorers
-            ]
+            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = []
+            for scorer in scorers:
+                try:
+                    if isinstance(scorer, ScorerWrapper):
+                        loaded_scorers.append(scorer.load_implementation(use_judgment=use_judgment))
+                    else:
+                        loaded_scorers.append(scorer)
+                except Exception as e:
+                    raise ValueError(f"Failed to load implementation for scorer {scorer}: {str(e)}")
+
+            # Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
+            if rules and any(isinstance(scorer, JudgevalScorer) for scorer in loaded_scorers):
+                raise ValueError("Cannot use Judgeval scorers (only API scorers) when using rules. Please either remove rules or use only APIJudgmentScorer types.")
+
+            # Convert ScorerWrapper in rules to their implementations
+            loaded_rules = None
+            if rules:
+                loaded_rules = []
+                for rule in rules:
+                    try:
+                        processed_conditions = []
+                        for condition in rule.conditions:
+                            # Convert metric if it's a ScorerWrapper
+                            if isinstance(condition.metric, ScorerWrapper):
+                                try:
+                                    condition_copy = condition.model_copy()
+                                    condition_copy.metric = condition.metric.load_implementation(use_judgment=use_judgment)
+                                    processed_conditions.append(condition_copy)
+                                except Exception as e:
+                                    raise ValueError(f"Failed to convert ScorerWrapper to implementation in rule '{rule.name}', condition metric '{condition.metric}': {str(e)}")
+                            else:
+                                processed_conditions.append(condition)
+                        
+                        # Create new rule with processed conditions
+                        new_rule = rule.model_copy()
+                        new_rule.conditions = processed_conditions
+                        loaded_rules.append(new_rule)
+                    except Exception as e:
+                        raise ValueError(f"Failed to process rule '{rule.name}': {str(e)}")
 
             eval = EvaluationRun(
                 log_results=log_results,
@@ -79,12 +133,15 @@ class JudgmentClient:
                 model=model,
                 aggregator=aggregator,
                 metadata=metadata,
-                judgment_api_key=self.judgment_api_key, 
+                judgment_api_key=self.judgment_api_key,
+                rules=loaded_rules,
                 organization_id=self.organization_id
             )
             return run_eval(eval, override)
         except ValueError as e:
             raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}")
+        except Exception as e:
+            raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
     
     def evaluate_dataset(
         self, 
@@ -96,17 +153,68 @@ class JudgmentClient:
         project_name: str = "",
         eval_run_name: str = "",
         log_results: bool = False,
-        use_judgment: bool = True
+        use_judgment: bool = True,
+        rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
         """
         Executes an evaluation of a `EvalDataset` using one or more `Scorer`s
+        
+        Args:
+            dataset (EvalDataset): The dataset containing examples to evaluate
+            scorers (List[Union[ScorerWrapper, JudgevalScorer]]): A list of scorers to use for evaluation
+            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            project_name (str): The name of the project the evaluation results belong to
+            eval_run_name (str): A name for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
+            use_judgment (bool): Whether to use Judgment API for evaluation
+            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
+            
+        Returns:
+            List[ScoringResult]: The results of the evaluation
         """
         try:
             # Load appropriate implementations for all scorers
-            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = [
-                scorer.load_implementation(use_judgment=use_judgment) if isinstance(scorer, ScorerWrapper) else scorer
-                for scorer in scorers
-            ]
+            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = []
+            for scorer in scorers:
+                try:
+                    if isinstance(scorer, ScorerWrapper):
+                        loaded_scorers.append(scorer.load_implementation(use_judgment=use_judgment))
+                    else:
+                        loaded_scorers.append(scorer)
+                except Exception as e:
+                    raise ValueError(f"Failed to load implementation for scorer {scorer}: {str(e)}")
+
+            # Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
+            if rules and any(isinstance(scorer, JudgevalScorer) for scorer in loaded_scorers):
+                raise ValueError("Cannot use Judgeval scorers (only API scorers) when using rules. Please either remove rules or use only APIJudgmentScorer types.")
+
+            # Convert ScorerWrapper in rules to their implementations
+            loaded_rules = None
+            if rules:
+                loaded_rules = []
+                for rule in rules:
+                    try:
+                        processed_conditions = []
+                        for condition in rule.conditions:
+                            # Convert metric if it's a ScorerWrapper
+                            if isinstance(condition.metric, ScorerWrapper):
+                                try:
+                                    condition_copy = condition.model_copy()
+                                    condition_copy.metric = condition.metric.load_implementation(use_judgment=use_judgment)
+                                    processed_conditions.append(condition_copy)
+                                except Exception as e:
+                                    raise ValueError(f"Failed to convert ScorerWrapper to implementation in rule '{rule.name}', condition metric '{condition.metric}': {str(e)}")
+                            else:
+                                processed_conditions.append(condition)
+                        
+                        # Create new rule with processed conditions
+                        new_rule = rule.model_copy()
+                        new_rule.conditions = processed_conditions
+                        loaded_rules.append(new_rule)
+                    except Exception as e:
+                        raise ValueError(f"Failed to process rule '{rule.name}': {str(e)}")
 
             evaluation_run = EvaluationRun(
                 log_results=log_results,
@@ -118,11 +226,14 @@ class JudgmentClient:
                 aggregator=aggregator,
                 metadata=metadata,
                 judgment_api_key=self.judgment_api_key,
+                rules=loaded_rules,
                 organization_id=self.organization_id
             )
             return run_eval(evaluation_run)
         except ValueError as e:
             raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}")
+        except Exception as e:
+            raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
 
     def create_dataset(self) -> EvalDataset:
         return self.eval_dataset_client.create_dataset()
@@ -364,9 +475,22 @@ class JudgmentClient:
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         override: bool = False,
+        rules: Optional[List[Rule]] = None
     ) -> None:
         """
         Asserts a test by running the evaluation and checking the results for success
+        
+        Args:
+            examples (List[Example]): The examples to evaluate
+            scorers (List[Union[APIJudgmentScorer, JudgevalScorer]]): A list of scorers to use for evaluation
+            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
+            project_name (str): The name of the project the evaluation results belong to
+            eval_run_name (str): A name for this evaluation run
+            override (bool): Whether to override an existing evaluation run with the same name
+            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
         """
         results = self.run_evaluation(
             examples=examples,
@@ -377,7 +501,8 @@ class JudgmentClient:
             log_results=log_results,
             project_name=project_name,
             eval_run_name=eval_run_name,
-            override=override
+            override=override,
+            rules=rules
         )
         
         assert_test(results)

judgeval/rules.py

@@ -0,0 +1,384 @@
+"""
+Rules system for Judgeval that enables alerts based on metric thresholds.
+"""
+
+from typing import Dict, List, Optional, Union, Any, Set, Tuple
+from pydantic import BaseModel, Field, field_validator, ConfigDict
+from enum import Enum
+from datetime import datetime
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+import time
+import uuid  # Add import for uuid module
+
+from judgeval.scorers import APIJudgmentScorer, JudgevalScorer
+from judgeval.scorers.judgeval_scorers import ScorerWrapper  # Import from the correct module
+
+class AlertStatus(str, Enum):
+    """Status of an alert evaluation."""
+    TRIGGERED = "triggered"
+    NOT_TRIGGERED = "not_triggered"
+
+class Operator(str, Enum):
+    """Comparison operators for conditions."""
+    GT = ">"
+    GTE = ">="
+    LT = "<"
+    LTE = "<="
+    EQ = "=="
+    NEQ = "!="
+
+class Condition(BaseModel):
+    """
+    A single metric condition.
+    
+    Example:
+        {
+            "metric": FaithfulnessScorer(threshold=0.7)  # Must be a scorer object: APIJudgmentScorer, JudgevalScorer, or ScorerWrapper
+            "operator": ">=",
+            "threshold": 0.7
+        }
+    """
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    
+    metric: Union[APIJudgmentScorer, JudgevalScorer, ScorerWrapper]  
+    operator: Operator
+    threshold: float
+
+    @property
+    def metric_name(self) -> str:
+        """Get the name of the metric for lookups in scores dictionary."""
+        if isinstance(self.metric, ScorerWrapper):
+            # Handle ScorerWrapper case specifically
+            return self.metric.scorer.score_type if hasattr(self.metric.scorer, 'score_type') else str(self.metric.scorer)
+        elif hasattr(self.metric, 'score_type'):
+            # Handle APIJudgmentScorer and JudgevalScorer which have score_type
+            return self.metric.score_type
+        elif hasattr(self.metric, '__name__'):
+            # Handle cases where metric has a __name__ attribute
+            return self.metric.__name__
+        # Fallback to string representation
+        return str(self.metric)
+
+    def evaluate(self, value: float) -> bool:
+        """Evaluate this condition against a value."""
+        if self.operator == Operator.GT:
+            return value > self.threshold
+        elif self.operator == Operator.GTE:
+            return value >= self.threshold
+        elif self.operator == Operator.LT:
+            return value < self.threshold
+        elif self.operator == Operator.LTE:
+            return value <= self.threshold
+        elif self.operator == Operator.EQ:
+            return value == self.threshold
+        elif self.operator == Operator.NEQ:
+            return value != self.threshold
+        else:
+            raise ValueError(f"Unknown operator: {self.operator}")
+
+class Rule(BaseModel):
+    """
+    Configuration for a single rule.
+    
+    Example:
+        {
+            "rule_id": "123e4567-e89b-12d3-a456-426614174000",
+            "name": "Quality Check",
+            "description": "Check if quality metrics meet thresholds",
+            "conditions": [
+                {"metric": FaithfulnessScorer(threshold=0.7), "operator": ">=", "threshold": 0.7},
+                {"metric": AnswerRelevancyScorer(threshold=0.8), "operator": ">=", "threshold": 0.8}
+            ],
+            "combine_type": "all"  # "all" or "any"
+        }
+    """
+    rule_id: str = Field(default_factory=lambda: str(uuid.uuid4()))  # Random UUID string as default value
+    name: str
+    description: Optional[str] = None
+    conditions: List[Condition]
+    combine_type: str = Field(..., pattern="^(all|any)$")  # all = AND, any = OR
+
+    def model_dump(self, **kwargs):
+        """
+        Custom serialization that properly handles condition serialization.
+        """
+        data = super().model_dump(**kwargs)
+        
+        # Special handling for conditions with complex metric objects
+        if "conditions" in data:
+            for i, condition in enumerate(data["conditions"]):
+                if "metric" in condition:
+                    # Get the actual metric object
+                    metric_obj = self.conditions[i].metric
+                    
+                    # Create standardized metric representation needed by server API
+                    metric_data = {
+                        "score_type": "",
+                        "threshold": 0.0
+                    }
+                    
+                    # First try to use object's own serialization methods
+                    if hasattr(metric_obj, "to_dict"):
+                        orig_data = metric_obj.to_dict()
+                        # Copy any existing fields
+                        for key, value in orig_data.items():
+                            metric_data[key] = value
+                    elif hasattr(metric_obj, "model_dump"):
+                        orig_data = metric_obj.model_dump()
+                        # Copy any existing fields
+                        for key, value in orig_data.items():
+                            metric_data[key] = value
+                    
+                    # If we already have data from original serialization methods but missing required fields
+                    if 'name' in metric_data and 'score_type' not in metric_data:
+                        metric_data['score_type'] = metric_data['name']
+                        
+                    # Ensure required fields have values by checking various sources
+                    if not metric_data['score_type']:
+                        # Try to get score_type from different possible attributes
+                        if hasattr(metric_obj, 'score_type'):
+                            metric_data['score_type'] = metric_obj.score_type
+                        elif hasattr(metric_obj, 'name'):
+                            metric_data['score_type'] = metric_obj.name
+                        else:
+                            # Last resort: use string representation
+                            metric_data['score_type'] = str(metric_obj)
+                    
+                    # Make sure threshold is set
+                    if not metric_data.get('threshold') and metric_data.get('threshold') != 0.0:
+                        if hasattr(metric_obj, 'threshold'):
+                            metric_data['threshold'] = metric_obj.threshold
+                        else:
+                            # Use condition threshold if metric doesn't have one
+                            metric_data['threshold'] = self.conditions[i].threshold
+                    
+                    # Update the condition with our properly serialized metric
+                    condition["metric"] = metric_data
+        
+        return data
+
+    @field_validator('conditions')
+    def validate_conditions_not_empty(cls, v):
+        if not v:
+            raise ValueError("Conditions list cannot be empty")
+        return v
+
+    @field_validator('combine_type')
+    def validate_combine_type(cls, v):
+        if v not in ["all", "any"]:
+            raise ValueError(f"combine_type must be 'all' or 'any', got: {v}")
+        return v
+
+
+class AlertResult(BaseModel):
+    """
+    Result of evaluating a rule.
+    
+    Example:
+        {
+            "status": "triggered",
+            "rule_name": "Quality Check",
+            "conditions_result": [
+                {"metric": "faithfulness", "value": 0.6, "threshold": 0.7, "passed": False},
+                {"metric": "relevancy", "value": 0.9, "threshold": 0.8, "passed": True}
+            ],
+            "rule_id": "123e4567-e89b-12d3-a456-426614174000",
+            "metadata": {
+                "example_id": "example_123",
+                "timestamp": "20240321_123456"
+            }
+        }
+    """
+    status: AlertStatus
+    rule_id: Optional[str] = None  # The unique identifier of the rule
+    rule_name: str
+    conditions_result: List[Dict[str, Any]]
+    metadata: Dict[str, Any] = {}
+    
+    @property
+    def example_id(self) -> Optional[str]:
+        """Get example_id from metadata for backward compatibility"""
+        return self.metadata.get("example_id")
+        
+    @property
+    def timestamp(self) -> Optional[str]:
+        """Get timestamp from metadata for backward compatibility"""
+        return self.metadata.get("timestamp")
+
+class RulesEngine:
+    """
+    Engine for evaluating rules and managing alerts.
+    
+    Example usage:
+        rules = {
+            "quality_check": Rule(
+                name="Quality Check",
+                conditions=[
+                    Condition(metric=FaithfulnessScorer(threshold=0.7), operator=">=", threshold=0.7),
+                    Condition(metric=AnswerRelevancyScorer(threshold=0.8), operator=">=", threshold=0.8)
+                ],
+                combine_type="all"
+            )
+        }
+        
+        engine = RulesEngine(rules)
+        scores = {"faithfulness": 0.8, "relevancy": 0.9}
+        alerts = engine.evaluate_rules(scores, example_metadata={
+            "example_id": "example_123",
+            "timestamp": "20240321_123456"
+        })
+    """
+    
+    def __init__(self, rules: Dict[str, Rule]):
+        """
+        Initialize the RulesEngine with rules.
+        
+        Args:
+            rules: Dictionary mapping rule IDs to rule configurations
+        """
+        self.rules = rules
+    
+    def evaluate_rules(self, scores: Dict[str, float], example_metadata: Optional[Dict[str, Any]] = None) -> Dict[str, AlertResult]:
+        """
+        Evaluate all rules against a set of scores.
+        Returns mapping of rule IDs to their alert results.
+        
+        Args:
+            scores: Dictionary of metric names to their score values
+            example_metadata: Optional dictionary containing example metadata (example_id, timestamp)
+        """
+        results = {}
+
+        for rule_id, rule in self.rules.items():
+            # Evaluate each condition
+            condition_results = []
+            passed_conditions = []
+            
+            for condition in rule.conditions:
+                # Get the metric name for lookup
+                metric_name = condition.metric_name
+                value = scores.get(metric_name)
+                if value is None:
+                    # Skip this condition instead of evaluating it as false
+                    condition_results.append({
+                        "metric": metric_name,
+                        "value": None,
+                        "threshold": condition.threshold,
+                        "operator": condition.operator,
+                        "passed": None,  # Using None to indicate the condition was skipped
+                        "skipped": True  # Add a flag to indicate this condition was skipped
+                    })
+                    continue  # Skip adding to passed_conditions
+                else:
+                    passed = condition.evaluate(value)
+                    condition_results.append({
+                        "metric": metric_name,
+                        "value": value,
+                        "threshold": condition.threshold,
+                        "operator": condition.operator,
+                        "passed": passed,
+                        "skipped": False  # Indicate this condition was evaluated
+                    })
+                    passed_conditions.append(passed)
+            
+            # Determine if alert should trigger - only consider conditions that weren't skipped
+            if not passed_conditions:
+                # If all conditions were skipped, the rule doesn't trigger
+                triggered = False
+            else:
+                triggered = all(passed_conditions) if rule.combine_type == "all" else any(passed_conditions)
+            
+            # Create alert result with example metadata
+            alert_result = AlertResult(
+                status=AlertStatus.TRIGGERED if triggered else AlertStatus.NOT_TRIGGERED,
+                rule_id=rule.rule_id,  # Include the rule's unique identifier
+                rule_name=rule.name,
+                conditions_result=condition_results
+            )
+            
+            # Add example metadata if provided
+            if example_metadata:
+                if "example_id" in example_metadata:
+                    alert_result.metadata["example_id"] = example_metadata["example_id"]
+                if "timestamp" in example_metadata:
+                    alert_result.metadata["timestamp"] = example_metadata["timestamp"]
+            
+            results[rule_id] = alert_result
+            
+        return results
+    
+    async def evaluate_rules_parallel(self, 
+                               example_scores: Dict[str, Dict[str, float]], 
+                               example_metadata: Dict[str, Dict[str, Any]],
+                               max_concurrent: int = 100) -> Dict[str, Dict[str, AlertResult]]:
+        """
+        Evaluate all rules against multiple examples in parallel.
+        
+        Args:
+            example_scores: Dictionary mapping example_ids to their score dictionaries
+            example_metadata: Dictionary mapping example_ids to their metadata
+            max_concurrent: Maximum number of concurrent evaluations
+            
+        Returns:
+            Dictionary mapping example_ids to dictionaries of rule_ids and their alert results
+        """
+        # Create semaphore to limit concurrent executions
+        semaphore = asyncio.Semaphore(max_concurrent)
+        results = {}
+        tasks = []
+        
+        # Create a task for each example
+        for example_id, scores in example_scores.items():
+            metadata = example_metadata.get(example_id, {})
+            task = self._evaluate_with_semaphore(
+                semaphore=semaphore,
+                example_id=example_id,
+                scores=scores,
+                metadata=metadata
+            )
+            tasks.append(task)
+        
+        # Run all tasks and collect results
+        example_results = await asyncio.gather(*tasks)
+        
+        # Organize results by example_id
+        for example_id, result in example_results:
+            results[example_id] = result
+            
+        return results
+    
+    async def _evaluate_with_semaphore(self, 
+                                semaphore: asyncio.Semaphore, 
+                                example_id: str, 
+                                scores: Dict[str, float], 
+                                metadata: Dict[str, Any]) -> Tuple[str, Dict[str, AlertResult]]:
+        """
+        Helper method to evaluate rules for an example with semaphore control.
+        
+        Args:
+            semaphore: Semaphore to control concurrency
+            example_id: ID of the example being evaluated
+            scores: Dictionary of scores for this example
+            metadata: Metadata for this example
+            
+        Returns:
+            Tuple of (example_id, rule_results)
+        """
+        async with semaphore:
+            # Run the evaluation in a thread pool to avoid blocking the event loop
+            # for CPU-bound operations
+            with ThreadPoolExecutor() as executor:
+                start_time = time.perf_counter()
+                rule_results = await asyncio.get_event_loop().run_in_executor(
+                    executor, 
+                    self.evaluate_rules,
+                    scores,
+                    metadata
+                )
+                end_time = time.perf_counter()
+                
+                # Could log performance metrics here if needed
+                # debug(f"Rule evaluation for example {example_id} took {end_time - start_time:.4f} seconds")
+                
+                return (example_id, rule_results)