judgeval 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

judgeval/data/example.py

@@ -5,7 +5,7 @@ Classes for representing examples in a dataset.
 
 from typing import TypeVar, Optional, Any, Dict, List
 from uuid import uuid4
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from enum import Enum
 from datetime import datetime
 import time
@@ -40,6 +40,13 @@ class Example(BaseModel):
     timestamp: Optional[str] = None
     trace_id: Optional[str] = None
 
+    @field_validator('input', 'actual_output', mode='before')
+    def convert_to_str(cls, value):
+        try:
+            return str(value)
+        except Exception:
+            return repr(value)
+    
     def __init__(self, **data):
         if 'example_id' not in data:
             data['example_id'] = str(uuid4())

judgeval/evaluation_run.py

@@ -6,6 +6,7 @@ from judgeval.scorers import JudgevalScorer, APIJudgmentScorer
 from judgeval.constants import ACCEPTABLE_MODELS
 from judgeval.common.logger import debug, error
 from judgeval.judges import JudgevalJudge
+from judgeval.rules import Rule
 
 class EvaluationRun(BaseModel):
     """
@@ -20,10 +21,12 @@ class EvaluationRun(BaseModel):
         aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
         metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run, e.g. comments, dataset name, purpose, etc.
         judgment_api_key (Optional[str]): The API key for running evaluations on the Judgment API
+        rules (Optional[List[Rule]]): Rules to evaluate against scoring results
     """
 
     # The user will specify whether they want log_results when they call run_eval
     log_results: bool = False  # NOTE: log_results has to be set first because it is used to validate project_name and eval_name
+    organization_id: Optional[str] = None
     project_name: Optional[str] = None
     eval_name: Optional[str] = None
     examples: List[Example]
@@ -34,6 +37,7 @@ class EvaluationRun(BaseModel):
     # API Key will be "" until user calls client.run_eval(), then API Key will be set
     judgment_api_key: Optional[str] = ""
     override: Optional[bool] = False
+    rules: Optional[List[Rule]] = None
     
     def model_dump(self, **kwargs):
         data = super().model_dump(**kwargs)
@@ -44,6 +48,11 @@ class EvaluationRun(BaseModel):
             else {"score_type": scorer.score_type, "threshold": scorer.threshold}
             for scorer in self.scorers
         ]
+
+        if self.rules:
+            # Process rules to ensure proper serialization
+              data["rules"] = [rule.model_dump() for rule in self.rules]
+            
         return data
 
     @field_validator('log_results', mode='before')

judgeval/judges/together_judge.py

@@ -14,7 +14,7 @@ BASE_CONVERSATION = [
 ]
 
 class TogetherJudge(JudgevalJudge):
-    def __init__(self, model: str = "QWEN", **kwargs):
+    def __init__(self, model: str = "Qwen/Qwen2.5-72B-Instruct-Turbo", **kwargs):
         debug(f"Initializing TogetherJudge with model={model}")
         self.model = model
         self.kwargs = kwargs

judgeval/judges/utils.py

@@ -39,7 +39,7 @@ def create_judge(
                     Please either set the `use_judgment` flag to True or use 
                     non-Judgment models."""
                 )
-            if m not in LITELLM_SUPPORTED_MODELS and m not in TOGETHER_SUPPORTED_MODELS:
+            if m not in ACCEPTABLE_MODELS:
                 raise InvalidJudgeModelError(f"Invalid judge model chosen: {m}")
         return MixtureOfJudges(models=model), True
     # If model is a string, check that it corresponds to a valid model

judgeval/judgment_client.py

@@ -15,7 +15,8 @@ from judgeval.scorers import (
     APIJudgmentScorer, 
     JudgevalScorer, 
     ClassifierScorer, 
-    ScorerWrapper
+    ScorerWrapper,
+    score,
 )
 from judgeval.evaluation_run import EvaluationRun
 from judgeval.run_evaluation import (
@@ -26,6 +27,7 @@ from judgeval.judges import JudgevalJudge
 from judgeval.constants import JUDGMENT_EVAL_FETCH_API_URL, JUDGMENT_EVAL_DELETE_API_URL, JUDGMENT_EVAL_DELETE_PROJECT_API_URL
 from judgeval.common.exceptions import JudgmentAPIError
 from pydantic import BaseModel
+from judgeval.rules import Rule
 
 class EvalRunRequestBody(BaseModel):
     eval_name: str
@@ -34,9 +36,10 @@ class EvalRunRequestBody(BaseModel):
 
 
 class JudgmentClient:
-    def __init__(self, judgment_api_key: str = os.getenv("JUDGMENT_API_KEY")):
+    def __init__(self, judgment_api_key: str = os.getenv("JUDGMENT_API_KEY"), organization_id: str = os.getenv("JUDGMENT_ORG_ID")):
         self.judgment_api_key = judgment_api_key
-        self.eval_dataset_client = EvalDatasetClient(judgment_api_key)
+        self.organization_id = organization_id
+        self.eval_dataset_client = EvalDatasetClient(judgment_api_key, organization_id)
         
         # Verify API key is valid
         result, response = self._validate_api_key()
@@ -57,17 +60,69 @@ class JudgmentClient:
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         override: bool = False,
-        use_judgment: bool = True
+        use_judgment: bool = True,
+        rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
         """
         Executes an evaluation of `Example`s using one or more `Scorer`s
+        
+        Args:
+            examples (List[Example]): The examples to evaluate
+            scorers (List[Union[ScorerWrapper, JudgevalScorer]]): A list of scorers to use for evaluation
+            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
+            project_name (str): The name of the project the evaluation results belong to
+            eval_run_name (str): A name for this evaluation run
+            override (bool): Whether to override an existing evaluation run with the same name
+            use_judgment (bool): Whether to use Judgment API for evaluation
+            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
+            
+        Returns:
+            List[ScoringResult]: The results of the evaluation
         """
         try:
             # Load appropriate implementations for all scorers
-            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = [
-                scorer.load_implementation(use_judgment=use_judgment) if isinstance(scorer, ScorerWrapper) else scorer
-                for scorer in scorers
-            ]
+            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = []
+            for scorer in scorers:
+                try:
+                    if isinstance(scorer, ScorerWrapper):
+                        loaded_scorers.append(scorer.load_implementation(use_judgment=use_judgment))
+                    else:
+                        loaded_scorers.append(scorer)
+                except Exception as e:
+                    raise ValueError(f"Failed to load implementation for scorer {scorer}: {str(e)}")
+
+            # Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
+            if rules and any(isinstance(scorer, JudgevalScorer) for scorer in loaded_scorers):
+                raise ValueError("Cannot use Judgeval scorers (only API scorers) when using rules. Please either remove rules or use only APIJudgmentScorer types.")
+
+            # Convert ScorerWrapper in rules to their implementations
+            loaded_rules = None
+            if rules:
+                loaded_rules = []
+                for rule in rules:
+                    try:
+                        processed_conditions = []
+                        for condition in rule.conditions:
+                            # Convert metric if it's a ScorerWrapper
+                            if isinstance(condition.metric, ScorerWrapper):
+                                try:
+                                    condition_copy = condition.model_copy()
+                                    condition_copy.metric = condition.metric.load_implementation(use_judgment=use_judgment)
+                                    processed_conditions.append(condition_copy)
+                                except Exception as e:
+                                    raise ValueError(f"Failed to convert ScorerWrapper to implementation in rule '{rule.name}', condition metric '{condition.metric}': {str(e)}")
+                            else:
+                                processed_conditions.append(condition)
+                        
+                        # Create new rule with processed conditions
+                        new_rule = rule.model_copy()
+                        new_rule.conditions = processed_conditions
+                        loaded_rules.append(new_rule)
+                    except Exception as e:
+                        raise ValueError(f"Failed to process rule '{rule.name}': {str(e)}")
 
             eval = EvaluationRun(
                 log_results=log_results,
@@ -78,11 +133,15 @@ class JudgmentClient:
                 model=model,
                 aggregator=aggregator,
                 metadata=metadata,
-                judgment_api_key=self.judgment_api_key
+                judgment_api_key=self.judgment_api_key,
+                rules=loaded_rules,
+                organization_id=self.organization_id
             )
             return run_eval(eval, override)
         except ValueError as e:
             raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}")
+        except Exception as e:
+            raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
     
     def evaluate_dataset(
         self, 
@@ -94,17 +153,68 @@ class JudgmentClient:
         project_name: str = "",
         eval_run_name: str = "",
         log_results: bool = False,
-        use_judgment: bool = True
+        use_judgment: bool = True,
+        rules: Optional[List[Rule]] = None
     ) -> List[ScoringResult]:
         """
         Executes an evaluation of a `EvalDataset` using one or more `Scorer`s
+        
+        Args:
+            dataset (EvalDataset): The dataset containing examples to evaluate
+            scorers (List[Union[ScorerWrapper, JudgevalScorer]]): A list of scorers to use for evaluation
+            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            project_name (str): The name of the project the evaluation results belong to
+            eval_run_name (str): A name for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
+            use_judgment (bool): Whether to use Judgment API for evaluation
+            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
+            
+        Returns:
+            List[ScoringResult]: The results of the evaluation
         """
         try:
             # Load appropriate implementations for all scorers
-            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = [
-                scorer.load_implementation(use_judgment=use_judgment) if isinstance(scorer, ScorerWrapper) else scorer
-                for scorer in scorers
-            ]
+            loaded_scorers: List[Union[JudgevalScorer, APIJudgmentScorer]] = []
+            for scorer in scorers:
+                try:
+                    if isinstance(scorer, ScorerWrapper):
+                        loaded_scorers.append(scorer.load_implementation(use_judgment=use_judgment))
+                    else:
+                        loaded_scorers.append(scorer)
+                except Exception as e:
+                    raise ValueError(f"Failed to load implementation for scorer {scorer}: {str(e)}")
+
+            # Prevent using JudgevalScorer with rules - only APIJudgmentScorer allowed with rules
+            if rules and any(isinstance(scorer, JudgevalScorer) for scorer in loaded_scorers):
+                raise ValueError("Cannot use Judgeval scorers (only API scorers) when using rules. Please either remove rules or use only APIJudgmentScorer types.")
+
+            # Convert ScorerWrapper in rules to their implementations
+            loaded_rules = None
+            if rules:
+                loaded_rules = []
+                for rule in rules:
+                    try:
+                        processed_conditions = []
+                        for condition in rule.conditions:
+                            # Convert metric if it's a ScorerWrapper
+                            if isinstance(condition.metric, ScorerWrapper):
+                                try:
+                                    condition_copy = condition.model_copy()
+                                    condition_copy.metric = condition.metric.load_implementation(use_judgment=use_judgment)
+                                    processed_conditions.append(condition_copy)
+                                except Exception as e:
+                                    raise ValueError(f"Failed to convert ScorerWrapper to implementation in rule '{rule.name}', condition metric '{condition.metric}': {str(e)}")
+                            else:
+                                processed_conditions.append(condition)
+                        
+                        # Create new rule with processed conditions
+                        new_rule = rule.model_copy()
+                        new_rule.conditions = processed_conditions
+                        loaded_rules.append(new_rule)
+                    except Exception as e:
+                        raise ValueError(f"Failed to process rule '{rule.name}': {str(e)}")
 
             evaluation_run = EvaluationRun(
                 log_results=log_results,
@@ -115,11 +225,15 @@ class JudgmentClient:
                 model=model,
                 aggregator=aggregator,
                 metadata=metadata,
-                judgment_api_key=self.judgment_api_key
+                judgment_api_key=self.judgment_api_key,
+                rules=loaded_rules,
+                organization_id=self.organization_id
             )
             return run_eval(evaluation_run)
         except ValueError as e:
             raise ValueError(f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}")
+        except Exception as e:
+            raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
 
     def create_dataset(self) -> EvalDataset:
         return self.eval_dataset_client.create_dataset()
@@ -189,9 +303,11 @@ class JudgmentClient:
         eval_run = requests.post(JUDGMENT_EVAL_FETCH_API_URL,
                                  headers={
                                     "Content-Type": "application/json",
-                                    "Authorization": f"Bearer {self.judgment_api_key}"
+                                    "Authorization": f"Bearer {self.judgment_api_key}",
+                                    "X-Organization-Id": self.organization_id
                                  },
-                                 json=eval_run_request_body.model_dump())
+                                 json=eval_run_request_body.model_dump(),
+                                 verify=False)
         if eval_run.status_code != requests.codes.ok:
             raise ValueError(f"Error fetching eval results: {eval_run.json()}")
 
@@ -222,7 +338,8 @@ class JudgmentClient:
                         json=eval_run_request_body.model_dump(),
                         headers={
                             "Content-Type": "application/json",
-                            "Authorization": f"Bearer {self.judgment_api_key}"
+                            "Authorization": f"Bearer {self.judgment_api_key}",
+                            "X-Organization-Id": self.organization_id
                         })
         if response.status_code != requests.codes.ok:
             raise ValueError(f"Error deleting eval results: {response.json()}")
@@ -241,11 +358,12 @@ class JudgmentClient:
         response = requests.delete(JUDGMENT_EVAL_DELETE_PROJECT_API_URL, 
                         json={
                             "project_name": project_name,
-                            "judgment_api_key": self.judgment_api_key
+                            "judgment_api_key": self.judgment_api_key,
                         },
                         headers={
                             "Content-Type": "application/json",
-                            "Authorization": f"Bearer {self.judgment_api_key}"
+                            "Authorization": f"Bearer {self.judgment_api_key}",
+                            "X-Organization-Id": self.organization_id
                         })
         if response.status_code != requests.codes.ok:
             raise ValueError(f"Error deleting eval results: {response.json()}")
@@ -261,7 +379,8 @@ class JudgmentClient:
                 "Content-Type": "application/json",
                 "Authorization": f"Bearer {self.judgment_api_key}",
             },
-            json={}  # Empty body now
+            json={},  # Empty body now
+            verify=False
         )
         if response.status_code == 200:
             return True, response.json()
@@ -283,7 +402,6 @@ class JudgmentClient:
         """
         request_body = {
             "slug": slug,
-            # "judgment_api_key": self.judgment_api_key
         }
         
         response = requests.post(
@@ -291,8 +409,10 @@ class JudgmentClient:
             json=request_body,
             headers={
                 "Content-Type": "application/json",
-                "Authorization": f"Bearer {self.judgment_api_key}"
-            }
+                "Authorization": f"Bearer {self.judgment_api_key}",
+                "X-Organization-Id": self.organization_id
+            },
+            verify=False
         )
         
         if response.status_code == 500:
@@ -325,7 +445,6 @@ class JudgmentClient:
             "name": scorer.name,
             "conversation": scorer.conversation,
             "options": scorer.options,
-            # "judgment_api_key": self.judgment_api_key, 
             "slug": slug
         }
         
@@ -334,8 +453,10 @@ class JudgmentClient:
             json=request_body,
             headers={
                 "Content-Type": "application/json",
-                "Authorization": f"Bearer {self.judgment_api_key}"
-            }
+                "Authorization": f"Bearer {self.judgment_api_key}",
+                "X-Organization-Id": self.organization_id
+            },
+            verify=False
         )
         
         if response.status_code == 500:
@@ -358,9 +479,22 @@ class JudgmentClient:
         project_name: str = "default_project",
         eval_run_name: str = "default_eval_run",
         override: bool = False,
+        rules: Optional[List[Rule]] = None
     ) -> None:
         """
         Asserts a test by running the evaluation and checking the results for success
+        
+        Args:
+            examples (List[Example]): The examples to evaluate
+            scorers (List[Union[APIJudgmentScorer, JudgevalScorer]]): A list of scorers to use for evaluation
+            model (Union[str, List[str], JudgevalJudge]): The model used as a judge when using LLM as a Judge
+            aggregator (Optional[str]): The aggregator to use for evaluation if using Mixture of Judges
+            metadata (Optional[Dict[str, Any]]): Additional metadata to include for this evaluation run
+            log_results (bool): Whether to log the results to the Judgment API
+            project_name (str): The name of the project the evaluation results belong to
+            eval_run_name (str): A name for this evaluation run
+            override (bool): Whether to override an existing evaluation run with the same name
+            rules (Optional[List[Rule]]): Rules to evaluate against scoring results
         """
         results = self.run_evaluation(
             examples=examples,
@@ -371,7 +505,8 @@ class JudgmentClient:
             log_results=log_results,
             project_name=project_name,
             eval_run_name=eval_run_name,
-            override=override
+            override=override,
+            rules=rules
         )
         
         assert_test(results)