judgeval 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

judgeval/judges/__init__.py

@@ -1,6 +1,6 @@
 from judgeval.judges.base_judge import JudgevalJudge
 from judgeval.judges.litellm_judge import LiteLLMJudge
 from judgeval.judges.together_judge import TogetherJudge
-from judgeval.judges.mixture_of_judges import MixtureOfJudges
 
-__all__ = ["JudgevalJudge", "LiteLLMJudge", "TogetherJudge", "MixtureOfJudges"]
+
+__all__ = ["JudgevalJudge", "LiteLLMJudge", "TogetherJudge"]

judgeval/judges/litellm_judge.py

@@ -1,21 +1,77 @@
 import pydantic
-from typing import List, Union, Mapping
+from typing import Dict, List, Union, Mapping, Any
 
+from judgeval.constants import ACCEPTABLE_MODELS
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import (
-    afetch_litellm_api_response,
-    fetch_litellm_api_response,
-)
-from judgeval.common.logger import judgeval_logger
-from judgeval.constants import DEFAULT_GPT_MODEL
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+
+try:
+    import litellm
+except ImportError:
+    raise ImportError(
+        "Litellm is not installed and required for the litellm judge. Please install it with `pip install litellm`."
+    )
+
+
+def fetch_litellm_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if response_format is not None:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from litellm")
+    return content
+
+
+async def afetch_litellm_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if model not in ACCEPTABLE_MODELS:
+        raise ValueError(
+            f"Model {model} is not in the list of supported models: {ACCEPTABLE_MODELS}."
+        )
+
+    if response_format is not None:
+        response = await litellm.acompletion(
+            model=model, messages=messages, response_format=response_format
+        )
+    else:
+        response = await litellm.acompletion(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from litellm")
+    return content
+
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
-]  # for string inputs, we need to add the user query to a base conversation, since LiteLLM only accepts a list of dictionaries as a chat history
+]
 
 
 class LiteLLMJudge(JudgevalJudge):
-    def __init__(self, model: str = DEFAULT_GPT_MODEL, **kwargs):
+    def __init__(self, model: str = JUDGMENT_DEFAULT_GPT_MODEL, **kwargs):
         self.model = model
         self.kwargs = kwargs
         super().__init__(model_name=model)
@@ -25,17 +81,19 @@ class LiteLLMJudge(JudgevalJudge):
         input: Union[str, List[Mapping[str, str]]],
         schema: Union[pydantic.BaseModel, None] = None,
     ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
             return fetch_litellm_api_response(
-                model=self.model, messages=convo, response_format=schema
+                model=self.model, messages=convo, response_format=response_format
             )
         elif isinstance(input, list):
+            messages = [dict(msg) for msg in input]
             return fetch_litellm_api_response(
-                model=self.model, messages=input, response_format=schema
+                model=self.model, messages=messages, response_format=response_format
             )
         else:
-            judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError(
                 f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
             )
@@ -45,19 +103,21 @@ class LiteLLMJudge(JudgevalJudge):
         input: Union[str, List[Mapping[str, str]]],
         schema: Union[pydantic.BaseModel, None] = None,
     ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
             response = await afetch_litellm_api_response(
-                model=self.model, messages=convo, response_format=schema
+                model=self.model, messages=convo, response_format=response_format
             )
             return response
         elif isinstance(input, list):
+            messages = [dict(msg) for msg in input]
             response = await afetch_litellm_api_response(
-                model=self.model, messages=input, response_format=schema
+                model=self.model, messages=messages, response_format=response_format
             )
             return response
         else:
-            judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError(
                 f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
             )

judgeval/judges/together_judge.py

@@ -3,15 +3,77 @@ Implementation of using TogetherAI inference for judges.
 """
 
 from pydantic import BaseModel
-from typing import List, Union
-
+from typing import Dict, List, Union, Any, cast
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import (
-    fetch_together_api_response,
-    afetch_together_api_response,
+from judgeval.logger import judgeval_logger
+from judgeval.env import (
+    JUDGMENT_DEFAULT_TOGETHER_MODEL,
+    TOGETHERAI_API_KEY,
+    TOGETHER_API_KEY,
 )
-from judgeval.common.logger import judgeval_logger
-from judgeval.constants import DEFAULT_TOGETHER_MODEL
+
+together_api_key = TOGETHERAI_API_KEY or TOGETHER_API_KEY
+if together_api_key:
+    try:
+        from together import Together, AsyncTogether  # type: ignore[import-untyped]
+
+        together_client = Together(api_key=together_api_key)
+        async_together_client = AsyncTogether(api_key=together_api_key)
+    except Exception:
+        pass
+
+
+def fetch_together_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if response_format is not None:
+        response = together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from TogetherAI")
+    return cast(str, content)
+
+
+async def afetch_together_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if response_format is not None:
+        response = await async_together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = await async_together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from TogetherAI")
+    return cast(str, content)
+
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
@@ -19,46 +81,52 @@ BASE_CONVERSATION = [
 
 
 class TogetherJudge(JudgevalJudge):
-    def __init__(self, model: str = DEFAULT_TOGETHER_MODEL, **kwargs):
+    def __init__(self, model: str = JUDGMENT_DEFAULT_TOGETHER_MODEL, **kwargs):
         self.model = model
         self.kwargs = kwargs
         super().__init__(model_name=model)
 
-    # TODO: Fix cost for generate and a_generate
     def generate(
-        self, input: Union[str, List[dict]], schema: Union[BaseModel, None] = None
+        self,
+        input: Union[str, List[Dict[str, str]]],
+        schema: Union[BaseModel, None] = None,
     ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
             return fetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, convo, response_format=response_format
             )
         elif isinstance(input, list):
-            convo = input
+            messages = [dict(msg) for msg in input]
             return fetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, messages, response_format=response_format
             )
         else:
             judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
     async def a_generate(
-        self, input: Union[str, List[dict]], schema: Union[BaseModel, None] = None
+        self,
+        input: Union[str, List[Dict[str, str]]],
+        schema: Union[BaseModel, None] = None,
     ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
             res = await afetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, convo, response_format=response_format
             )
             return res
         elif isinstance(input, list):
-            convo = input
+            messages = [dict(msg) for msg in input]
             res = await afetch_together_api_response(
-                self.model, convo, response_format=schema
+                self.model, messages, response_format=response_format
             )
             return res
         else:
-            judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
     def load_model(self) -> str:

judgeval/judges/utils.py

@@ -3,22 +3,21 @@ This module contains utility functions for judge models.
 """
 
 import litellm
-from typing import Optional, Union, Tuple, List
+from typing import Optional, Union, Tuple
 
-from judgeval.common.exceptions import InvalidJudgeModelError
-from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges
-from judgeval.constants import DEFAULT_GPT_MODEL
+from judgeval.exceptions import InvalidJudgeModelError
+from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
 from judgeval.constants import (
     TOGETHER_SUPPORTED_MODELS,
     JUDGMENT_SUPPORTED_MODELS,
-    ACCEPTABLE_MODELS,
 )
 
 LITELLM_SUPPORTED_MODELS = set(litellm.model_list)
 
 
 def create_judge(
-    model: Optional[Union[str, List[str], JudgevalJudge]] = None,
+    model: Optional[Union[str, JudgevalJudge]] = None,
 ) -> Tuple[JudgevalJudge, bool]:
     """
     Creates a judge model from string(s) or a judgeval judge object.
@@ -31,28 +30,15 @@ def create_judge(
     If no model is provided, uses GPT4o as the default judge.
     """
     if model is None:  # default option
-        return LiteLLMJudge(model=DEFAULT_GPT_MODEL), True
+        return LiteLLMJudge(model=JUDGMENT_DEFAULT_GPT_MODEL), True
     if not isinstance(model, (str, list, JudgevalJudge)):
         raise InvalidJudgeModelError(
             f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead."
         )
     # If model is already a valid judge type, return it and mark native
-    if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges)):
+    if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge)):
         return model, True
 
-    # Either string or List[str]
-    if isinstance(model, list):
-        for m in model:
-            if m in JUDGMENT_SUPPORTED_MODELS:
-                raise NotImplementedError(
-                    """Judgment models are not yet supported for local scoring.
-                    Please either set the `use_judgment` flag to True or use 
-                    non-Judgment models."""
-                )
-            if m not in ACCEPTABLE_MODELS:
-                raise InvalidJudgeModelError(f"Invalid judge model chosen: {m}")
-        return MixtureOfJudges(models=model), True
-    # If model is a string, check that it corresponds to a valid model
     if model in LITELLM_SUPPORTED_MODELS:
         return LiteLLMJudge(model=model), True
     if model in TOGETHER_SUPPORTED_MODELS:

judgeval/{common/logger.py → logger.py}

@@ -1,10 +1,9 @@
-# logger.py
-
 import logging
 import sys
-import os
 
-# ANSI escape sequences
+from judgeval.env import JUDGMENT_NO_COLOR
+from judgeval.utils.decorators import use_once
+
 RESET = "\033[0m"
 RED = "\033[31m"
 YELLOW = "\033[33m"
@@ -38,8 +37,9 @@ class ColorFormatter(logging.Formatter):
         return message
 
 
+@use_once
 def _setup_judgeval_logger():
-    use_color = sys.stdout.isatty() and os.getenv("NO_COLOR") is None
+    use_color = sys.stdout.isatty() and JUDGMENT_NO_COLOR is None
     handler = logging.StreamHandler(sys.stdout)
     handler.setLevel(logging.DEBUG)
     handler.setFormatter(
@@ -56,5 +56,7 @@ def _setup_judgeval_logger():
     return logger
 
 
-# Global logger you can import elsewhere
 judgeval_logger = _setup_judgeval_logger()
+
+
+__all__ = ("judgeval_logger",)

judgeval/scorers/__init__.py

@@ -1,8 +1,6 @@
 from judgeval.scorers.api_scorer import APIScorerConfig
 from judgeval.scorers.base_scorer import BaseScorer
 from judgeval.scorers.judgeval_scorers.api_scorers import (
-    ExecutionOrderScorer,
-    HallucinationScorer,
     FaithfulnessScorer,
     AnswerRelevancyScorer,
     AnswerCorrectnessScorer,
@@ -17,8 +15,6 @@ __all__ = [
     "APIScorerConfig",
     "BaseScorer",
     "PromptScorer",
-    "ExecutionOrderScorer",
-    "HallucinationScorer",
     "FaithfulnessScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",

judgeval/scorers/agent_scorer.py

@@ -1,21 +1,17 @@
 from judgeval.scorers.base_scorer import BaseScorer
-from judgeval.data import Trace
+from judgeval.data.judgment_types import Trace as JudgmentTrace
 from typing import List, Optional
 from abc import abstractmethod
 
-from judgeval.common.logger import warning, error
 
-
-class AgentScorer(BaseScorer):
+class TraceScorer(BaseScorer):
     @abstractmethod
     async def a_score_trace(
-        self, trace: Trace, tools: Optional[List] = None, *args, **kwargs
+        self, trace: JudgmentTrace, tools: Optional[List] = None, *args, **kwargs
     ) -> float:
         """
         Asynchronously measures the score on a trace
         """
-        warning("Attempting to call unimplemented a_score_trace method")
-        error("a_score_trace method not implemented")
         raise NotImplementedError(
             "You must implement the `a_score_trace` method in your custom scorer"
         )

judgeval/scorers/api_scorer.py

@@ -4,11 +4,12 @@ Judgment Scorer class.
 Scores `Example`s using ready-made Judgment evaluators.
 """
 
+from __future__ import annotations
+
 from pydantic import BaseModel, field_validator
 from typing import List
-from judgeval.data import ExampleParams
-from judgeval.constants import APIScorerType, UNBOUNDED_SCORERS
-from judgeval.common.logger import judgeval_logger
+from judgeval.constants import UNBOUNDED_SCORERS, APIScorerType
+from judgeval.data.example import ExampleParams
 
 
 class APIScorerConfig(BaseModel):
@@ -28,9 +29,10 @@ class APIScorerConfig(BaseModel):
     name: str = ""
     threshold: float = 0.5
     strict_mode: bool = False
-    required_params: List[
-        ExampleParams
-    ] = []  # This is used to check if the example has the required parameters before running the scorer
+
+    # This is used to check if the example has the required parameters before running the scorer
+    required_params: List[ExampleParams] = []
+
     kwargs: dict = {}
 
     @field_validator("threshold")
@@ -42,17 +44,11 @@ class APIScorerConfig(BaseModel):
         score_type = info.data.get("score_type")
         if score_type in UNBOUNDED_SCORERS:
             if v < 0:
-                judgeval_logger.error(
-                    f"Threshold for {score_type} must be greater than 0, got: {v}"
-                )
                 raise ValueError(
                     f"Threshold for {score_type} must be greater than 0, got: {v}"
                 )
         else:
             if not 0 <= v <= 1:
-                judgeval_logger.error(
-                    f"Threshold for {score_type} must be between 0 and 1, got: {v}"
-                )
                 raise ValueError(
                     f"Threshold for {score_type} must be between 0 and 1, got: {v}"
                 )
@@ -61,7 +57,6 @@ class APIScorerConfig(BaseModel):
     @field_validator("name", mode="after")
     @classmethod
     def set_name_to_score_type_if_none(cls, v, info):
-        """Set name to score_type if not provided"""
         if v is None:
             return info.data.get("score_type")
         return v

judgeval/scorers/base_scorer.py

@@ -2,6 +2,7 @@
 Base class for all scorers.
 """
 
+from __future__ import annotations
 from typing import Dict, Optional
 
 from pydantic import BaseModel
@@ -19,44 +20,63 @@ class BaseScorer(BaseModel):
     where none of Judgment's scorers are suitable.
     """
 
-    score_type: str  # type of your scorer (Faithfulness, PromptScorer)
-    threshold: float = (
-        0.5  # The threshold to pass a test while using this scorer as a scorer
-    )
-    name: Optional[str] = (
-        None  # name of your scorer (Faithfulness, PromptScorer-randomslug)
-    )
-    class_name: Optional[str] = None  # The name of the class of the scorer
-    score: Optional[float] = None  # The float score of the scorer run on the test case
+    # type of your scorer (Faithfulness, PromptScorer)
+    score_type: str
+
+    # The threshold to pass a test while using this scorer as a scorer
+    threshold: float = 0.5
+
+    # name of your scorer (Faithfulness, PromptScorer-randomslug)
+    name: Optional[str] = None
+
+    # The name of the class of the scorer
+    class_name: Optional[str] = None
+
+    # The float score of the scorer run on the test case
+    score: Optional[float] = None
+
     score_breakdown: Optional[Dict] = None
     reason: Optional[str] = ""
-    using_native_model: Optional[bool] = None  # Whether the model is a native model
-    success: Optional[bool] = None  # Whether the test case passed or failed
-    model: Optional[str] = None  # The name of the model used to evaluate the test case
-    model_client: Optional[Any] = Field(
-        default=None, exclude=True
-    )  # The model used to evaluate the test case
-    strict_mode: bool = False  # Whether to run the scorer in strict mode
-    error: Optional[str] = None  # The error message if the scorer failed
-    additional_metadata: Optional[Dict] = None  # Additional metadata for the scorer
-    user: Optional[str] = None  # The user ID of the scorer
-    server_hosted: bool = False  # Whether the scorer is enabled for e2b
+
+    # Whether the model is a native model
+    using_native_model: Optional[bool] = None
+
+    # Whether the test case passed or failed
+    success: Optional[bool] = None
+
+    # The name of the model used to evaluate the test case
+    model: Optional[str] = None
+
+    # The model used to evaluate the test case
+    model_client: Optional[Any] = Field(default=None, exclude=True)
+
+    # Whether to run the scorer in strict mode
+    strict_mode: bool = False
+
+    # The error message if the scorer failed
+    error: Optional[str] = None
+
+    # Additional metadata for the scorer
+    additional_metadata: Optional[Dict] = None
+
+    # The user ID of the scorer
+    user: Optional[str] = None
+
+    # Whether the scorer is hosted on the server
+    server_hosted: bool = False
 
     @model_validator(mode="after")
-    @classmethod
-    def enforce_strict_threshold(cls, data: "BaseScorer"):
-        if data.strict_mode:
-            data.threshold = 1.0
-        return data
+    def enforce_strict_threshold(self):
+        if self.strict_mode:
+            self.threshold = 1.0
+        return self
 
     @model_validator(mode="after")
-    @classmethod
-    def default_name(cls, m: "BaseScorer") -> "BaseScorer":
-        # Always set class_name to the string name of the class
-        m.class_name = m.__class__.__name__
-        if not m.name:
-            m.name = m.class_name
-        return m
+    def default_name(self):
+        self.class_name = self.__class__.__name__
+        if not self.name:
+            self.name = self.class_name
+        return self
 
     def _add_model(self, model: str):
         """

judgeval/scorers/example_scorer.py

@@ -2,18 +2,16 @@ from judgeval.scorers.base_scorer import BaseScorer
 from judgeval.data import Example
 from typing import List
 from pydantic import Field
-from judgeval.common.logger import judgeval_logger
 
 
 class ExampleScorer(BaseScorer):
-    score_type: str = "Custom"  # default to custom score type
+    score_type: str = "Custom"
     required_params: List[str] = Field(default_factory=list)
 
     async def a_score_example(self, example: Example, *args, **kwargs) -> float:
         """
         Asynchronously measures the score on a single example
         """
-        judgeval_logger.error("a_score_example method not implemented")
         raise NotImplementedError(
             "You must implement the `a_score_example` method in your custom scorer"
         )

judgeval/scorers/judgeval_scorers/api_scorers/__init__.py

@@ -1,9 +1,3 @@
-from judgeval.scorers.judgeval_scorers.api_scorers.execution_order import (
-    ExecutionOrderScorer,
-)
-from judgeval.scorers.judgeval_scorers.api_scorers.hallucination import (
-    HallucinationScorer,
-)
 from judgeval.scorers.judgeval_scorers.api_scorers.faithfulness import (
     FaithfulnessScorer,
 )
@@ -28,18 +22,10 @@ from judgeval.scorers.judgeval_scorers.api_scorers.tool_dependency import (
 )
 
 __all__ = [
-    "ExecutionOrderScorer",
-    "JSONCorrectnessScorer",
-    "SummarizationScorer",
-    "HallucinationScorer",
     "FaithfulnessScorer",
-    "ContextualRelevancyScorer",
-    "ContextualPrecisionScorer",
-    "ContextualRecallScorer",
     "AnswerRelevancyScorer",
     "AnswerCorrectnessScorer",
     "InstructionAdherenceScorer",
-    "GroundednessScorer",
     "DerailmentScorer",
     "ToolOrderScorer",
     "PromptScorer",