judgeval 0.0.11__py3-none-any.whl → 0.22.2__py3-none-any.whl

judgeval/integrations/openlit/__init__.py

@@ -0,0 +1,50 @@
+from abc import ABC
+from judgeval.tracer import Tracer
+from judgeval.logger import judgeval_logger
+from judgeval.utils.url import url_for
+
+
+try:
+    import openlit  # type: ignore
+except ImportError:
+    raise ImportError(
+        "Openlit is not installed and required for the openlit integration. Please install it with `pip install openlit`."
+    )
+
+
+class Openlit(ABC):
+    @staticmethod
+    def initialize(
+        **kwargs,
+    ):
+        tracer = Tracer.get_instance()
+        if not tracer or not tracer._initialized:
+            raise ValueError(
+                "Openlit must be initialized after the tracer has been initialized. Please create the Tracer instance first before initializing Openlit."
+            )
+
+        api_key = tracer.api_key
+        organization_id = tracer.organization_id
+        project_name = tracer.project_name
+
+        project_id = Tracer._resolve_project_id(project_name, api_key, organization_id)
+        if not project_id:
+            judgeval_logger.warning(
+                f"Project {project_name} not found. Please create it first at https://app.judgmentlabs.ai/org/{organization_id}/projects."
+            )
+            return
+
+        openlit.init(
+            service_name=project_name,
+            otlp_endpoint=url_for("/otel"),
+            otlp_headers={
+                "Authorization": f"Bearer {api_key}",
+                "X-Organization-Id": organization_id,
+                "X-Project-Id": project_id,
+            },
+            tracer=tracer.get_tracer(),
+            **kwargs,
+        )
+
+
+__all__ = ["Openlit"]

judgeval/judges/__init__.py

@@ -1,7 +1,6 @@
-from pydantic import BaseModel
 from judgeval.judges.base_judge import JudgevalJudge
 from judgeval.judges.litellm_judge import LiteLLMJudge
 from judgeval.judges.together_judge import TogetherJudge
-from judgeval.judges.mixture_of_judges import MixtureOfJudges
 
-__all__ = ["JudgevalJudge", "LiteLLMJudge", "TogetherJudge", "MixtureOfJudges"]
+
+__all__ = ["JudgevalJudge", "LiteLLMJudge", "TogetherJudge"]

judgeval/judges/base_judge.py

@@ -3,7 +3,7 @@ Implements the base class for all Judgeval Judge models.
 """
 
 from abc import ABC, abstractmethod
-from typing import Optional, List
+from typing import Optional
 
 
 class JudgevalJudge(ABC):
@@ -37,8 +37,7 @@ class JudgevalJudge(ABC):
             A string.
         """
         pass
-    
+
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str:
         pass
-    

judgeval/judges/litellm_judge.py

@@ -1,47 +1,127 @@
 import pydantic
-from typing import List, Union, Mapping
+from typing import Dict, List, Union, Mapping, Any
 
-from judgeval import *
+from judgeval.constants import ACCEPTABLE_MODELS
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import afetch_litellm_api_response, fetch_litellm_api_response
-from judgeval.common.logger import debug, error
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+
+try:
+    import litellm
+except ImportError:
+    raise ImportError(
+        "Litellm is not installed and required for the litellm judge. Please install it with `pip install litellm`."
+    )
+
+
+def fetch_litellm_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if response_format is not None:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from litellm")
+    return content
+
+
+async def afetch_litellm_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if model not in ACCEPTABLE_MODELS:
+        raise ValueError(
+            f"Model {model} is not in the list of supported models: {ACCEPTABLE_MODELS}."
+        )
+
+    if response_format is not None:
+        response = await litellm.acompletion(
+            model=model, messages=messages, response_format=response_format
+        )
+    else:
+        response = await litellm.acompletion(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from litellm")
+    return content
+
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
-]  # for string inputs, we need to add the user query to a base conversation, since LiteLLM only accepts a list of dictionaries as a chat history
+]
 
 
 class LiteLLMJudge(JudgevalJudge):
-    def __init__(self, model: str = "gpt-4o-mini", **kwargs):
-        debug(f"Initializing LiteLLMJudge with model={model}")
+    def __init__(self, model: str = JUDGMENT_DEFAULT_GPT_MODEL, **kwargs):
         self.model = model
         self.kwargs = kwargs
         super().__init__(model_name=model)
 
-    def generate(self, input: Union[str, List[Mapping[str, str]]], schema: pydantic.BaseModel = None) -> str:
-        debug(f"Generating response for input type: {type(input)}")
+    def generate(
+        self,
+        input: Union[str, List[Mapping[str, str]]],
+        schema: Union[pydantic.BaseModel, None] = None,
+    ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            return fetch_litellm_api_response(model=self.model, messages=convo, response_format=schema)
+            return fetch_litellm_api_response(
+                model=self.model, messages=convo, response_format=response_format
+            )
         elif isinstance(input, list):
-            return fetch_litellm_api_response(model=self.model, messages=input, response_format=schema)
+            messages = [dict(msg) for msg in input]
+            return fetch_litellm_api_response(
+                model=self.model, messages=messages, response_format=response_format
+            )
         else:
-            error(f"Invalid input type received: {type(input)}")
-            raise TypeError(f"Input must be a string or a list of dictionaries. Input type of: {type(input)}")
+            raise TypeError(
+                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
+            )
+
+    async def a_generate(
+        self,
+        input: Union[str, List[Mapping[str, str]]],
+        schema: Union[pydantic.BaseModel, None] = None,
+    ) -> str:
+        response_format = schema.model_json_schema() if schema else None
 
-    async def a_generate(self, input: Union[str, List[Mapping[str, str]]], schema: pydantic.BaseModel = None) -> str:
-        debug(f"Async generating response for input type: {type(input)}")
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            response = await afetch_litellm_api_response(model=self.model, messages=convo, response_format=schema)
+            response = await afetch_litellm_api_response(
+                model=self.model, messages=convo, response_format=response_format
+            )
             return response
         elif isinstance(input, list):
-            response = await afetch_litellm_api_response(model=self.model, messages=input, response_format=schema)
+            messages = [dict(msg) for msg in input]
+            response = await afetch_litellm_api_response(
+                model=self.model, messages=messages, response_format=response_format
+            )
             return response
         else:
-            error(f"Invalid input type received: {type(input)}")
-            raise TypeError(f"Input must be a string or a list of dictionaries. Input type of: {type(input)}")
-    
+            raise TypeError(
+                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
+            )
+
     def load_model(self):
         return self.model
 

judgeval/judges/together_judge.py

@@ -3,48 +3,130 @@ Implementation of using TogetherAI inference for judges.
 """
 
 from pydantic import BaseModel
-from typing import List, Union, Mapping
-from judgeval.common.logger import debug, error
-
+from typing import Dict, List, Union, Any, cast
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import fetch_together_api_response, afetch_together_api_response
+from judgeval.logger import judgeval_logger
+from judgeval.env import (
+    JUDGMENT_DEFAULT_TOGETHER_MODEL,
+    TOGETHERAI_API_KEY,
+    TOGETHER_API_KEY,
+)
+
+together_api_key = TOGETHERAI_API_KEY or TOGETHER_API_KEY
+if together_api_key:
+    try:
+        from together import Together, AsyncTogether  # type: ignore[import-untyped]
+
+        together_client = Together(api_key=together_api_key)
+        async_together_client = AsyncTogether(api_key=together_api_key)
+    except Exception:
+        pass
+
+
+def fetch_together_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if response_format is not None:
+        response = together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from TogetherAI")
+    return cast(str, content)
+
+
+async def afetch_together_api_response(
+    model: str,
+    messages: List[Dict[str, str]],
+    response_format: Union[Dict[str, Any], None] = None,
+) -> str:
+    if not messages:
+        raise ValueError("Messages cannot be empty")
+
+    if response_format is not None:
+        response = await async_together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+            response_format=response_format,
+        )
+    else:
+        response = await async_together_client.chat.completions.create(
+            model=model,
+            messages=messages,
+        )
+
+    content = response.choices[0].message.content  # type: ignore[attr-defined]
+    if content is None:
+        raise ValueError("Received empty response from TogetherAI")
+    return cast(str, content)
+
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
 ]
 
+
 class TogetherJudge(JudgevalJudge):
-    def __init__(self, model: str = "QWEN", **kwargs):
-        debug(f"Initializing TogetherJudge with model={model}")
+    def __init__(self, model: str = JUDGMENT_DEFAULT_TOGETHER_MODEL, **kwargs):
         self.model = model
         self.kwargs = kwargs
         super().__init__(model_name=model)
 
-    # TODO: Fix cost for generate and a_generate
-    def generate(self, input: Union[str, List[Mapping[str, str]]], schema: BaseModel = None) -> str:
-        debug(f"Generating response for input type: {type(input)}")
+    def generate(
+        self,
+        input: Union[str, List[Dict[str, str]]],
+        schema: Union[BaseModel, None] = None,
+    ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            return fetch_together_api_response(self.model, convo, response_format=schema)
+            return fetch_together_api_response(
+                self.model, convo, response_format=response_format
+            )
         elif isinstance(input, list):
-            convo = input
-            return fetch_together_api_response(self.model, convo, response_format=schema)
+            messages = [dict(msg) for msg in input]
+            return fetch_together_api_response(
+                self.model, messages, response_format=response_format
+            )
         else:
-            error(f"Invalid input type received: {type(input)}")
+            judgeval_logger.error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
-    async def a_generate(self, input: Union[str, List[dict]], schema: BaseModel = None) -> str:
-        debug(f"Async generating response for input type: {type(input)}")
+    async def a_generate(
+        self,
+        input: Union[str, List[Dict[str, str]]],
+        schema: Union[BaseModel, None] = None,
+    ) -> str:
+        response_format = schema.model_json_schema() if schema else None
+
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            res = await afetch_together_api_response(self.model, convo, response_format=schema)
+            res = await afetch_together_api_response(
+                self.model, convo, response_format=response_format
+            )
             return res
         elif isinstance(input, list):
-            convo = input
-            res = await afetch_together_api_response(self.model, convo, response_format=schema)
+            messages = [dict(msg) for msg in input]
+            res = await afetch_together_api_response(
+                self.model, messages, response_format=response_format
+            )
             return res
         else:
-            error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
     def load_model(self) -> str:
@@ -52,4 +134,3 @@ class TogetherJudge(JudgevalJudge):
 
     def get_model_name(self) -> str:
         return self.model
-    

judgeval/judges/utils.py

@@ -1,48 +1,44 @@
 """
 This module contains utility functions for judge models.
 """
+
 import litellm
-from typing import Optional, Union, Tuple, List
+from typing import Optional, Union, Tuple
 
-from judgeval.common.exceptions import InvalidJudgeModelError
-from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges
-from judgeval.constants import TOGETHER_SUPPORTED_MODELS, JUDGMENT_SUPPORTED_MODELS, ACCEPTABLE_MODELS
+from judgeval.exceptions import InvalidJudgeModelError
+from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge
+from judgeval.env import JUDGMENT_DEFAULT_GPT_MODEL
+from judgeval.constants import (
+    TOGETHER_SUPPORTED_MODELS,
+    JUDGMENT_SUPPORTED_MODELS,
+)
 
 LITELLM_SUPPORTED_MODELS = set(litellm.model_list)
 
+
 def create_judge(
-    model: Optional[Union[str, List[str], JudgevalJudge]] = None) -> Tuple[JudgevalJudge, bool]:
+    model: Optional[Union[str, JudgevalJudge]] = None,
+) -> Tuple[JudgevalJudge, bool]:
     """
     Creates a judge model from string(s) or a judgeval judge object.
 
     If `model` is a single string, it is assumed to be a judge model name.
     If `model` is a list of strings, it is assumed to be a list of judge model names (for MixtureOfJudges).
-    If `model` is a judgeval judge object, it is returned as is.    
+    If `model` is a judgeval judge object, it is returned as is.
 
     Returns a tuple of (initialized judgevalBaseLLM, using_native_model boolean)
     If no model is provided, uses GPT4o as the default judge.
     """
     if model is None:  # default option
-        return LiteLLMJudge(model="gpt-4o"), True
+        return LiteLLMJudge(model=JUDGMENT_DEFAULT_GPT_MODEL), True
     if not isinstance(model, (str, list, JudgevalJudge)):
-        raise InvalidJudgeModelError(f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead.")
+        raise InvalidJudgeModelError(
+            f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead."
+        )
     # If model is already a valid judge type, return it and mark native
-    if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges)):
-        return model, True 
-
-    # Either string or List[str]
-    if isinstance(model, list):
-        for m in model:
-            if m in JUDGMENT_SUPPORTED_MODELS:
-                raise NotImplementedError(
-                    """Judgment models are not yet supported for local scoring.
-                    Please either set the `use_judgment` flag to True or use 
-                    non-Judgment models."""
-                )
-            if m not in LITELLM_SUPPORTED_MODELS and m not in TOGETHER_SUPPORTED_MODELS:
-                raise InvalidJudgeModelError(f"Invalid judge model chosen: {m}")
-        return MixtureOfJudges(models=model), True
-    # If model is a string, check that it corresponds to a valid model
+    if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge)):
+        return model, True
+
     if model in LITELLM_SUPPORTED_MODELS:
         return LiteLLMJudge(model=model), True
     if model in TOGETHER_SUPPORTED_MODELS:

judgeval/logger.py

@@ -0,0 +1,62 @@
+import logging
+import sys
+
+from judgeval.env import JUDGMENT_NO_COLOR
+from judgeval.utils.decorators.use_once import use_once
+
+RESET = "\033[0m"
+RED = "\033[31m"
+YELLOW = "\033[33m"
+BLUE = "\033[34m"
+GRAY = "\033[90m"
+
+
+class ColorFormatter(logging.Formatter):
+    """
+    Wrap the final formatted log record in ANSI color codes based on level.
+    """
+
+    COLORS = {
+        logging.DEBUG: GRAY,
+        logging.INFO: GRAY,
+        logging.WARNING: YELLOW,
+        logging.ERROR: RED,
+        logging.CRITICAL: RED,
+    }
+
+    def __init__(self, fmt=None, datefmt=None, use_color=True):
+        super().__init__(fmt=fmt, datefmt=datefmt)
+        self.use_color = use_color and sys.stdout.isatty()
+
+    def format(self, record):
+        message = super().format(record)
+        if self.use_color:
+            color = self.COLORS.get(record.levelno, "")
+            if color:
+                message = f"{color}{message}{RESET}"
+        return message
+
+
+@use_once
+def _setup_judgeval_logger():
+    use_color = sys.stdout.isatty() and JUDGMENT_NO_COLOR is None
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(
+        ColorFormatter(
+            fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+            use_color=use_color,
+        )
+    )
+
+    logger = logging.getLogger("judgeval")
+    logger.setLevel(logging.DEBUG)
+    logger.addHandler(handler)
+    return logger
+
+
+judgeval_logger = _setup_judgeval_logger()
+
+
+__all__ = ("judgeval_logger",)