judgeval 0.0.44__py3-none-any.whl → 0.0.46__py3-none-any.whl

judgeval/judges/base_judge.py

@@ -37,8 +37,7 @@ class JudgevalJudge(ABC):
             A string.
         """
         pass
-    
+
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str:
         pass
-    

judgeval/judges/litellm_judge.py

@@ -1,9 +1,11 @@
 import pydantic
 from typing import List, Union, Mapping
 
-from judgeval import *
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import afetch_litellm_api_response, fetch_litellm_api_response
+from judgeval.common.utils import (
+    afetch_litellm_api_response,
+    fetch_litellm_api_response,
+)
 from judgeval.common.logger import debug, error
 
 BASE_CONVERSATION = [
@@ -18,30 +20,50 @@ class LiteLLMJudge(JudgevalJudge):
         self.kwargs = kwargs
         super().__init__(model_name=model)
 
-    def generate(self, input: Union[str, List[Mapping[str, str]]], schema: pydantic.BaseModel = None) -> str:
+    def generate(
+        self,
+        input: Union[str, List[Mapping[str, str]]],
+        schema: pydantic.BaseModel = None,
+    ) -> str:
         debug(f"Generating response for input type: {type(input)}")
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            return fetch_litellm_api_response(model=self.model, messages=convo, response_format=schema)
+            return fetch_litellm_api_response(
+                model=self.model, messages=convo, response_format=schema
+            )
         elif isinstance(input, list):
-            return fetch_litellm_api_response(model=self.model, messages=input, response_format=schema)
+            return fetch_litellm_api_response(
+                model=self.model, messages=input, response_format=schema
+            )
         else:
             error(f"Invalid input type received: {type(input)}")
-            raise TypeError(f"Input must be a string or a list of dictionaries. Input type of: {type(input)}")
+            raise TypeError(
+                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
+            )
 
-    async def a_generate(self, input: Union[str, List[Mapping[str, str]]], schema: pydantic.BaseModel = None) -> str:
+    async def a_generate(
+        self,
+        input: Union[str, List[Mapping[str, str]]],
+        schema: pydantic.BaseModel = None,
+    ) -> str:
         debug(f"Async generating response for input type: {type(input)}")
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            response = await afetch_litellm_api_response(model=self.model, messages=convo, response_format=schema)
+            response = await afetch_litellm_api_response(
+                model=self.model, messages=convo, response_format=schema
+            )
             return response
         elif isinstance(input, list):
-            response = await afetch_litellm_api_response(model=self.model, messages=input, response_format=schema)
+            response = await afetch_litellm_api_response(
+                model=self.model, messages=input, response_format=schema
+            )
             return response
         else:
             error(f"Invalid input type received: {type(input)}")
-            raise TypeError(f"Input must be a string or a list of dictionaries. Input type of: {type(input)}")
-    
+            raise TypeError(
+                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
+            )
+
     def load_model(self):
         return self.model
 

judgeval/judges/mixture_of_judges.py

@@ -3,23 +3,24 @@ Implementation for Mixture of Judges model through Judgeval
 
 Enables client to use multiple models to generate responses and then aggregate them into a single response.
 """
-from judgeval import *
+
 import pydantic
-from typing import List, Union, Mapping
+from typing import List, Union
 from judgeval.judges import JudgevalJudge
 from judgeval.common.utils import (
-    get_completion_multiple_models, 
-    get_chat_completion, 
-    aget_completion_multiple_models, 
-    aget_chat_completion
+    get_completion_multiple_models,
+    get_chat_completion,
+    aget_completion_multiple_models,
+    aget_chat_completion,
 )
 from judgeval.common.logger import debug, error
 
+
 def build_dynamic_mixture_prompt(
-        judge_responses: List[str], 
-        custom_system_prompt: str = None, 
-        custom_conversation_history: List[Mapping] = None
-    ) -> List[Mapping]:
+    judge_responses: List[str],
+    custom_system_prompt: str | None = None,
+    custom_conversation_history: List[dict] | None = None,
+) -> List[dict]:
     """
     Dynamically builds a prompt to mix judge responses together for the Mixture of Judges model.
 
@@ -29,10 +30,15 @@ def build_dynamic_mixture_prompt(
     Args:
         judge_responses (List[str]): List of responses from individual judges to be synthesized
         custom_system_prompt (str, optional): Custom system prompt to override the default one. Defaults to None.
-        custom_conversation_history (List[Mapping], optional): Custom conversation history to override the default one. Defaults to None.
+        custom_conversation_history (List[dict], optional): Custom conversation history to override the default one. Defaults to None.
     """
-    formatted_responses = "\n".join([f"# Judge {i + 1}'s response: #\n{response}" for i, response in enumerate(judge_responses)])
-    
+    formatted_responses = "\n".join(
+        [
+            f"# Judge {i + 1}'s response: #\n{response}"
+            for i, response in enumerate(judge_responses)
+        ]
+    )
+
     # This is the default prompt for the Mixture of Judges model
     """
     You are tasked with synthesizing responses from multiple expert judges. You will receive N individual answers on the same topic. Your job is to:
@@ -51,75 +57,99 @@ def build_dynamic_mixture_prompt(
 
     default_conversation = [  # inject the judge responses into the default prompt
         {
-            'role': 'system',
-            'content': 'You are tasked with synthesizing responses from multiple expert judges. You will receive N individual answers on the same topic. Your job is to:\n1. Analyze and compare the key points, patterns, and agreements between the answers.\n2. Identify the consensus by focusing on areas where most or all of the answers align. Consider common reasoning and frequently mentioned conclusions.\n3. Condense the responses into a single, coherent, and concise answer that represents the collective judgment of the group.\n4. When opinions differ or contradict, highlight the most supported viewpoint while briefly acknowledging the dissenting perspectives.\n5. Ensure the final answer is balanced and clear, providing a comprehensive summary that captures the wisdom of all judges while avoiding repetition.\n\n**IMPORTANT**: IF THE JUDGE RESPONSES ARE IN JSON FORMAT, YOU MUST RESPOND USING THE SAME JSON FORMAT THAT THE RESPONSES ARE IN. If the judge responses are in JSON, you MUST RESPOND IN VALID JSON FORMAT. '
-        }, 
+            "role": "system",
+            "content": "You are tasked with synthesizing responses from multiple expert judges. You will receive N individual answers on the same topic. Your job is to:\n1. Analyze and compare the key points, patterns, and agreements between the answers.\n2. Identify the consensus by focusing on areas where most or all of the answers align. Consider common reasoning and frequently mentioned conclusions.\n3. Condense the responses into a single, coherent, and concise answer that represents the collective judgment of the group.\n4. When opinions differ or contradict, highlight the most supported viewpoint while briefly acknowledging the dissenting perspectives.\n5. Ensure the final answer is balanced and clear, providing a comprehensive summary that captures the wisdom of all judges while avoiding repetition.\n\n**IMPORTANT**: IF THE JUDGE RESPONSES ARE IN JSON FORMAT, YOU MUST RESPOND USING THE SAME JSON FORMAT THAT THE RESPONSES ARE IN. If the judge responses are in JSON, you MUST RESPOND IN VALID JSON FORMAT. ",
+        },
         {
-            'role': 'user',
-            'content': '## Start of Judge Responses ## \n# Judge 1\'s response: #\n{\n"claims": [\n{\n"claim": "A 30-day full refund is offered.",\n"quote": "We offer a 30-day full refund at no extra cost."\n},\n{\n"claim": "The 30-day full refund comes at no extra cost.",\n"quote": "We offer a 30-day full refund at no extra cost."\n}\n]\n}\n\n# Judge 2\'s response: #\n{\n    "claims": [\n        {\n            "claim": "A full refund is offered within 30 days.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}\n# Judge 3\'s response: #\n {\n    "claims": [\n        {\n            "claim": "A 30-day full refund is offered.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}\n## End of Judge Responses ##\nSynthesized response:'
-        }, 
+            "role": "user",
+            "content": '## Start of Judge Responses ## \n# Judge 1\'s response: #\n{\n"claims": [\n{\n"claim": "A 30-day full refund is offered.",\n"quote": "We offer a 30-day full refund at no extra cost."\n},\n{\n"claim": "The 30-day full refund comes at no extra cost.",\n"quote": "We offer a 30-day full refund at no extra cost."\n}\n]\n}\n\n# Judge 2\'s response: #\n{\n    "claims": [\n        {\n            "claim": "A full refund is offered within 30 days.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}\n# Judge 3\'s response: #\n {\n    "claims": [\n        {\n            "claim": "A 30-day full refund is offered.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}\n## End of Judge Responses ##\nSynthesized response:',
+        },
         {
-            'role': 'assistant',
-            'content': 'The consensus among the judges is clear and unanimous. All three judges agree that a 30-day full refund is offered, and this refund is available at no extra cost. This conclusion is consistently supported by their statements, as each of their claims is directly quoted as: "We offer a 30-day full refund at no extra cost." There are no dissenting perspectives or opposing views provided in any of the responses, indicating complete alignment on this topic.\n\nJSON:\n{\n    "claims": [\n        {\n            "claim": "A full refund is offered within 30 days.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}'
+            "role": "assistant",
+            "content": 'The consensus among the judges is clear and unanimous. All three judges agree that a 30-day full refund is offered, and this refund is available at no extra cost. This conclusion is consistently supported by their statements, as each of their claims is directly quoted as: "We offer a 30-day full refund at no extra cost." There are no dissenting perspectives or opposing views provided in any of the responses, indicating complete alignment on this topic.\n\nJSON:\n{\n    "claims": [\n        {\n            "claim": "A full refund is offered within 30 days.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}',
         },
         {
-            'role': 'user',
-            'content': "## Start of Judge Responses ##\n# Judge 1's response: # \nThe capital of France is Paris.\n\n# Judge 2's response: #\nThe capital of France is Paris.\n\n# Judge 3's response: # \nThe capital of France is Paris. It's one of the most popular tourist destinations in the world, known for its art, culture, and history. It's also famous for its iconic landmarks such as the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral.\n\n## End of Judge Responses ##\nSynthesized response:"
+            "role": "user",
+            "content": "## Start of Judge Responses ##\n# Judge 1's response: # \nThe capital of France is Paris.\n\n# Judge 2's response: #\nThe capital of France is Paris.\n\n# Judge 3's response: # \nThe capital of France is Paris. It's one of the most popular tourist destinations in the world, known for its art, culture, and history. It's also famous for its iconic landmarks such as the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral.\n\n## End of Judge Responses ##\nSynthesized response:",
         },
         {
-            'role': 'assistant',
-            'content': "The capital of France is Paris. It is widely recognized as one of the world's most popular tourist destinations, celebrated for its rich art, culture, and history. Paris is renowned for its iconic landmarks, including the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral."
-        }, 
+            "role": "assistant",
+            "content": "The capital of France is Paris. It is widely recognized as one of the world's most popular tourist destinations, celebrated for its rich art, culture, and history. Paris is renowned for its iconic landmarks, including the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral.",
+        },
         {
-            'role': 'user',
-            'content': f'## Start of Judge Responses ##\n{formatted_responses}\n## End of Judge Responses ##\nSynthesized response:\n'
-        }
+            "role": "user",
+            "content": f"## Start of Judge Responses ##\n{formatted_responses}\n## End of Judge Responses ##\nSynthesized response:\n",
+        },
     ]
-    
+
     # If a custom system prompt is provided, validate and use it
     if custom_system_prompt is not None:
         if not isinstance(custom_system_prompt, str):
-            error(f"TypeError: Custom system prompt must be a string. Received: {type(custom_system_prompt)}.")
-            raise TypeError(f"Custom system prompt must be a string. Received: {type(custom_system_prompt)}.")
+            error(
+                f"TypeError: Custom system prompt must be a string. Received: {type(custom_system_prompt)}."
+            )
+            raise TypeError(
+                f"Custom system prompt must be a string. Received: {type(custom_system_prompt)}."
+            )
         if not custom_system_prompt:
             error("ValueError: Custom system prompt cannot be empty")
             raise ValueError("Custom system prompt cannot be empty")
         # Override the default system prompt, but also add special instructions for handling JSON
-        default_conversation[0]['content'] = custom_system_prompt + "\n\n**IMPORTANT**: IF THE JUDGE RESPONSES ARE IN JSON FORMAT, YOU MUST RESPOND USING THE SAME JSON FORMAT THAT THE RESPONSES ARE IN. If the judge responses are in JSON, you MUST RESPOND IN VALID JSON FORMAT."
-    
+        default_conversation[0]["content"] = (
+            custom_system_prompt
+            + "\n\n**IMPORTANT**: IF THE JUDGE RESPONSES ARE IN JSON FORMAT, YOU MUST RESPOND USING THE SAME JSON FORMAT THAT THE RESPONSES ARE IN. If the judge responses are in JSON, you MUST RESPOND IN VALID JSON FORMAT."
+        )
+
     # If a custom conversation history is provided, append the judge responses to it
     if custom_conversation_history is not None:
         # Validate custom conversation history format
         for message in custom_conversation_history:
             if not isinstance(message, dict):
-                error(f"TypeError: Custom conversation history must be a list of dictionaries. Received: {message}.")
-                raise TypeError(f"Custom conversation history must be a list of dictionaries. Received: {message}.")
-            
-            if 'role' not in message or 'content' not in message:
+                error(
+                    f"TypeError: Custom conversation history must be a list of dictionaries. Received: {message}."
+                )
+                raise TypeError(
+                    f"Custom conversation history must be a list of dictionaries. Received: {message}."
+                )
+
+            if "role" not in message or "content" not in message:
                 error("ValueError: Each message must have 'role' and 'content' keys")
                 raise ValueError("Each message must have 'role' and 'content' keys")
-            
-            if not isinstance(message['role'], str) or not isinstance(message['content'], str):
-                error(f"TypeError: Message role and content must be strings. Received: {type(message['role'])}, {type(message['content'])}.")
-                raise TypeError(f"Message role and content must be strings. Received: {type(message['role'])}, {type(message['content'])}.")
-            
-            if message['role'] not in ['system', 'user', 'assistant']:
-                error(f"ValueError: Message role must be one of: 'system', 'user', 'assistant'. Received: {message['role']}.")
-                raise ValueError(f"Message role must be one of: 'system', 'user', 'assistant'. Received: {message['role']}.")
-            
+
+            if not isinstance(message["role"], str) or not isinstance(
+                message["content"], str
+            ):
+                error(
+                    f"TypeError: Message role and content must be strings. Received: {type(message['role'])}, {type(message['content'])}."
+                )
+                raise TypeError(
+                    f"Message role and content must be strings. Received: {type(message['role'])}, {type(message['content'])}."
+                )
+
+            if message["role"] not in ["system", "user", "assistant"]:
+                error(
+                    f"ValueError: Message role must be one of: 'system', 'user', 'assistant'. Received: {message['role']}."
+                )
+                raise ValueError(
+                    f"Message role must be one of: 'system', 'user', 'assistant'. Received: {message['role']}."
+                )
+
         judge_responses_prompt = {
-            'role': 'user', 
-            'content': f'## Start of Judge Responses ##\n{formatted_responses}\n## End of Judge Responses ##\nSynthesized response:\n'
+            "role": "user",
+            "content": f"## Start of Judge Responses ##\n{formatted_responses}\n## End of Judge Responses ##\nSynthesized response:\n",
         }
         return custom_conversation_history + [judge_responses_prompt]
-    
+
     # Otherwise return the default conversation with system prompt and examples
     # No customization, return the default conversation with system prompt and examples
     return default_conversation
 
+
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
 ]  # for string inputs, we need to add the user query to a base conversation, since LiteLLM only accepts a list of dictionaries as a chat history
+
+
 class MixtureOfJudges(JudgevalJudge):
     """
     IMPORTANT: When supplying custom prompts and conversation histories for aggregation, supply them in the following format:
@@ -134,10 +164,17 @@ class MixtureOfJudges(JudgevalJudge):
         ]
     }
     """
-    def __init__(self, 
-                 models: List[str] = ['QWEN', 'LLAMA3_70B_INSTRUCT_TURBO', 'MISTRAL_8x22B_INSTRUCT'],
-                 aggregator: str = 'gpt-4.1', 
-                 **kwargs):
+
+    def __init__(
+        self,
+        models: List[str] = [
+            "QWEN",
+            "LLAMA3_70B_INSTRUCT_TURBO",
+            "MISTRAL_8x22B_INSTRUCT",
+        ],
+        aggregator: str = "gpt-4.1",
+        **kwargs,
+    ):
         """
         `models` are the individual judge models to be used for generating responses.
         `aggregator` is the model that will aggregate the responses from the individual judges.
@@ -150,11 +187,12 @@ class MixtureOfJudges(JudgevalJudge):
         super().__init__(model_name=models)
 
     def generate(
-            self, 
-            input: Union[str, List[Mapping[str, str]]], 
-            response_schema: pydantic.BaseModel = None, 
-            aggregation_schema: pydantic.BaseModel = None,
-            **kwargs) -> str:
+        self,
+        input: Union[str, List[dict]],
+        response_schema: pydantic.BaseModel = None,
+        aggregation_schema: pydantic.BaseModel = None,
+        **kwargs,
+    ) -> str:
         """
         Args:
             input (Union[str, List[Mapping[str, str]]]): Input query or conversation history to the model.
@@ -163,7 +201,7 @@ class MixtureOfJudges(JudgevalJudge):
             kwargs: Additional keyword arguments.
         """
         debug(f"Generating response for input type: {type(input)}")
-        
+
         # Convert input to conversation format if needed
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
@@ -171,20 +209,26 @@ class MixtureOfJudges(JudgevalJudge):
             convo = input
         else:
             error(f"Invalid input type received: {type(input)}")
-            raise TypeError(f"Input must be a string or a list of dictionaries. Input type of: {type(input)}")
+            raise TypeError(
+                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
+            )
 
         try:
             responses = get_completion_multiple_models(
                 models=self.models,
                 messages=[convo] * len(self.models),
-                response_formats=[response_schema] * len(self.models)
+                response_formats=[response_schema] * len(self.models),
             )
         except Exception as e:
             error(f"Error getting completions from multiple models: {str(e)}")
             raise
 
-        compiled_mixture_prompt = build_dynamic_mixture_prompt(responses, self.kwargs.get('custom_prompt'), self.kwargs.get('custom_conversation'))
-        
+        compiled_mixture_prompt = build_dynamic_mixture_prompt(
+            responses,
+            self.kwargs.get("custom_prompt"),
+            self.kwargs.get("custom_conversation"),
+        )
+
         try:
             mixed_response = get_chat_completion(
                 model_type=self.aggregator,
@@ -194,16 +238,16 @@ class MixtureOfJudges(JudgevalJudge):
         except Exception as e:
             error(f"Error getting chat completion from aggregator: {str(e)}")
             raise
-            
+
         return mixed_response
 
     async def a_generate(
-            self, 
-            input: Union[str, List[Mapping[str, str]]], 
-            response_schema: pydantic.BaseModel = None,
-            aggregation_schema: pydantic.BaseModel = None,
-            **kwargs
-        ) -> str:
+        self,
+        input: Union[str, List[dict]],
+        response_schema: pydantic.BaseModel = None,
+        aggregation_schema: pydantic.BaseModel = None,
+        **kwargs,
+    ) -> str:
         """
         Args:
             input (Union[str, List[Mapping[str, str]]]): Input query or conversation history to the model.
@@ -212,7 +256,7 @@ class MixtureOfJudges(JudgevalJudge):
             kwargs: Additional keyword arguments.
         """
         debug(f"Generating response for input type: {type(input)}")
-        
+
         # Convert input to conversation format if needed
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
@@ -220,20 +264,26 @@ class MixtureOfJudges(JudgevalJudge):
             convo = input
         else:
             error(f"Invalid input type received: {type(input)}")
-            raise TypeError(f"Input must be a string or a list of dictionaries. Input type of: {type(input)}")
+            raise TypeError(
+                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
+            )
 
         try:
             responses = await aget_completion_multiple_models(
                 models=self.models,
                 messages=[convo] * len(self.models),
-                response_formats=[response_schema] * len(self.models)
+                response_formats=[response_schema] * len(self.models),
             )
         except Exception as e:
             error(f"Error getting async completions from multiple models: {str(e)}")
             raise
 
-        compiled_mixture_prompt = build_dynamic_mixture_prompt(responses, self.kwargs.get('custom_prompt'), self.kwargs.get('custom_conversation'))
-        
+        compiled_mixture_prompt = build_dynamic_mixture_prompt(
+            responses,
+            self.kwargs.get("custom_prompt"),
+            self.kwargs.get("custom_conversation"),
+        )
+
         try:
             mixed_response = await aget_chat_completion(
                 model_type=self.aggregator,
@@ -243,9 +293,9 @@ class MixtureOfJudges(JudgevalJudge):
         except Exception as e:
             error(f"Error getting async chat completion from aggregator: {str(e)}")
             raise
-            
+
         return mixed_response
-    
+
     def load_model(self):
         return self.models
 

judgeval/judges/together_judge.py

@@ -3,16 +3,20 @@ Implementation of using TogetherAI inference for judges.
 """
 
 from pydantic import BaseModel
-from typing import List, Union, Mapping
+from typing import List, Union
 from judgeval.common.logger import debug, error
 
 from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import fetch_together_api_response, afetch_together_api_response
+from judgeval.common.utils import (
+    fetch_together_api_response,
+    afetch_together_api_response,
+)
 
 BASE_CONVERSATION = [
     {"role": "system", "content": "You are a helpful assistant."},
 ]
 
+
 class TogetherJudge(JudgevalJudge):
     def __init__(self, model: str = "Qwen/Qwen2.5-72B-Instruct-Turbo", **kwargs):
         debug(f"Initializing TogetherJudge with model={model}")
@@ -21,27 +25,37 @@ class TogetherJudge(JudgevalJudge):
         super().__init__(model_name=model)
 
     # TODO: Fix cost for generate and a_generate
-    def generate(self, input: Union[str, List[Mapping[str, str]]], schema: BaseModel = None) -> str:
+    def generate(self, input: Union[str, List[dict]], schema: BaseModel = None) -> str:
         debug(f"Generating response for input type: {type(input)}")
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            return fetch_together_api_response(self.model, convo, response_format=schema)
+            return fetch_together_api_response(
+                self.model, convo, response_format=schema
+            )
         elif isinstance(input, list):
             convo = input
-            return fetch_together_api_response(self.model, convo, response_format=schema)
+            return fetch_together_api_response(
+                self.model, convo, response_format=schema
+            )
         else:
             error(f"Invalid input type received: {type(input)}")
             raise TypeError("Input must be a string or a list of dictionaries.")
 
-    async def a_generate(self, input: Union[str, List[dict]], schema: BaseModel = None) -> str:
+    async def a_generate(
+        self, input: Union[str, List[dict]], schema: BaseModel = None
+    ) -> str:
         debug(f"Async generating response for input type: {type(input)}")
         if isinstance(input, str):
             convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-            res = await afetch_together_api_response(self.model, convo, response_format=schema)
+            res = await afetch_together_api_response(
+                self.model, convo, response_format=schema
+            )
             return res
         elif isinstance(input, list):
             convo = input
-            res = await afetch_together_api_response(self.model, convo, response_format=schema)
+            res = await afetch_together_api_response(
+                self.model, convo, response_format=schema
+            )
             return res
         else:
             error(f"Invalid input type received: {type(input)}")
@@ -52,4 +66,3 @@ class TogetherJudge(JudgevalJudge):
 
     def get_model_name(self) -> str:
         return self.model
-    

judgeval/judges/utils.py

@@ -1,23 +1,30 @@
 """
 This module contains utility functions for judge models.
 """
+
 import litellm
 from typing import Optional, Union, Tuple, List
 
 from judgeval.common.exceptions import InvalidJudgeModelError
 from judgeval.judges import JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges
-from judgeval.constants import TOGETHER_SUPPORTED_MODELS, JUDGMENT_SUPPORTED_MODELS, ACCEPTABLE_MODELS
+from judgeval.constants import (
+    TOGETHER_SUPPORTED_MODELS,
+    JUDGMENT_SUPPORTED_MODELS,
+    ACCEPTABLE_MODELS,
+)
 
 LITELLM_SUPPORTED_MODELS = set(litellm.model_list)
 
+
 def create_judge(
-    model: Optional[Union[str, List[str], JudgevalJudge]] = None) -> Tuple[JudgevalJudge, bool]:
+    model: Optional[Union[str, List[str], JudgevalJudge]] = None,
+) -> Tuple[JudgevalJudge, bool]:
     """
     Creates a judge model from string(s) or a judgeval judge object.
 
     If `model` is a single string, it is assumed to be a judge model name.
     If `model` is a list of strings, it is assumed to be a list of judge model names (for MixtureOfJudges).
-    If `model` is a judgeval judge object, it is returned as is.    
+    If `model` is a judgeval judge object, it is returned as is.
 
     Returns a tuple of (initialized judgevalBaseLLM, using_native_model boolean)
     If no model is provided, uses GPT4o as the default judge.
@@ -25,10 +32,12 @@ def create_judge(
     if model is None:  # default option
         return LiteLLMJudge(model="gpt-4.1"), True
     if not isinstance(model, (str, list, JudgevalJudge)):
-        raise InvalidJudgeModelError(f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead.")
+        raise InvalidJudgeModelError(
+            f"Model must be a string, list of strings, or a judgeval judge object. Got: {type(model)} instead."
+        )
     # If model is already a valid judge type, return it and mark native
     if isinstance(model, (JudgevalJudge, LiteLLMJudge, TogetherJudge, MixtureOfJudges)):
-        return model, True 
+        return model, True
 
     # Either string or List[str]
     if isinstance(model, list):