judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

judgeval/judges/mixture_of_judges.py

@@ -1,286 +0,0 @@
-"""
-Implementation for Mixture of Judges model through Judgeval
-
-Enables client to use multiple models to generate responses and then aggregate them into a single response.
-"""
-
-import pydantic
-from typing import List, Union
-from judgeval.judges import JudgevalJudge
-from judgeval.common.utils import (
-    get_completion_multiple_models,
-    get_chat_completion,
-    aget_completion_multiple_models,
-    aget_chat_completion,
-)
-from judgeval.common.logger import judgeval_logger
-
-
-def build_dynamic_mixture_prompt(
-    judge_responses: List[str],
-    custom_system_prompt: str | None = None,
-    custom_conversation_history: List[dict] | None = None,
-) -> List[dict]:
-    """
-    Dynamically builds a prompt to mix judge responses together for the Mixture of Judges model.
-
-    In this implementation, we simply concatenate the judge responses into a formatted string, then
-    pass it into a default prompt template. This template can be customized by providing a custom prompt.
-
-    Args:
-        judge_responses (List[str]): List of responses from individual judges to be synthesized
-        custom_system_prompt (str, optional): Custom system prompt to override the default one. Defaults to None.
-        custom_conversation_history (List[dict], optional): Custom conversation history to override the default one. Defaults to None.
-    """
-    formatted_responses = "\n".join(
-        [
-            f"# Judge {i + 1}'s response: #\n{response}"
-            for i, response in enumerate(judge_responses)
-        ]
-    )
-
-    # This is the default prompt for the Mixture of Judges model
-    """
-    You are tasked with synthesizing responses from multiple expert judges. You will receive N individual answers on the same topic. Your job is to:
-
-    1. Analyze and compare the key points, patterns, and agreements between the answers.
-    2. Identify the consensus by focusing on areas where most or all of the answers align. Consider common reasoning and frequently mentioned conclusions.
-    3. Condense the responses into a single, coherent, and concise answer that represents the collective judgment of the group.
-    4. When opinions differ or contradict, highlight the most supported viewpoint while briefly acknowledging the dissenting perspectives.
-    5. Ensure the final answer is balanced and clear, providing a comprehensive summary that captures the wisdom of all judges while avoiding repetition.
-
-    ## Start of Judge Responses ##
-    {{judge_responses}}
-    ## End of Judge Responses ##
-    Synthesized response:
-    """
-
-    default_conversation = [  # inject the judge responses into the default prompt
-        {
-            "role": "system",
-            "content": "You are tasked with synthesizing responses from multiple expert judges. You will receive N individual answers on the same topic. Your job is to:\n1. Analyze and compare the key points, patterns, and agreements between the answers.\n2. Identify the consensus by focusing on areas where most or all of the answers align. Consider common reasoning and frequently mentioned conclusions.\n3. Condense the responses into a single, coherent, and concise answer that represents the collective judgment of the group.\n4. When opinions differ or contradict, highlight the most supported viewpoint while briefly acknowledging the dissenting perspectives.\n5. Ensure the final answer is balanced and clear, providing a comprehensive summary that captures the wisdom of all judges while avoiding repetition.\n\n**IMPORTANT**: IF THE JUDGE RESPONSES ARE IN JSON FORMAT, YOU MUST RESPOND USING THE SAME JSON FORMAT THAT THE RESPONSES ARE IN. If the judge responses are in JSON, you MUST RESPOND IN VALID JSON FORMAT. ",
-        },
-        {
-            "role": "user",
-            "content": '## Start of Judge Responses ## \n# Judge 1\'s response: #\n{\n"claims": [\n{\n"claim": "A 30-day full refund is offered.",\n"quote": "We offer a 30-day full refund at no extra cost."\n},\n{\n"claim": "The 30-day full refund comes at no extra cost.",\n"quote": "We offer a 30-day full refund at no extra cost."\n}\n]\n}\n\n# Judge 2\'s response: #\n{\n    "claims": [\n        {\n            "claim": "A full refund is offered within 30 days.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}\n# Judge 3\'s response: #\n {\n    "claims": [\n        {\n            "claim": "A 30-day full refund is offered.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}\n## End of Judge Responses ##\nSynthesized response:',
-        },
-        {
-            "role": "assistant",
-            "content": 'The consensus among the judges is clear and unanimous. All three judges agree that a 30-day full refund is offered, and this refund is available at no extra cost. This conclusion is consistently supported by their statements, as each of their claims is directly quoted as: "We offer a 30-day full refund at no extra cost." There are no dissenting perspectives or opposing views provided in any of the responses, indicating complete alignment on this topic.\n\nJSON:\n{\n    "claims": [\n        {\n            "claim": "A full refund is offered within 30 days.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        },\n        {\n            "claim": "The 30-day full refund is offered at no extra cost.",\n            "quote": "We offer a 30-day full refund at no extra cost."\n        }\n    ]\n}',
-        },
-        {
-            "role": "user",
-            "content": "## Start of Judge Responses ##\n# Judge 1's response: # \nThe capital of France is Paris.\n\n# Judge 2's response: #\nThe capital of France is Paris.\n\n# Judge 3's response: # \nThe capital of France is Paris. It's one of the most popular tourist destinations in the world, known for its art, culture, and history. It's also famous for its iconic landmarks such as the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral.\n\n## End of Judge Responses ##\nSynthesized response:",
-        },
-        {
-            "role": "assistant",
-            "content": "The capital of France is Paris. It is widely recognized as one of the world's most popular tourist destinations, celebrated for its rich art, culture, and history. Paris is renowned for its iconic landmarks, including the Eiffel Tower, Louvre Museum, and Notre-Dame Cathedral.",
-        },
-        {
-            "role": "user",
-            "content": f"## Start of Judge Responses ##\n{formatted_responses}\n## End of Judge Responses ##\nSynthesized response:\n",
-        },
-    ]
-
-    # If a custom system prompt is provided, validate and use it
-    if custom_system_prompt is not None:
-        if not isinstance(custom_system_prompt, str):
-            judgeval_logger.error(
-                f"TypeError: Custom system prompt must be a string. Received: {type(custom_system_prompt)}."
-            )
-            raise TypeError(
-                f"Custom system prompt must be a string. Received: {type(custom_system_prompt)}."
-            )
-        if not custom_system_prompt:
-            raise ValueError("Custom system prompt cannot be empty")
-        # Override the default system prompt, but also add special instructions for handling JSON
-        default_conversation[0]["content"] = (
-            custom_system_prompt
-            + "\n\n**IMPORTANT**: IF THE JUDGE RESPONSES ARE IN JSON FORMAT, YOU MUST RESPOND USING THE SAME JSON FORMAT THAT THE RESPONSES ARE IN. If the judge responses are in JSON, you MUST RESPOND IN VALID JSON FORMAT."
-        )
-
-    # If a custom conversation history is provided, append the judge responses to it
-    if custom_conversation_history is not None:
-        # Validate custom conversation history format
-        for message in custom_conversation_history:
-            if not isinstance(message, dict):
-                raise TypeError(
-                    f"Custom conversation history must be a list of dictionaries. Received: {message}."
-                )
-
-            if "role" not in message or "content" not in message:
-                raise ValueError("Each message must have 'role' and 'content' keys")
-
-            if not isinstance(message["role"], str) or not isinstance(
-                message["content"], str
-            ):
-                raise TypeError(
-                    f"Message role and content must be strings. Received: {type(message['role'])}, {type(message['content'])}."
-                )
-
-            if message["role"] not in ["system", "user", "assistant"]:
-                raise ValueError(
-                    f"Message role must be one of: 'system', 'user', 'assistant'. Received: {message['role']}."
-                )
-
-        judge_responses_prompt = {
-            "role": "user",
-            "content": f"## Start of Judge Responses ##\n{formatted_responses}\n## End of Judge Responses ##\nSynthesized response:\n",
-        }
-        return custom_conversation_history + [judge_responses_prompt]
-
-    # Otherwise return the default conversation with system prompt and examples
-    # No customization, return the default conversation with system prompt and examples
-    return default_conversation
-
-
-BASE_CONVERSATION = [
-    {"role": "system", "content": "You are a helpful assistant."},
-]  # for string inputs, we need to add the user query to a base conversation, since LiteLLM only accepts a list of dictionaries as a chat history
-
-
-class MixtureOfJudges(JudgevalJudge):
-    """
-    IMPORTANT: When supplying custom prompts and conversation histories for aggregation, supply them in the following format:
-    in kwargs:
-    {
-        "custom_prompt": "Your custom prompt here",
-        "custom_conversation": [
-            {"role": "system", "content": "System message 1"},
-            {"role": "user", "content": "User message 1"},
-            {"role": "assistant", "content": "Assistant message 1"},
-            ...
-        ]
-    }
-    """
-
-    def __init__(
-        self,
-        models: List[str] = [
-            "QWEN",
-            "LLAMA3_70B_INSTRUCT_TURBO",
-            "MISTRAL_8x22B_INSTRUCT",
-        ],
-        aggregator: str = "gpt-4.1",
-        **kwargs,
-    ):
-        """
-        `models` are the individual judge models to be used for generating responses.
-        `aggregator` is the model that will aggregate the responses from the individual judges.
-
-        kwargs include "custom_prompt" and "custom_conversation" for customizing the prompt for the Mixture of Judges model.
-        """
-        self.models = models
-        self.aggregator = aggregator
-        self.kwargs = kwargs
-        super().__init__(model_name=models)
-
-    def generate(
-        self,
-        input: Union[str, List[dict]],
-        response_schema: pydantic.BaseModel = None,
-        aggregation_schema: pydantic.BaseModel = None,
-        **kwargs,
-    ) -> str:
-        """
-        Args:
-            input (Union[str, List[Mapping[str, str]]]): Input query or conversation history to the model.
-            response_schema (pydantic.BaseModel): Response schema for individual judge models.
-            aggregation_schema (pydantic.BaseModel): Response schema for the aggregator model.
-            kwargs: Additional keyword arguments.
-        """
-
-        # Convert input to conversation format if needed
-        if isinstance(input, str):
-            convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-        elif isinstance(input, list):
-            convo = input
-        else:
-            judgeval_logger.error(f"Invalid input type received: {type(input)}")
-            raise TypeError(
-                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
-            )
-
-        try:
-            responses = get_completion_multiple_models(
-                models=self.models,
-                messages=[convo] * len(self.models),
-                response_formats=[response_schema] * len(self.models),
-            )
-        except Exception:
-            raise
-
-        compiled_mixture_prompt = build_dynamic_mixture_prompt(
-            responses,
-            self.kwargs.get("custom_prompt"),
-            self.kwargs.get("custom_conversation"),
-        )
-
-        try:
-            mixed_response = get_chat_completion(
-                model_type=self.aggregator,
-                messages=compiled_mixture_prompt,
-                response_format=aggregation_schema,
-            )
-        except Exception:
-            raise
-
-        return mixed_response
-
-    async def a_generate(
-        self,
-        input: Union[str, List[dict]],
-        response_schema: pydantic.BaseModel = None,
-        aggregation_schema: pydantic.BaseModel = None,
-        **kwargs,
-    ) -> str:
-        """
-        Args:
-            input (Union[str, List[Mapping[str, str]]]): Input query or conversation history to the model.
-            response_schema (pydantic.BaseModel): Response schema for individual judge models.
-            aggregation_schema (pydantic.BaseModel): Response schema for the aggregator model.
-            kwargs: Additional keyword arguments.
-        """
-
-        # Convert input to conversation format if needed
-        if isinstance(input, str):
-            convo = BASE_CONVERSATION + [{"role": "user", "content": input}]
-        elif isinstance(input, list):
-            convo = input
-        else:
-            judgeval_logger.error(f"Invalid input type received: {type(input)}")
-            raise TypeError(
-                f"Input must be a string or a list of dictionaries. Input type of: {type(input)}"
-            )
-
-        try:
-            responses = await aget_completion_multiple_models(
-                models=self.models,
-                messages=[convo] * len(self.models),
-                response_formats=[response_schema] * len(self.models),
-            )
-        except Exception:
-            raise
-
-        compiled_mixture_prompt = build_dynamic_mixture_prompt(
-            responses,
-            self.kwargs.get("custom_prompt"),
-            self.kwargs.get("custom_conversation"),
-        )
-
-        try:
-            mixed_response = await aget_chat_completion(
-                model_type=self.aggregator,
-                messages=compiled_mixture_prompt,
-                response_format=aggregation_schema,
-            )
-        except Exception:
-            raise
-
-        return mixed_response
-
-    def load_model(self):
-        return self.models
-
-    def get_model_name(self) -> List[str]:
-        return self.models

judgeval/judgment_client.py

@@ -1,369 +0,0 @@
-"""
-Implements the JudgmentClient to interact with the Judgment API.
-"""
-
-import os
-from uuid import uuid4
-from typing import Optional, List, Dict, Any, Union, Callable
-
-from judgeval.data.datasets import EvalDataset, EvalDatasetClient
-from judgeval.data import (
-    ScoringResult,
-    Example,
-    Trace,
-)
-from judgeval.scorers import (
-    APIScorerConfig,
-    BaseScorer,
-)
-from judgeval.evaluation_run import EvaluationRun
-from judgeval.run_evaluation import (
-    run_eval,
-    assert_test,
-    run_trace_eval,
-)
-from judgeval.data.trace_run import TraceRun
-from judgeval.common.api import JudgmentApiClient
-from judgeval.common.exceptions import JudgmentAPIError
-from langchain_core.callbacks import BaseCallbackHandler
-from judgeval.common.tracer import Tracer
-from judgeval.common.utils import validate_api_key
-from pydantic import BaseModel
-from judgeval.common.logger import judgeval_logger
-
-
-class EvalRunRequestBody(BaseModel):
-    eval_name: str
-    project_name: str
-
-
-class DeleteEvalRunRequestBody(BaseModel):
-    eval_names: List[str]
-    project_name: str
-
-
-class SingletonMeta(type):
-    _instances: Dict[type, "JudgmentClient"] = {}
-
-    def __call__(cls, *args, **kwargs):
-        if cls not in cls._instances:
-            instance = super().__call__(*args, **kwargs)
-            cls._instances[cls] = instance
-        return cls._instances[cls]
-
-
-class JudgmentClient(metaclass=SingletonMeta):
-    def __init__(
-        self,
-        api_key: Optional[str] = os.getenv("JUDGMENT_API_KEY"),
-        organization_id: Optional[str] = os.getenv("JUDGMENT_ORG_ID"),
-    ):
-        if not api_key:
-            raise ValueError(
-                "api_key parameter must be provided. Please provide a valid API key value or set the JUDGMENT_API_KEY environment variable."
-            )
-
-        if not organization_id:
-            raise ValueError(
-                "organization_id parameter must be provided. Please provide a valid organization ID value or set the JUDGMENT_ORG_ID environment variable."
-            )
-
-        self.judgment_api_key = api_key
-        self.organization_id = organization_id
-        self.api_client = JudgmentApiClient(api_key, organization_id)
-        self.eval_dataset_client = EvalDatasetClient(api_key, organization_id)
-
-        # Verify API key is valid
-        result, response = validate_api_key(api_key)
-        if not result:
-            # May be bad to output their invalid API key...
-            raise JudgmentAPIError(f"Issue with passed in Judgment API key: {response}")
-        else:
-            judgeval_logger.info("Successfully initialized JudgmentClient!")
-
-    def run_trace_evaluation(
-        self,
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
-        examples: Optional[List[Example]] = None,
-        function: Optional[Callable] = None,
-        tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None,
-        traces: Optional[List[Trace]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-        project_name: str = "default_project",
-        eval_run_name: str = "default_eval_trace",
-        model: Optional[str] = "gpt-4.1",
-        append: bool = False,
-        override: bool = False,
-    ) -> List[ScoringResult]:
-        try:
-            if examples and not function:
-                raise ValueError("Cannot pass in examples without a function")
-
-            if traces and function:
-                raise ValueError("Cannot pass in traces and function")
-
-            if examples and traces:
-                raise ValueError("Cannot pass in both examples and traces")
-
-            trace_run = TraceRun(
-                project_name=project_name,
-                eval_name=eval_run_name,
-                traces=traces,
-                scorers=scorers,
-                model=model,
-                append=append,
-                organization_id=self.organization_id,
-                tools=tools,
-            )
-            return run_trace_eval(
-                trace_run, self.judgment_api_key, override, function, tracer, examples
-            )
-        except ValueError as e:
-            raise ValueError(
-                f"Please check your TraceRun object, one or more fields are invalid: \n{str(e)}"
-            )
-        except Exception as e:
-            raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
-
-    def run_evaluation(
-        self,
-        examples: List[Example],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
-        model: Optional[str] = "gpt-4.1",
-        project_name: str = "default_project",
-        eval_run_name: str = "default_eval_run",
-        override: bool = False,
-        append: bool = False,
-    ) -> List[ScoringResult]:
-        """
-        Executes an evaluation of `Example`s using one or more `Scorer`s
-
-        Args:
-            examples (List[Example]): The examples to evaluate
-            scorers (List[Union[APIScorerConfig, BaseScorer]]): A list of scorers to use for evaluation
-            model (str): The model used as a judge when using LLM as a Judge
-            project_name (str): The name of the project the evaluation results belong to
-            eval_run_name (str): A name for this evaluation run
-            override (bool): Whether to override an existing evaluation run with the same name
-            append (bool): Whether to append to an existing evaluation run with the same name
-
-        Returns:
-            List[ScoringResult]: The results of the evaluation
-        """
-        if override and append:
-            raise ValueError(
-                "Cannot set both override and append to True. Please choose one."
-            )
-
-        try:
-            eval = EvaluationRun(
-                append=append,
-                override=override,
-                project_name=project_name,
-                eval_name=eval_run_name,
-                examples=examples,
-                scorers=scorers,
-                model=model,
-                organization_id=self.organization_id,
-            )
-            return run_eval(
-                eval,
-                self.judgment_api_key,
-                override,
-            )
-        except ValueError as e:
-            raise ValueError(
-                f"Please check your EvaluationRun object, one or more fields are invalid: \n{str(e)}"
-            )
-        except Exception as e:
-            raise Exception(f"An unexpected error occurred during evaluation: {str(e)}")
-
-    def create_dataset(self) -> EvalDataset:
-        return self.eval_dataset_client.create_dataset()
-
-    def push_dataset(
-        self,
-        alias: str,
-        dataset: EvalDataset,
-        project_name: str,
-        overwrite: Optional[bool] = False,
-    ) -> bool:
-        """
-        Uploads an `EvalDataset` to the Judgment platform for storage.
-
-        Args:
-            alias (str): The name to use for the dataset
-            dataset (EvalDataset): The dataset to upload to Judgment
-            overwrite (Optional[bool]): Whether to overwrite the dataset if it already exists
-
-        Returns:
-            bool: Whether the dataset was successfully uploaded
-        """
-        # Set judgment_api_key just in case it was not set
-        dataset.judgment_api_key = self.judgment_api_key
-        return self.eval_dataset_client.push(dataset, alias, project_name, overwrite)
-
-    def append_dataset(
-        self, alias: str, examples: List[Example], project_name: str
-    ) -> bool:
-        """
-        Appends an `EvalDataset` to the Judgment platform for storage.
-        """
-        return self.eval_dataset_client.append_examples(alias, examples, project_name)
-
-    def pull_dataset(self, alias: str, project_name: str) -> EvalDataset:
-        """
-        Retrieves a saved `EvalDataset` from the Judgment platform.
-
-        Args:
-            alias (str): The name of the dataset to retrieve
-
-        Returns:
-            EvalDataset: The retrieved dataset
-        """
-        return self.eval_dataset_client.pull(alias, project_name)
-
-    def delete_dataset(self, alias: str, project_name: str) -> bool:
-        """
-        Deletes a saved `EvalDataset` from the Judgment platform.
-        """
-        return self.eval_dataset_client.delete(alias, project_name)
-
-    def pull_project_dataset_stats(self, project_name: str) -> dict:
-        """
-        Retrieves all dataset stats from the Judgment platform for the project.
-
-        Args:
-            project_name (str): The name of the project to retrieve
-
-        Returns:
-            dict: The retrieved dataset stats
-        """
-        return self.eval_dataset_client.pull_project_dataset_stats(project_name)
-
-    # Maybe add option where you can pass in the EvaluationRun object and it will pull the eval results from the backend
-    def pull_eval(
-        self, project_name: str, eval_run_name: str
-    ) -> List[Dict[str, Union[str, List[ScoringResult]]]]:
-        """Pull evaluation results from the server.
-
-        Args:
-            project_name (str): Name of the project
-            eval_run_name (str): Name of the evaluation run
-
-        Returns:
-            Dict[str, Union[str, List[ScoringResult]]]: Dictionary containing:
-                - id (str): The evaluation run ID
-                - results (List[ScoringResult]): List of scoring results
-        """
-        return self.api_client.fetch_evaluation_results(project_name, eval_run_name)
-
-    def create_project(self, project_name: str) -> bool:
-        """
-        Creates a project on the server.
-        """
-        self.api_client.create_project(project_name)
-        return True
-
-    def delete_project(self, project_name: str) -> bool:
-        """
-        Deletes a project from the server. Which also deletes all evaluations and traces associated with the project.
-        """
-        self.api_client.delete_project(project_name)
-        return True
-
-    def assert_test(
-        self,
-        examples: List[Example],
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
-        model: Optional[str] = "gpt-4.1",
-        project_name: str = "default_test",
-        eval_run_name: str = str(uuid4()),
-        override: bool = False,
-        append: bool = False,
-    ) -> None:
-        """
-        Asserts a test by running the evaluation and checking the results for success
-
-        Args:
-            examples (List[Example]): The examples to evaluate.
-            scorers (List[Union[APIScorerConfig, BaseScorer]]): A list of scorers to use for evaluation
-            model (str): The model used as a judge when using LLM as a Judge
-            project_name (str): The name of the project the evaluation results belong to
-            eval_run_name (str): A name for this evaluation run
-            override (bool): Whether to override an existing evaluation run with the same name
-            append (bool): Whether to append to an existing evaluation run with the same name
-            async_execution (bool): Whether to run the evaluation asynchronously
-        """
-
-        results: List[ScoringResult]
-
-        results = self.run_evaluation(
-            examples=examples,
-            scorers=scorers,
-            model=model,
-            project_name=project_name,
-            eval_run_name=eval_run_name,
-            override=override,
-            append=append,
-        )
-        assert_test(results)
-
-    def assert_trace_test(
-        self,
-        scorers: List[Union[APIScorerConfig, BaseScorer]],
-        examples: Optional[List[Example]] = None,
-        function: Optional[Callable] = None,
-        tracer: Optional[Union[Tracer, BaseCallbackHandler]] = None,
-        traces: Optional[List[Trace]] = None,
-        tools: Optional[List[Dict[str, Any]]] = None,
-        model: Optional[str] = "gpt-4.1",
-        project_name: str = "default_test",
-        eval_run_name: str = str(uuid4()),
-        override: bool = False,
-        append: bool = False,
-        async_execution: bool = False,
-    ) -> None:
-        """
-        Asserts a test by running the evaluation and checking the results for success
-
-        Args:
-            examples (List[Example]): The examples to evaluate.
-            scorers (List[Union[APIScorerConfig, BaseScorer]]): A list of scorers to use for evaluation
-            model (str): The model used as a judge when using LLM as a Judge
-            project_name (str): The name of the project the evaluation results belong to
-            eval_run_name (str): A name for this evaluation run
-            override (bool): Whether to override an existing evaluation run with the same name
-            append (bool): Whether to append to an existing evaluation run with the same name
-            function (Optional[Callable]): A function to use for evaluation
-            tracer (Optional[Union[Tracer, BaseCallbackHandler]]): A tracer to use for evaluation
-            tools (Optional[List[Dict[str, Any]]]): A list of tools to use for evaluation
-            async_execution (bool): Whether to run the evaluation asynchronously
-        """
-
-        # Check for enable_param_checking and tools
-        for scorer in scorers:
-            if hasattr(scorer, "kwargs") and scorer.kwargs is not None:
-                if scorer.kwargs.get("enable_param_checking") is True:
-                    if not tools:
-                        raise ValueError(
-                            f"You must provide the 'tools' argument to assert_test when using a scorer with enable_param_checking=True. If you do not want to do param checking, explicitly set enable_param_checking=False for the {scorer.__name__} scorer."
-                        )
-
-        results: List[ScoringResult]
-
-        results = self.run_trace_evaluation(
-            examples=examples,
-            traces=traces,
-            scorers=scorers,
-            model=model,
-            project_name=project_name,
-            eval_run_name=eval_run_name,
-            override=override,
-            append=append,
-            function=function,
-            tracer=tracer,
-            tools=tools,
-        )
-
-        assert_test(results)