deepeval 3.6.6__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.6.6"
+__version__: str = "3.6.7"

deepeval/benchmarks/equity_med_qa/equity_med_qa.py

@@ -121,6 +121,7 @@ class EquityMedQA(DeepEvalBaseBenchmark):
         score = metric.measure(
             LLMTestCase(input=golden.input, actual_output=prediction),
             _show_indicator=False,
+            _log_metric_to_confident=False,
         )
         flipped_score = (
             1 - metric.score if metric.score in [0, 1] else metric.score

deepeval/cli/main.py

@@ -328,6 +328,31 @@ def set_debug(
         "--trace-flush/--no-trace-flush",
         help="Enable / disable  CONFIDENT_TRACE_FLUSH.",
     ),
+    trace_sample_rate: Optional[float] = typer.Option(
+        None,
+        "--trace-sample-rate",
+        help="Set CONFIDENT_TRACE_SAMPLE_RATE.",
+    ),
+    metric_logging_verbose: Optional[bool] = typer.Option(
+        None,
+        "--metric-logging-verbose/--no-metric-logging-verbose",
+        help="Enable / disable CONFIDENT_METRIC_LOGGING_VERBOSE.",
+    ),
+    metric_logging_flush: Optional[bool] = typer.Option(
+        None,
+        "--metric-logging-flush/--no-metric-logging-flush",
+        help="Enable / disable CONFIDENT_METRIC_LOGGING_FLUSH.",
+    ),
+    metric_logging_sample_rate: Optional[float] = typer.Option(
+        None,
+        "--metric-logging-sample-rate",
+        help="Set CONFIDENT_METRIC_LOGGING_SAMPLE_RATE.",
+    ),
+    metric_logging_enabled: Optional[bool] = typer.Option(
+        None,
+        "--metric-logging-enabled/--no-metric-logging-enabled",
+        help="Enable / disable CONFIDENT_METRIC_LOGGING_ENABLED.",
+    ),
     # Advanced / potentially surprising
     error_reporting: Optional[bool] = typer.Option(
         None,
@@ -387,6 +412,20 @@ def set_debug(
             settings.CONFIDENT_TRACE_ENVIRONMENT = trace_env
         if trace_flush is not None:
             settings.CONFIDENT_TRACE_FLUSH = trace_flush
+        if trace_sample_rate is not None:
+            settings.CONFIDENT_TRACE_SAMPLE_RATE = trace_sample_rate
+
+        # Confident metrics
+        if metric_logging_verbose is not None:
+            settings.CONFIDENT_METRIC_LOGGING_VERBOSE = metric_logging_verbose
+        if metric_logging_flush is not None:
+            settings.CONFIDENT_METRIC_LOGGING_FLUSH = metric_logging_flush
+        if metric_logging_sample_rate is not None:
+            settings.CONFIDENT_METRIC_LOGGING_SAMPLE_RATE = (
+                metric_logging_sample_rate
+            )
+        if metric_logging_enabled is not None:
+            settings.CONFIDENT_METRIC_LOGGING_ENABLED = metric_logging_enabled
 
         # Advanced
         if error_reporting is not None:
@@ -438,6 +477,8 @@ def unset_debug(
         settings.LOG_LEVEL = "info"
         settings.CONFIDENT_TRACE_ENVIRONMENT = "development"
         settings.CONFIDENT_TRACE_VERBOSE = True
+        settings.CONFIDENT_METRIC_LOGGING_VERBOSE = True
+        settings.CONFIDENT_METRIC_LOGGING_ENABLED = True
 
         # Clear optional toggles/overrides
         settings.DEEPEVAL_VERBOSE_MODE = None
@@ -449,6 +490,7 @@ def unset_debug(
         settings.GRPC_TRACE = None
 
         settings.CONFIDENT_TRACE_FLUSH = None
+        settings.CONFIDENT_METRIC_LOGGING_FLUSH = None
 
         settings.ERROR_REPORTING = None
         settings.IGNORE_DEEPEVAL_ERRORS = None

deepeval/confident/api.py

@@ -87,6 +87,7 @@ class Endpoints(Enum):
     DATASET_ALIAS_QUEUE_ENDPOINT = "/v1/datasets/:alias/queue"
 
     TEST_RUN_ENDPOINT = "/v1/test-run"
+    METRIC_DATA_ENDPOINT = "/v1/metric-data"
     TRACES_ENDPOINT = "/v1/traces"
     ANNOTATIONS_ENDPOINT = "/v1/annotations"
     PROMPTS_VERSION_ID_ENDPOINT = "/v1/prompts/:alias/versions/:versionId"

deepeval/config/settings.py

@@ -337,10 +337,17 @@ class Settings(BaseSettings):
     SKIP_DEEPEVAL_MISSING_PARAMS: Optional[bool] = None
     DEEPEVAL_VERBOSE_MODE: Optional[bool] = None
     ENABLE_DEEPEVAL_CACHE: Optional[bool] = None
+
     CONFIDENT_TRACE_FLUSH: Optional[bool] = None
     CONFIDENT_TRACE_ENVIRONMENT: Optional[str] = "development"
     CONFIDENT_TRACE_VERBOSE: Optional[bool] = True
-    CONFIDENT_SAMPLE_RATE: Optional[float] = 1.0
+    CONFIDENT_TRACE_SAMPLE_RATE: Optional[float] = 1.0
+
+    CONFIDENT_METRIC_LOGGING_FLUSH: Optional[bool] = None
+    CONFIDENT_METRIC_LOGGING_VERBOSE: Optional[bool] = True
+    CONFIDENT_METRIC_LOGGING_SAMPLE_RATE: Optional[float] = 1.0
+    CONFIDENT_METRIC_LOGGING_ENABLED: Optional[bool] = True
+
     OTEL_EXPORTER_OTLP_ENDPOINT: Optional[AnyUrl] = None
 
     #
@@ -355,6 +362,12 @@ class Settings(BaseSettings):
         None  # per-attempt timeout. Set 0/None to disable
     )
 
+    #
+    # Async Document Pipelines
+    #
+
+    DEEPEVAL_MAX_CONCURRENT_DOC_PROCESSING: conint(ge=1) = 2
+
     #
     # Async Task Configuration
     #
@@ -484,7 +497,8 @@ class Settings(BaseSettings):
         "OPENAI_COST_PER_INPUT_TOKEN",
         "OPENAI_COST_PER_OUTPUT_TOKEN",
         "TEMPERATURE",
-        "CONFIDENT_SAMPLE_RATE",
+        "CONFIDENT_TRACE_SAMPLE_RATE",
+        "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE",
         mode="before",
     )
     @classmethod
@@ -496,13 +510,17 @@ class Settings(BaseSettings):
             return None
         return float(v)
 
-    @field_validator("CONFIDENT_SAMPLE_RATE")
+    @field_validator(
+        "CONFIDENT_TRACE_SAMPLE_RATE", "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE"
+    )
     @classmethod
     def _validate_sample_rate(cls, v):
         if v is None:
             return None
         if not (0.0 <= float(v) <= 1.0):
-            raise ValueError("CONFIDENT_SAMPLE_RATE must be between 0 and 1")
+            raise ValueError(
+                "CONFIDENT_TRACE_SAMPLE_RATE or CONFIDENT_METRIC_LOGGING_SAMPLE_RATE must be between 0 and 1"
+            )
         return float(v)
 
     @field_validator("DEEPEVAL_DEFAULT_SAVE", mode="before")

deepeval/constants.py

@@ -9,9 +9,16 @@ LOGIN_PROMPT = "\n✨👀 Looking for a place for your LLM test data to live
 
 CONFIDENT_TRACE_VERBOSE = "CONFIDENT_TRACE_VERBOSE"
 CONFIDENT_TRACE_FLUSH = "CONFIDENT_TRACE_FLUSH"
-CONFIDENT_SAMPLE_RATE = "CONFIDENT_SAMPLE_RATE"
+CONFIDENT_TRACE_SAMPLE_RATE = "CONFIDENT_TRACE_SAMPLE_RATE"
 CONFIDENT_TRACE_ENVIRONMENT = "CONFIDENT_TRACE_ENVIRONMENT"
 CONFIDENT_TRACING_ENABLED = "CONFIDENT_TRACING_ENABLED"
+
+CONFIDENT_METRIC_LOGGING_VERBOSE = "CONFIDENT_METRIC_LOGGING_VERBOSE"
+CONFIDENT_METRIC_LOGGING_FLUSH = "CONFIDENT_METRIC_LOGGING_FLUSH"
+CONFIDENT_METRIC_LOGGING_SAMPLE_RATE = "CONFIDENT_METRIC_LOGGING_SAMPLE_RATE"
+CONFIDENT_METRIC_LOGGING_ENABLED = "CONFIDENT_METRIC_LOGGING_ENABLED"
+
+
 CONFIDENT_OPEN_BROWSER = "CONFIDENT_OPEN_BROWSER"
 CONFIDENT_TEST_CASE_BATCH_SIZE = "CONFIDENT_TEST_CASE_BATCH_SIZE"
 

deepeval/dataset/dataset.py

@@ -49,7 +49,7 @@ from deepeval.utils import (
 from deepeval.test_run import (
     global_test_run_manager,
 )
-from deepeval.openai.utils import openai_test_case_pairs
+
 from deepeval.tracing import trace_manager
 from deepeval.tracing.tracing import EVAL_DUMMY_SPAN_NAME
 
@@ -1248,16 +1248,7 @@ class EvaluationDataset:
                         display_config.file_output_dir,
                     )
 
-            # update hyperparameters
-            test_run = global_test_run_manager.get_test_run()
-            if len(openai_test_case_pairs) > 0:
-                raw_hyperparameters = openai_test_case_pairs[-1].hyperparameters
-                test_run.hyperparameters = process_hyperparameters(
-                    raw_hyperparameters
-                )
-
-            # clean up
-            openai_test_case_pairs.clear()
+            # save test run
             global_test_run_manager.save_test_run(TEMP_FILE_PATH)
 
             # sandwich end trace for OTEL

deepeval/dataset/utils.py

@@ -120,7 +120,7 @@ def format_turns(turns: List[Turn]) -> str:
         }
         res.append(cur_turn)
     try:
-        return json.dumps(res)
+        return json.dumps(res, ensure_ascii=False)
     except Exception as e:
         raise ValueError(f"Error serializing turns: {e}")
 

deepeval/evaluate/evaluate.py

@@ -28,7 +28,10 @@ from deepeval.evaluate.utils import (
 from deepeval.dataset import Golden
 from deepeval.prompt import Prompt
 from deepeval.test_case.utils import check_valid_test_cases_type
-from deepeval.test_run.hyperparameters import process_hyperparameters
+from deepeval.test_run.hyperparameters import (
+    process_hyperparameters,
+    process_prompts,
+)
 from deepeval.test_run.test_run import TEMP_FILE_PATH
 from deepeval.utils import (
     get_or_create_event_loop,
@@ -267,6 +270,7 @@ def evaluate(
 
         test_run = global_test_run_manager.get_test_run()
         test_run.hyperparameters = process_hyperparameters(hyperparameters)
+        test_run.prompts = process_prompts(hyperparameters)
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
         res = global_test_run_manager.wrap_up_test_run(
             run_duration, display_table=False

deepeval/evaluate/execute.py

@@ -61,6 +61,7 @@ from deepeval.test_case import (
     ConversationalTestCase,
     MLLMTestCase,
 )
+from deepeval.test_case.api import create_api_test_case
 from deepeval.test_run import (
     global_test_run_manager,
     LLMApiTestCase,
@@ -80,15 +81,18 @@ from deepeval.evaluate.utils import (
     create_api_trace,
     create_metric_data,
     create_test_result,
-    create_api_test_case,
     count_metrics_in_trace,
     extract_trace_test_results,
 )
 from deepeval.utils import add_pbar, update_pbar, custom_console
-from deepeval.openai.utils import openai_test_case_pairs
 from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.config.settings import get_settings
-
+from deepeval.test_run import TEMP_FILE_PATH
+from deepeval.confident.api import is_confident
+from deepeval.test_run.hyperparameters import (
+    process_hyperparameters,
+    process_prompts,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -902,6 +906,7 @@ def execute_agentic_test_cases(
                         trace_api.agent_spans.append(api_span)
                     elif isinstance(span, LlmSpan):
                         trace_api.llm_spans.append(api_span)
+                        log_prompt(span, test_run_manager)
                     elif isinstance(span, RetrieverSpan):
                         trace_api.retriever_spans.append(api_span)
                     elif isinstance(span, ToolSpan):
@@ -1284,6 +1289,7 @@ async def _a_execute_agentic_test_case(
             verbose_mode=verbose_mode,
             progress=progress,
             pbar_eval_id=pbar_eval_id,
+            test_run_manager=test_run_manager,
             _use_bar_indicator=_use_bar_indicator,
         )
         child_tasks = [dfs(child) for child in span.children]
@@ -1291,7 +1297,18 @@ async def _a_execute_agentic_test_case(
             await asyncio.gather(*child_tasks)
 
     test_start_time = time.perf_counter()
-    await dfs(current_trace.root_spans[0])
+    if current_trace and current_trace.root_spans:
+        await dfs(current_trace.root_spans[0])
+    else:
+        if (
+            logger.isEnabledFor(logging.DEBUG)
+            and get_settings().DEEPEVAL_VERBOSE_MODE
+        ):
+            logger.debug(
+                "Skipping DFS: empty trace or no root spans (trace=%s)",
+                current_trace.uuid if current_trace else None,
+            )
+
     test_end_time = time.perf_counter()
     run_duration = test_end_time - test_start_time
 
@@ -1313,6 +1330,7 @@ async def _a_execute_span_test_case(
     verbose_mode: Optional[bool],
     progress: Optional[Progress],
     pbar_eval_id: Optional[int],
+    test_run_manager: Optional[TestRunManager],
     _use_bar_indicator: bool,
 ):
     api_span: BaseApiSpan = trace_manager._convert_span_to_api_span(span)
@@ -1320,6 +1338,7 @@ async def _a_execute_span_test_case(
         trace_api.agent_spans.append(api_span)
     elif isinstance(span, LlmSpan):
         trace_api.llm_spans.append(api_span)
+        log_prompt(span, test_run_manager)
     elif isinstance(span, RetrieverSpan):
         trace_api.retriever_spans.append(api_span)
     elif isinstance(span, ToolSpan):
@@ -1568,6 +1587,7 @@ def execute_agentic_test_cases_from_loop(
                         trace_api.agent_spans.append(api_span)
                     elif isinstance(span, LlmSpan):
                         trace_api.llm_spans.append(api_span)
+                        log_prompt(span, test_run_manager)
                     elif isinstance(span, RetrieverSpan):
                         trace_api.retriever_spans.append(api_span)
                     elif isinstance(span, ToolSpan):
@@ -1748,6 +1768,7 @@ def execute_agentic_test_cases_from_loop(
         local_trace_manager.evaluating = False
         local_trace_manager.traces_to_evaluate_order.clear()
         local_trace_manager.traces_to_evaluate.clear()
+        local_trace_manager.trace_uuid_to_golden.clear()
 
 
 def a_execute_agentic_test_cases_from_loop(
@@ -1950,12 +1971,12 @@ def a_execute_agentic_test_cases_from_loop(
                     return
 
                 try:
+                    current_tasks = set()
                     # Find tasks that were created during this run but we didn’t track
                     current_tasks = loop.run_until_complete(_snapshot_tasks())
                 except RuntimeError:
                     # this might happen if the loop is already closing
-                    # nothing we can do
-                    return
+                    pass
 
                 leftovers = [
                     t
@@ -1965,9 +1986,6 @@ def a_execute_agentic_test_cases_from_loop(
                     and not t.done()
                 ]
 
-                if not leftovers:
-                    return
-
                 if get_settings().DEEPEVAL_DEBUG_ASYNC:
                     logger.warning(
                         "[deepeval] %d stray task(s) not tracked; cancelling...",
@@ -1978,20 +1996,21 @@ def a_execute_agentic_test_cases_from_loop(
                         name = t.get_name()
                         logger.warning("  - STRAY %s meta=%s", name, meta)
 
-                for t in leftovers:
-                    t.cancel()
+                if leftovers:
+                    for t in leftovers:
+                        t.cancel()
 
-                # Drain strays so they don’t leak into the next iteration
-                try:
-                    loop.run_until_complete(
-                        asyncio.gather(*leftovers, return_exceptions=True)
-                    )
-                except RuntimeError:
-                    # If the loop is closing here, just continue
-                    if get_settings().DEEPEVAL_DEBUG_ASYNC:
-                        logger.warning(
-                            "[deepeval] failed to drain stray tasks because loop is closing"
+                    # Drain strays so they don’t leak into the next iteration
+                    try:
+                        loop.run_until_complete(
+                            asyncio.gather(*leftovers, return_exceptions=True)
                         )
+                    except RuntimeError:
+                        # If the loop is closing here, just continue
+                        if get_settings().DEEPEVAL_DEBUG_ASYNC:
+                            logger.warning(
+                                "[deepeval] failed to drain stray tasks because loop is closing"
+                            )
 
         # Evaluate traces
         if trace_manager.traces_to_evaluate:
@@ -2014,25 +2033,6 @@ def a_execute_agentic_test_cases_from_loop(
                     pbar_id=pbar_id,
                 )
             )
-        elif openai_test_case_pairs:
-            loop.run_until_complete(
-                _evaluate_test_case_pairs(
-                    test_case_pairs=openai_test_case_pairs,
-                    test_run=test_run,
-                    test_run_manager=test_run_manager,
-                    test_results=test_results,
-                    ignore_errors=error_config.ignore_errors,
-                    skip_on_missing_params=error_config.skip_on_missing_params,
-                    show_indicator=display_config.show_indicator,
-                    verbose_mode=display_config.verbose_mode,
-                    throttle_value=async_config.throttle_value,
-                    max_concurrent=async_config.max_concurrent,
-                    _use_bar_indicator=_use_bar_indicator,
-                    _is_assert_test=_is_assert_test,
-                    progress=progress,
-                    pbar_id=pbar_id,
-                )
-            )
         elif trace_manager.integration_traces_to_evaluate:
             loop.run_until_complete(
                 _a_evaluate_traces(
@@ -2106,6 +2106,7 @@ def a_execute_agentic_test_cases_from_loop(
         local_trace_manager.evaluating = False
         local_trace_manager.traces_to_evaluate_order.clear()
         local_trace_manager.traces_to_evaluate.clear()
+        local_trace_manager.trace_uuid_to_golden.clear()
 
 
 async def _a_evaluate_traces(
@@ -2132,8 +2133,26 @@ async def _a_evaluate_traces(
             return await func(*args, **kwargs)
 
     eval_tasks = []
-    for count, trace in enumerate(traces_to_evaluate):
-        golden = goldens[count]
+    # Here, we will work off a fixed-set copy to avoid surprises from potential
+    # mid-iteration mutation
+    traces_snapshot = list(traces_to_evaluate or [])
+
+    for count, trace in enumerate(traces_snapshot):
+        # Prefer the explicit mapping from trace -> golden captured at trace creation.
+        golden = trace_manager.trace_uuid_to_golden.get(trace.uuid)
+        if not golden:
+            # trace started during evaluation_loop but the CURRENT_GOLDEN was
+            # not set for some reason. We can’t map it to a golden, so the best
+            # we can do is skip evaluation for this trace.
+            if (
+                logger.isEnabledFor(logging.DEBUG)
+                and get_settings().DEEPEVAL_VERBOSE_MODE
+            ):
+                logger.debug(
+                    "Skipping trace %s: no golden association found during evaluation_loop ",
+                    trace.uuid,
+                )
+            continue
         with capture_evaluation_run("golden"):
             task = execute_evals_with_semaphore(
                 func=_a_execute_agentic_test_case,
@@ -2225,6 +2244,7 @@ def _execute_metric(
             test_case,
             _show_indicator=show_metric_indicator,
             _in_component=in_component,
+            _log_metric_to_confident=False,
         )
     except MissingTestCaseParamsError as e:
         if error_config.skip_on_missing_params:
@@ -2259,3 +2279,38 @@ def _execute_metric(
             metric.success = False
         else:
             raise
+
+
+def log_prompt(
+    llm_span: LlmSpan,
+    test_run_manager: TestRunManager,
+):
+    prompt = llm_span.prompt
+    if prompt is None:
+        return
+
+    span_hyperparameters = {}
+    prompt_version = prompt.version if is_confident() else None
+    key = f"{prompt.alias}_{prompt_version}"
+    span_hyperparameters[key] = prompt
+
+    test_run = test_run_manager.get_test_run()
+    if test_run.prompts is None:
+        test_run.prompts = []
+    if test_run.hyperparameters is None:
+        test_run.hyperparameters = {}
+
+    if key not in test_run.hyperparameters:
+        test_run.hyperparameters.update(
+            process_hyperparameters(span_hyperparameters, False)
+        )
+        existing_prompt_keys = {
+            f"{p.alias}_{p.version}" for p in test_run.prompts
+        }
+        new_prompts = process_prompts(span_hyperparameters)
+        for new_prompt in new_prompts:
+            new_prompt_key = f"{new_prompt.alias}_{new_prompt.version}"
+            if new_prompt_key not in existing_prompt_keys:
+                test_run.prompts.append(new_prompt)
+
+    global_test_run_manager.save_test_run(TEMP_FILE_PATH)

deepeval/evaluate/utils.py

@@ -28,7 +28,6 @@ from deepeval.evaluate.types import TestResult
 from deepeval.tracing.api import TraceApi, BaseApiSpan, TraceSpanApiStatus
 from deepeval.tracing.tracing import BaseSpan, Trace
 from deepeval.tracing.types import TraceSpanStatus
-from deepeval.constants import PYTEST_RUN_TEST_NAME
 from deepeval.tracing.utils import (
     perf_counter_to_datetime,
     to_zod_compatible_iso,
@@ -133,121 +132,6 @@ def create_test_result(
             )
 
 
-def create_api_turn(turn: Turn, index: int) -> TurnApi:
-    return TurnApi(
-        role=turn.role,
-        content=turn.content,
-        user_id=turn.user_id,
-        retrievalContext=turn.retrieval_context,
-        toolsCalled=turn.tools_called,
-        additionalMetadata=turn.additional_metadata,
-        order=index,
-    )
-
-
-def create_api_test_case(
-    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
-    trace: Optional[TraceApi] = None,
-    index: Optional[int] = None,
-) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
-    if isinstance(test_case, ConversationalTestCase):
-        order = (
-            test_case._dataset_rank
-            if test_case._dataset_rank is not None
-            else index
-        )
-        if test_case.name:
-            name = test_case.name
-        else:
-            name = os.getenv(
-                PYTEST_RUN_TEST_NAME, f"conversational_test_case_{order}"
-            )
-
-        api_test_case = ConversationalApiTestCase(
-            name=name,
-            success=True,
-            metricsData=[],
-            runDuration=0,
-            evaluationCost=None,
-            order=order,
-            scenario=test_case.scenario,
-            expectedOutcome=test_case.expected_outcome,
-            userDescription=test_case.user_description,
-            context=test_case.context,
-            tags=test_case.tags,
-            comments=test_case.comments,
-            additionalMetadata=test_case.additional_metadata,
-        )
-        api_test_case.turns = [
-            create_api_turn(
-                turn=turn,
-                index=index,
-            )
-            for index, turn in enumerate(test_case.turns)
-        ]
-
-        return api_test_case
-    else:
-        order = (
-            test_case._dataset_rank
-            if test_case._dataset_rank is not None
-            else index
-        )
-
-        success = True
-        if test_case.name is not None:
-            name = test_case.name
-        else:
-            name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
-        metrics_data = []
-
-        if isinstance(test_case, LLMTestCase):
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input=test_case.input,
-                actualOutput=test_case.actual_output,
-                expectedOutput=test_case.expected_output,
-                context=test_case.context,
-                retrievalContext=test_case.retrieval_context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                tags=test_case.tags,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-                trace=trace,
-            )
-        elif isinstance(test_case, MLLMTestCase):
-            api_test_case = LLMApiTestCase(
-                name=name,
-                input="",
-                multimodalInput=test_case.input,
-                multimodalActualOutput=test_case.actual_output,
-                multimodalExpectedOutput=test_case.expected_output,
-                multimodalRetrievalContext=test_case.retrieval_context,
-                multimodalContext=test_case.context,
-                toolsCalled=test_case.tools_called,
-                expectedTools=test_case.expected_tools,
-                tokenCost=test_case.token_cost,
-                completionTime=test_case.completion_time,
-                success=success,
-                metricsData=metrics_data,
-                runDuration=None,
-                evaluationCost=None,
-                order=order,
-                additionalMetadata=test_case.additional_metadata,
-                comments=test_case.comments,
-            )
-        # llm_test_case_lookup_map[instance_id] = api_test_case
-        return api_test_case
-
-
 def create_api_trace(trace: Trace, golden: Golden) -> TraceApi:
     return TraceApi(
         uuid=trace.uuid,
@@ -309,6 +193,26 @@ def validate_assert_test_inputs(
             "Both 'test_case' and 'metrics' must be provided together."
         )
 
+    if test_case and metrics:
+        if isinstance(test_case, LLMTestCase) and not all(
+            isinstance(metric, BaseMetric) for metric in metrics
+        ):
+            raise ValueError(
+                "All 'metrics' for an 'LLMTestCase' must be instances of 'BaseMetric' only."
+            )
+        if isinstance(test_case, ConversationalTestCase) and not all(
+            isinstance(metric, BaseConversationalMetric) for metric in metrics
+        ):
+            raise ValueError(
+                "All 'metrics' for an 'ConversationalTestCase' must be instances of 'BaseConversationalMetric' only."
+            )
+        if isinstance(test_case, MLLMTestCase) and not all(
+            isinstance(metric, BaseMultimodalMetric) for metric in metrics
+        ):
+            raise ValueError(
+                "All 'metrics' for an 'MLLMTestCase' must be instances of 'BaseMultimodalMetric' only."
+            )
+
     if not ((golden and observed_callback) or (test_case and metrics)):
         raise ValueError(
             "You must provide either ('golden' + 'observed_callback') or ('test_case' + 'metrics')."

deepeval/integrations/crewai/__init__.py

@@ -1,3 +1,8 @@
 from .handler import instrument_crewai
+from .subs import (
+    DeepEvalCrew as Crew,
+    DeepEvalAgent as Agent,
+    DeepEvalLLM as LLM,
+)
 
-__all__ = ["instrument_crewai"]
+__all__ = ["instrument_crewai", "Crew", "Agent", "LLM"]

deepeval/integrations/crewai/handler.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 
 
 try:
-    from crewai.utilities.events.base_event_listener import BaseEventListener
+    from crewai.events import BaseEventListener
     from crewai.events import (
         CrewKickoffStartedEvent,
         CrewKickoffCompletedEvent,