deepeval 3.7.3__py3-none-any.whl → 3.7.5__py3-none-any.whl

deepeval/evaluate/utils.py

@@ -16,7 +16,6 @@ from deepeval.metrics import (
 from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
-    MLLMTestCase,
 )
 from deepeval.test_run import (
     LLMApiTestCase,
@@ -129,17 +128,14 @@ def create_test_result(
             turns=api_test_case.turns,
         )
     else:
-        multimodal = (
-            api_test_case.multimodal_input is not None
-            and api_test_case.multimodal_input_actual_output is not None
-        )
+        multimodal = api_test_case.images_mapping
         if multimodal:
             return TestResult(
                 name=name,
                 success=api_test_case.success,
                 metrics_data=api_test_case.metrics_data,
-                input=api_test_case.multimodal_input,
-                actual_output=api_test_case.multimodal_input_actual_output,
+                input=api_test_case.input,
+                actual_output=api_test_case.actual_output,
                 conversational=False,
                 multimodal=True,
                 additional_metadata=api_test_case.additional_metadata,
@@ -222,9 +218,9 @@ def validate_assert_test_inputs(
         )
 
     if test_case and metrics:
-        if isinstance(test_case, LLMTestCase) and not all(
-            isinstance(metric, BaseMetric) for metric in metrics
-        ):
+        if (
+            isinstance(test_case, LLMTestCase) and not test_case.multimodal
+        ) and not all(isinstance(metric, BaseMetric) for metric in metrics):
             raise ValueError(
                 "All 'metrics' for an 'LLMTestCase' must be instances of 'BaseMetric' only."
             )
@@ -234,11 +230,17 @@ def validate_assert_test_inputs(
             raise ValueError(
                 "All 'metrics' for an 'ConversationalTestCase' must be instances of 'BaseConversationalMetric' only."
             )
-        if isinstance(test_case, MLLMTestCase) and not all(
-            isinstance(metric, BaseMultimodalMetric) for metric in metrics
+        if (
+            isinstance(test_case, LLMTestCase) and test_case.multimodal
+        ) and not all(
+            (
+                isinstance(metric, BaseMultimodalMetric)
+                or isinstance(metric, BaseMetric)
+            )
+            for metric in metrics
         ):
             raise ValueError(
-                "All 'metrics' for an 'MLLMTestCase' must be instances of 'BaseMultimodalMetric' only."
+                "All 'metrics' for multi-modal LLMTestCase must be instances of 'BaseMultimodalMetric' only."
             )
 
     if not ((golden and observed_callback) or (test_case and metrics)):
@@ -251,9 +253,7 @@ def validate_evaluate_inputs(
     goldens: Optional[List] = None,
     observed_callback: Optional[Callable] = None,
     test_cases: Optional[
-        Union[
-            List[LLMTestCase], List[ConversationalTestCase], List[MLLMTestCase]
-        ]
+        Union[List[LLMTestCase], List[ConversationalTestCase]]
     ] = None,
     metrics: Optional[
         Union[
@@ -292,9 +292,10 @@ def validate_evaluate_inputs(
     if test_cases and metrics:
         for test_case in test_cases:
             for metric in metrics:
-                if isinstance(test_case, LLMTestCase) and not isinstance(
-                    metric, BaseMetric
-                ):
+                if (
+                    isinstance(test_case, LLMTestCase)
+                    and not test_case.multimodal
+                ) and not isinstance(metric, BaseMetric):
                     raise ValueError(
                         f"Metric {metric.__name__} is not a valid metric for LLMTestCase."
                     )
@@ -305,11 +306,14 @@ def validate_evaluate_inputs(
                     raise ValueError(
                         f"Metric {metric.__name__} is not a valid metric for ConversationalTestCase."
                     )
-                if isinstance(test_case, MLLMTestCase) and not isinstance(
-                    metric, BaseMultimodalMetric
+                if (
+                    isinstance(test_case, LLMTestCase) and test_case.multimodal
+                ) and not (
+                    isinstance(metric, BaseMultimodalMetric)
+                    or isinstance(metric, BaseMetric)
                 ):
                     raise ValueError(
-                        f"Metric {metric.__name__} is not a valid metric for MLLMTestCase."
+                        f"Metric {metric.__name__} is not a valid metric for multi-modal LLMTestCase."
                     )
 
 

deepeval/integrations/pydantic_ai/agent.py

@@ -1,12 +1,29 @@
 import warnings
+from typing import TYPE_CHECKING, Any
 
 try:
-    from pydantic_ai.agent import Agent
+    from pydantic_ai.agent import Agent as _BaseAgent
 
     is_pydantic_ai_installed = True
-except:
+except ImportError:
     is_pydantic_ai_installed = False
 
+    class _BaseAgent:
+        """Dummy fallback so imports don't crash when pydantic-ai is missing."""
+
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            # No-op: for compatibility
+            pass
+
+
+if TYPE_CHECKING:
+    # For type checkers: use the real Agent if available.
+    from pydantic_ai.agent import Agent  # type: ignore[unused-ignore]
+else:
+    # At runtime we always have some base: real Agent or our dummy.
+    # This is just to avoid blow-ups.
+    Agent = _BaseAgent
+
 
 class DeepEvalPydanticAIAgent(Agent):
 

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -1,40 +1,58 @@
+from __future__ import annotations
+
 import json
 import logging
 import os
 from time import perf_counter
-from typing import Literal, Optional, List
+from typing import Any, List, Optional, TYPE_CHECKING
 
 from deepeval.config.settings import get_settings
 from deepeval.confident.api import get_confident_api_key
 from deepeval.metrics.base_metric import BaseMetric
 from deepeval.prompt import Prompt
 from deepeval.tracing.context import current_trace_context
-from deepeval.tracing.types import Trace
-from deepeval.tracing.otel.utils import to_hex_string
-from deepeval.tracing.tracing import trace_manager
-from deepeval.tracing.otel.utils import normalize_pydantic_ai_messages
 from deepeval.tracing.otel.exporter import ConfidentSpanExporter
-
+from deepeval.tracing.otel.test_exporter import test_exporter
+from deepeval.tracing.otel.utils import (
+    normalize_pydantic_ai_messages,
+    to_hex_string,
+)
+from deepeval.tracing.perf_epoch_bridge import init_clock_bridge
+from deepeval.tracing.tracing import trace_manager
+from deepeval.tracing.types import (
+    AgentSpan,
+    Trace,
+    TraceSpanStatus,
+    ToolCall,
+)
 
 logger = logging.getLogger(__name__)
 
-
 try:
-    from pydantic_ai.models.instrumented import InstrumentationSettings
-    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+    # Optional dependencies
+    from opentelemetry.sdk.trace import (
+        ReadableSpan as _ReadableSpan,
+        SpanProcessor as _SpanProcessor,
+        TracerProvider,
+    )
     from opentelemetry.sdk.trace.export import BatchSpanProcessor
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
-    from opentelemetry.sdk.trace import ReadableSpan
+    from pydantic_ai.models.instrumented import (
+        InstrumentationSettings as _BaseInstrumentationSettings,
+    )
 
     dependency_installed = True
 except ImportError as e:
+    dependency_installed = False
+
+    # Preserve previous behavior: only log when verbose mode is enabled.
     if get_settings().DEEPEVAL_VERBOSE_MODE:
         if isinstance(e, ModuleNotFoundError):
             logger.warning(
                 "Optional tracing dependency not installed: %s",
-                e.name,
+                getattr(e, "name", repr(e)),
                 stacklevel=2,
             )
         else:
@@ -43,26 +61,47 @@ except ImportError as e:
                 e,
                 stacklevel=2,
             )
-    dependency_installed = False
+
+    # Dummy fallbacks so imports and class definitions don't crash when
+    # optional deps are missing. Actual use is still guarded by
+    # is_dependency_installed().
+    class _BaseInstrumentationSettings:
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            pass
+
+    class _SpanProcessor:
+        def __init__(self, *args: Any, **kwargs: Any) -> None:
+            pass
+
+        def on_start(self, span: Any, parent_context: Any) -> None:
+            pass
+
+        def on_end(self, span: Any) -> None:
+            pass
+
+    class _ReadableSpan:
+        pass
 
 
-def is_dependency_installed():
+def is_dependency_installed() -> bool:
     if not dependency_installed:
         raise ImportError(
-            "Dependencies are not installed. Please install it with `pip install pydantic-ai opentelemetry-sdk opentelemetry-exporter-otlp-proto-http`."
+            "Dependencies are not installed. Please install it with "
+            "`pip install pydantic-ai opentelemetry-sdk "
+            "opentelemetry-exporter-otlp-proto-http`."
         )
     return True
 
 
-from deepeval.tracing.types import AgentSpan
-from deepeval.confident.api import get_confident_api_key
-from deepeval.prompt import Prompt
-from deepeval.tracing.otel.test_exporter import test_exporter
-from deepeval.tracing.context import current_trace_context
-from deepeval.tracing.types import Trace
-from deepeval.tracing.otel.utils import to_hex_string
-from deepeval.tracing.types import TraceSpanStatus, ToolCall
-from deepeval.tracing.perf_epoch_bridge import init_clock_bridge
+if TYPE_CHECKING:
+    # For type checkers, use real types
+    from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor
+    from pydantic_ai.models.instrumented import InstrumentationSettings
+else:
+    # At runtime we always have something to subclass / annotate with
+    InstrumentationSettings = _BaseInstrumentationSettings
+    SpanProcessor = _SpanProcessor
+    ReadableSpan = _ReadableSpan
 
 # OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
 OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"

deepeval/metrics/__init__.py

@@ -42,6 +42,16 @@ from .mcp_use_metric.mcp_use_metric import MCPUseMetric
 from .turn_relevancy.turn_relevancy import (
     TurnRelevancyMetric,
 )
+from .turn_faithfulness.turn_faithfulness import TurnFaithfulnessMetric
+from .turn_contextual_precision.turn_contextual_precision import (
+    TurnContextualPrecisionMetric,
+)
+from .turn_contextual_recall.turn_contextual_recall import (
+    TurnContextualRecallMetric,
+)
+from .turn_contextual_relevancy.turn_contextual_relevancy import (
+    TurnContextualRelevancyMetric,
+)
 from .conversation_completeness.conversation_completeness import (
     ConversationCompletenessMetric,
 )
@@ -55,12 +65,6 @@ from .multimodal_metrics import (
     ImageCoherenceMetric,
     ImageHelpfulnessMetric,
     ImageReferenceMetric,
-    MultimodalContextualRecallMetric,
-    MultimodalContextualRelevancyMetric,
-    MultimodalContextualPrecisionMetric,
-    MultimodalAnswerRelevancyMetric,
-    MultimodalFaithfulnessMetric,
-    MultimodalToolCorrectnessMetric,
     MultimodalGEval,
 )
 
@@ -119,17 +123,15 @@ __all__ = [
     # Conversational metrics
     "TurnRelevancyMetric",
     "ConversationCompletenessMetric",
+    "TurnFaithfulnessMetric",
+    "TurnContextualPrecisionMetric",
+    "TurnContextualRecallMetric",
+    "TurnContextualRelevancyMetric",
     # Multimodal metrics
     "TextToImageMetric",
     "ImageEditingMetric",
     "ImageCoherenceMetric",
     "ImageHelpfulnessMetric",
     "ImageReferenceMetric",
-    "MultimodalContextualRecallMetric",
-    "MultimodalContextualRelevancyMetric",
-    "MultimodalContextualPrecisionMetric",
-    "MultimodalAnswerRelevancyMetric",
-    "MultimodalFaithfulnessMetric",
-    "MultimodalToolCorrectnessMetric",
     "MultimodalGEval",
 ]

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -1,16 +1,17 @@
 from typing import Optional, List, Type, Union
 
-from deepeval.utils import get_or_create_event_loop, prettify_list
+from deepeval.utils import (
+    get_or_create_event_loop,
+    prettify_list,
+)
 from deepeval.metrics.utils import (
     construct_verbose_logs,
     trimAndLoadJson,
     check_llm_test_case_params,
+    check_mllm_test_case_params,
     initialize_model,
 )
-from deepeval.test_case import (
-    LLMTestCase,
-    LLMTestCaseParams,
-)
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams, MLLMImage
 from deepeval.metrics import BaseMetric
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.answer_relevancy.template import AnswerRelevancyTemplate
@@ -53,7 +54,14 @@ class AnswerRelevancyMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -70,14 +78,17 @@ class AnswerRelevancyMetric(BaseMetric):
                     )
                 )
             else:
+                input = test_case.input
+                actual_output = test_case.actual_output
+
                 self.statements: List[str] = self._generate_statements(
-                    test_case.actual_output
+                    actual_output, multimodal
                 )
                 self.verdicts: List[AnswerRelevancyVerdict] = (
-                    self._generate_verdicts(test_case.input)
+                    self._generate_verdicts(input, multimodal)
                 )
                 self.score = self._calculate_score()
-                self.reason = self._generate_reason(test_case.input)
+                self.reason = self._generate_reason(input, multimodal)
                 self.success = self.score >= self.threshold
                 self.verbose_logs = construct_verbose_logs(
                     self,
@@ -101,7 +112,14 @@ class AnswerRelevancyMetric(BaseMetric):
         _in_component: bool = False,
         _log_metric_to_confident: bool = True,
     ) -> float:
-        check_llm_test_case_params(test_case, self._required_params, self)
+
+        multimodal = test_case.multimodal
+        if multimodal:
+            check_mllm_test_case_params(
+                test_case, self._required_params, None, None, self, self.model
+            )
+        else:
+            check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
         with metric_progress_indicator(
@@ -110,14 +128,17 @@ class AnswerRelevancyMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
+            input = test_case.input
+            actual_output = test_case.actual_output
+
             self.statements: List[str] = await self._a_generate_statements(
-                test_case.actual_output
+                actual_output, multimodal
             )
             self.verdicts: List[AnswerRelevancyVerdict] = (
-                await self._a_generate_verdicts(test_case.input)
+                await self._a_generate_verdicts(input, multimodal)
             )
             self.score = self._calculate_score()
-            self.reason = await self._a_generate_reason(test_case.input)
+            self.reason = await self._a_generate_reason(input, multimodal)
             self.success = self.score >= self.threshold
             self.verbose_logs = construct_verbose_logs(
                 self,
@@ -133,7 +154,7 @@ class AnswerRelevancyMetric(BaseMetric):
                 )
             return self.score
 
-    async def _a_generate_reason(self, input: str) -> str:
+    async def _a_generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -146,7 +167,9 @@ class AnswerRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             input=input,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
+
         if self.using_native_model:
             res, cost = await self.model.a_generate(
                 prompt, schema=AnswerRelevancyScoreReason
@@ -164,7 +187,7 @@ class AnswerRelevancyMetric(BaseMetric):
                 data = trimAndLoadJson(res, self)
                 return data["reason"]
 
-    def _generate_reason(self, input: str) -> str:
+    def _generate_reason(self, input: str, multimodal: bool) -> str:
         if self.include_reason is False:
             return None
 
@@ -177,6 +200,7 @@ class AnswerRelevancyMetric(BaseMetric):
             irrelevant_statements=irrelevant_statements,
             input=input,
             score=format(self.score, ".2f"),
+            multimodal=multimodal,
         )
 
         if self.using_native_model:
@@ -197,14 +221,13 @@ class AnswerRelevancyMetric(BaseMetric):
                 return data["reason"]
 
     async def _a_generate_verdicts(
-        self, input: str
+        self, input: str, multimodal: bool
     ) -> List[AnswerRelevancyVerdict]:
         if len(self.statements) == 0:
             return []
 
         prompt = self.evaluation_template.generate_verdicts(
-            input=input,
-            statements=self.statements,
+            input=input, statements=self.statements, multimodal=multimodal
         )
 
         if self.using_native_model:
@@ -224,14 +247,16 @@ class AnswerRelevancyMetric(BaseMetric):
                     AnswerRelevancyVerdict(**item) for item in data["verdicts"]
                 ]
 
-    def _generate_verdicts(self, input: str) -> List[AnswerRelevancyVerdict]:
+    def _generate_verdicts(
+        self, input: str, multimodal: bool
+    ) -> List[AnswerRelevancyVerdict]:
         if len(self.statements) == 0:
             return []
 
         prompt = self.evaluation_template.generate_verdicts(
-            input=input,
-            statements=self.statements,
+            input=input, statements=self.statements, multimodal=multimodal
         )
+
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Verdicts)
             self.evaluation_cost += cost
@@ -250,44 +275,64 @@ class AnswerRelevancyMetric(BaseMetric):
     async def _a_generate_statements(
         self,
         actual_output: str,
+        multimodal: bool,
     ) -> List[str]:
         prompt = self.evaluation_template.generate_statements(
-            actual_output=actual_output,
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = await self.model.a_generate(prompt, schema=Statements)
             self.evaluation_cost += cost
-            return res.statements
+            statements: List[str] = res.statements + [
+                ele for ele in actual_output if isinstance(ele, MLLMImage)
+            ]
+            return statements
         else:
             try:
                 res: Statements = await self.model.a_generate(
                     prompt, schema=Statements
                 )
-                return res.statements
+                statements: List[str] = res.statements + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
             except TypeError:
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
-                return data["statements"]
+                statements = data["statements"] + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
 
     def _generate_statements(
         self,
         actual_output: str,
+        multimodal: bool,
     ) -> List[str]:
         prompt = self.evaluation_template.generate_statements(
-            actual_output=actual_output,
+            actual_output=actual_output, multimodal=multimodal
         )
         if self.using_native_model:
             res, cost = self.model.generate(prompt, schema=Statements)
             self.evaluation_cost += cost
-            return res.statements
+            statements = res.statements + [
+                ele for ele in actual_output if isinstance(ele, MLLMImage)
+            ]
+            return statements
         else:
             try:
                 res: Statements = self.model.generate(prompt, schema=Statements)
-                return res.statements
+                statements = res.statements + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
-                return data["statements"]
+                statements = data["statements"] + [
+                    ele for ele in actual_output if isinstance(ele, MLLMImage)
+                ]
+                return statements
 
     def _calculate_score(self):
         number_of_verdicts = len(self.verdicts)