deepeval 3.7.9__py3-none-any.whl → 3.8.1__py3-none-any.whl

deepeval/integrations/langchain/utils.py

@@ -145,6 +145,7 @@ def enter_current_context(
     progress: Optional[Progress] = None,
     pbar_callback_id: Optional[int] = None,
     uuid_str: Optional[str] = None,
+    fallback_trace_uuid: Optional[str] = None,
 ) -> BaseSpan:
     start_time = perf_counter()
     observe_kwargs = observe_kwargs or {}
@@ -159,12 +160,27 @@ def enter_current_context(
     parent_uuid: Optional[str] = None
 
     if parent_span:
-        parent_uuid = parent_span.uuid
-        trace_uuid = parent_span.trace_uuid
-    else:
+        # Validate that the parent span's trace is still active
+        if parent_span.trace_uuid in trace_manager.active_traces:
+            parent_uuid = parent_span.uuid
+            trace_uuid = parent_span.trace_uuid
+        else:
+            # Parent span references a dead trace - treat as if no parent
+            parent_span = None
+
+    if not parent_span:
         current_trace = current_trace_context.get()
-        if current_trace:
+        # IMPORTANT: Verify trace is still active, not just in context
+        # (a previous failed async operation might leave a dead trace in context)
+        if current_trace and current_trace.uuid in trace_manager.active_traces:
             trace_uuid = current_trace.uuid
+        elif (
+            fallback_trace_uuid
+            and fallback_trace_uuid in trace_manager.active_traces
+        ):
+            # In async contexts, ContextVar may not propagate. Use the fallback trace_uuid
+            # provided by the CallbackHandler to avoid creating duplicate traces.
+            trace_uuid = fallback_trace_uuid
         else:
             trace = trace_manager.start_new_trace(
                 metric_collection=metric_collection
@@ -258,11 +274,13 @@ def exit_current_context(
 
     current_span = current_span_context.get()
 
+    # In async contexts (LangChain/LangGraph), context variables don't propagate
+    # reliably across task boundaries. Fall back to direct span lookup.
     if not current_span or current_span.uuid != uuid_str:
-        print(
-            f"Error: Current span in context does not match the span being exited. Expected UUID: {uuid_str}, Got: {current_span.uuid if current_span else 'None'}"
-        )
-        return
+        current_span = trace_manager.get_span_by_uuid(uuid_str)
+        if not current_span:
+            # Span already removed or never existed
+            return
 
     current_span.end_time = end_time
     if exc_type is not None:
@@ -295,7 +313,12 @@ def exit_current_context(
         else:
             current_span_context.set(None)
     else:
+        # Try context first, then fall back to direct trace lookup for async contexts
         current_trace = current_trace_context.get()
+        if not current_trace and current_span.trace_uuid:
+            current_trace = trace_manager.get_trace_by_uuid(
+                current_span.trace_uuid
+            )
         if current_span.status == TraceSpanStatus.ERRORED and current_trace:
             current_trace.status = TraceSpanStatus.ERRORED
         if current_trace and current_trace.uuid == current_span.trace_uuid:

deepeval/key_handler.py

@@ -162,6 +162,13 @@ class ModelKeyValues(Enum):
     VLLM_API_KEY = "VLLM_API_KEY"
     VLLM_MODEL_NAME = "VLLM_MODEL_NAME"
 
+    # OpenRouter
+    USE_OPENROUTER_MODEL = "USE_OPENROUTER_MODEL"
+    OPENROUTER_MODEL_NAME = "OPENROUTER_MODEL_NAME"
+    OPENROUTER_COST_PER_INPUT_TOKEN = "OPENROUTER_COST_PER_INPUT_TOKEN"
+    OPENROUTER_COST_PER_OUTPUT_TOKEN = "OPENROUTER_COST_PER_OUTPUT_TOKEN"
+    OPENROUTER_API_KEY = "OPENROUTER_API_KEY"
+
 
 class EmbeddingKeyValues(Enum):
     # Azure OpenAI
@@ -174,7 +181,7 @@ class EmbeddingKeyValues(Enum):
     USE_LOCAL_EMBEDDINGS = "USE_LOCAL_EMBEDDINGS"
     LOCAL_EMBEDDING_MODEL_NAME = "LOCAL_EMBEDDING_MODEL_NAME"
     LOCAL_EMBEDDING_BASE_URL = "LOCAL_EMBEDDING_BASE_URL"
-    LOCAL_EMBEDDING_API_KEY = "LOCAL_EMBEDDING_API_KEY"
+    LOCAL_EMBEDDING_API_KEY = ("LOCAL_EMBEDDING_API_KEY",)
 
 
 class KeyFileHandler:

deepeval/metrics/contextual_recall/contextual_recall.py

@@ -23,6 +23,7 @@ from deepeval.metrics.contextual_recall.schema import (
     ContextualRecallVerdict,
     Verdicts,
     ContextualRecallScoreReason,
+    VerdictWithExpectedOutput,
 )
 from deepeval.metrics.api import metric_data_manager
 
@@ -93,7 +94,7 @@ class ContextualRecallMetric(BaseMetric):
                 expected_output = test_case.expected_output
                 retrieval_context = test_case.retrieval_context
 
-                self.verdicts: List[ContextualRecallVerdict] = (
+                self.verdicts: List[VerdictWithExpectedOutput] = (
                     self._generate_verdicts(
                         expected_output, retrieval_context, multimodal
                     )
@@ -144,7 +145,7 @@ class ContextualRecallMetric(BaseMetric):
             expected_output = test_case.expected_output
             retrieval_context = test_case.retrieval_context
 
-            self.verdicts: List[ContextualRecallVerdict] = (
+            self.verdicts: List[VerdictWithExpectedOutput] = (
                 await self._a_generate_verdicts(
                     expected_output, retrieval_context, multimodal
                 )
@@ -241,13 +242,13 @@ class ContextualRecallMetric(BaseMetric):
         expected_output: str,
         retrieval_context: List[str],
         multimodal: bool,
-    ) -> List[ContextualRecallVerdict]:
+    ) -> List[VerdictWithExpectedOutput]:
         prompt = self.evaluation_template.generate_verdicts(
             expected_output=expected_output,
             retrieval_context=retrieval_context,
             multimodal=multimodal,
         )
-        return await a_generate_with_schema_and_extract(
+        verdicts = await a_generate_with_schema_and_extract(
             metric=self,
             prompt=prompt,
             schema_cls=Verdicts,
@@ -256,19 +257,28 @@ class ContextualRecallMetric(BaseMetric):
                 ContextualRecallVerdict(**item) for item in data["verdicts"]
             ],
         )
+        final_verdicts = []
+        for verdict in verdicts:
+            new_verdict = VerdictWithExpectedOutput(
+                verdict=verdict.verdict,
+                reason=verdict.reason,
+                expected_output=expected_output,
+            )
+            final_verdicts.append(new_verdict)
+        return final_verdicts
 
     def _generate_verdicts(
         self,
         expected_output: str,
         retrieval_context: List[str],
         multimodal: bool,
-    ) -> List[ContextualRecallVerdict]:
+    ) -> List[VerdictWithExpectedOutput]:
         prompt = self.evaluation_template.generate_verdicts(
             expected_output=expected_output,
             retrieval_context=retrieval_context,
             multimodal=multimodal,
         )
-        return generate_with_schema_and_extract(
+        verdicts = generate_with_schema_and_extract(
             metric=self,
             prompt=prompt,
             schema_cls=Verdicts,
@@ -277,6 +287,15 @@ class ContextualRecallMetric(BaseMetric):
                 ContextualRecallVerdict(**item) for item in data["verdicts"]
             ],
         )
+        final_verdicts = []
+        for verdict in verdicts:
+            new_verdict = VerdictWithExpectedOutput(
+                verdict=verdict.verdict,
+                reason=verdict.reason,
+                expected_output=expected_output,
+            )
+            final_verdicts.append(new_verdict)
+        return final_verdicts
 
     def is_successful(self) -> bool:
         if self.error is not None:

deepeval/metrics/contextual_recall/schema.py

@@ -7,6 +7,12 @@ class ContextualRecallVerdict(BaseModel):
     reason: str
 
 
+class VerdictWithExpectedOutput(BaseModel):
+    verdict: str
+    reason: str
+    expected_output: str
+
+
 class Verdicts(BaseModel):
     verdicts: List[ContextualRecallVerdict]
 

deepeval/metrics/conversational_g_eval/conversational_g_eval.py

@@ -2,6 +2,7 @@
 
 from openai.types.chat.chat_completion import ChatCompletion
 from typing import Optional, List, Tuple, Union, Dict, Type
+from rich.console import Console
 import math
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.metrics.g_eval.utils import (
@@ -11,6 +12,8 @@ from deepeval.metrics.g_eval.utils import (
     format_rubrics,
     validate_and_sort_rubrics,
     validate_criteria_and_evaluation_steps,
+    CONVERSATIONAL_G_EVAL_API_PARAMS,
+    construct_geval_upload_payload,
 )
 from deepeval.test_case import (
     TurnParams,
@@ -33,6 +36,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 import deepeval.metrics.conversational_g_eval.schema as cgschema
 from deepeval.metrics.api import metric_data_manager
+from deepeval.confident.api import Api, Endpoints, HttpMethods
 
 
 class ConversationalGEval(BaseConversationalMetric):
@@ -412,6 +416,37 @@ class ConversationalGEval(BaseConversationalMetric):
                 self.success = False
         return self.success
 
+    def upload(self):
+        api = Api()
+
+        payload = construct_geval_upload_payload(
+            name=self.name,
+            evaluation_params=self.evaluation_params,
+            g_eval_api_params=CONVERSATIONAL_G_EVAL_API_PARAMS,
+            criteria=self.criteria,
+            evaluation_steps=self.evaluation_steps,
+            multi_turn=True,
+            rubric=self.rubric,
+        )
+
+        data, _ = api.send_request(
+            method=HttpMethods.POST,
+            endpoint=Endpoints.METRICS_ENDPOINT,
+            body=payload,
+        )
+
+        metric_id = data.get("id")
+        self.metric_id = metric_id
+        console = Console()
+
+        if metric_id:
+            console.print(
+                "[rgb(5,245,141)]✓[/rgb(5,245,141)] Metric uploaded successfully "
+                f"(id: [bold]{metric_id}[/bold])"
+            )
+
+        return data
+
     @property
     def __name__(self):
         if self._include_g_eval_suffix:

deepeval/metrics/g_eval/g_eval.py

@@ -1,7 +1,7 @@
 """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
 import asyncio
-
+from rich.console import Console
 from typing import Optional, List, Tuple, Union, Type
 from deepeval.metrics import BaseMetric
 from deepeval.test_case import (
@@ -32,9 +32,12 @@ from deepeval.metrics.g_eval.utils import (
     validate_criteria_and_evaluation_steps,
     number_evaluation_steps,
     get_score_range,
+    construct_geval_upload_payload,
+    G_EVAL_API_PARAMS,
 )
 from deepeval.metrics.api import metric_data_manager
 from deepeval.config.settings import get_settings
+from deepeval.confident.api import Api, Endpoints, HttpMethods
 
 
 class GEval(BaseMetric):
@@ -408,6 +411,37 @@ class GEval(BaseMetric):
                 self.success = False
         return self.success
 
+    def upload(self):
+        api = Api()
+
+        payload = construct_geval_upload_payload(
+            name=self.name,
+            evaluation_params=self.evaluation_params,
+            g_eval_api_params=G_EVAL_API_PARAMS,
+            criteria=self.criteria,
+            evaluation_steps=self.evaluation_steps,
+            multi_turn=False,
+            rubric=self.rubric,
+        )
+
+        data, _ = api.send_request(
+            method=HttpMethods.POST,
+            endpoint=Endpoints.METRICS_ENDPOINT,
+            body=payload,
+        )
+
+        metric_id = data.get("id")
+        self.metric_id = metric_id
+        console = Console()
+
+        if metric_id:
+            console.print(
+                "[rgb(5,245,141)]✓[/rgb(5,245,141)] Metric uploaded successfully "
+                f"(id: [bold]{metric_id}[/bold])"
+            )
+
+        return data
+
     @property
     def __name__(self):
         if self._include_g_eval_suffix:

deepeval/metrics/g_eval/utils.py

@@ -52,6 +52,71 @@ CONVERSATIONAL_G_EVAL_PARAMS = {
     TurnParams.SCENARIO: "Scenario",
 }
 
+G_EVAL_API_PARAMS = {
+    LLMTestCaseParams.INPUT: "input",
+    LLMTestCaseParams.ACTUAL_OUTPUT: "actualOutput",
+    LLMTestCaseParams.EXPECTED_OUTPUT: "expectedOutput",
+    LLMTestCaseParams.CONTEXT: "context",
+    LLMTestCaseParams.RETRIEVAL_CONTEXT: "retrievalContext",
+    LLMTestCaseParams.EXPECTED_TOOLS: "expectedTools",
+    LLMTestCaseParams.TOOLS_CALLED: "toolsCalled",
+}
+
+CONVERSATIONAL_G_EVAL_API_PARAMS = {
+    TurnParams.ROLE: "role",
+    TurnParams.CONTENT: "content",
+    TurnParams.SCENARIO: "scenario",
+    TurnParams.EXPECTED_OUTCOME: "expectedOutcome",
+    TurnParams.RETRIEVAL_CONTEXT: "retrievalContext",
+    TurnParams.TOOLS_CALLED: "toolsCalled",
+}
+
+
+def construct_geval_upload_payload(
+    name: str,
+    evaluation_params: List[LLMTestCaseParams],
+    g_eval_api_params: Dict,
+    criteria: Optional[str] = None,
+    evaluation_steps: Optional[List[str]] = None,
+    multi_turn: bool = False,
+    rubric: Optional[List[Rubric]] = None,
+) -> Dict:
+    if not evaluation_params:
+        raise ValueError("GEval requires at least one evaluation parameter.")
+
+    unsupported_params = [
+        param for param in evaluation_params if param not in g_eval_api_params
+    ]
+    if unsupported_params:
+        raise ValueError(
+            "Unsupported evaluation params for GEval upload: "
+            + ", ".join(param.name for param in unsupported_params)
+        )
+
+    payload = {
+        "name": name,
+        "evaluationParams": [
+            g_eval_api_params[param] for param in evaluation_params
+        ],
+        "multiTurn": multi_turn,
+    }
+
+    if criteria is not None:
+        payload["criteria"] = criteria
+    else:
+        payload["evaluationSteps"] = evaluation_steps
+
+    if rubric is not None:
+        payload["rubric"] = [
+            {
+                "scoreRange": list(r.score_range),
+                "expectedOutcome": r.expected_outcome,
+            }
+            for r in rubric
+        ]
+
+    return payload
+
 
 def validate_criteria_and_evaluation_steps(
     criteria: Optional[str] = None,

deepeval/metrics/multimodal_metrics/image_coherence/image_coherence.py

@@ -85,7 +85,12 @@ class ImageCoherenceMetric(BaseMetric):
                 self.contexts_below = []
                 self.scores = []
                 self.reasons = []
-                for image_index in self.get_image_indices(actual_output):
+                image_indices = self.get_image_indices(actual_output)
+                if not image_indices:
+                    raise ValueError(
+                        f"The test case must have atleast one image in the `actual_output` to calculate {self.__name__} score"
+                    )
+                for image_index in image_indices:
                     context_above, context_below = self.get_image_context(
                         image_index, actual_output
                     )
@@ -188,6 +193,10 @@ class ImageCoherenceMetric(BaseMetric):
 
             tasks = []
             image_indices = self.get_image_indices(actual_output)
+            if not image_indices:
+                raise ValueError(
+                    f"The test case must have atleast one image in the `actual_output` to calculate {self.__name__} score"
+                )
             for image_index in image_indices:
                 context_above, context_below = self.get_image_context(
                     image_index, actual_output

deepeval/metrics/multimodal_metrics/image_helpfulness/image_helpfulness.py

@@ -86,7 +86,12 @@ class ImageHelpfulnessMetric(BaseMetric):
                 self.contexts_below = []
                 self.scores = []
                 self.reasons = []
-                for image_index in self.get_image_indices(actual_output):
+                image_indices = self.get_image_indices(actual_output)
+                if not image_indices:
+                    raise ValueError(
+                        f"The test case must have atleast one image in the `actual_output` to calculate {self.__name__} score"
+                    )
+                for image_index in image_indices:
                     context_above, context_below = self.get_image_context(
                         image_index, actual_output
                     )
@@ -189,6 +194,10 @@ class ImageHelpfulnessMetric(BaseMetric):
 
             tasks = []
             image_indices = self.get_image_indices(actual_output)
+            if not image_indices:
+                raise ValueError(
+                    f"The test case must have atleast one image in the `actual_output` to calculate {self.__name__} score"
+                )
             for image_index in image_indices:
                 context_above, context_below = self.get_image_context(
                     image_index, actual_output

deepeval/metrics/multimodal_metrics/image_reference/image_reference.py

@@ -86,7 +86,12 @@ class ImageReferenceMetric(BaseMetric):
                 self.contexts_below = []
                 self.scores = []
                 self.reasons = []
-                for image_index in self.get_image_indices(actual_output):
+                image_indices = self.get_image_indices(actual_output)
+                if not image_indices:
+                    raise ValueError(
+                        f"The test case must have atleast one image in the `actual_output` to calculate {self.__name__} score"
+                    )
+                for image_index in image_indices:
                     context_above, context_below = self.get_image_context(
                         image_index, actual_output
                     )
@@ -189,6 +194,10 @@ class ImageReferenceMetric(BaseMetric):
 
             tasks = []
             image_indices = self.get_image_indices(actual_output)
+            if not image_indices:
+                raise ValueError(
+                    f"The test case must have atleast one image in the `actual_output` to calculate {self.__name__} score"
+                )
             for image_index in image_indices:
                 context_above, context_below = self.get_image_context(
                     image_index, actual_output

deepeval/metrics/utils.py

@@ -312,7 +312,7 @@ def check_llm_test_case_params(
                 if isinstance(ele, MLLMImage):
                     count += 1
             if count != actual_output_image_count:
-                error_str = f"Unable to evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
+                error_str = f"Can only evaluate test cases with '{actual_output_image_count}' output images using the '{metric.__name__}' metric. `{count}` found."
                 raise ValueError(error_str)
 
     if isinstance(test_case, LLMTestCase) is False:

deepeval/models/__init__.py

@@ -16,6 +16,7 @@ from deepeval.models.llms import (
     GrokModel,
     DeepSeekModel,
     PortkeyModel,
+    OpenRouterModel,
 )
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
@@ -44,4 +45,5 @@ __all__ = [
     "LocalEmbeddingModel",
     "OllamaEmbeddingModel",
     "PortkeyModel",
+    "OpenRouterModel",
 ]

deepeval/models/llms/__init__.py

@@ -10,6 +10,7 @@ from .kimi_model import KimiModel
 from .grok_model import GrokModel
 from .deepseek_model import DeepSeekModel
 from .portkey_model import PortkeyModel
+from .openrouter_model import OpenRouterModel
 
 __all__ = [
     "AzureOpenAIModel",
@@ -24,4 +25,5 @@ __all__ = [
     "GrokModel",
     "DeepSeekModel",
     "PortkeyModel",
+    "OpenRouterModel",
 ]

deepeval/models/llms/amazon_bedrock_model.py

@@ -14,6 +14,7 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.test_case import MLLMImage
+from deepeval.errors import DeepEvalError
 from deepeval.utils import check_if_multimodal, convert_to_multi_modal_array
 from deepeval.models import DeepEvalBaseLLM
 from deepeval.models.llms.constants import BEDROCK_MODELS_DATA
@@ -155,27 +156,28 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
 
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, BaseModel], float]:
+    ) -> Tuple[Union[str, BaseModel], Optional[float]]:
         return safe_asyncio_run(self.a_generate(prompt, schema))
 
     @retry_bedrock
     async def a_generate(
         self, prompt: str, schema: Optional[BaseModel] = None
-    ) -> Tuple[Union[str, BaseModel], float]:
+    ) -> Tuple[Union[str, BaseModel], Optional[float]]:
         if check_if_multimodal(prompt):
             prompt = convert_to_multi_modal_array(input=prompt)
             payload = self.generate_payload(prompt)
         else:
             payload = self.get_converse_request_body(prompt)
 
-        payload = self.get_converse_request_body(prompt)
         client = await self._ensure_client()
         response = await client.converse(
             modelId=self.get_model_name(),
             messages=payload["messages"],
             inferenceConfig=payload["inferenceConfig"],
         )
-        message = response["output"]["message"]["content"][0]["text"]
+
+        message = self._extract_text_from_converse_response(response)
+
         cost = self.calculate_cost(
             response["usage"]["inputTokens"],
             response["usage"]["outputTokens"],
@@ -206,7 +208,7 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
                 try:
                     image_raw_bytes = base64.b64decode(element.dataBase64)
                 except Exception:
-                    raise ValueError(
+                    raise DeepEvalError(
                         f"Invalid base64 data in MLLMImage: {element._id}"
                     )
 
@@ -294,6 +296,46 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
     # Helpers
     ###############################################
 
+    @staticmethod
+    def _extract_text_from_converse_response(response: dict) -> str:
+        try:
+            content = response["output"]["message"]["content"]
+        except Exception as e:
+            raise DeepEvalError(
+                "Missing output.message.content in Bedrock response"
+            ) from e
+
+        # Collect any text blocks (ignore reasoning/tool blocks)
+        text_parts = []
+        for block in content:
+            if isinstance(block, dict) and "text" in block:
+                v = block.get("text")
+                if isinstance(v, str) and v.strip():
+                    text_parts.append(v)
+
+        if text_parts:
+            # join in case there are multiple text blocks
+            return "\n".join(text_parts)
+
+        # No text blocks present; raise an actionable error
+        keys = []
+        for b in content:
+            if isinstance(b, dict):
+                keys.append(list(b.keys()))
+            else:
+                keys.append(type(b).__name__)
+
+        stop_reason = (
+            response.get("stopReason")
+            or response.get("output", {}).get("stopReason")
+            or response.get("output", {}).get("message", {}).get("stopReason")
+        )
+
+        raise DeepEvalError(
+            f"Bedrock response contained no text content blocks. "
+            f"content keys={keys}, stopReason={stop_reason}"
+        )
+
     def get_converse_request_body(self, prompt: str) -> dict:
 
         return {
@@ -303,11 +345,14 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
             },
         }
 
-    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+    def calculate_cost(
+        self, input_tokens: int, output_tokens: int
+    ) -> Optional[float]:
         if self.model_data.input_price and self.model_data.output_price:
             input_cost = input_tokens * self.model_data.input_price
             output_cost = output_tokens * self.model_data.output_price
             return input_cost + output_cost
+        return None
 
     def load_model(self):
         pass