deepeval 3.7.9__py3-none-any.whl → 3.8.0__py3-none-any.whl

deepeval/integrations/langchain/utils.py

@@ -145,6 +145,7 @@ def enter_current_context(
     progress: Optional[Progress] = None,
     pbar_callback_id: Optional[int] = None,
     uuid_str: Optional[str] = None,
+    fallback_trace_uuid: Optional[str] = None,
 ) -> BaseSpan:
     start_time = perf_counter()
     observe_kwargs = observe_kwargs or {}
@@ -159,12 +160,27 @@ def enter_current_context(
     parent_uuid: Optional[str] = None
 
     if parent_span:
-        parent_uuid = parent_span.uuid
-        trace_uuid = parent_span.trace_uuid
-    else:
+        # Validate that the parent span's trace is still active
+        if parent_span.trace_uuid in trace_manager.active_traces:
+            parent_uuid = parent_span.uuid
+            trace_uuid = parent_span.trace_uuid
+        else:
+            # Parent span references a dead trace - treat as if no parent
+            parent_span = None
+
+    if not parent_span:
         current_trace = current_trace_context.get()
-        if current_trace:
+        # IMPORTANT: Verify trace is still active, not just in context
+        # (a previous failed async operation might leave a dead trace in context)
+        if current_trace and current_trace.uuid in trace_manager.active_traces:
             trace_uuid = current_trace.uuid
+        elif (
+            fallback_trace_uuid
+            and fallback_trace_uuid in trace_manager.active_traces
+        ):
+            # In async contexts, ContextVar may not propagate. Use the fallback trace_uuid
+            # provided by the CallbackHandler to avoid creating duplicate traces.
+            trace_uuid = fallback_trace_uuid
         else:
             trace = trace_manager.start_new_trace(
                 metric_collection=metric_collection
@@ -258,11 +274,13 @@ def exit_current_context(
 
     current_span = current_span_context.get()
 
+    # In async contexts (LangChain/LangGraph), context variables don't propagate
+    # reliably across task boundaries. Fall back to direct span lookup.
     if not current_span or current_span.uuid != uuid_str:
-        print(
-            f"Error: Current span in context does not match the span being exited. Expected UUID: {uuid_str}, Got: {current_span.uuid if current_span else 'None'}"
-        )
-        return
+        current_span = trace_manager.get_span_by_uuid(uuid_str)
+        if not current_span:
+            # Span already removed or never existed
+            return
 
     current_span.end_time = end_time
     if exc_type is not None:
@@ -295,7 +313,12 @@ def exit_current_context(
         else:
             current_span_context.set(None)
     else:
+        # Try context first, then fall back to direct trace lookup for async contexts
         current_trace = current_trace_context.get()
+        if not current_trace and current_span.trace_uuid:
+            current_trace = trace_manager.get_trace_by_uuid(
+                current_span.trace_uuid
+            )
         if current_span.status == TraceSpanStatus.ERRORED and current_trace:
             current_trace.status = TraceSpanStatus.ERRORED
         if current_trace and current_trace.uuid == current_span.trace_uuid:

deepeval/key_handler.py

@@ -162,6 +162,13 @@ class ModelKeyValues(Enum):
     VLLM_API_KEY = "VLLM_API_KEY"
     VLLM_MODEL_NAME = "VLLM_MODEL_NAME"
 
+    # OpenRouter
+    USE_OPENROUTER_MODEL = "USE_OPENROUTER_MODEL"
+    OPENROUTER_MODEL_NAME = "OPENROUTER_MODEL_NAME"
+    OPENROUTER_COST_PER_INPUT_TOKEN = "OPENROUTER_COST_PER_INPUT_TOKEN"
+    OPENROUTER_COST_PER_OUTPUT_TOKEN = "OPENROUTER_COST_PER_OUTPUT_TOKEN"
+    OPENROUTER_API_KEY = "OPENROUTER_API_KEY"
+
 
 class EmbeddingKeyValues(Enum):
     # Azure OpenAI
@@ -174,7 +181,7 @@ class EmbeddingKeyValues(Enum):
     USE_LOCAL_EMBEDDINGS = "USE_LOCAL_EMBEDDINGS"
     LOCAL_EMBEDDING_MODEL_NAME = "LOCAL_EMBEDDING_MODEL_NAME"
     LOCAL_EMBEDDING_BASE_URL = "LOCAL_EMBEDDING_BASE_URL"
-    LOCAL_EMBEDDING_API_KEY = "LOCAL_EMBEDDING_API_KEY"
+    LOCAL_EMBEDDING_API_KEY = ("LOCAL_EMBEDDING_API_KEY",)
 
 
 class KeyFileHandler:

deepeval/metrics/conversational_g_eval/conversational_g_eval.py

@@ -2,6 +2,7 @@
 
 from openai.types.chat.chat_completion import ChatCompletion
 from typing import Optional, List, Tuple, Union, Dict, Type
+from rich.console import Console
 import math
 from deepeval.metrics import BaseConversationalMetric
 from deepeval.metrics.g_eval.utils import (
@@ -11,6 +12,8 @@ from deepeval.metrics.g_eval.utils import (
     format_rubrics,
     validate_and_sort_rubrics,
     validate_criteria_and_evaluation_steps,
+    CONVERSATIONAL_G_EVAL_API_PARAMS,
+    construct_geval_upload_payload,
 )
 from deepeval.test_case import (
     TurnParams,
@@ -33,6 +36,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.indicator import metric_progress_indicator
 import deepeval.metrics.conversational_g_eval.schema as cgschema
 from deepeval.metrics.api import metric_data_manager
+from deepeval.confident.api import Api, Endpoints, HttpMethods
 
 
 class ConversationalGEval(BaseConversationalMetric):
@@ -412,6 +416,37 @@ class ConversationalGEval(BaseConversationalMetric):
                 self.success = False
         return self.success
 
+    def upload(self):
+        api = Api()
+
+        payload = construct_geval_upload_payload(
+            name=self.name,
+            evaluation_params=self.evaluation_params,
+            g_eval_api_params=CONVERSATIONAL_G_EVAL_API_PARAMS,
+            criteria=self.criteria,
+            evaluation_steps=self.evaluation_steps,
+            multi_turn=True,
+            rubric=self.rubric,
+        )
+
+        data, _ = api.send_request(
+            method=HttpMethods.POST,
+            endpoint=Endpoints.METRICS_ENDPOINT,
+            body=payload,
+        )
+
+        metric_id = data.get("id")
+        self.metric_id = metric_id
+        console = Console()
+
+        if metric_id:
+            console.print(
+                "[rgb(5,245,141)]✓[/rgb(5,245,141)] Metric uploaded successfully "
+                f"(id: [bold]{metric_id}[/bold])"
+            )
+
+        return data
+
     @property
     def __name__(self):
         if self._include_g_eval_suffix:

deepeval/metrics/g_eval/g_eval.py

@@ -1,7 +1,7 @@
 """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
 import asyncio
-
+from rich.console import Console
 from typing import Optional, List, Tuple, Union, Type
 from deepeval.metrics import BaseMetric
 from deepeval.test_case import (
@@ -32,9 +32,12 @@ from deepeval.metrics.g_eval.utils import (
     validate_criteria_and_evaluation_steps,
     number_evaluation_steps,
     get_score_range,
+    construct_geval_upload_payload,
+    G_EVAL_API_PARAMS,
 )
 from deepeval.metrics.api import metric_data_manager
 from deepeval.config.settings import get_settings
+from deepeval.confident.api import Api, Endpoints, HttpMethods
 
 
 class GEval(BaseMetric):
@@ -408,6 +411,37 @@ class GEval(BaseMetric):
                 self.success = False
         return self.success
 
+    def upload(self):
+        api = Api()
+
+        payload = construct_geval_upload_payload(
+            name=self.name,
+            evaluation_params=self.evaluation_params,
+            g_eval_api_params=G_EVAL_API_PARAMS,
+            criteria=self.criteria,
+            evaluation_steps=self.evaluation_steps,
+            multi_turn=False,
+            rubric=self.rubric,
+        )
+
+        data, _ = api.send_request(
+            method=HttpMethods.POST,
+            endpoint=Endpoints.METRICS_ENDPOINT,
+            body=payload,
+        )
+
+        metric_id = data.get("id")
+        self.metric_id = metric_id
+        console = Console()
+
+        if metric_id:
+            console.print(
+                "[rgb(5,245,141)]✓[/rgb(5,245,141)] Metric uploaded successfully "
+                f"(id: [bold]{metric_id}[/bold])"
+            )
+
+        return data
+
     @property
     def __name__(self):
         if self._include_g_eval_suffix:

deepeval/metrics/g_eval/utils.py

@@ -52,6 +52,71 @@ CONVERSATIONAL_G_EVAL_PARAMS = {
     TurnParams.SCENARIO: "Scenario",
 }
 
+G_EVAL_API_PARAMS = {
+    LLMTestCaseParams.INPUT: "input",
+    LLMTestCaseParams.ACTUAL_OUTPUT: "actualOutput",
+    LLMTestCaseParams.EXPECTED_OUTPUT: "expectedOutput",
+    LLMTestCaseParams.CONTEXT: "context",
+    LLMTestCaseParams.RETRIEVAL_CONTEXT: "retrievalContext",
+    LLMTestCaseParams.EXPECTED_TOOLS: "expectedTools",
+    LLMTestCaseParams.TOOLS_CALLED: "toolsCalled",
+}
+
+CONVERSATIONAL_G_EVAL_API_PARAMS = {
+    TurnParams.ROLE: "role",
+    TurnParams.CONTENT: "content",
+    TurnParams.SCENARIO: "scenario",
+    TurnParams.EXPECTED_OUTCOME: "expectedOutcome",
+    TurnParams.RETRIEVAL_CONTEXT: "retrievalContext",
+    TurnParams.TOOLS_CALLED: "toolsCalled",
+}
+
+
+def construct_geval_upload_payload(
+    name: str,
+    evaluation_params: List[LLMTestCaseParams],
+    g_eval_api_params: Dict,
+    criteria: Optional[str] = None,
+    evaluation_steps: Optional[List[str]] = None,
+    multi_turn: bool = False,
+    rubric: Optional[List[Rubric]] = None,
+) -> Dict:
+    if not evaluation_params:
+        raise ValueError("GEval requires at least one evaluation parameter.")
+
+    unsupported_params = [
+        param for param in evaluation_params if param not in g_eval_api_params
+    ]
+    if unsupported_params:
+        raise ValueError(
+            "Unsupported evaluation params for GEval upload: "
+            + ", ".join(param.name for param in unsupported_params)
+        )
+
+    payload = {
+        "name": name,
+        "evaluationParams": [
+            g_eval_api_params[param] for param in evaluation_params
+        ],
+        "multiTurn": multi_turn,
+    }
+
+    if criteria is not None:
+        payload["criteria"] = criteria
+    else:
+        payload["evaluationSteps"] = evaluation_steps
+
+    if rubric is not None:
+        payload["rubric"] = [
+            {
+                "scoreRange": list(r.score_range),
+                "expectedOutcome": r.expected_outcome,
+            }
+            for r in rubric
+        ]
+
+    return payload
+
 
 def validate_criteria_and_evaluation_steps(
     criteria: Optional[str] = None,

deepeval/models/__init__.py

@@ -16,6 +16,7 @@ from deepeval.models.llms import (
     GrokModel,
     DeepSeekModel,
     PortkeyModel,
+    OpenRouterModel,
 )
 from deepeval.models.embedding_models import (
     OpenAIEmbeddingModel,
@@ -44,4 +45,5 @@ __all__ = [
     "LocalEmbeddingModel",
     "OllamaEmbeddingModel",
     "PortkeyModel",
+    "OpenRouterModel",
 ]

deepeval/models/llms/__init__.py

@@ -10,6 +10,7 @@ from .kimi_model import KimiModel
 from .grok_model import GrokModel
 from .deepseek_model import DeepSeekModel
 from .portkey_model import PortkeyModel
+from .openrouter_model import OpenRouterModel
 
 __all__ = [
     "AzureOpenAIModel",
@@ -24,4 +25,5 @@ __all__ = [
     "GrokModel",
     "DeepSeekModel",
     "PortkeyModel",
+    "OpenRouterModel",
 ]

deepeval/models/llms/constants.py

@@ -3,6 +3,11 @@ from typing import Any, Callable, Union
 from deepeval.models.base_model import DeepEvalModelData
 
 
+DEFAULT_GPT_MODEL = "gpt-4.1"
+# OpenRouter uses provider/model format (e.g., "openai/gpt-4", "anthropic/claude-3-opus")
+# DeepEval does not validate OpenRouter model strings.
+DEFAULT_OPENROUTER_MODEL = f"openai/{DEFAULT_GPT_MODEL}"
+
 ModelDataFactory = Callable[[], DeepEvalModelData]
 ModelDataValue = Union[DeepEvalModelData, ModelDataFactory]
 
@@ -366,6 +371,24 @@ OPENAI_MODELS_DATA = ModelDataRegistry(
             input_price=1.25 / 1e6,
             output_price=10.00 / 1e6,
         ),
+        "gpt-5.1": make_model_data(
+            supports_log_probs=False,
+            supports_multimodal=True,
+            supports_structured_outputs=True,
+            supports_json=False,
+            supports_temperature=False,
+            input_price=1.25 / 1e6,
+            output_price=10.00 / 1e6,
+        ),
+        "gpt-5.2": make_model_data(
+            supports_log_probs=False,
+            supports_multimodal=True,
+            supports_structured_outputs=True,
+            supports_json=False,
+            supports_temperature=False,
+            input_price=1.75 / 1e6,
+            output_price=14.00 / 1e6,
+        ),
     }
 )
 

deepeval/models/llms/openai_model.py

@@ -24,14 +24,13 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.models.llms.constants import (
+    DEFAULT_GPT_MODEL,
     OPENAI_MODELS_DATA,
 )
 
 
 retry_openai = create_retry_decorator(PS.OPENAI)
 
-default_gpt_model = "gpt-4.1"
-
 
 def _request_timeout_seconds() -> float:
     timeout = float(get_settings().DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS or 0)
@@ -70,7 +69,7 @@ class GPTModel(DeepEvalBaseLLM):
 
         model = model or settings.OPENAI_MODEL_NAME
         if model is None:
-            model = default_gpt_model
+            model = DEFAULT_GPT_MODEL
 
         cost_per_input_token = (
             cost_per_input_token
@@ -377,7 +376,9 @@ class GPTModel(DeepEvalBaseLLM):
     # Utilities #
     #############
 
-    def calculate_cost(self, input_tokens: int, output_tokens: int) -> float:
+    def calculate_cost(
+        self, input_tokens: int, output_tokens: int
+    ) -> Optional[float]:
         if self.model_data.input_price and self.model_data.output_price:
             input_cost = input_tokens * self.model_data.input_price
             output_cost = output_tokens * self.model_data.output_price