deepeval 3.6.8__py3-none-any.whl → 3.7.0__py3-none-any.whl

deepeval/telemetry.py

@@ -3,12 +3,12 @@ import logging
 import os
 import socket
 import sys
-from threading import Event
 import uuid
 import sentry_sdk
 from enum import Enum
 from typing import List, Dict
 import requests
+from deepeval.config.settings import get_settings
 from deepeval.constants import LOGIN_PROMPT, HIDDEN_DIR, KEY_FILE
 from posthog import Posthog
 
@@ -34,7 +34,7 @@ TELEMETRY_PATH = os.path.join(HIDDEN_DIR, TELEMETRY_DATA_FILE)
 
 
 def telemetry_opt_out():
-    return os.getenv("DEEPEVAL_TELEMETRY_OPT_OUT") == "1"
+    return get_settings().DEEPEVAL_TELEMETRY_OPT_OUT
 
 
 def blocked_by_firewall():
@@ -131,7 +131,7 @@ if not telemetry_opt_out():
 
 
 if (
-    os.getenv("ERROR_REPORTING") == "1"
+    get_settings().ERROR_REPORTING
     and not blocked_by_firewall()
     and not telemetry_opt_out()
 ):

deepeval/test_case/llm_test_case.py

@@ -1,5 +1,4 @@
 from pydantic import (
-    ConfigDict,
     Field,
     BaseModel,
     model_validator,
@@ -11,6 +10,8 @@ from enum import Enum
 import json
 import uuid
 
+from deepeval.utils import make_model_config
+
 from deepeval.test_case.mcp import (
     MCPServer,
     MCPPromptCall,
@@ -156,7 +157,7 @@ class ToolCall(BaseModel):
 
 
 class LLMTestCase(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     input: str
     actual_output: Optional[str] = Field(

deepeval/test_run/api.py

@@ -1,8 +1,9 @@
-from pydantic import BaseModel, Field, ConfigDict
+from pydantic import BaseModel, Field
 from typing import Optional, List, Union, Dict
 
 from deepeval.test_case import MLLMImage, ToolCall
 from deepeval.tracing.api import TraceApi, MetricData
+from deepeval.utils import make_model_config
 
 
 class LLMApiTestCase(BaseModel):
@@ -49,7 +50,7 @@ class LLMApiTestCase(BaseModel):
     comments: Optional[str] = Field(None)
     trace: Optional[TraceApi] = Field(None)
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config = make_model_config(arbitrary_types_allowed=True)
     # metric_collection: Optional[str] = Field(None, alias="metricCollection")
 
     def update_metric_data(self, metric_data: MetricData):

deepeval/test_run/cache.py

@@ -6,6 +6,8 @@ from typing import List, Optional, Union, Dict, Union
 from enum import Enum
 from pydantic import BaseModel, Field
 
+from deepeval.utils import make_model_config
+
 from deepeval.test_case import LLMTestCaseParams, LLMTestCase, ToolCallParams
 from deepeval.test_run.api import MetricData
 from deepeval.utils import (
@@ -20,6 +22,8 @@ TEMP_CACHE_FILE_NAME = f"{HIDDEN_DIR}/.temp-deepeval-cache.json"
 
 
 class MetricConfiguration(BaseModel):
+    model_config = make_model_config(arbitrary_types_allowed=True)
+
     ##### Required fields #####
     threshold: float
     evaluation_model: Optional[str] = None
@@ -36,9 +40,6 @@ class MetricConfiguration(BaseModel):
         Union[List[LLMTestCaseParams], List[ToolCallParams]]
     ] = None
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class CachedMetricData(BaseModel):
     metric_data: MetricData

deepeval/test_run/test_run.py

@@ -463,19 +463,29 @@ class TestRunManager:
                     mode="r",
                     flags=portalocker.LOCK_SH | portalocker.LOCK_NB,
                 ) as file:
-                    self.test_run = self.test_run.load(file)
+                    loaded = self.test_run.load(file)
+                    # only overwrite if loading actually worked
+                    self.test_run = loaded
             except (
                 FileNotFoundError,
+                json.JSONDecodeError,
                 portalocker.exceptions.LockException,
             ) as e:
-                print(f"Error loading test run from disk: {e}", file=sys.stderr)
-                self.test_run = None
+                print(
+                    f"Warning: Could not load test run from disk: {e}",
+                    file=sys.stderr,
+                )
 
         return self.test_run
 
     def save_test_run(self, path: str, save_under_key: Optional[str] = None):
         if self.save_to_disk:
             try:
+                # ensure parent directory exists
+                parent = os.path.dirname(path)
+                if parent:
+                    os.makedirs(parent, exist_ok=True)
+
                 with portalocker.Lock(path, mode="w") as file:
                     if save_under_key:
                         try:
@@ -533,10 +543,19 @@ class TestRunManager:
                     self.test_run.save(file)
             except (
                 FileNotFoundError,
+                json.JSONDecodeError,
                 portalocker.exceptions.LockException,
             ) as e:
-                print(f"Error updating test run to disk: {e}", file=sys.stderr)
-                self.test_run = None
+                print(
+                    f"Warning: Could not update test run on disk: {e}",
+                    file=sys.stderr,
+                )
+                if self.test_run is None:
+                    # guarantee a valid in-memory run so the update can proceed.
+                    # never destroy in-memory state on I/O failure.
+                    self.create_test_run()
+                self.test_run.add_test_case(api_test_case)
+                self.test_run.set_dataset_properties(test_case)
         else:
             if self.test_run is None:
                 self.create_test_run()

deepeval/tracing/api.py

@@ -1,8 +1,9 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union, Literal, Any
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, Field
 
 from deepeval.test_case import ToolCall
+from deepeval.utils import make_model_config
 
 
 class SpanApiType(Enum):
@@ -27,7 +28,7 @@ class PromptApi(BaseModel):
 
 
 class MetricData(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     name: str
     threshold: float
@@ -42,6 +43,10 @@ class MetricData(BaseModel):
 
 
 class BaseApiSpan(BaseModel):
+    model_config = make_model_config(
+        use_enum_values=True, validate_assignment=True
+    )
+
     uuid: str
     name: str = None
     status: TraceSpanApiStatus
@@ -96,12 +101,12 @@ class BaseApiSpan(BaseModel):
     metric_collection: Optional[str] = Field(None, alias="metricCollection")
     metrics_data: Optional[List[MetricData]] = Field(None, alias="metricsData")
 
-    class Config:
-        use_enum_values = True
-        validate_assignment = True
-
 
 class TraceApi(BaseModel):
+    model_config = make_model_config(
+        use_enum_values=True, validate_assignment=True
+    )
+
     uuid: str
     base_spans: Optional[List[BaseApiSpan]] = Field(None, alias="baseSpans")
     agent_spans: Optional[List[BaseApiSpan]] = Field(None, alias="agentSpans")
@@ -139,7 +144,3 @@ class TraceApi(BaseModel):
 
     # Don't serialize these
     confident_api_key: Optional[str] = Field(None, exclude=True)
-
-    class Config:
-        use_enum_values = True
-        validate_assignment = True

deepeval/tracing/otel/exporter.py

@@ -493,6 +493,17 @@ class ConfidentSpanExporter(SpanExporter):
             output_token_count = span.attributes.get(
                 "confident.llm.output_token_count"
             )
+
+            # fallback to gen ai attributes if not found in confident attributes
+            if not input_token_count:
+                input_token_count = span.attributes.get(
+                    "gen_ai.usage.input_tokens"
+                )
+            if not output_token_count:
+                output_token_count = span.attributes.get(
+                    "gen_ai.usage.output_tokens"
+                )
+
             cost_per_input_token = span.attributes.get(
                 "confident.llm.cost_per_input_token"
             )

deepeval/tracing/patchers.py

@@ -1,6 +1,8 @@
-from openai import OpenAI
 import functools
 
+from anthropic import Anthropic
+from openai import OpenAI
+
 from deepeval.tracing.context import update_current_span, update_llm_span
 from deepeval.tracing.context import current_span_context
 from deepeval.tracing.types import LlmSpan
@@ -82,3 +84,102 @@ def patch_openai_client(client: OpenAI):
                 return response
 
             setattr(current_obj, method_name, wrapped_method)
+
+
+def patch_anthropic_client(client: Anthropic):
+    """
+    Patch an Anthropic client instance to add tracing capabilities.
+
+    Args:
+        client: An instance of Anthropic client to patch
+    """
+    original_methods = {}
+
+    methods_to_patch = [
+        "messages.create",
+    ]
+
+    for method_path in methods_to_patch:
+        parts = method_path.split(".")
+        current_obj = client
+
+        for part in parts[:-1]:
+            if not hasattr(current_obj, part):
+                print(f"Warning: Cannot find {part} in the path {method_path}")
+                continue
+            current_obj = getattr(current_obj, part)
+
+        method_name = parts[-1]
+        if not hasattr(current_obj, method_name):
+            print(
+                f"Warning: Cannot find method {method_name} in the path {method_path}"
+            )
+            continue
+
+        method = getattr(current_obj, method_name)
+
+        if callable(method) and not isinstance(method, type):
+            original_methods[method_path] = method
+
+            @functools.wraps(method)
+            def wrapped_method(*args, original_method=method, **kwargs):
+                current_span = current_span_context.get()
+                response = original_method(*args, **kwargs)
+
+                if isinstance(current_span, LlmSpan):
+                    model = kwargs.get("model", None)
+                    if model is None:
+                        raise ValueError("model not found in client")
+
+                    current_span.model = model
+
+                    output = None
+                    try:
+                        if (
+                            hasattr(response, "content")
+                            and response.content
+                            and len(response.content) > 0
+                        ):
+                            for block in response.content:
+                                if hasattr(block, "text"):
+                                    output = block.text
+                                    break
+                    except Exception:
+                        pass
+
+                    input_token_count = None
+                    output_token_count = None
+                    try:
+                        if hasattr(response, "usage"):
+                            usage = response.usage
+                            # usage can be a dict or an object with attributes
+                            if isinstance(usage, dict):
+                                input_token_count = usage.get(
+                                    "input_tokens", None
+                                )
+                                output_token_count = usage.get(
+                                    "output_tokens", None
+                                )
+                            else:
+                                input_token_count = getattr(
+                                    usage, "input_tokens", None
+                                )
+                                output_token_count = getattr(
+                                    usage, "output_tokens", None
+                                )
+                    except Exception:
+                        pass
+
+                    update_current_span(
+                        input=kwargs.get("messages", "INPUT_MESSAGE_NOT_FOUND"),
+                        output=output if output else "OUTPUT_MESSAGE_NOT_FOUND",
+                    )
+                    update_llm_span(
+                        input_token_count=input_token_count,
+                        output_token_count=output_token_count,
+                    )
+                return response
+
+            setattr(current_obj, method_name, wrapped_method)
+
+    return original_methods

deepeval/tracing/trace_context.py

@@ -1,13 +1,15 @@
-from typing import Optional, List, Dict, Any
 from contextvars import ContextVar
 from contextlib import contextmanager
 from dataclasses import dataclass
+from typing import Optional, List, Dict, Any
 
-from .tracing import trace_manager
-from .context import current_trace_context, update_current_trace
-from deepeval.prompt import Prompt
 from deepeval.metrics import BaseMetric
+from deepeval.prompt import Prompt
 from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import current_trace_context, update_current_trace
+from deepeval.tracing.tracing import trace_manager
+from deepeval.tracing.types import TraceWorkerStatus
+from deepeval.tracing.utils import is_async_context
 
 
 @dataclass
@@ -59,6 +61,13 @@ def trace(
     metrics: Optional[List[BaseMetric]] = None,
     metric_collection: Optional[str] = None,
 ):
+    if is_async_context():
+        trace_manager._print_trace_status(
+            message="Warning: Detected use of the synchronous 'trace' context manager within an async method",
+            trace_worker_status=TraceWorkerStatus.WARNING,
+            description="Wrapping an async method with the synchronous 'trace' context manager may lead to unexpected behavior.",
+        )
+
     current_trace = current_trace_context.get()
 
     if not current_trace:

deepeval/tracing/tracing.py

@@ -19,6 +19,7 @@ import random
 import atexit
 import queue
 import uuid
+from anthropic import Anthropic
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
@@ -38,7 +39,10 @@ from deepeval.tracing.api import (
     TraceSpanApiStatus,
 )
 from deepeval.telemetry import capture_send_trace
-from deepeval.tracing.patchers import patch_openai_client
+from deepeval.tracing.patchers import (
+    patch_anthropic_client,
+    patch_openai_client,
+)
 from deepeval.tracing.types import (
     AgentSpan,
     BaseSpan,
@@ -111,6 +115,7 @@ class TraceManager:
 
         self.sampling_rate = settings.CONFIDENT_TRACE_SAMPLE_RATE
         validate_sampling_rate(self.sampling_rate)
+        self.anthropic_client = None
         self.openai_client = None
         self.tracing_enabled = True
 
@@ -149,6 +154,7 @@ class TraceManager:
         environment: Optional[str] = None,
         sampling_rate: Optional[float] = None,
         confident_api_key: Optional[str] = None,
+        anthropic_client: Optional[Anthropic] = None,
         openai_client: Optional[OpenAI] = None,
         tracing_enabled: Optional[bool] = None,
     ) -> None:
@@ -165,6 +171,9 @@ class TraceManager:
         if openai_client is not None:
             self.openai_client = openai_client
             patch_openai_client(openai_client)
+        if anthropic_client is not None:
+            self.anthropic_client = anthropic_client
+            patch_anthropic_client(anthropic_client)
         if tracing_enabled is not None:
             self.tracing_enabled = tracing_enabled
 

deepeval/tracing/types.py

@@ -1,9 +1,11 @@
 from enum import Enum
 from dataclasses import dataclass, field
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ConfigDict
 from typing import Any, Dict, List, Optional, Union, Literal
 from rich.progress import Progress
 
+from deepeval.utils import make_model_config
+
 from deepeval.prompt.prompt import Prompt
 from deepeval.test_case.llm_test_case import ToolCall
 from deepeval.test_case import LLMTestCase
@@ -55,6 +57,8 @@ class LlmOutput(BaseModel):
 
 
 class BaseSpan(BaseModel):
+    model_config = make_model_config(arbitrary_types_allowed=True)
+
     uuid: str
     status: TraceSpanStatus
     children: List["BaseSpan"] = Field(default_factory=list)
@@ -90,9 +94,6 @@ class BaseSpan(BaseModel):
         None, serialization_alias="expectedTools"
     )
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class AgentSpan(BaseSpan):
     name: str
@@ -125,7 +126,7 @@ class LlmSpan(BaseSpan):
     # output_metadata: Optional[Dict[str, Any]] = Field(None, serialization_alias="outputMetadata")
 
     # for serializing `prompt`
-    model_config = {"arbitrary_types_allowed": True}
+    model_config = make_model_config(arbitrary_types_allowed=True)
 
 
 class RetrieverSpan(BaseSpan):
@@ -140,6 +141,8 @@ class ToolSpan(BaseSpan):
 
 
 class Trace(BaseModel):
+    model_config = make_model_config(arbitrary_types_allowed=True)
+
     uuid: str = Field(serialization_alias="uuid")
     status: TraceSpanStatus
     root_spans: List[BaseSpan] = Field(serialization_alias="rootSpans")
@@ -174,9 +177,6 @@ class Trace(BaseModel):
         None, serialization_alias="expectedTools"
     )
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class TraceAttributes(BaseModel):
     name: Optional[str] = None

deepeval/tracing/utils.py

@@ -1,3 +1,4 @@
+import asyncio
 import os
 from typing import Dict, Any
 from datetime import datetime, timezone
@@ -191,3 +192,11 @@ def prepare_tool_call_input_parameters(output: Any) -> Dict[str, Any]:
     if res and not isinstance(res, dict):
         res = {"output": res}
     return res
+
+
+def is_async_context() -> bool:
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False

deepeval/utils.py

@@ -21,7 +21,6 @@ from pydantic import BaseModel
 from rich.progress import Progress
 from rich.console import Console, Theme
 
-from deepeval.confident.api import set_confident_api_key
 from deepeval.config.settings import get_settings
 from deepeval.config.utils import (
     get_env_bool,
@@ -29,6 +28,48 @@ from deepeval.config.utils import (
 )
 
 
+#####################
+# Pydantic Compat   #
+#####################
+
+import pydantic
+
+PYDANTIC_V2 = pydantic.VERSION.startswith("2")
+
+
+def make_model_config(**kwargs):
+    """
+    Create a model configuration that works with both Pydantic v1 and v2.
+
+    Usage in a model (Pydantic v2 style):
+        class MyModel(BaseModel):
+            model_config = make_model_config(arbitrary_types_allowed=True)
+            field: str
+
+    This will work correctly in both v1 and v2:
+    - In v2: Returns ConfigDict(**kwargs)
+    - In v1: Returns a Config class with the attributes set
+
+    Args:
+        **kwargs: Configuration options (e.g., use_enum_values=True, arbitrary_types_allowed=True)
+
+    Returns:
+        ConfigDict (v2) or Config class (v1)
+    """
+    if PYDANTIC_V2:
+        from pydantic import ConfigDict
+
+        return ConfigDict(**kwargs)
+    else:
+        # For Pydantic v1, create an inner Config class
+        class Config:
+            pass
+
+        for key, value in kwargs.items():
+            setattr(Config, key, value)
+        return Config
+
+
 ###############
 # Local Types #
 ###############
@@ -232,6 +273,7 @@ def login(api_key: str):
         raise ValueError("Unable to login, please provide a non-empty api key.")
 
     from rich import print
+    from deepeval.confident.api import set_confident_api_key
 
     set_confident_api_key(api_key)
     print(
@@ -751,7 +793,7 @@ custom_console = Console(theme=my_theme)
 
 
 def format_error_text(
-    exc: BaseException, *, with_stack: bool | None = None
+    exc: BaseException, *, with_stack: Optional[bool] = None
 ) -> str:
     if with_stack is None:
         with_stack = logging.getLogger("deepeval").isEnabledFor(logging.DEBUG)

{deepeval-3.6.8.dist-info → deepeval-3.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.8
+Version: 3.7.0
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -25,7 +25,7 @@ Requires-Dist: opentelemetry-api (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: portalocker
-Requires-Dist: posthog (>=6.3.0,<7.0.0)
+Requires-Dist: posthog (>=5.4.0,<6.0.0)
 Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.10.1,<3.0.0)
 Requires-Dist: pyfiglet