deepeval 3.6.9__py3-none-any.whl → 3.7.1__py3-none-any.whl

deepeval/test_case/mllm_test_case.py

@@ -11,33 +11,50 @@ from deepeval.test_case import ToolCall
 
 @dataclass
 class MLLMImage:
-    url: str
+    dataBase64: Optional[str] = None
+    mimeType: Optional[str] = None
+    url: Optional[str] = None
     local: Optional[bool] = None
-    filename: Optional[str] = field(default=None, init=False, repr=False)
-    mimeType: Optional[str] = field(default=None, init=False, repr=False)
-    dataBase64: Optional[str] = field(default=None, init=False, repr=False)
+    filename: Optional[str] = None
 
     def __post_init__(self):
-        is_local = self.is_local_path(self.url)
-        if self.local is not None:
-            assert self.local == is_local, "Local path mismatch"
-        else:
-            self.local = is_local
-
-        # compute filename, mime_type, and Base64 data
-        if self.local:
-            path = self.process_url(self.url)
-            self.filename = os.path.basename(path)
-            self.mimeType = (
-                mimetypes.guess_type(path)[0] or "application/octet-stream"
+
+        if self.url and self.dataBase64:
+            raise ValueError(
+                "You cannot provide both 'url' and 'dataBase64' at the same time when creating an MLLMImage."
+            )
+
+        if not self.url and not self.dataBase64:
+            raise ValueError(
+                "You must provide either a 'url' or both 'dataBase64' and 'mimeType' to create an MLLMImage."
             )
-            with open(path, "rb") as f:
-                raw = f.read()
-            self.dataBase64 = base64.b64encode(raw).decode("ascii")
+
+        if self.dataBase64 is not None:
+            if self.mimeType is None:
+                raise ValueError(
+                    "mimeType must be provided when initializing from Base64 data."
+                )
         else:
-            self.filename = None
-            self.mimeType = None
-            self.dataBase64 = None
+            is_local = self.is_local_path(self.url)
+            if self.local is not None:
+                assert self.local == is_local, "Local path mismatch"
+            else:
+                self.local = is_local
+
+            # compute filename, mime_type, and Base64 data
+            if self.local:
+                path = self.process_url(self.url)
+                self.filename = os.path.basename(path)
+                self.mimeType = (
+                    mimetypes.guess_type(path)[0] or "application/octet-stream"
+                )
+                with open(path, "rb") as f:
+                    raw = f.read()
+                self.dataBase64 = base64.b64encode(raw).decode("ascii")
+            else:
+                self.filename = None
+                self.mimeType = None
+                self.dataBase64 = None
 
     @staticmethod
     def process_url(url: str) -> str:
@@ -69,6 +86,12 @@ class MLLMImage:
             return os.path.exists(path)
         return False
 
+    def as_data_uri(self) -> Optional[str]:
+        """Return the image as a data URI string, if Base64 data is available."""
+        if not self.dataBase64 or not self.mimeType:
+            return None
+        return f"data:{self.mimeType};base64,{self.dataBase64}"
+
 
 class MLLMTestCaseParams(Enum):
     INPUT = "input"

deepeval/test_run/api.py

@@ -1,8 +1,9 @@
-from pydantic import BaseModel, Field, ConfigDict
+from pydantic import BaseModel, Field
 from typing import Optional, List, Union, Dict
 
 from deepeval.test_case import MLLMImage, ToolCall
 from deepeval.tracing.api import TraceApi, MetricData
+from deepeval.utils import make_model_config
 
 
 class LLMApiTestCase(BaseModel):
@@ -49,7 +50,7 @@ class LLMApiTestCase(BaseModel):
     comments: Optional[str] = Field(None)
     trace: Optional[TraceApi] = Field(None)
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config = make_model_config(arbitrary_types_allowed=True)
     # metric_collection: Optional[str] = Field(None, alias="metricCollection")
 
     def update_metric_data(self, metric_data: MetricData):

deepeval/test_run/cache.py

@@ -1,25 +1,44 @@
-import portalocker
+import logging
 import sys
 import json
 import os
-from typing import List, Optional, Union, Dict, Union
+from typing import List, Optional, Dict, Union
 from enum import Enum
 from pydantic import BaseModel, Field
 
+from deepeval.utils import make_model_config
+
 from deepeval.test_case import LLMTestCaseParams, LLMTestCase, ToolCallParams
 from deepeval.test_run.api import MetricData
 from deepeval.utils import (
     delete_file_if_exists,
+    is_read_only_env,
     serialize,
 )
 from deepeval.metrics import BaseMetric
 from deepeval.constants import HIDDEN_DIR
 
+
+logger = logging.getLogger(__name__)
+
+
+portalocker = None
+if not is_read_only_env():
+    try:
+        import portalocker
+    except Exception as e:
+        logger.warning("failed to import portalocker: %s", e)
+else:
+    logger.warning("READ_ONLY filesystem: skipping disk cache for test runs.")
+
+
 CACHE_FILE_NAME = f"{HIDDEN_DIR}/.deepeval-cache.json"
 TEMP_CACHE_FILE_NAME = f"{HIDDEN_DIR}/.temp-deepeval-cache.json"
 
 
 class MetricConfiguration(BaseModel):
+    model_config = make_model_config(arbitrary_types_allowed=True)
+
     ##### Required fields #####
     threshold: float
     evaluation_model: Optional[str] = None
@@ -36,9 +55,6 @@ class MetricConfiguration(BaseModel):
         Union[List[LLMTestCaseParams], List[ToolCallParams]]
     ] = None
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class CachedMetricData(BaseModel):
     metric_data: MetricData
@@ -96,7 +112,7 @@ class TestRunCacheManager:
     def get_cached_test_case(
         self, test_case: LLMTestCase, hyperparameters: Union[Dict, None]
     ) -> Union[CachedTestCase, None]:
-        if self.disable_write_cache:
+        if self.disable_write_cache or portalocker is None:
             return None
 
         cached_test_run = self.get_cached_test_run()
@@ -121,7 +137,7 @@ class TestRunCacheManager:
         hyperparameters: Union[Dict, None],
         to_temp: bool = False,
     ):
-        if self.disable_write_cache:
+        if self.disable_write_cache or portalocker is None:
             return
         cache_dict = {
             LLMTestCaseParams.INPUT.value: test_case.input,
@@ -141,7 +157,7 @@ class TestRunCacheManager:
     def set_cached_test_run(
         self, cached_test_run: CachedTestRun, temp: bool = False
     ):
-        if self.disable_write_cache:
+        if self.disable_write_cache or portalocker is None:
             return
 
         if temp:
@@ -150,7 +166,7 @@ class TestRunCacheManager:
             self.cached_test_run = cached_test_run
 
     def save_cached_test_run(self, to_temp: bool = False):
-        if self.disable_write_cache:
+        if self.disable_write_cache or portalocker is None:
             return
 
         if to_temp:
@@ -177,7 +193,7 @@ class TestRunCacheManager:
                 )
 
     def create_cached_test_run(self, temp: bool = False):
-        if self.disable_write_cache:
+        if self.disable_write_cache or portalocker is None:
             return
 
         cached_test_run = CachedTestRun()
@@ -187,7 +203,7 @@ class TestRunCacheManager:
     def get_cached_test_run(
         self, from_temp: bool = False
     ) -> Union[CachedTestRun, None]:
-        if self.disable_write_cache:
+        if self.disable_write_cache or portalocker is None:
             return
 
         should_create_cached_test_run = False
@@ -208,7 +224,7 @@ class TestRunCacheManager:
                     try:
                         data = json.loads(content)
                         self.temp_cached_test_run = CachedTestRun.load(data)
-                    except Exception as e:
+                    except Exception:
                         should_create_cached_test_run = True
             except portalocker.exceptions.LockException as e:
                 print(
@@ -216,6 +232,9 @@ class TestRunCacheManager:
                     file=sys.stderr,
                 )
 
+            if should_create_cached_test_run:
+                self.create_cached_test_run(temp=from_temp)
+
             return self.temp_cached_test_run
         else:
             if self.cached_test_run:
@@ -249,6 +268,9 @@ class TestRunCacheManager:
             return self.cached_test_run
 
     def wrap_up_cached_test_run(self):
+        if portalocker is None:
+            return
+
         if self.disable_write_cache:
             # Clear cache if write cache is disabled
             delete_file_if_exists(self.cache_file_name)
@@ -329,7 +351,7 @@ class Cache:
                             if criteria_value != cached_criteria_value:
                                 return False
                             continue
-                    except:
+                    except Exception:
                         # For non-GEval
                         continue
 

deepeval/test_run/hyperparameters.py

@@ -33,7 +33,11 @@ def process_hyperparameters(
             )
 
         if isinstance(value, Prompt):
-            prompt_key = f"{value.alias}_{value.version}"
+            try:
+                prompt_key = f"{value.alias}_{value.version}"
+            except AttributeError:
+                prompt_key = f"{value.alias}_00.00.01"
+
             if value._prompt_version_id is not None and value.type is not None:
                 processed_hyperparameters[key] = PromptApi(
                     id=value._prompt_version_id,

deepeval/test_run/test_run.py

@@ -6,11 +6,11 @@ from typing import Any, Optional, List, Dict, Union, Tuple
 import shutil
 import sys
 import datetime
-import portalocker
 from rich.table import Table
 from rich.console import Console
 from rich import print
 
+
 from deepeval.metrics import BaseMetric
 from deepeval.confident.api import Api, Endpoints, HttpMethods, is_confident
 from deepeval.test_run.api import (
@@ -25,6 +25,7 @@ from deepeval.test_case import LLMTestCase, ConversationalTestCase, MLLMTestCase
 from deepeval.utils import (
     delete_file_if_exists,
     get_is_running_deepeval,
+    is_read_only_env,
     open_browser,
     shorten,
     format_turn,
@@ -42,6 +43,21 @@ from rich.panel import Panel
 from rich.columns import Columns
 
 
+portalocker = None
+if not is_read_only_env():
+    try:
+        import portalocker
+    except Exception as e:
+        print(
+            f"Warning: failed to import portalocker: {e}",
+            file=sys.stderr,
+        )
+else:
+    print(
+        "Warning: DeepEval is configured for read only environment. Test runs will not be written to disk."
+    )
+
+
 TEMP_FILE_PATH = f"{HIDDEN_DIR}/.temp_test_run_data.json"
 LATEST_TEST_RUN_FILE_PATH = f"{HIDDEN_DIR}/.latest_test_run.json"
 LATEST_TEST_RUN_DATA_KEY = "testRunData"
@@ -456,26 +472,36 @@ class TestRunManager:
         if self.test_run is None:
             self.create_test_run(identifier=identifier)
 
-        if self.save_to_disk:
+        if portalocker and self.save_to_disk:
             try:
                 with portalocker.Lock(
                     self.temp_file_path,
                     mode="r",
                     flags=portalocker.LOCK_SH | portalocker.LOCK_NB,
                 ) as file:
-                    self.test_run = self.test_run.load(file)
+                    loaded = self.test_run.load(file)
+                    # only overwrite if loading actually worked
+                    self.test_run = loaded
             except (
                 FileNotFoundError,
+                json.JSONDecodeError,
                 portalocker.exceptions.LockException,
             ) as e:
-                print(f"Error loading test run from disk: {e}", file=sys.stderr)
-                self.test_run = None
+                print(
+                    f"Warning: Could not load test run from disk: {e}",
+                    file=sys.stderr,
+                )
 
         return self.test_run
 
     def save_test_run(self, path: str, save_under_key: Optional[str] = None):
-        if self.save_to_disk:
+        if portalocker and self.save_to_disk:
             try:
+                # ensure parent directory exists
+                parent = os.path.dirname(path)
+                if parent:
+                    os.makedirs(parent, exist_ok=True)
+
                 with portalocker.Lock(path, mode="w") as file:
                     if save_under_key:
                         try:
@@ -495,11 +521,14 @@ class TestRunManager:
                 pass
 
     def save_final_test_run_link(self, link: str):
-        try:
-            with portalocker.Lock(LATEST_TEST_RUN_FILE_PATH, mode="w") as file:
-                json.dump({LATEST_TEST_RUN_LINK_KEY: link}, file)
-        except portalocker.exceptions.LockException:
-            pass
+        if portalocker:
+            try:
+                with portalocker.Lock(
+                    LATEST_TEST_RUN_FILE_PATH, mode="w"
+                ) as file:
+                    json.dump({LATEST_TEST_RUN_LINK_KEY: link}, file)
+            except portalocker.exceptions.LockException:
+                pass
 
     def update_test_run(
         self,
@@ -513,7 +542,7 @@ class TestRunManager:
         ):
             return
 
-        if self.save_to_disk:
+        if portalocker and self.save_to_disk:
             try:
                 with portalocker.Lock(
                     self.temp_file_path,
@@ -533,10 +562,19 @@ class TestRunManager:
                     self.test_run.save(file)
             except (
                 FileNotFoundError,
+                json.JSONDecodeError,
                 portalocker.exceptions.LockException,
             ) as e:
-                print(f"Error updating test run to disk: {e}", file=sys.stderr)
-                self.test_run = None
+                print(
+                    f"Warning: Could not update test run on disk: {e}",
+                    file=sys.stderr,
+                )
+                if self.test_run is None:
+                    # guarantee a valid in-memory run so the update can proceed.
+                    # never destroy in-memory state on I/O failure.
+                    self.create_test_run()
+                self.test_run.add_test_case(api_test_case)
+                self.test_run.set_dataset_properties(test_case)
         else:
             if self.test_run is None:
                 self.create_test_run()

deepeval/tracing/api.py

@@ -1,8 +1,9 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union, Literal, Any
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, Field
 
 from deepeval.test_case import ToolCall
+from deepeval.utils import make_model_config
 
 
 class SpanApiType(Enum):
@@ -27,7 +28,7 @@ class PromptApi(BaseModel):
 
 
 class MetricData(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     name: str
     threshold: float
@@ -42,6 +43,10 @@ class MetricData(BaseModel):
 
 
 class BaseApiSpan(BaseModel):
+    model_config = make_model_config(
+        use_enum_values=True, validate_assignment=True
+    )
+
     uuid: str
     name: str = None
     status: TraceSpanApiStatus
@@ -96,12 +101,12 @@ class BaseApiSpan(BaseModel):
     metric_collection: Optional[str] = Field(None, alias="metricCollection")
     metrics_data: Optional[List[MetricData]] = Field(None, alias="metricsData")
 
-    class Config:
-        use_enum_values = True
-        validate_assignment = True
-
 
 class TraceApi(BaseModel):
+    model_config = make_model_config(
+        use_enum_values=True, validate_assignment=True
+    )
+
     uuid: str
     base_spans: Optional[List[BaseApiSpan]] = Field(None, alias="baseSpans")
     agent_spans: Optional[List[BaseApiSpan]] = Field(None, alias="agentSpans")
@@ -139,7 +144,3 @@ class TraceApi(BaseModel):
 
     # Don't serialize these
     confident_api_key: Optional[str] = Field(None, exclude=True)
-
-    class Config:
-        use_enum_values = True
-        validate_assignment = True

deepeval/tracing/otel/exporter.py

@@ -493,6 +493,17 @@ class ConfidentSpanExporter(SpanExporter):
             output_token_count = span.attributes.get(
                 "confident.llm.output_token_count"
             )
+
+            # fallback to gen ai attributes if not found in confident attributes
+            if not input_token_count:
+                input_token_count = span.attributes.get(
+                    "gen_ai.usage.input_tokens"
+                )
+            if not output_token_count:
+                output_token_count = span.attributes.get(
+                    "gen_ai.usage.output_tokens"
+                )
+
             cost_per_input_token = span.attributes.get(
                 "confident.llm.cost_per_input_token"
             )

deepeval/tracing/patchers.py

@@ -1,6 +1,8 @@
-from openai import OpenAI
 import functools
 
+from anthropic import Anthropic
+from openai import OpenAI
+
 from deepeval.tracing.context import update_current_span, update_llm_span
 from deepeval.tracing.context import current_span_context
 from deepeval.tracing.types import LlmSpan
@@ -82,3 +84,102 @@ def patch_openai_client(client: OpenAI):
                 return response
 
             setattr(current_obj, method_name, wrapped_method)
+
+
+def patch_anthropic_client(client: Anthropic):
+    """
+    Patch an Anthropic client instance to add tracing capabilities.
+
+    Args:
+        client: An instance of Anthropic client to patch
+    """
+    original_methods = {}
+
+    methods_to_patch = [
+        "messages.create",
+    ]
+
+    for method_path in methods_to_patch:
+        parts = method_path.split(".")
+        current_obj = client
+
+        for part in parts[:-1]:
+            if not hasattr(current_obj, part):
+                print(f"Warning: Cannot find {part} in the path {method_path}")
+                continue
+            current_obj = getattr(current_obj, part)
+
+        method_name = parts[-1]
+        if not hasattr(current_obj, method_name):
+            print(
+                f"Warning: Cannot find method {method_name} in the path {method_path}"
+            )
+            continue
+
+        method = getattr(current_obj, method_name)
+
+        if callable(method) and not isinstance(method, type):
+            original_methods[method_path] = method
+
+            @functools.wraps(method)
+            def wrapped_method(*args, original_method=method, **kwargs):
+                current_span = current_span_context.get()
+                response = original_method(*args, **kwargs)
+
+                if isinstance(current_span, LlmSpan):
+                    model = kwargs.get("model", None)
+                    if model is None:
+                        raise ValueError("model not found in client")
+
+                    current_span.model = model
+
+                    output = None
+                    try:
+                        if (
+                            hasattr(response, "content")
+                            and response.content
+                            and len(response.content) > 0
+                        ):
+                            for block in response.content:
+                                if hasattr(block, "text"):
+                                    output = block.text
+                                    break
+                    except Exception:
+                        pass
+
+                    input_token_count = None
+                    output_token_count = None
+                    try:
+                        if hasattr(response, "usage"):
+                            usage = response.usage
+                            # usage can be a dict or an object with attributes
+                            if isinstance(usage, dict):
+                                input_token_count = usage.get(
+                                    "input_tokens", None
+                                )
+                                output_token_count = usage.get(
+                                    "output_tokens", None
+                                )
+                            else:
+                                input_token_count = getattr(
+                                    usage, "input_tokens", None
+                                )
+                                output_token_count = getattr(
+                                    usage, "output_tokens", None
+                                )
+                    except Exception:
+                        pass
+
+                    update_current_span(
+                        input=kwargs.get("messages", "INPUT_MESSAGE_NOT_FOUND"),
+                        output=output if output else "OUTPUT_MESSAGE_NOT_FOUND",
+                    )
+                    update_llm_span(
+                        input_token_count=input_token_count,
+                        output_token_count=output_token_count,
+                    )
+                return response
+
+            setattr(current_obj, method_name, wrapped_method)
+
+    return original_methods

deepeval/tracing/trace_context.py

@@ -1,13 +1,15 @@
-from typing import Optional, List, Dict, Any
 from contextvars import ContextVar
 from contextlib import contextmanager
 from dataclasses import dataclass
+from typing import Optional, List, Dict, Any
 
-from .tracing import trace_manager
-from .context import current_trace_context, update_current_trace
-from deepeval.prompt import Prompt
 from deepeval.metrics import BaseMetric
+from deepeval.prompt import Prompt
 from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import current_trace_context, update_current_trace
+from deepeval.tracing.tracing import trace_manager
+from deepeval.tracing.types import TraceWorkerStatus
+from deepeval.tracing.utils import is_async_context
 
 
 @dataclass
@@ -59,6 +61,13 @@ def trace(
     metrics: Optional[List[BaseMetric]] = None,
     metric_collection: Optional[str] = None,
 ):
+    if is_async_context():
+        trace_manager._print_trace_status(
+            message="Warning: Detected use of the synchronous 'trace' context manager within an async method",
+            trace_worker_status=TraceWorkerStatus.WARNING,
+            description="Wrapping an async method with the synchronous 'trace' context manager may lead to unexpected behavior.",
+        )
+
     current_trace = current_trace_context.get()
 
     if not current_trace:

deepeval/tracing/tracing.py

@@ -19,6 +19,7 @@ import random
 import atexit
 import queue
 import uuid
+from anthropic import Anthropic
 from openai import OpenAI
 from rich.console import Console
 from rich.progress import Progress
@@ -38,7 +39,10 @@ from deepeval.tracing.api import (
     TraceSpanApiStatus,
 )
 from deepeval.telemetry import capture_send_trace
-from deepeval.tracing.patchers import patch_openai_client
+from deepeval.tracing.patchers import (
+    patch_anthropic_client,
+    patch_openai_client,
+)
 from deepeval.tracing.types import (
     AgentSpan,
     BaseSpan,
@@ -111,6 +115,7 @@ class TraceManager:
 
         self.sampling_rate = settings.CONFIDENT_TRACE_SAMPLE_RATE
         validate_sampling_rate(self.sampling_rate)
+        self.anthropic_client = None
         self.openai_client = None
         self.tracing_enabled = True
 
@@ -139,7 +144,7 @@ class TraceManager:
 
     def mask(self, data: Any):
         if self.custom_mask_fn is not None:
-            self.custom_mask_fn(data)
+            return self.custom_mask_fn(data)
         else:
             return data
 
@@ -149,6 +154,7 @@ class TraceManager:
         environment: Optional[str] = None,
         sampling_rate: Optional[float] = None,
         confident_api_key: Optional[str] = None,
+        anthropic_client: Optional[Anthropic] = None,
         openai_client: Optional[OpenAI] = None,
         tracing_enabled: Optional[bool] = None,
     ) -> None:
@@ -165,6 +171,9 @@ class TraceManager:
         if openai_client is not None:
             self.openai_client = openai_client
             patch_openai_client(openai_client)
+        if anthropic_client is not None:
+            self.anthropic_client = anthropic_client
+            patch_anthropic_client(anthropic_client)
         if tracing_enabled is not None:
             self.tracing_enabled = tracing_enabled