deepeval 3.6.9__py3-none-any.whl → 3.7.1__py3-none-any.whl

deepeval/model_integrations/utils.py

@@ -0,0 +1,116 @@
+import json
+import uuid
+from typing import Any, List, Optional
+
+from deepeval.model_integrations.types import InputParameters, OutputParameters
+from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import (
+    current_span_context,
+    current_trace_context,
+    update_current_span,
+    update_llm_span,
+)
+from deepeval.tracing.trace_context import current_llm_context
+from deepeval.tracing.types import ToolSpan, TraceSpanStatus
+from deepeval.utils import shorten, len_long
+
+
+def _update_all_attributes(
+    input_parameters: InputParameters,
+    output_parameters: OutputParameters,
+    expected_tools: List[ToolCall],
+    expected_output: str,
+    context: List[str],
+    retrieval_context: List[str],
+):
+    """Update span and trace attributes with input/output parameters."""
+    update_current_span(
+        input=input_parameters.input or input_parameters.messages or "NA",
+        output=output_parameters.output or "NA",
+        tools_called=output_parameters.tools_called,
+        # attributes to be added
+        expected_output=expected_output,
+        expected_tools=expected_tools,
+        context=context,
+        retrieval_context=retrieval_context,
+    )
+
+    llm_context = current_llm_context.get()
+
+    update_llm_span(
+        input_token_count=output_parameters.prompt_tokens,
+        output_token_count=output_parameters.completion_tokens,
+        prompt=llm_context.prompt,
+    )
+
+    if output_parameters.tools_called:
+        create_child_tool_spans(output_parameters)
+
+    __update_input_and_output_of_current_trace(
+        input_parameters, output_parameters
+    )
+
+
+def __update_input_and_output_of_current_trace(
+    input_parameters: InputParameters, output_parameters: OutputParameters
+):
+
+    current_trace = current_trace_context.get()
+    if current_trace:
+        if current_trace.input is None:
+            current_trace.input = (
+                input_parameters.input or input_parameters.messages
+            )
+        if current_trace.output is None:
+            current_trace.output = output_parameters.output
+
+    return
+
+
+def create_child_tool_spans(output_parameters: OutputParameters):
+    if output_parameters.tools_called is None:
+        return
+
+    current_span = current_span_context.get()
+    for tool_called in output_parameters.tools_called:
+        tool_span = ToolSpan(
+            **{
+                "uuid": str(uuid.uuid4()),
+                "trace_uuid": current_span.trace_uuid,
+                "parent_uuid": current_span.uuid,
+                "start_time": current_span.start_time,
+                "end_time": current_span.start_time,
+                "status": TraceSpanStatus.SUCCESS,
+                "children": [],
+                "name": tool_called.name,
+                "input": tool_called.input_parameters,
+                "output": None,
+                "metrics": None,
+                "description": tool_called.description,
+            }
+        )
+        current_span.children.append(tool_span)
+
+
+_URL_MAX = 200
+_JSON_MAX = max(
+    len_long(), 400
+)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
+
+
+def compact_dump(value: Any) -> str:
+    try:
+        dumped = json.dumps(
+            value, ensure_ascii=False, default=str, separators=(",", ":")
+        )
+    except Exception:
+        dumped = repr(value)
+    return shorten(dumped, max_len=_JSON_MAX)
+
+
+def fmt_url(url: Optional[str]) -> str:
+    if not url:
+        return ""
+    if url.startswith("data:"):
+        return "[data-uri]"
+    return shorten(url, max_len=_URL_MAX)

deepeval/models/base_model.py

@@ -68,7 +68,9 @@ class DeepEvalBaseLLM(ABC):
         Returns:
             A list of strings.
         """
-        raise AttributeError
+        raise NotImplementedError(
+            "batch_generate is not implemented for this model"
+        )
 
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str:

deepeval/models/llms/gemini_model.py

@@ -1,7 +1,6 @@
 from pydantic import BaseModel
-from google.genai import types
+from google.genai import types, Client
 from typing import Optional, Dict
-from google import genai
 
 from deepeval.models.retry_policy import (
     create_retry_decorator,
@@ -9,7 +8,8 @@ from deepeval.models.retry_policy import (
 from deepeval.key_handler import ModelKeyValues, KEY_FILE_HANDLER
 from deepeval.models.base_model import DeepEvalBaseLLM
 from deepeval.constants import ProviderSlug as PS
-
+from google.oauth2 import service_account
+import json
 
 default_gemini_model = "gemini-1.5-pro"
 
@@ -52,6 +52,7 @@ class GeminiModel(DeepEvalBaseLLM):
         api_key: Optional[str] = None,
         project: Optional[str] = None,
         location: Optional[str] = None,
+        service_account_key: Optional[Dict[str, str]] = None,
         temperature: float = 0,
         generation_kwargs: Optional[Dict] = None,
         **kwargs,
@@ -75,6 +76,17 @@ class GeminiModel(DeepEvalBaseLLM):
         self.use_vertexai = KEY_FILE_HANDLER.fetch_data(
             ModelKeyValues.GOOGLE_GENAI_USE_VERTEXAI
         )
+        if service_account_key:
+            self.service_account_key = service_account_key
+        else:
+            service_account_key_data = KEY_FILE_HANDLER.fetch_data(
+                ModelKeyValues.GOOGLE_SERVICE_ACCOUNT_KEY
+            )
+            if service_account_key_data is None:
+                self.service_account_key = None
+            elif isinstance(service_account_key_data, str):
+                self.service_account_key = json.loads(service_account_key_data)
+
         if temperature < 0:
             raise ValueError("Temperature must be >= 0.")
         self.temperature = temperature
@@ -117,10 +129,20 @@ class GeminiModel(DeepEvalBaseLLM):
                 )
 
             # Create client for Vertex AI
-            self.client = genai.Client(
+            self.client = Client(
                 vertexai=True,
                 project=self.project,
                 location=self.location,
+                credentials=(
+                    service_account.Credentials.from_service_account_info(
+                        self.service_account_key,
+                        scopes=[
+                            "https://www.googleapis.com/auth/cloud-platform"
+                        ],
+                    )
+                    if self.service_account_key
+                    else None
+                ),
                 **self.kwargs,
             )
         else:
@@ -130,7 +152,7 @@ class GeminiModel(DeepEvalBaseLLM):
                     "or set it in your DeepEval configuration."
                 )
             # Create client for Gemini API
-            self.client = genai.Client(api_key=self.api_key, **self.kwargs)
+            self.client = Client(api_key=self.api_key, **self.kwargs)
 
         # Configure default model generation settings
         self.model_safety_settings = [

deepeval/openai/__init__.py

@@ -15,5 +15,7 @@ except ImportError:
 
 if OpenAI or AsyncOpenAI:
     from deepeval.openai.patch import patch_openai_classes
+    from deepeval.telemetry import capture_tracing_integration
 
-    patch_openai_classes()
+    with capture_tracing_integration("openai"):
+        patch_openai_classes()

deepeval/openai/extractors.py

@@ -4,13 +4,13 @@ from typing import Any, Union, Dict
 from openai.types.responses import Response
 
 from deepeval.test_case.llm_test_case import ToolCall
+
+from deepeval.model_integrations.types import InputParameters, OutputParameters
 from deepeval.openai.utils import (
     render_response_input,
     stringify_multimodal_content,
     render_messages,
 )
-from deepeval.openai.types import InputParameters, OutputParameters
-from deepeval.tracing.types import Message
 
 
 # guarding against errors to be compatible with legacy APIs

deepeval/openai/utils.py

@@ -1,6 +1,6 @@
 import json
 import uuid
-from typing import Any, Dict, List, Optional, Iterable
+from typing import Any, Dict, List, Iterable
 
 from openai.types.chat.chat_completion_message_param import (
     ChatCompletionMessageParam,
@@ -8,32 +8,8 @@ from openai.types.chat.chat_completion_message_param import (
 
 from deepeval.tracing.types import ToolSpan, TraceSpanStatus
 from deepeval.tracing.context import current_span_context
-from deepeval.utils import shorten, len_long
-from deepeval.openai.types import OutputParameters
-
-
-_URL_MAX = 200
-_JSON_MAX = max(
-    len_long(), 400
-)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
-
-
-def _compact_dump(value: Any) -> str:
-    try:
-        dumped = json.dumps(
-            value, ensure_ascii=False, default=str, separators=(",", ":")
-        )
-    except Exception:
-        dumped = repr(value)
-    return shorten(dumped, max_len=_JSON_MAX)
-
-
-def _fmt_url(url: Optional[str]) -> str:
-    if not url:
-        return ""
-    if url.startswith("data:"):
-        return "[data-uri]"
-    return shorten(url, max_len=_URL_MAX)
+from deepeval.model_integrations.types import OutputParameters
+from deepeval.model_integrations.utils import compact_dump, fmt_url
 
 
 def create_child_tool_spans(output_parameters: OutputParameters):
@@ -111,7 +87,7 @@ def stringify_multimodal_content(content: Any) -> str:
                 url = image_url
             else:
                 url = (image_url or {}).get("url") or content.get("url")
-            return f"[image:{_fmt_url(url)}]"
+            return f"[image:{fmt_url(url)}]"
 
         # Responses API variants
         if t == "input_text":
@@ -122,14 +98,14 @@ def stringify_multimodal_content(content: Any) -> str:
                 url = image_url
             else:
                 url = (image_url or {}).get("url") or content.get("url")
-            return f"[image:{_fmt_url(url)}]"
+            return f"[image:{fmt_url(url)}]"
 
         # readability for other input_* types we don't currently handle
         if t and t.startswith("input_"):
             return f"[{t}]"
 
     # unknown dicts and types returned as shortened JSON
-    return _compact_dump(content)
+    return compact_dump(content)
 
 
 def render_messages(
@@ -228,7 +204,7 @@ def _render_content(content: Dict[str, Any], indent: int = 0) -> str:
             lines.append(f"{prefix}{key}:")
             lines.append(_render_content(value, indent + 1))
         elif isinstance(value, list):
-            lines.append(f"{prefix}{key}: {_compact_dump(value)}")
+            lines.append(f"{prefix}{key}: {compact_dump(value)}")
         else:
             lines.append(f"{prefix}{key}: {value}")
 

deepeval/openai_agents/callback_handler.py

@@ -1,13 +1,21 @@
+from time import perf_counter
+
 from deepeval.tracing.tracing import (
     Observer,
     current_span_context,
     trace_manager,
 )
-from deepeval.openai_agents.extractors import *
+from deepeval.openai_agents.extractors import (
+    update_span_properties,
+    update_trace_properties_from_span_data,
+)
 from deepeval.tracing.context import current_trace_context
 from deepeval.tracing.utils import make_json_serializable
-from time import perf_counter
-from deepeval.tracing.types import TraceSpanStatus
+from deepeval.tracing.types import (
+    BaseSpan,
+    LlmSpan,
+    TraceSpanStatus,
+)
 
 try:
     from agents.tracing import Span, Trace, TracingProcessor
@@ -18,6 +26,7 @@ try:
         GenerationSpanData,
         GuardrailSpanData,
         HandoffSpanData,
+        MCPListToolsSpanData,
         ResponseSpanData,
         SpanData,
     )

deepeval/prompt/api.py

@@ -1,8 +1,10 @@
-from pydantic import BaseModel, Field, AliasChoices
+from pydantic import BaseModel, Field, AliasChoices, ConfigDict
 from enum import Enum
 from typing import List, Optional
 from pydantic import TypeAdapter
 
+from deepeval.utils import make_model_config
+
 ###################################
 # Model Settings
 ###################################
@@ -92,6 +94,8 @@ class SchemaDataType(Enum):
 
 
 class OutputSchemaField(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     id: str
     type: SchemaDataType
     name: str
@@ -102,9 +106,6 @@ class OutputSchemaField(BaseModel):
         validation_alias=AliasChoices("parent_id", "parentId"),
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class OutputSchema(BaseModel):
     fields: Optional[List[OutputSchemaField]] = None
@@ -187,6 +188,10 @@ class PromptHttpResponse(BaseModel):
 
 
 class PromptPushRequest(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
+    model_config = ConfigDict(use_enum_values=True)
+
     alias: str
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
@@ -203,11 +208,10 @@ class PromptPushRequest(BaseModel):
         default=None, serialization_alias="outputType"
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class PromptUpdateRequest(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
     interpolation_type: PromptInterpolationType = Field(
@@ -223,9 +227,6 @@ class PromptUpdateRequest(BaseModel):
         default=None, serialization_alias="outputType"
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class PromptApi(BaseModel):
     id: str

deepeval/prompt/prompt.py

@@ -1,3 +1,8 @@
+import logging
+import time
+import json
+import os
+
 from enum import Enum
 from typing import Optional, List, Dict, Type, Literal
 from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
@@ -7,9 +12,10 @@ import json
 import os
 from pydantic import BaseModel, ValidationError
 import asyncio
-import portalocker
 import threading
 
+from deepeval.utils import make_model_config, is_read_only_env
+
 from deepeval.prompt.api import (
     PromptHttpResponse,
     PromptMessage,
@@ -22,9 +28,6 @@ from deepeval.prompt.api import (
     ModelSettings,
     OutputSchema,
     OutputType,
-    ReasoningEffort,
-    Verbosity,
-    ModelProvider,
 )
 from deepeval.prompt.utils import (
     interpolate_text,
@@ -34,6 +37,18 @@ from deepeval.prompt.utils import (
 from deepeval.confident.api import Api, Endpoints, HttpMethods
 from deepeval.constants import HIDDEN_DIR
 
+
+logger = logging.getLogger(__name__)
+
+portalocker = None
+if not is_read_only_env():
+    try:
+        import portalocker
+    except Exception as e:
+        logger.warning("failed to import portalocker: %s", e)
+else:
+    logger.warning("READ_ONLY filesystem: skipping disk cache for prompts.")
+
 CACHE_FILE_NAME = f"{HIDDEN_DIR}/.deepeval-prompt-cache.json"
 VERSION_CACHE_KEY = "version"
 LABEL_CACHE_KEY = "label"
@@ -77,6 +92,8 @@ class CustomEncoder(json.JSONEncoder):
 
 
 class CachedPrompt(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     alias: str
     version: str
     label: Optional[str] = None
@@ -89,9 +106,6 @@ class CachedPrompt(BaseModel):
     output_type: Optional[OutputType]
     output_schema: Optional[OutputSchema]
 
-    class Config:
-        use_enum_values = True
-
 
 class Prompt:
 
@@ -164,7 +178,7 @@ class Prompt:
             content = f.read()
         try:
             data = json.loads(content)
-        except:
+        except (json.JSONDecodeError, TypeError):
             self.text_template = content
             return content
 
@@ -202,7 +216,6 @@ class Prompt:
                     "Unable to interpolate empty prompt template. Please pull a prompt from Confident AI or set template manually to continue."
                 )
 
-            print("@@@@@")
             return interpolate_text(interpolation_type, text_template, **kwargs)
 
         elif prompt_type == PromptType.LIST:
@@ -247,7 +260,7 @@ class Prompt:
         version: Optional[str] = None,
         label: Optional[str] = None,
     ) -> Optional[CachedPrompt]:
-        if not os.path.exists(CACHE_FILE_NAME):
+        if portalocker is None or not os.path.exists(CACHE_FILE_NAME):
             return None
 
         try:
@@ -295,13 +308,12 @@ class Prompt:
         output_type: Optional[OutputType] = None,
         output_schema: Optional[OutputSchema] = None,
     ):
-        if not self.alias:
+        if portalocker is None or not self.alias:
             return
 
-        # Ensure directory exists
-        os.makedirs(HIDDEN_DIR, exist_ok=True)
-
         try:
+            # Ensure directory exists
+            os.makedirs(HIDDEN_DIR, exist_ok=True)
             # Use r+ mode if file exists, w mode if it doesn't
             mode = "r+" if os.path.exists(CACHE_FILE_NAME) else "w"
 
@@ -480,7 +492,7 @@ class Prompt:
                             cached_prompt.output_schema
                         )
                     return
-            except:
+            except Exception:
                 pass
 
         api = Api()

deepeval/simulator/template.py

@@ -112,7 +112,7 @@ class ConversationSimulatorTemplate:
             ]
             Example JSON Output:
             {{
-                "is_complete": False,
+                "is_complete": false,
                 "reason": "The assistant explained how to forget password but ahas not confirmed that the user successfully set a new password."
             }}
 

deepeval/telemetry.py

@@ -3,12 +3,12 @@ import logging
 import os
 import socket
 import sys
-from threading import Event
 import uuid
 import sentry_sdk
 from enum import Enum
 from typing import List, Dict
 import requests
+from deepeval.config.settings import get_settings
 from deepeval.constants import LOGIN_PROMPT, HIDDEN_DIR, KEY_FILE
 from posthog import Posthog
 
@@ -34,7 +34,7 @@ TELEMETRY_PATH = os.path.join(HIDDEN_DIR, TELEMETRY_DATA_FILE)
 
 
 def telemetry_opt_out():
-    return os.getenv("DEEPEVAL_TELEMETRY_OPT_OUT") == "1"
+    return get_settings().DEEPEVAL_TELEMETRY_OPT_OUT
 
 
 def blocked_by_firewall():
@@ -131,7 +131,7 @@ if not telemetry_opt_out():
 
 
 if (
-    os.getenv("ERROR_REPORTING") == "1"
+    get_settings().ERROR_REPORTING
     and not blocked_by_firewall()
     and not telemetry_opt_out()
 ):

deepeval/test_case/__init__.py

@@ -10,7 +10,7 @@ from .conversational_test_case import (
     TurnParams,
 )
 from .mllm_test_case import MLLMTestCase, MLLMTestCaseParams, MLLMImage
-from .arena_test_case import ArenaTestCase
+from .arena_test_case import ArenaTestCase, Contestant
 from .mcp import (
     MCPServer,
     MCPPromptCall,
@@ -35,4 +35,5 @@ __all__ = [
     "MLLMTestCaseParams",
     "MLLMImage",
     "ArenaTestCase",
+    "Contestant",
 ]

deepeval/test_case/arena_test_case.py

@@ -1,20 +1,31 @@
+from typing import List, Dict, Optional, Union
 from dataclasses import dataclass
-from typing import List, Dict
+from pydantic import BaseModel
+
 from deepeval.test_case import (
     LLMTestCase,
 )
+from deepeval.prompt import Prompt
+
+
+class Contestant(BaseModel):
+    name: str
+    test_case: LLMTestCase
+    hyperparameters: Optional[Dict[str, Union[str, int, float, Prompt]]] = None
+
+    model_config = {"arbitrary_types_allowed": True}
 
 
 @dataclass
 class ArenaTestCase:
-    contestants: Dict[str, LLMTestCase]
+    contestants: List[Contestant]
 
     def __post_init__(self):
-        contestant_names = list(self.contestants.keys())
+        contestant_names = [contestant.name for contestant in self.contestants]
         if len(contestant_names) != len(set(contestant_names)):
             raise ValueError("All contestant names must be unique.")
 
-        cases = list(self.contestants.values())
+        cases = [contestant.test_case for contestant in self.contestants]
         ref_input = cases[0].input
         for case in cases[1:]:
             if case.input != ref_input:

deepeval/test_case/llm_test_case.py

@@ -1,5 +1,4 @@
 from pydantic import (
-    ConfigDict,
     Field,
     BaseModel,
     model_validator,
@@ -11,6 +10,8 @@ from enum import Enum
 import json
 import uuid
 
+from deepeval.utils import make_model_config
+
 from deepeval.test_case.mcp import (
     MCPServer,
     MCPPromptCall,
@@ -156,7 +157,7 @@ class ToolCall(BaseModel):
 
 
 class LLMTestCase(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     input: str
     actual_output: Optional[str] = Field(