deepeval 3.6.9__tar.gz → 3.7.1__tar.gz

{deepeval-3.6.9 → deepeval-3.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.9
+Version: 3.7.1
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -25,14 +25,14 @@ Requires-Dist: opentelemetry-api (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.24.0,<2.0.0)
 Requires-Dist: opentelemetry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: portalocker
-Requires-Dist: posthog (>=6.3.0,<7.0.0)
+Requires-Dist: posthog (>=5.4.0,<6.0.0)
 Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.10.1,<3.0.0)
 Requires-Dist: pyfiglet
 Requires-Dist: pytest
 Requires-Dist: pytest-asyncio
 Requires-Dist: pytest-repeat
-Requires-Dist: pytest-rerunfailures (>=12.0,<13.0)
+Requires-Dist: pytest-rerunfailures
 Requires-Dist: pytest-xdist
 Requires-Dist: python-dotenv (>=1.1.1,<2.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)

{deepeval-3.6.9 → deepeval-3.7.1}/deepeval/__init__.py

@@ -102,9 +102,5 @@ def update_warning_opt_in():
     return os.getenv("DEEPEVAL_UPDATE_WARNING_OPT_IN") == "1"
 
 
-def is_read_only_env():
-    return os.getenv("DEEPEVAL_FILE_SYSTEM") == "READ_ONLY"
-
-
 if update_warning_opt_in():
     check_for_update()

deepeval-3.7.1/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.7.1"

deepeval-3.7.1/deepeval/anthropic/__init__.py

@@ -0,0 +1,19 @@
+try:
+    import anthropic  # noqa: F401
+except ImportError:
+    raise ModuleNotFoundError(
+        "Please install anthropic to use this feature: 'pip install anthropic'"
+    )
+
+try:
+    from anthropic import Anthropic, AsyncAnthropic  # noqa: F401
+except ImportError:
+    Anthropic = None  # type: ignore
+    AsyncAnthropic = None  # type: ignore
+
+if Anthropic or AsyncAnthropic:
+    from deepeval.anthropic.patch import patch_anthropic_classes
+    from deepeval.telemetry import capture_tracing_integration
+
+    with capture_tracing_integration("anthropic"):
+        patch_anthropic_classes()

deepeval-3.7.1/deepeval/anthropic/extractors.py

@@ -0,0 +1,94 @@
+from anthropic.types.message import Message
+from anthropic.types import ToolUseBlock
+from typing import Any, Dict
+
+from deepeval.anthropic.utils import (
+    render_messages_anthropic,
+    stringify_anthropic_content,
+)
+from deepeval.model_integrations.types import InputParameters, OutputParameters
+from deepeval.test_case.llm_test_case import ToolCall
+
+
+def safe_extract_input_parameters(kwargs: Dict[str, Any]) -> InputParameters:
+    # guarding against errors to be compatible with legacy APIs
+    try:
+        return extract_messages_api_input_parameters(kwargs)
+    except:
+        return InputParameters(model="NA")
+
+
+def extract_messages_api_input_parameters(
+    kwargs: Dict[str, Any],
+) -> InputParameters:
+    model = kwargs.get("model")
+    tools = kwargs.get("tools")
+    messages = kwargs.get("messages")
+    tool_descriptions = (
+        {tool["name"]: tool["description"] for tool in tools}
+        if tools is not None
+        else None
+    )
+
+    input_argument = ""
+    user_messages = []
+    for message in messages:
+        role = message["role"]
+        if role == "user":
+            user_messages.append(message["content"])
+    if len(user_messages) > 0:
+        input_argument = user_messages[0]
+
+    return InputParameters(
+        model=model,
+        input=stringify_anthropic_content(input_argument),
+        messages=render_messages_anthropic(messages),
+        tools=tools,
+        tool_descriptions=tool_descriptions,
+    )
+
+
+def safe_extract_output_parameters(
+    message_response: Message,
+    input_parameters: InputParameters,
+) -> OutputParameters:
+    # guarding against errors to be compatible with legacy APIs
+    try:
+        return extract_messages_api_output_parameters(
+            message_response, input_parameters
+        )
+    except:
+        return OutputParameters()
+
+
+def extract_messages_api_output_parameters(
+    message_response: Message,
+    input_parameters: InputParameters,
+) -> OutputParameters:
+    output = str(message_response.content[0].text)
+    prompt_tokens = message_response.usage.input_tokens
+    completion_tokens = message_response.usage.output_tokens
+
+    tools_called = None
+    anthropic_tool_calls = [
+        block
+        for block in message_response.content
+        if isinstance(block, ToolUseBlock)
+    ]
+    if anthropic_tool_calls:
+        tools_called = []
+        tool_descriptions = input_parameters.tool_descriptions or {}
+        for tool_call in anthropic_tool_calls:
+            tools_called.append(
+                ToolCall(
+                    name=tool_call.name,
+                    input_parameters=tool_call.input,
+                    description=tool_descriptions.get(tool_call.name),
+                )
+            )
+    return OutputParameters(
+        output=output,
+        prompt_tokens=prompt_tokens,
+        completion_tokens=completion_tokens,
+        tools_called=tools_called,
+    )

deepeval-3.7.1/deepeval/anthropic/patch.py

@@ -0,0 +1,169 @@
+from typing import Callable
+from functools import wraps
+
+from deepeval.anthropic.extractors import (
+    safe_extract_input_parameters,
+    safe_extract_output_parameters,
+    InputParameters,
+)
+from deepeval.model_integrations.utils import _update_all_attributes
+from deepeval.tracing import observe
+from deepeval.tracing.trace_context import current_llm_context
+
+_ORIGINAL_METHODS = {}
+_ANTHROPIC_PATCHED = False
+
+
+def patch_anthropic_classes():
+    """
+    Monkey patch Anthropic resource classes directly.
+    """
+    global _ANTHROPIC_PATCHED
+
+    # Single guard - if already patched, return immediately
+    if _ANTHROPIC_PATCHED:
+        return
+
+    try:
+        from anthropic.resources.messages import Messages, AsyncMessages
+
+        # Store original methods before patching
+        if hasattr(Messages, "create"):
+            _ORIGINAL_METHODS["Messages.create"] = Messages.create
+            Messages.create = _create_sync_wrapper(Messages.create)
+
+        if hasattr(AsyncMessages, "create"):
+            _ORIGINAL_METHODS["AsyncMessages.create"] = AsyncMessages.create
+            AsyncMessages.create = _create_async_wrapper(AsyncMessages.create)
+
+    except ImportError:
+        pass
+
+    _ANTHROPIC_PATCHED = True
+
+
+def _create_sync_wrapper(original_method):
+    """
+    Create a wrapper for sync methods - called ONCE during patching.
+    """
+
+    @wraps(original_method)
+    def method_wrapper(self, *args, **kwargs):
+        bound_method = original_method.__get__(self, type(self))
+        patched = _patch_sync_anthropic_client_method(
+            original_method=bound_method
+        )
+        return patched(*args, **kwargs)
+
+    return method_wrapper
+
+
+def _create_async_wrapper(original_method):
+    """
+    Create a wrapper for sync methods - called ONCE during patching.
+    """
+
+    @wraps(original_method)
+    def method_wrapper(self, *args, **kwargs):
+        bound_method = original_method.__get__(self, type(self))
+        patched = _patch_async_anthropic_client_method(
+            original_method=bound_method
+        )
+        return patched(*args, **kwargs)
+
+    return method_wrapper
+
+
+def _patch_sync_anthropic_client_method(original_method: Callable):
+    @wraps(original_method)
+    def patched_sync_anthropic_method(*args, **kwargs):
+        input_parameters: InputParameters = safe_extract_input_parameters(
+            kwargs
+        )
+        llm_context = current_llm_context.get()
+
+        @observe(
+            type="llm",
+            model=input_parameters.model,
+            metrics=llm_context.metrics,
+            metric_collection=llm_context.metric_collection,
+        )
+        def llm_generation(*args, **kwargs):
+            messages_api_response = original_method(*args, **kwargs)
+            output_parameters = safe_extract_output_parameters(
+                messages_api_response, input_parameters
+            )
+            _update_all_attributes(
+                input_parameters,
+                output_parameters,
+                llm_context.expected_tools,
+                llm_context.expected_output,
+                llm_context.context,
+                llm_context.retrieval_context,
+            )
+            return messages_api_response
+
+        return llm_generation(*args, **kwargs)
+
+    return patched_sync_anthropic_method
+
+
+def _patch_async_anthropic_client_method(original_method: Callable):
+    @wraps(original_method)
+    async def patched_async_anthropic_method(*args, **kwargs):
+        input_parameters: InputParameters = safe_extract_input_parameters(
+            kwargs
+        )
+        llm_context = current_llm_context.get()
+
+        @observe(
+            type="llm",
+            model=input_parameters.model,
+            metrics=llm_context.metrics,
+            metric_collection=llm_context.metric_collection,
+        )
+        async def llm_generation(*args, **kwargs):
+            messages_api_response = await original_method(*args, **kwargs)
+            output_parameters = safe_extract_output_parameters(
+                messages_api_response, input_parameters
+            )
+            _update_all_attributes(
+                input_parameters,
+                output_parameters,
+                llm_context.expected_tools,
+                llm_context.expected_output,
+                llm_context.context,
+                llm_context.retrieval_context,
+            )
+            return messages_api_response
+
+        return await llm_generation(*args, **kwargs)
+
+    return patched_async_anthropic_method
+
+
+def unpatch_anthropic_classes():
+    """
+    Restore Anthropic resource classes to their original state.
+    """
+    global _ANTHROPIC_PATCHED
+
+    # If not patched, nothing to do
+    if not _ANTHROPIC_PATCHED:
+        return
+
+    try:
+        from anthropic.resources.messages import Messages, AsyncMessages
+
+        # Restore original methods for Messages
+        if hasattr(Messages, "create"):
+            Messages.create = _ORIGINAL_METHODS["Messages.create"]
+
+        if hasattr(AsyncMessages, "create"):
+            AsyncMessages.create = _ORIGINAL_METHODS["AsyncMessages.create"]
+
+    except ImportError:
+        pass
+
+    # Reset the patched flag
+    _ANTHROPIC_PATCHED = False

deepeval-3.7.1/deepeval/anthropic/utils.py

@@ -0,0 +1,225 @@
+from typing import Any, Iterable, List
+
+from anthropic.types import Message
+
+from deepeval.model_integrations.utils import compact_dump, fmt_url
+from deepeval.utils import shorten
+
+
+def stringify_anthropic_content(content: Any) -> str:
+    """
+    Return a short, human-readable summary string for an Anthropic-style multimodal `content` value.
+
+    This is used to populate span summaries, such as `InputParameters.input`. It never raises and
+    never returns huge blobs.
+
+    Notes:
+    - Data URIs and base64 content are redacted to "[data-uri]" or "[base64:...]".
+    - Output is capped via `deepeval.utils.shorten` (configurable through settings).
+    - Fields that are not explicitly handled are returned as size-capped JSON dumps
+    - This string is for display/summary only, not intended to be parsable.
+
+    Args:
+        content: The value of an Anthropic message `content`, may be a str or list of content blocks,
+                 or any nested structure.
+
+    Returns:
+        A short, readable `str` summary.
+    """
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, (bytes, bytearray)):
+        return f"[bytes:{len(content)}]"
+
+    # list of content blocks for Anthropic Messages API
+    if isinstance(content, list):
+        parts: List[str] = []
+        for block in content:
+            s = stringify_anthropic_content(block)
+            if s:
+                parts.append(s)
+        return "\n".join(parts)
+
+    # dict shapes for Anthropic Messages API
+    if isinstance(content, dict):
+        t = content.get("type")
+
+        # Text block
+        if t == "text":
+            return str(content.get("text", ""))
+
+        # Image block
+        if t == "image":
+            source = content.get("source", {})
+            source_type = source.get("type")
+            if source_type == "base64":
+                media_type = source.get("media_type", "unknown")
+                data = source.get("data", "")
+                data_preview = data[:20] if data else ""
+                return f"[image:{media_type}:base64:{data_preview}...]"
+            elif source_type == "url":
+                url = source.get("url", "")
+                return f"[image:{fmt_url(url)}]"
+            else:
+                return f"[image:{source_type or 'unknown'}]"
+
+        # Tool use block (in assistant messages)
+        if t == "tool_use":
+            tool_name = content.get("name", "unknown")
+            tool_id = content.get("id", "")
+            tool_input = content.get("input", {})
+            input_str = compact_dump(tool_input) if tool_input else ""
+            return f"[tool_use:{tool_name}:{tool_id}:{input_str}]"
+
+        # Tool result block (in user messages)
+        if t == "tool_result":
+            tool_id = content.get("tool_use_id", "")
+            tool_content = content.get("content")
+            content_str = (
+                stringify_anthropic_content(tool_content)
+                if tool_content
+                else ""
+            )
+            is_error = content.get("is_error", False)
+            error_flag = ":error" if is_error else ""
+            return f"[tool_result:{tool_id}{error_flag}:{content_str}]"
+
+        # Document block (for PDFs and other documents)
+        if t == "document":
+            source = content.get("source", {})
+            source_type = source.get("type")
+            if source_type == "base64":
+                media_type = source.get("media_type", "unknown")
+                return f"[document:{media_type}:base64]"
+            elif source_type == "url":
+                url = source.get("url", "")
+                return f"[document:{fmt_url(url)}]"
+            else:
+                return f"[document:{source_type or 'unknown'}]"
+
+        # Thinking block (for extended thinking models)
+        if t == "thinking":
+            thinking_text = content.get("thinking", "")
+            return {
+                "role": "thinking",
+                "content": shorten(thinking_text, max_len=100),
+            }
+
+        # readability for other block types we don't currently handle
+        if t:
+            return f"[{t}]"
+
+    # unknown dicts and types returned as shortened JSON
+    return compact_dump(content)
+
+
+def render_messages_anthropic(
+    messages: Iterable[Message],
+):
+    """
+    Extracts and normalizes tool calls and tool results from Anthropic API messages
+    for observability/logging purposes.
+
+    Args:
+        messages: Iterable of message dictionaries in Anthropic API format
+
+    Returns:
+        List of normalized message objects suitable for logging/observability
+    """
+    messages_list = []
+
+    for message in messages:
+        role = message.get("role")
+        content = message.get("content")
+
+        if role == "assistant":
+            if isinstance(content, str):
+                messages_list.append(
+                    {
+                        "role": role,
+                        "content": content,
+                    }
+                )
+            elif isinstance(content, list):
+                for block in content:
+                    block_type = block.get("type")
+                    if block_type == "text":
+                        messages_list.append(
+                            {
+                                "role": role,
+                                "content": block.get("text", ""),
+                            }
+                        )
+                    elif block_type == "tool_use":
+                        messages_list.append(
+                            {
+                                "id": block.get("id", ""),
+                                "call_id": block.get("id", ""),
+                                "name": block.get("name", ""),
+                                "type": "function",
+                                "arguments": block.get("input", {}),
+                            }
+                        )
+
+        elif role == "user":
+            if isinstance(content, str):
+                messages_list.append(
+                    {
+                        "role": role,
+                        "content": content,
+                    }
+                )
+            elif isinstance(content, list):
+                for block in content:
+                    block_type = block.get("type")
+                    if block_type == "text":
+                        messages_list.append(
+                            {
+                                "role": role,
+                                "content": block.get("text", ""),
+                            }
+                        )
+                    elif block_type == "image":
+                        messages_list.append(
+                            {
+                                "role": role,
+                                "content": "[Image content]",
+                                "image_source": block.get("source", {}),
+                            }
+                        )
+                    elif block_type == "tool_result":
+                        tool_content = block.get("content", "")
+                        if isinstance(tool_content, list):
+                            output_parts = []
+                            for tool_content_block in tool_content:
+                                if isinstance(tool_content_block, dict):
+                                    if tool_content_block.get("type") == "text":
+                                        output_parts.append(
+                                            tool_content_block.get("text", "")
+                                        )
+                                else:
+                                    output_parts.append(str(tool_content_block))
+                            output = "\n".join(output_parts)
+                        else:
+                            output = tool_content
+
+                        messages_list.append(
+                            {
+                                "call_id": block.get("tool_use_id", ""),
+                                "type": "tool",
+                                "output": output,
+                                "is_error": block.get("is_error", False),
+                            }
+                        )
+
+        elif role == "system":
+            messages_list.append(
+                {
+                    "role": role,
+                    "content": content,
+                }
+            )
+
+    return messages_list

{deepeval-3.6.9 → deepeval-3.7.1}/deepeval/benchmarks/drop/drop.py

@@ -1,6 +1,10 @@
+import logging
+
 from typing import List, Optional, Dict, Union
 from tqdm import tqdm
 
+from deepeval.config.settings import get_settings
+from deepeval.errors import DeepEvalError
 from deepeval.dataset import Golden
 from deepeval.benchmarks.base_benchmark import (
     DeepEvalBaseBenchmark,
@@ -17,6 +21,8 @@ from deepeval.benchmarks.schema import (
 )
 from deepeval.telemetry import capture_benchmark_run
 
+
+logger = logging.getLogger(__name__)
 DELIMITER = ","
 
 
@@ -164,7 +170,7 @@ class DROP(DeepEvalBaseBenchmark):
     def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
         # Define prompt template
         assert (
-            self.shots_dataset != None
+            self.shots_dataset is not None
         ), "Example dataset is empty. Call load_benchmark."
         prompt: dict = DROPTemplate.generate_output(
             train_set=self.shots_dataset,
@@ -206,7 +212,7 @@ class DROP(DeepEvalBaseBenchmark):
     ) -> List[Dict]:
         # Define prompt template
         assert (
-            self.shots_dataset != None
+            self.shots_dataset is not None
         ), "Example dataset is empty. Call load_benchmark."
 
         prompts = []
@@ -215,7 +221,6 @@ class DROP(DeepEvalBaseBenchmark):
             prompt: dict = DROPTemplate.generate_output(
                 train_set=self.shots_dataset,
                 input=golden.input,
-                type=golden.context[0],
                 n_shots=self.n_shots,
             )
             prompts.append(prompt)
@@ -228,23 +233,44 @@ class DROP(DeepEvalBaseBenchmark):
                 schema = DROPStringSchema
             schemas.append(schema)
 
+        effective_batch_size = len(goldens)
+        model_name = getattr(
+            model, "get_model_name", lambda: type(model).__name__
+        )()
+
         try:
             responses: List[
                 Union[DROPNumberSchema, DROPDateSchema, DROPStringSchema]
             ] = model.batch_generate(prompts=prompts, schemas=schemas)
             predictions = [str(res.answer) for res in responses]
-        except TypeError:
-            prompts = [
-                prompt
-                + "Output should be of type {type}. No explanation needed.".format(
-                    type=type
-                )
-                for prompt in prompts
-            ]
-            predictions = model.batch_generate(prompts)
+        except (AttributeError, NotImplementedError) as e:
+            logger.error(
+                "DROP: model %s does not implement batch_generate. Batch evaluation "
+                "(effective batch_size=%s) requires a batch-capable model. "
+                "Use a model that implements batch_generate(prompts, schemas) or run with batch_size=0/None.",
+                model_name,
+                effective_batch_size,
+                exc_info=get_settings().DEEPEVAL_LOG_STACK_TRACES,
+            )
+            raise DeepEvalError(
+                "Model does not implement batch_generate. Use a batch-capable model or set batch_size=0/None."
+            ) from e
+
+        except TypeError as e:
+            logger.error(
+                "DROP: model %s does not support schema-aware batch generation "
+                "(batch_generate(prompts, schemas)). DROP requires structured outputs "
+                "for number/date/span. Use a model that supports schemas or run with batch_size=0/None.",
+                model_name,
+                exc_info=get_settings().DEEPEVAL_LOG_STACK_TRACES,
+            )
+            raise DeepEvalError(
+                "Model does not support schema-aware batch generation required by DROP. "
+                "Use batch_generate(prompts, schemas) or set batch_size=0/None."
+            ) from e
 
-        if len(predictions) is not len(goldens):
-            raise ValueError(
+        if len(predictions) != effective_batch_size:
+            raise DeepEvalError(
                 "Custom `batch_generate` method did not return the same number of generations as the number of prompts."
             )
 

{deepeval-3.6.9 → deepeval-3.7.1}/deepeval/benchmarks/ifeval/ifeval.py

@@ -1,8 +1,8 @@
-from pydantic.config import ConfigDict
 from deepeval.benchmarks.base_benchmark import (
     DeepEvalBaseBenchmark,
     DeepEvalBaseBenchmarkResult,
 )
+from deepeval.utils import make_model_config
 from typing import List, Optional, Dict, Any, Tuple
 from tqdm import tqdm
 import re
@@ -19,7 +19,7 @@ from deepeval.telemetry import capture_benchmark_run
 
 
 class IFEvalResult(DeepEvalBaseBenchmarkResult):
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config = make_model_config(arbitrary_types_allowed=True)
     instruction_breakdown: dict[str, Any]
     predictions: "pd.DataFrame"
 

{deepeval-3.6.9 → deepeval-3.7.1}/deepeval/cli/main.py

@@ -1484,6 +1484,11 @@ def set_gemini_model_env(
     google_cloud_location: Optional[str] = typer.Option(
         None, "--location", help="Google Cloud location"
     ),
+    google_service_account_key: Optional[str] = typer.Option(
+        None,
+        "--service-account-key",
+        help="Google Service Account Key for Gemini",
+    ),
     save: Optional[str] = typer.Option(
         None,
         "--save",
@@ -1513,6 +1518,8 @@ def set_gemini_model_env(
             settings.GOOGLE_CLOUD_PROJECT = google_cloud_project
         if google_cloud_location:
             settings.GOOGLE_CLOUD_LOCATION = google_cloud_location
+        if google_service_account_key:
+            settings.GOOGLE_SERVICE_ACCOUNT_KEY = google_service_account_key
         if model_name:
             settings.GEMINI_MODEL_NAME = model_name