deepeval 3.6.9__py3-none-any.whl → 3.7.0__py3-none-any.whl

deepeval/metrics/pattern_match/pattern_match.py

@@ -0,0 +1,103 @@
+import re
+from typing import List
+
+from deepeval.metrics.indicator import metric_progress_indicator
+from deepeval.metrics.utils import (
+    check_llm_test_case_params,
+    construct_verbose_logs,
+)
+from deepeval.metrics.api import metric_data_manager
+from deepeval.metrics import BaseMetric
+from deepeval.test_case import LLMTestCase, LLMTestCaseParams
+
+
+class PatternMatchMetric(BaseMetric):
+    _required_params: List[LLMTestCaseParams] = [
+        LLMTestCaseParams.INPUT,
+        LLMTestCaseParams.ACTUAL_OUTPUT,
+    ]
+
+    def __init__(
+        self,
+        pattern: str,
+        ignore_case: bool = False,
+        threshold: float = 1.0,
+        verbose_mode: bool = False,
+    ):
+        self.pattern = pattern.strip()
+        self.ignore_case = ignore_case
+        self.verbose_mode = verbose_mode
+        self.threshold = threshold
+
+        flags = re.IGNORECASE if ignore_case else 0
+        try:
+            self._compiled_pattern = re.compile(self.pattern, flags)
+        except re.error as e:
+            raise ValueError(f"Invalid regex pattern: {pattern} — {e}")
+
+    def measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
+    ) -> float:
+        check_llm_test_case_params(test_case, self._required_params, self)
+
+        with metric_progress_indicator(
+            self, _show_indicator=_show_indicator, _in_component=_in_component
+        ):
+            actual = test_case.actual_output.strip()
+            full_match = self._compiled_pattern.fullmatch(actual)
+
+            self.score = 1.0 if full_match else 0.0
+            self.reason = (
+                f"The actual output fully matches the pattern."
+                if full_match
+                else f"The actual output does not match the pattern."
+            )
+            self.success = self.score >= self.threshold
+
+            if self.verbose_mode:
+                self.verbose_logs = construct_verbose_logs(
+                    self,
+                    steps=[
+                        f"Pattern: {self.pattern}",
+                        f"Actual: {actual}",
+                        f"Score: {self.score:.2f}",
+                        f"Reason: {self.reason}",
+                    ],
+                )
+
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
+
+            return self.score
+
+    async def a_measure(
+        self,
+        test_case: LLMTestCase,
+        _show_indicator: bool = True,
+        _in_component: bool = False,
+    ) -> float:
+        return self.measure(
+            test_case,
+            _show_indicator=_show_indicator,
+            _in_component=_in_component,
+        )
+
+    def is_successful(self) -> bool:
+        if self.error is not None:
+            self.success = False
+        else:
+            try:
+                self.success = self.score >= self.threshold
+            except:
+                self.success = False
+        return self.success
+
+    @property
+    def __name__(self):
+        return "Pattern Match"

deepeval/metrics/task_completion/task_completion.py

@@ -36,6 +36,11 @@ class TaskCompletionMetric(BaseMetric):
         strict_mode: bool = False,
         verbose_mode: bool = False,
     ):
+        if task is None:
+            self._is_task_provided = False
+        else:
+            self._is_task_provided = True
+
         self.task = task
         self.threshold = 1 if strict_mode else threshold
         self.model, self.using_native_model = initialize_model(model)
@@ -73,7 +78,8 @@ class TaskCompletionMetric(BaseMetric):
                 )
             else:
                 task, self.outcome = self._extract_task_and_outcome(test_case)
-                self.task = task if self.task is None else self.task
+                if self.task is None or not self._is_task_provided:
+                    self.task = task
                 self.verdict, self.reason = self._generate_verdicts()
                 self.score = self._calculate_score()
                 self.success = self.score >= self.threshold
@@ -108,7 +114,8 @@ class TaskCompletionMetric(BaseMetric):
             task, self.outcome = await self._a_extract_task_and_outcome(
                 test_case
             )
-            self.task = task if self.task is None else self.task
+            if self.task is None or not self._is_task_provided:
+                self.task = task
             self.verdict, self.reason = await self._a_generate_verdicts()
             self.score = self._calculate_score()
             self.success = self.score >= self.threshold

deepeval/model_integrations/utils.py

@@ -0,0 +1,116 @@
+import json
+import uuid
+from typing import Any, List, Optional
+
+from deepeval.model_integrations.types import InputParameters, OutputParameters
+from deepeval.test_case.llm_test_case import ToolCall
+from deepeval.tracing.context import (
+    current_span_context,
+    current_trace_context,
+    update_current_span,
+    update_llm_span,
+)
+from deepeval.tracing.trace_context import current_llm_context
+from deepeval.tracing.types import ToolSpan, TraceSpanStatus
+from deepeval.utils import shorten, len_long
+
+
+def _update_all_attributes(
+    input_parameters: InputParameters,
+    output_parameters: OutputParameters,
+    expected_tools: List[ToolCall],
+    expected_output: str,
+    context: List[str],
+    retrieval_context: List[str],
+):
+    """Update span and trace attributes with input/output parameters."""
+    update_current_span(
+        input=input_parameters.input or input_parameters.messages or "NA",
+        output=output_parameters.output or "NA",
+        tools_called=output_parameters.tools_called,
+        # attributes to be added
+        expected_output=expected_output,
+        expected_tools=expected_tools,
+        context=context,
+        retrieval_context=retrieval_context,
+    )
+
+    llm_context = current_llm_context.get()
+
+    update_llm_span(
+        input_token_count=output_parameters.prompt_tokens,
+        output_token_count=output_parameters.completion_tokens,
+        prompt=llm_context.prompt,
+    )
+
+    if output_parameters.tools_called:
+        create_child_tool_spans(output_parameters)
+
+    __update_input_and_output_of_current_trace(
+        input_parameters, output_parameters
+    )
+
+
+def __update_input_and_output_of_current_trace(
+    input_parameters: InputParameters, output_parameters: OutputParameters
+):
+
+    current_trace = current_trace_context.get()
+    if current_trace:
+        if current_trace.input is None:
+            current_trace.input = (
+                input_parameters.input or input_parameters.messages
+            )
+        if current_trace.output is None:
+            current_trace.output = output_parameters.output
+
+    return
+
+
+def create_child_tool_spans(output_parameters: OutputParameters):
+    if output_parameters.tools_called is None:
+        return
+
+    current_span = current_span_context.get()
+    for tool_called in output_parameters.tools_called:
+        tool_span = ToolSpan(
+            **{
+                "uuid": str(uuid.uuid4()),
+                "trace_uuid": current_span.trace_uuid,
+                "parent_uuid": current_span.uuid,
+                "start_time": current_span.start_time,
+                "end_time": current_span.start_time,
+                "status": TraceSpanStatus.SUCCESS,
+                "children": [],
+                "name": tool_called.name,
+                "input": tool_called.input_parameters,
+                "output": None,
+                "metrics": None,
+                "description": tool_called.description,
+            }
+        )
+        current_span.children.append(tool_span)
+
+
+_URL_MAX = 200
+_JSON_MAX = max(
+    len_long(), 400
+)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
+
+
+def compact_dump(value: Any) -> str:
+    try:
+        dumped = json.dumps(
+            value, ensure_ascii=False, default=str, separators=(",", ":")
+        )
+    except Exception:
+        dumped = repr(value)
+    return shorten(dumped, max_len=_JSON_MAX)
+
+
+def fmt_url(url: Optional[str]) -> str:
+    if not url:
+        return ""
+    if url.startswith("data:"):
+        return "[data-uri]"
+    return shorten(url, max_len=_URL_MAX)

deepeval/models/base_model.py

@@ -68,7 +68,9 @@ class DeepEvalBaseLLM(ABC):
         Returns:
             A list of strings.
         """
-        raise AttributeError
+        raise NotImplementedError(
+            "batch_generate is not implemented for this model"
+        )
 
     @abstractmethod
     def get_model_name(self, *args, **kwargs) -> str:

deepeval/openai/__init__.py

@@ -15,5 +15,7 @@ except ImportError:
 
 if OpenAI or AsyncOpenAI:
     from deepeval.openai.patch import patch_openai_classes
+    from deepeval.telemetry import capture_tracing_integration
 
-    patch_openai_classes()
+    with capture_tracing_integration("openai"):
+        patch_openai_classes()

deepeval/openai/extractors.py

@@ -4,13 +4,13 @@ from typing import Any, Union, Dict
 from openai.types.responses import Response
 
 from deepeval.test_case.llm_test_case import ToolCall
+
+from deepeval.model_integrations.types import InputParameters, OutputParameters
 from deepeval.openai.utils import (
     render_response_input,
     stringify_multimodal_content,
     render_messages,
 )
-from deepeval.openai.types import InputParameters, OutputParameters
-from deepeval.tracing.types import Message
 
 
 # guarding against errors to be compatible with legacy APIs

deepeval/openai/utils.py

@@ -1,6 +1,6 @@
 import json
 import uuid
-from typing import Any, Dict, List, Optional, Iterable
+from typing import Any, Dict, List, Iterable
 
 from openai.types.chat.chat_completion_message_param import (
     ChatCompletionMessageParam,
@@ -8,32 +8,8 @@ from openai.types.chat.chat_completion_message_param import (
 
 from deepeval.tracing.types import ToolSpan, TraceSpanStatus
 from deepeval.tracing.context import current_span_context
-from deepeval.utils import shorten, len_long
-from deepeval.openai.types import OutputParameters
-
-
-_URL_MAX = 200
-_JSON_MAX = max(
-    len_long(), 400
-)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
-
-
-def _compact_dump(value: Any) -> str:
-    try:
-        dumped = json.dumps(
-            value, ensure_ascii=False, default=str, separators=(",", ":")
-        )
-    except Exception:
-        dumped = repr(value)
-    return shorten(dumped, max_len=_JSON_MAX)
-
-
-def _fmt_url(url: Optional[str]) -> str:
-    if not url:
-        return ""
-    if url.startswith("data:"):
-        return "[data-uri]"
-    return shorten(url, max_len=_URL_MAX)
+from deepeval.model_integrations.types import OutputParameters
+from deepeval.model_integrations.utils import compact_dump, fmt_url
 
 
 def create_child_tool_spans(output_parameters: OutputParameters):
@@ -111,7 +87,7 @@ def stringify_multimodal_content(content: Any) -> str:
                 url = image_url
             else:
                 url = (image_url or {}).get("url") or content.get("url")
-            return f"[image:{_fmt_url(url)}]"
+            return f"[image:{fmt_url(url)}]"
 
         # Responses API variants
         if t == "input_text":
@@ -122,14 +98,14 @@ def stringify_multimodal_content(content: Any) -> str:
                 url = image_url
             else:
                 url = (image_url or {}).get("url") or content.get("url")
-            return f"[image:{_fmt_url(url)}]"
+            return f"[image:{fmt_url(url)}]"
 
         # readability for other input_* types we don't currently handle
         if t and t.startswith("input_"):
             return f"[{t}]"
 
     # unknown dicts and types returned as shortened JSON
-    return _compact_dump(content)
+    return compact_dump(content)
 
 
 def render_messages(
@@ -228,7 +204,7 @@ def _render_content(content: Dict[str, Any], indent: int = 0) -> str:
             lines.append(f"{prefix}{key}:")
             lines.append(_render_content(value, indent + 1))
         elif isinstance(value, list):
-            lines.append(f"{prefix}{key}: {_compact_dump(value)}")
+            lines.append(f"{prefix}{key}: {compact_dump(value)}")
         else:
             lines.append(f"{prefix}{key}: {value}")
 

deepeval/prompt/api.py

@@ -1,8 +1,10 @@
-from pydantic import BaseModel, Field, AliasChoices
+from pydantic import BaseModel, Field, AliasChoices, ConfigDict
 from enum import Enum
 from typing import List, Optional
 from pydantic import TypeAdapter
 
+from deepeval.utils import make_model_config
+
 ###################################
 # Model Settings
 ###################################
@@ -92,6 +94,8 @@ class SchemaDataType(Enum):
 
 
 class OutputSchemaField(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     id: str
     type: SchemaDataType
     name: str
@@ -102,9 +106,6 @@ class OutputSchemaField(BaseModel):
         validation_alias=AliasChoices("parent_id", "parentId"),
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class OutputSchema(BaseModel):
     fields: Optional[List[OutputSchemaField]] = None
@@ -187,6 +188,10 @@ class PromptHttpResponse(BaseModel):
 
 
 class PromptPushRequest(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
+    model_config = ConfigDict(use_enum_values=True)
+
     alias: str
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
@@ -203,11 +208,10 @@ class PromptPushRequest(BaseModel):
         default=None, serialization_alias="outputType"
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class PromptUpdateRequest(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
     interpolation_type: PromptInterpolationType = Field(
@@ -223,9 +227,6 @@ class PromptUpdateRequest(BaseModel):
         default=None, serialization_alias="outputType"
     )
 
-    class Config:
-        use_enum_values = True
-
 
 class PromptApi(BaseModel):
     id: str

deepeval/prompt/prompt.py

@@ -5,11 +5,13 @@ from rich.console import Console
 import time
 import json
 import os
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel, ValidationError, ConfigDict
 import asyncio
 import portalocker
 import threading
 
+from deepeval.utils import make_model_config
+
 from deepeval.prompt.api import (
     PromptHttpResponse,
     PromptMessage,
@@ -77,6 +79,8 @@ class CustomEncoder(json.JSONEncoder):
 
 
 class CachedPrompt(BaseModel):
+    model_config = make_model_config(use_enum_values=True)
+
     alias: str
     version: str
     label: Optional[str] = None
@@ -89,9 +93,6 @@ class CachedPrompt(BaseModel):
     output_type: Optional[OutputType]
     output_schema: Optional[OutputSchema]
 
-    class Config:
-        use_enum_values = True
-
 
 class Prompt:
 

deepeval/telemetry.py

@@ -3,12 +3,12 @@ import logging
 import os
 import socket
 import sys
-from threading import Event
 import uuid
 import sentry_sdk
 from enum import Enum
 from typing import List, Dict
 import requests
+from deepeval.config.settings import get_settings
 from deepeval.constants import LOGIN_PROMPT, HIDDEN_DIR, KEY_FILE
 from posthog import Posthog
 
@@ -34,7 +34,7 @@ TELEMETRY_PATH = os.path.join(HIDDEN_DIR, TELEMETRY_DATA_FILE)
 
 
 def telemetry_opt_out():
-    return os.getenv("DEEPEVAL_TELEMETRY_OPT_OUT") == "1"
+    return get_settings().DEEPEVAL_TELEMETRY_OPT_OUT
 
 
 def blocked_by_firewall():
@@ -131,7 +131,7 @@ if not telemetry_opt_out():
 
 
 if (
-    os.getenv("ERROR_REPORTING") == "1"
+    get_settings().ERROR_REPORTING
     and not blocked_by_firewall()
     and not telemetry_opt_out()
 ):

deepeval/test_case/llm_test_case.py

@@ -1,5 +1,4 @@
 from pydantic import (
-    ConfigDict,
     Field,
     BaseModel,
     model_validator,
@@ -11,6 +10,8 @@ from enum import Enum
 import json
 import uuid
 
+from deepeval.utils import make_model_config
+
 from deepeval.test_case.mcp import (
     MCPServer,
     MCPPromptCall,
@@ -156,7 +157,7 @@ class ToolCall(BaseModel):
 
 
 class LLMTestCase(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     input: str
     actual_output: Optional[str] = Field(

deepeval/test_run/api.py

@@ -1,8 +1,9 @@
-from pydantic import BaseModel, Field, ConfigDict
+from pydantic import BaseModel, Field
 from typing import Optional, List, Union, Dict
 
 from deepeval.test_case import MLLMImage, ToolCall
 from deepeval.tracing.api import TraceApi, MetricData
+from deepeval.utils import make_model_config
 
 
 class LLMApiTestCase(BaseModel):
@@ -49,7 +50,7 @@ class LLMApiTestCase(BaseModel):
     comments: Optional[str] = Field(None)
     trace: Optional[TraceApi] = Field(None)
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config = make_model_config(arbitrary_types_allowed=True)
     # metric_collection: Optional[str] = Field(None, alias="metricCollection")
 
     def update_metric_data(self, metric_data: MetricData):

deepeval/test_run/cache.py

@@ -6,6 +6,8 @@ from typing import List, Optional, Union, Dict, Union
 from enum import Enum
 from pydantic import BaseModel, Field
 
+from deepeval.utils import make_model_config
+
 from deepeval.test_case import LLMTestCaseParams, LLMTestCase, ToolCallParams
 from deepeval.test_run.api import MetricData
 from deepeval.utils import (
@@ -20,6 +22,8 @@ TEMP_CACHE_FILE_NAME = f"{HIDDEN_DIR}/.temp-deepeval-cache.json"
 
 
 class MetricConfiguration(BaseModel):
+    model_config = make_model_config(arbitrary_types_allowed=True)
+
     ##### Required fields #####
     threshold: float
     evaluation_model: Optional[str] = None
@@ -36,9 +40,6 @@ class MetricConfiguration(BaseModel):
         Union[List[LLMTestCaseParams], List[ToolCallParams]]
     ] = None
 
-    class Config:
-        arbitrary_types_allowed = True
-
 
 class CachedMetricData(BaseModel):
     metric_data: MetricData

deepeval/test_run/test_run.py

@@ -463,19 +463,29 @@ class TestRunManager:
                     mode="r",
                     flags=portalocker.LOCK_SH | portalocker.LOCK_NB,
                 ) as file:
-                    self.test_run = self.test_run.load(file)
+                    loaded = self.test_run.load(file)
+                    # only overwrite if loading actually worked
+                    self.test_run = loaded
             except (
                 FileNotFoundError,
+                json.JSONDecodeError,
                 portalocker.exceptions.LockException,
             ) as e:
-                print(f"Error loading test run from disk: {e}", file=sys.stderr)
-                self.test_run = None
+                print(
+                    f"Warning: Could not load test run from disk: {e}",
+                    file=sys.stderr,
+                )
 
         return self.test_run
 
     def save_test_run(self, path: str, save_under_key: Optional[str] = None):
         if self.save_to_disk:
             try:
+                # ensure parent directory exists
+                parent = os.path.dirname(path)
+                if parent:
+                    os.makedirs(parent, exist_ok=True)
+
                 with portalocker.Lock(path, mode="w") as file:
                     if save_under_key:
                         try:
@@ -533,10 +543,19 @@ class TestRunManager:
                     self.test_run.save(file)
             except (
                 FileNotFoundError,
+                json.JSONDecodeError,
                 portalocker.exceptions.LockException,
             ) as e:
-                print(f"Error updating test run to disk: {e}", file=sys.stderr)
-                self.test_run = None
+                print(
+                    f"Warning: Could not update test run on disk: {e}",
+                    file=sys.stderr,
+                )
+                if self.test_run is None:
+                    # guarantee a valid in-memory run so the update can proceed.
+                    # never destroy in-memory state on I/O failure.
+                    self.create_test_run()
+                self.test_run.add_test_case(api_test_case)
+                self.test_run.set_dataset_properties(test_case)
         else:
             if self.test_run is None:
                 self.create_test_run()

deepeval/tracing/api.py

@@ -1,8 +1,9 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union, Literal, Any
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, Field
 
 from deepeval.test_case import ToolCall
+from deepeval.utils import make_model_config
 
 
 class SpanApiType(Enum):
@@ -27,7 +28,7 @@ class PromptApi(BaseModel):
 
 
 class MetricData(BaseModel):
-    model_config = ConfigDict(extra="ignore")
+    model_config = make_model_config(extra="ignore")
 
     name: str
     threshold: float
@@ -42,6 +43,10 @@ class MetricData(BaseModel):
 
 
 class BaseApiSpan(BaseModel):
+    model_config = make_model_config(
+        use_enum_values=True, validate_assignment=True
+    )
+
     uuid: str
     name: str = None
     status: TraceSpanApiStatus
@@ -96,12 +101,12 @@ class BaseApiSpan(BaseModel):
     metric_collection: Optional[str] = Field(None, alias="metricCollection")
     metrics_data: Optional[List[MetricData]] = Field(None, alias="metricsData")
 
-    class Config:
-        use_enum_values = True
-        validate_assignment = True
-
 
 class TraceApi(BaseModel):
+    model_config = make_model_config(
+        use_enum_values=True, validate_assignment=True
+    )
+
     uuid: str
     base_spans: Optional[List[BaseApiSpan]] = Field(None, alias="baseSpans")
     agent_spans: Optional[List[BaseApiSpan]] = Field(None, alias="agentSpans")
@@ -139,7 +144,3 @@ class TraceApi(BaseModel):
 
     # Don't serialize these
     confident_api_key: Optional[str] = Field(None, exclude=True)
-
-    class Config:
-        use_enum_values = True
-        validate_assignment = True