deepeval 3.6.1__tar.gz → 3.6.2__tar.gz

{deepeval-3.6.1 → deepeval-3.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.1
+Version: 3.6.2
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -359,7 +359,7 @@ for golden in dataset.goldens:
 
 @pytest.mark.parametrize(
     "test_case",
-    dataset,
+    dataset.test_cases,
 )
 def test_customer_chatbot(test_case: LLMTestCase):
     answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5)

{deepeval-3.6.1 → deepeval-3.6.2}/README.md

@@ -309,7 +309,7 @@ for golden in dataset.goldens:
 
 @pytest.mark.parametrize(
     "test_case",
-    dataset,
+    dataset.test_cases,
 )
 def test_customer_chatbot(test_case: LLMTestCase):
     answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5)

deepeval-3.6.2/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.6.2"

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/evaluate/execute.py

@@ -836,7 +836,13 @@ def execute_agentic_test_cases(
                 ):
                     if asyncio.iscoroutinefunction(observed_callback):
                         loop = get_or_create_event_loop()
-                        loop.run_until_complete(observed_callback(golden.input))
+                        coro = observed_callback(golden.input)
+                        loop.run_until_complete(
+                            asyncio.wait_for(
+                                coro,
+                                timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                            )
+                        )
                     else:
                         observed_callback(golden.input)
                     current_trace: Trace = current_trace_context.get()
@@ -1190,7 +1196,10 @@ async def _a_execute_agentic_test_case(
             _pbar_callback_id=pbar_tags_id,
         ):
             if asyncio.iscoroutinefunction(observed_callback):
-                await observed_callback(golden.input)
+                await asyncio.wait_for(
+                    observed_callback(golden.input),
+                    timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                )
             else:
                 observed_callback(golden.input)
             current_trace: Trace = current_trace_context.get()

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/metrics/hallucination/hallucination.py

@@ -30,7 +30,7 @@ class HallucinationMetric(BaseMetric):
         threshold: float = 0.5,
         model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         include_reason: bool = True,
-        async_mode: bool = False,
+        async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
         evaluation_template: Type[

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/metrics/tool_correctness/tool_correctness.py

@@ -152,19 +152,14 @@ class ToolCorrectnessMetric(BaseMetric):
 
     # Calculate score
     def _calculate_score(self):
-        if self.should_exact_match:
+        # Fix: handle empty expected_tools to avoid ZeroDivisionError
+        if len(self.expected_tools) == 0:
+            score = 1.0 if len(self.tools_called) == 0 else 0.0
+        elif self.should_exact_match:
             score = self._calculate_exact_match_score()
         elif self.should_consider_ordering:
             _, weighted_length = self._compute_weighted_lcs()
-            if (
-                len(self.tools_called) == len(self.expected_tools)
-                and len(self.expected_tools) == 0
-            ):
-                score = 1.0
-            elif len(self.expected_tools) == 0:
-                score = 0.0
-            else:
-                score = weighted_length / len(self.expected_tools)
+            score = weighted_length / len(self.expected_tools)
         else:
             score = self._calculate_non_exact_match_score()
         return 0 if self.strict_mode and score < self.threshold else score

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/test_case/llm_test_case.py

@@ -1,4 +1,5 @@
 from pydantic import (
+    ConfigDict,
     Field,
     BaseModel,
     model_validator,
@@ -151,6 +152,8 @@ class ToolCall(BaseModel):
 
 
 class LLMTestCase(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
     input: str
     actual_output: Optional[str] = Field(
         default=None,

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/tracing/otel/utils.py

@@ -1,11 +1,16 @@
+import json
+
 from typing import List, Optional, Tuple, Any
-from deepeval.tracing.types import Trace, LLMTestCase, ToolCall
-from deepeval.tracing import trace_manager, BaseSpan
 from opentelemetry.sdk.trace.export import ReadableSpan
-import json
 
+from deepeval.evaluate.utils import create_api_test_case
+from deepeval.test_run.api import LLMApiTestCase
+from deepeval.test_run.test_run import global_test_run_manager
+from deepeval.tracing.types import Trace, LLMTestCase, ToolCall
+from deepeval.tracing import trace_manager, BaseSpan
 from deepeval.tracing.utils import make_json_serializable
 
+
 GEN_AI_OPERATION_NAMES = ["chat", "generate_content", "task_completion"]
 
 
@@ -107,12 +112,12 @@ def check_llm_input_from_gen_ai_attributes(
         input = json.loads(span.attributes.get("gen_ai.input.messages"))
         input = _flatten_input(input)
 
-    except Exception as e:
+    except Exception:
         pass
     try:
         output = json.loads(span.attributes.get("gen_ai.output.messages"))
         output = _flatten_input(output)
-    except Exception as e:
+    except Exception:
         pass
 
     if input is None and output is None:
@@ -126,7 +131,7 @@ def check_llm_input_from_gen_ai_attributes(
                     and last_event.get("event.name") == "gen_ai.choice"
                 ):
                     output = last_event
-        except Exception as e:
+        except Exception:
             pass
 
     return input, output
@@ -181,7 +186,7 @@ def _flatten_input(input: list) -> list:
                         }
                     )
             return result
-        except Exception as e:
+        except Exception:
             return input
 
     return input
@@ -192,7 +197,7 @@ def check_tool_name_from_gen_ai_attributes(span: ReadableSpan) -> Optional[str]:
         gen_ai_tool_name = span.attributes.get("gen_ai.tool.name")
         if gen_ai_tool_name:
             return gen_ai_tool_name
-    except Exception as e:
+    except Exception:
         pass
 
     return None
@@ -205,7 +210,7 @@ def check_tool_input_parameters_from_gen_ai_attributes(
         tool_arguments = span.attributes.get("tool_arguments")
         if tool_arguments:
             return json.loads(tool_arguments)
-    except Exception as e:
+    except Exception:
         pass
 
     return None
@@ -224,7 +229,7 @@ def check_span_type_from_gen_ai_attributes(span: ReadableSpan):
 
         elif gen_ai_tool_name:
             return "tool"
-    except Exception as e:
+    except Exception:
         pass
 
     return "base"
@@ -235,7 +240,7 @@ def check_model_from_gen_ai_attributes(span: ReadableSpan):
         gen_ai_request_model_name = span.attributes.get("gen_ai.request.model")
         if gen_ai_request_model_name:
             return gen_ai_request_model_name
-    except Exception as e:
+    except Exception:
         pass
 
     return None
@@ -286,7 +291,7 @@ def prepare_trace_llm_test_case(span: ReadableSpan) -> Optional[LLMTestCase]:
                     tools_called.append(
                         ToolCall.model_validate_json(tool_call_json_str)
                     )
-                except Exception as e:
+                except Exception:
                     pass
 
     _expected_tools = span.attributes.get(
@@ -299,7 +304,7 @@ def prepare_trace_llm_test_case(span: ReadableSpan) -> Optional[LLMTestCase]:
                     expected_tools.append(
                         ToolCall.model_validate_json(tool_call_json_str)
                     )
-                except Exception as e:
+                except Exception:
                     pass
 
     test_case.tools_called = tools_called
@@ -328,12 +333,6 @@ def parse_list_of_strings(context: List[str]) -> List[str]:
     return parsed_context
 
 
-from deepeval.evaluate.utils import create_api_test_case
-from deepeval.test_run.api import LLMApiTestCase
-from deepeval.test_run.test_run import global_test_run_manager
-from typing import Optional
-
-
 def post_test_run(traces: List[Trace], test_run_id: Optional[str]):
     # Accept single trace or list of traces
     if isinstance(traces, Trace):
@@ -442,7 +441,7 @@ def check_pydantic_ai_agent_input_output(
 def check_tool_output(span: ReadableSpan):
     try:
         return span.attributes.get("tool_response")
-    except Exception as e:
+    except Exception:
         pass
     return None
 

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/tracing/tracing.py

@@ -208,7 +208,13 @@ class TraceManager:
                 else:
                     # print(f"Ending trace: {trace.root_spans}")
                     self.environment = Environment.TESTING
-                    trace.root_spans = [trace.root_spans[0].children[0]]
+                    if (
+                        trace.root_spans
+                        and len(trace.root_spans) > 0
+                        and trace.root_spans[0].children
+                        and len(trace.root_spans[0].children) > 0
+                    ):
+                        trace.root_spans = [trace.root_spans[0].children[0]]
                     for root_span in trace.root_spans:
                         root_span.parent_uuid = None
 

{deepeval-3.6.1 → deepeval-3.6.2}/deepeval/tracing/utils.py

@@ -1,15 +1,12 @@
 import os
-import time
 import inspect
 import json
 import sys
-import difflib
 from datetime import datetime, timezone
 from enum import Enum
 from time import perf_counter
-import time
 from collections import deque
-from typing import Any, Dict, Optional, Sequence, Callable
+from typing import Any, Dict, Optional
 
 from deepeval.constants import CONFIDENT_TRACING_ENABLED
 
@@ -189,8 +186,8 @@ def perf_counter_to_datetime(perf_counter_value: float) -> datetime:
 def replace_self_with_class_name(obj):
     try:
         return f"<{obj.__class__.__name__}>"
-    except:
-        return f"<self>"
+    except Exception:
+        return "<self>"
 
 
 def get_deepeval_trace_mode() -> Optional[str]:

{deepeval-3.6.1 → deepeval-3.6.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "deepeval"
-version = "3.6.1"
+version = "3.6.2"
 description = "The LLM Evaluation Framework"
 authors = ["Jeffrey Ip <jeffreyip@confident-ai.com>"]
 license = "Apache-2.0"

deepeval-3.6.1/deepeval/_version.py

@@ -1 +0,0 @@
-__version__: str = "3.6.1"