deepeval 3.6.1__tar.gz → 3.6.3__tar.gz

{deepeval-3.6.1 → deepeval-3.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.1
+Version: 3.6.3
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0
@@ -359,7 +359,7 @@ for golden in dataset.goldens:
 
 @pytest.mark.parametrize(
     "test_case",
-    dataset,
+    dataset.test_cases,
 )
 def test_customer_chatbot(test_case: LLMTestCase):
     answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5)

{deepeval-3.6.1 → deepeval-3.6.3}/README.md

@@ -309,7 +309,7 @@ for golden in dataset.goldens:
 
 @pytest.mark.parametrize(
     "test_case",
-    dataset,
+    dataset.test_cases,
 )
 def test_customer_chatbot(test_case: LLMTestCase):
     answer_relevancy_metric = AnswerRelevancyMetric(threshold=0.5)

deepeval-3.6.3/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.6.3"

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/confident/api.py

@@ -90,6 +90,7 @@ class Endpoints(Enum):
     TRACES_ENDPOINT = "/v1/traces"
     ANNOTATIONS_ENDPOINT = "/v1/annotations"
     PROMPTS_VERSION_ID_ENDPOINT = "/v1/prompts/:alias/versions/:versionId"
+    PROMPTS_LABEL_ENDPOINT = "/v1/prompts/:alias/labels/:label"
     PROMPTS_ENDPOINT = "/v1/prompts"
     PROMPTS_VERSIONS_ENDPOINT = "/v1/prompts/:alias/versions"
     SIMULATE_ENDPOINT = "/v1/simulate"

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/evaluate/execute.py

@@ -836,7 +836,13 @@ def execute_agentic_test_cases(
                 ):
                     if asyncio.iscoroutinefunction(observed_callback):
                         loop = get_or_create_event_loop()
-                        loop.run_until_complete(observed_callback(golden.input))
+                        coro = observed_callback(golden.input)
+                        loop.run_until_complete(
+                            asyncio.wait_for(
+                                coro,
+                                timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                            )
+                        )
                     else:
                         observed_callback(golden.input)
                     current_trace: Trace = current_trace_context.get()
@@ -1190,7 +1196,10 @@ async def _a_execute_agentic_test_case(
             _pbar_callback_id=pbar_tags_id,
         ):
             if asyncio.iscoroutinefunction(observed_callback):
-                await observed_callback(golden.input)
+                await asyncio.wait_for(
+                    observed_callback(golden.input),
+                    timeout=settings.DEEPEVAL_PER_TASK_TIMEOUT_SECONDS,
+                )
             else:
                 observed_callback(golden.input)
             current_trace: Trace = current_trace_context.get()

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/metrics/g_eval/g_eval.py

@@ -97,7 +97,8 @@ class GEval(BaseMetric):
                     test_case, _additional_context=_additional_context
                 )
                 self.score = (
-                    float(g_score) / self.score_range_span
+                    (float(g_score) - self.score_range[0])
+                    / self.score_range_span
                     if not self.strict_mode
                     else int(g_score)
                 )
@@ -140,7 +141,7 @@ class GEval(BaseMetric):
                 test_case, _additional_context=_additional_context
             )
             self.score = (
-                float(g_score) / self.score_range_span
+                (float(g_score) - self.score_range[0]) / self.score_range_span
                 if not self.strict_mode
                 else int(g_score)
             )

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/metrics/hallucination/hallucination.py

@@ -30,7 +30,7 @@ class HallucinationMetric(BaseMetric):
         threshold: float = 0.5,
         model: Optional[Union[str, DeepEvalBaseLLM]] = None,
         include_reason: bool = True,
-        async_mode: bool = False,
+        async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
         evaluation_template: Type[

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/metrics/tool_correctness/tool_correctness.py

@@ -1,4 +1,4 @@
-from typing import List, Union, Dict
+from typing import List, Dict
 
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.utils import (
@@ -299,7 +299,7 @@ class ToolCorrectnessMetric(BaseMetric):
     def is_successful(self) -> bool:
         try:
             self.success = self.score >= self.threshold
-        except:
+        except (AttributeError, TypeError):
             self.success = False
         return self.success
 

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/models/llms/amazon_bedrock_model.py

@@ -9,7 +9,7 @@ from deepeval.models.retry_policy import (
     sdk_retries_for,
 )
 from deepeval.models import DeepEvalBaseLLM
-from deepeval.models.llms.utils import trim_and_load_json
+from deepeval.models.llms.utils import trim_and_load_json, safe_asyncio_run
 from deepeval.constants import ProviderSlug as PS
 
 # check aiobotocore availability
@@ -40,7 +40,6 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
         region_name: str,
         aws_access_key_id: Optional[str] = None,
         aws_secret_access_key: Optional[str] = None,
-        temperature: float = 0,
         input_token_cost: float = 0,
         output_token_cost: float = 0,
         generation_kwargs: Optional[Dict] = None,
@@ -53,13 +52,9 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
         self.region_name = region_name
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
-        self.temperature = temperature
         self.input_token_cost = input_token_cost
         self.output_token_cost = output_token_cost
 
-        if self.temperature < 0:
-            raise ValueError("Temperature must be >= 0.")
-
         # prepare aiobotocore session, config, and async exit stack
         self._session = get_session()
         self._exit_stack = AsyncExitStack()
@@ -75,7 +70,7 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
     def generate(
         self, prompt: str, schema: Optional[BaseModel] = None
     ) -> Tuple[Union[str, Dict], float]:
-        return asyncio.run(self.a_generate(prompt, schema))
+        return safe_asyncio_run(self.a_generate(prompt, schema))
 
     @retry_bedrock
     async def a_generate(
@@ -142,34 +137,11 @@ class AmazonBedrockModel(DeepEvalBaseLLM):
     ###############################################
 
     def get_converse_request_body(self, prompt: str) -> dict:
-        # Inline parameter translation with defaults
-        param_mapping = {
-            "max_tokens": "maxTokens",
-            "top_p": "topP",
-            "top_k": "topK",
-            "stop_sequences": "stopSequences",
-        }
-
-        # Start with defaults for required parameters
-        translated_kwargs = {
-            "maxTokens": self.generation_kwargs.get("max_tokens", 1000),
-            "topP": self.generation_kwargs.get("top_p", 0),
-        }
-
-        # Add any other parameters from generation_kwargs
-        for key, value in self.generation_kwargs.items():
-            if key not in [
-                "max_tokens",
-                "top_p",
-            ]:  # Skip already handled defaults
-                aws_key = param_mapping.get(key, key)
-                translated_kwargs[aws_key] = value
 
         return {
             "messages": [{"role": "user", "content": [{"text": prompt}]}],
             "inferenceConfig": {
-                "temperature": self.temperature,
-                **translated_kwargs,
+                **self.generation_kwargs,
             },
         }
 

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/models/llms/openai_model.py

@@ -204,7 +204,6 @@ models_requiring_temperature_1 = [
     "gpt-5-mini-2025-08-07",
     "gpt-5-nano",
     "gpt-5-nano-2025-08-07",
-    "gpt-5-chat-latest",
 ]
 
 

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/models/llms/utils.py

@@ -1,6 +1,7 @@
 from typing import Dict
 import re
 import json
+import asyncio
 
 
 def trim_and_load_json(
@@ -20,3 +21,24 @@ def trim_and_load_json(
         raise ValueError(error_str)
     except Exception as e:
         raise Exception(f"An unexpected error occurred: {str(e)}")
+
+
+def safe_asyncio_run(coro):
+    """
+    Run an async coroutine safely.
+    Falls back to run_until_complete if already in a running event loop.
+    """
+    try:
+        return asyncio.run(coro)
+    except RuntimeError:
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                future = asyncio.ensure_future(coro)
+                return loop.run_until_complete(future)
+            else:
+                return loop.run_until_complete(coro)
+        except Exception as inner_e:
+            raise
+    except Exception as e:
+        raise

{deepeval-3.6.1 → deepeval-3.6.3}/deepeval/prompt/api.py

@@ -45,6 +45,8 @@ class PromptVersionsHttpResponse(BaseModel):
 
 class PromptHttpResponse(BaseModel):
     id: str
+    version: str
+    label: Optional[str] = None
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
     interpolation_type: PromptInterpolationType = Field(