deepeval 4.3.2__tar.gz → 4.3.4__tar.gz

{deepeval-4.3.2 → deepeval-4.3.4}/PKG-INFO

@@ -1,15 +1,14 @@
 Metadata-Version: 2.4
 Name: deepeval
-Version: 4.3.2
+Version: 4.3.4
 Summary: The LLM Evaluation Framework
 License: Apache-2.0
 License-File: LICENSE.md
 Author: Jeffrey Ip
 Author-email: jeffreyip@confident-ai.com
-Requires-Python: >=3.9,<4.0
+Requires-Python: >=3.10,<4.0
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -18,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: inspect
 Requires-Dist: aiohttp
 Requires-Dist: click (>=8.0.0,<8.4.0)
+Requires-Dist: confident-trace (>=0.1.14,<0.2.0)
 Requires-Dist: grpcio (>=1.67.1,<2.0.0)
 Requires-Dist: jinja2
 Requires-Dist: nest_asyncio
@@ -114,7 +114,7 @@ Description-Content-Type: text/markdown
 
 **DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating large-language model systems. It is similar to Pytest but specialized for unit testing LLM apps. DeepEval incorporates the latest research to run evals via metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which use LLM-as-a-judge and other NLP models that run **locally on your machine**.
 
-Whether you're building AI agents, multi-turn chatbots, voice agents, RAG pipelines, or anything else, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
+Whether you're building AI agents, voice agents, multi-turn chatbots, RAG pipelines, or anything else, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
 
 - **LLM apps end-to-end** as black boxes
 - **Complete AI agent trajectories** across every decision and action
@@ -155,17 +155,6 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
-  - <details>
-    <summary><b>Multi-Turn Metrics</b></summary>
-
-    - [Knowledge Retention](https://deepeval.com/docs/metrics-knowledge-retention) — evaluate whether the chatbot retains factual information throughout a conversation
-    - [Conversation Completeness](https://deepeval.com/docs/metrics-conversation-completeness) — measure whether the chatbot satisfies user needs throughout a conversation
-    - [Turn Relevancy](https://deepeval.com/docs/metrics-turn-relevancy) — evaluate whether the chatbot generates consistently relevant responses throughout a conversation
-    - [Turn Faithfulness](https://deepeval.com/docs/metrics-turn-faithfulness) — check if the chatbot's responses are factually grounded in retrieval context across turns
-    - [Role Adherence](https://deepeval.com/docs/metrics-role-adherence) — evaluate whether the chatbot adheres to its assigned role throughout a conversation
-
-    </details>
-
   - <details>
     <summary><b>Voice Agent Metrics</b></summary>
 
@@ -177,6 +166,17 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
+  - <details>
+    <summary><b>Multi-Turn Metrics</b></summary>
+
+    - [Knowledge Retention](https://deepeval.com/docs/metrics-knowledge-retention) — evaluate whether the chatbot retains factual information throughout a conversation
+    - [Conversation Completeness](https://deepeval.com/docs/metrics-conversation-completeness) — measure whether the chatbot satisfies user needs throughout a conversation
+    - [Turn Relevancy](https://deepeval.com/docs/metrics-turn-relevancy) — evaluate whether the chatbot generates consistently relevant responses throughout a conversation
+    - [Turn Faithfulness](https://deepeval.com/docs/metrics-turn-faithfulness) — check if the chatbot's responses are factually grounded in retrieval context across turns
+    - [Role Adherence](https://deepeval.com/docs/metrics-role-adherence) — evaluate whether the chatbot adheres to its assigned role throughout a conversation
+
+    </details>
+
   - <details>
     <summary><b>RAG Metrics</b></summary>
 
@@ -223,7 +223,7 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
 - 🎯 Supports both end-to-end and component-level LLM evaluation.
 - 🧩 Build your own custom metrics that are automatically integrated with DeepEval's ecosystem.
-- 🔮 Generate both single and multi-turn synthetic datasets for evaluation.
+- 🔮 Generate single and multi-turn synthetic datasets, and simulate live voice calls against your voice agent.
 - 🔗 Integrates seamlessly with **ANY** CI/CD environment.
 - 🧬 Optimize prompts automatically based on evaluation results.
 - 🏆 Easily benchmark **ANY** LLM on popular LLM benchmarks in [under 10 lines of code.](https://deepeval.com/docs/benchmarks-introduction?utm_source=GitHub), including MMLU, HellaSwag, DROP, BIG-Bench Hard, TruthfulQA, HumanEval, GSM8K.
@@ -232,7 +232,7 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
 # 🔌 Integrations
 
-DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. The same traces can go to **Confident AI** once you're live, where evals run on every production trace.
+DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more — and into voice stacks like Vapi, ElevenLabs, LiveKit, and Pipecat. The same traces can go to **Confident AI** once you're live, where evals run on every production trace.
 
 ## Frameworks
 
@@ -249,13 +249,20 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 - [OpenAI Agents](https://www.deepeval.com/integrations/frameworks/openai-agents?utm_source=GitHub) — evaluate OpenAI Agents end-to-end in under a minute
 - [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate AI agents and RAG applications built with LlamaIndex
 
+## Voice Agent Connectors
+
+- [Vapi](https://www.deepeval.com/integrations/voice-agent-connectors/vapi?utm_source=GitHub) — simulate calls to a Vapi assistant over its WebSocket transport
+- [ElevenLabs](https://www.deepeval.com/integrations/voice-agent-connectors/elevenlabs?utm_source=GitHub) — simulate calls to an ElevenLabs conversational agent
+- [LiveKit](https://www.deepeval.com/integrations/voice-agent-connectors/livekit?utm_source=GitHub) — join your agent's LiveKit room and trade audio over WebRTC
+- [Pipecat](https://www.deepeval.com/integrations/voice-agent-connectors/pipecat?utm_source=GitHub) — connect to your Pipecat pipeline over its WebSocket transport
+
 ## ☁️ Platform + Ecosystem
 
 [Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section&ref_page=github/readme) is AI monitoring for applications in production. DeepEval tests what you know to test before you ship; Confident AI finds what you didn't, and it's model- and framework-agnostic.
 
 - Failing traces surface before anyone has written an eval for them, clustered into failure modes, with user sentiment and behavior across traces.
 - Failing traces route to the right annotators and become goldens, so production issues turn into your next eval.
-- For enterprise teams, one system for AI quality: governance as release gates and native red teaming.
+- For enterprise teams, one system for AI quality: evals, monitoring, human annotation, and voice-native observability, with governance as release gates.
 - Prefer to stay in your editor? Inspect failing traces and pull the datasets built from them in Claude Code or Cursor through Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
 
 <p align="center">
@@ -266,7 +273,7 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 
 # 🤖 Vibe-Coder QuickStart
 
-Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your AI agent, multi-turn chatbot, voice agent, RAG pipeline, or any other LLM app, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
+Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your AI agent, voice agent, multi-turn chatbot, RAG pipeline, or any other LLM app, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
 
 [Start with the 5-minute vibe-coder guide](https://deepeval.com/docs/vibe-coder-quickstart?utm_source=GitHub).
 
@@ -278,7 +285,7 @@ Let's pretend your LLM application is a customer support AI agent; here's how De
 
 ## Installation
 
-Deepeval works with **Python>=3.9+**.
+Deepeval works with **Python>=3.10+**.
 
 ```
 pip install -U deepeval
@@ -670,7 +677,7 @@ print(answer_relevancy_metric.score)
 print(answer_relevancy_metric.reason)
 ```
 
-Note that some metrics are built for specific use cases, such as AI agents, multi-turn chatbots, voice agents, RAG, and more. Make sure to use our docs to pick the right one for your use case.
+Note that some metrics are built for specific use cases, such as AI agents, voice agents, multi-turn chatbots, RAG, and more. Make sure to use our docs to pick the right one for your use case.
 
 ## A Note on Env Variables (.env / .env.local)
 

{deepeval-4.3.2 → deepeval-4.3.4}/README.md

@@ -61,7 +61,7 @@
 
 **DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating large-language model systems. It is similar to Pytest but specialized for unit testing LLM apps. DeepEval incorporates the latest research to run evals via metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which use LLM-as-a-judge and other NLP models that run **locally on your machine**.
 
-Whether you're building AI agents, multi-turn chatbots, voice agents, RAG pipelines, or anything else, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
+Whether you're building AI agents, voice agents, multi-turn chatbots, RAG pipelines, or anything else, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
 
 - **LLM apps end-to-end** as black boxes
 - **Complete AI agent trajectories** across every decision and action
@@ -102,17 +102,6 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
-  - <details>
-    <summary><b>Multi-Turn Metrics</b></summary>
-
-    - [Knowledge Retention](https://deepeval.com/docs/metrics-knowledge-retention) — evaluate whether the chatbot retains factual information throughout a conversation
-    - [Conversation Completeness](https://deepeval.com/docs/metrics-conversation-completeness) — measure whether the chatbot satisfies user needs throughout a conversation
-    - [Turn Relevancy](https://deepeval.com/docs/metrics-turn-relevancy) — evaluate whether the chatbot generates consistently relevant responses throughout a conversation
-    - [Turn Faithfulness](https://deepeval.com/docs/metrics-turn-faithfulness) — check if the chatbot's responses are factually grounded in retrieval context across turns
-    - [Role Adherence](https://deepeval.com/docs/metrics-role-adherence) — evaluate whether the chatbot adheres to its assigned role throughout a conversation
-
-    </details>
-
   - <details>
     <summary><b>Voice Agent Metrics</b></summary>
 
@@ -124,6 +113,17 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
+  - <details>
+    <summary><b>Multi-Turn Metrics</b></summary>
+
+    - [Knowledge Retention](https://deepeval.com/docs/metrics-knowledge-retention) — evaluate whether the chatbot retains factual information throughout a conversation
+    - [Conversation Completeness](https://deepeval.com/docs/metrics-conversation-completeness) — measure whether the chatbot satisfies user needs throughout a conversation
+    - [Turn Relevancy](https://deepeval.com/docs/metrics-turn-relevancy) — evaluate whether the chatbot generates consistently relevant responses throughout a conversation
+    - [Turn Faithfulness](https://deepeval.com/docs/metrics-turn-faithfulness) — check if the chatbot's responses are factually grounded in retrieval context across turns
+    - [Role Adherence](https://deepeval.com/docs/metrics-role-adherence) — evaluate whether the chatbot adheres to its assigned role throughout a conversation
+
+    </details>
+
   - <details>
     <summary><b>RAG Metrics</b></summary>
 
@@ -170,7 +170,7 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
 - 🎯 Supports both end-to-end and component-level LLM evaluation.
 - 🧩 Build your own custom metrics that are automatically integrated with DeepEval's ecosystem.
-- 🔮 Generate both single and multi-turn synthetic datasets for evaluation.
+- 🔮 Generate single and multi-turn synthetic datasets, and simulate live voice calls against your voice agent.
 - 🔗 Integrates seamlessly with **ANY** CI/CD environment.
 - 🧬 Optimize prompts automatically based on evaluation results.
 - 🏆 Easily benchmark **ANY** LLM on popular LLM benchmarks in [under 10 lines of code.](https://deepeval.com/docs/benchmarks-introduction?utm_source=GitHub), including MMLU, HellaSwag, DROP, BIG-Bench Hard, TruthfulQA, HumanEval, GSM8K.
@@ -179,7 +179,7 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
 # 🔌 Integrations
 
-DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. The same traces can go to **Confident AI** once you're live, where evals run on every production trace.
+DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more — and into voice stacks like Vapi, ElevenLabs, LiveKit, and Pipecat. The same traces can go to **Confident AI** once you're live, where evals run on every production trace.
 
 ## Frameworks
 
@@ -196,13 +196,20 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 - [OpenAI Agents](https://www.deepeval.com/integrations/frameworks/openai-agents?utm_source=GitHub) — evaluate OpenAI Agents end-to-end in under a minute
 - [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate AI agents and RAG applications built with LlamaIndex
 
+## Voice Agent Connectors
+
+- [Vapi](https://www.deepeval.com/integrations/voice-agent-connectors/vapi?utm_source=GitHub) — simulate calls to a Vapi assistant over its WebSocket transport
+- [ElevenLabs](https://www.deepeval.com/integrations/voice-agent-connectors/elevenlabs?utm_source=GitHub) — simulate calls to an ElevenLabs conversational agent
+- [LiveKit](https://www.deepeval.com/integrations/voice-agent-connectors/livekit?utm_source=GitHub) — join your agent's LiveKit room and trade audio over WebRTC
+- [Pipecat](https://www.deepeval.com/integrations/voice-agent-connectors/pipecat?utm_source=GitHub) — connect to your Pipecat pipeline over its WebSocket transport
+
 ## ☁️ Platform + Ecosystem
 
 [Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section&ref_page=github/readme) is AI monitoring for applications in production. DeepEval tests what you know to test before you ship; Confident AI finds what you didn't, and it's model- and framework-agnostic.
 
 - Failing traces surface before anyone has written an eval for them, clustered into failure modes, with user sentiment and behavior across traces.
 - Failing traces route to the right annotators and become goldens, so production issues turn into your next eval.
-- For enterprise teams, one system for AI quality: governance as release gates and native red teaming.
+- For enterprise teams, one system for AI quality: evals, monitoring, human annotation, and voice-native observability, with governance as release gates.
 - Prefer to stay in your editor? Inspect failing traces and pull the datasets built from them in Claude Code or Cursor through Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
 
 <p align="center">
@@ -213,7 +220,7 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 
 # 🤖 Vibe-Coder QuickStart
 
-Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your AI agent, multi-turn chatbot, voice agent, RAG pipeline, or any other LLM app, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
+Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your AI agent, voice agent, multi-turn chatbot, RAG pipeline, or any other LLM app, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
 
 [Start with the 5-minute vibe-coder guide](https://deepeval.com/docs/vibe-coder-quickstart?utm_source=GitHub).
 
@@ -225,7 +232,7 @@ Let's pretend your LLM application is a customer support AI agent; here's how De
 
 ## Installation
 
-Deepeval works with **Python>=3.9+**.
+Deepeval works with **Python>=3.10+**.
 
 ```
 pip install -U deepeval
@@ -617,7 +624,7 @@ print(answer_relevancy_metric.score)
 print(answer_relevancy_metric.reason)
 ```
 
-Note that some metrics are built for specific use cases, such as AI agents, multi-turn chatbots, voice agents, RAG, and more. Make sure to use our docs to pick the right one for your use case.
+Note that some metrics are built for specific use cases, such as AI agents, voice agents, multi-turn chatbots, RAG, and more. Make sure to use our docs to pick the right one for your use case.
 
 ## A Note on Env Variables (.env / .env.local)
 

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/__init__.py

@@ -55,20 +55,16 @@ def _expose_public_api() -> None:
     telemetry = _telemetry
 
     def instrument(*args, **kwargs):
-        """Set up Confident AI's OTel backend.
+        """Trace whatever OpenInference instrumentors run for deepeval.
 
-        Configures a TracerProvider, attaches deepeval's OpenInference span
-        interceptor, and routes spans through the context-aware processor
-        (REST when a deepeval trace context is active or an evaluation is
-        running, OTLP otherwise). Pair with any community OpenInference
-        instrumentor (e.g. ``GoogleADKInstrumentor``, ``OpenAIInstrumentor``)
-        to capture framework-specific telemetry.
+        Starts confident-trace, whose tracer provider any community
+        OpenInference instrumentor (e.g. ``OpenAIInstrumentor``) traces on.
+        deepeval evaluates the spans it collects, and confident-trace sends
+        the rest to Confident AI.
 
         Accepts the same trace-level kwargs as
         ``deepeval.integrations.openinference.instrument_openinference``:
-        ``api_key``, ``name``, ``thread_id``, ``user_id``, ``metadata``,
-        ``tags``, ``environment``, ``metric_collection``, ``test_case_id``,
-        ``turn_id``. Span-level config goes on ``with next_*_span(...)``
+        ``name``, ``thread_id``, ``metadata``, ``tags``, ``environment``. Span-level config goes on ``with next_*_span(...)``
         / ``update_current_span(...)``.
         """
         from deepeval.integrations.openinference import (

deepeval-4.3.4/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "4.3.4"

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/anthropic/__init__.py

@@ -12,9 +12,9 @@ except ImportError:
     AsyncAnthropic = None  # type: ignore
 
 if Anthropic or AsyncAnthropic:
-    from deepeval.anthropic.patch import patch_anthropic_classes
     from deepeval.telemetry import capture_tracing_integration
     from deepeval.tracing.integrations import Integration
+    from deepeval.tracing.otel.setup import ensure_confident_trace
 
     with capture_tracing_integration(Integration.ANTHROPIC):
-        patch_anthropic_classes()
+        ensure_confident_trace(("anthropic",))

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/config/settings.py

@@ -990,6 +990,14 @@ class Settings(BaseSettings):
         None,
         description="Enable detailed internal tracing of metric and model methods inside @observe spans.",
     )
+    DEEPEVAL_AUTO_TRACE: Optional[bool] = Field(
+        True,
+        description="Trace every installed library confident-trace supports while evals_iterator or deepeval test run runs. Set to false to trace only @observe code and the integrations you set up.",
+    )
+    DEEPEVAL_GLOBAL_TRACER_PROVIDER: Optional[bool] = Field(
+        False,
+        description="Install deepeval's tracer provider as the global OpenTelemetry provider when none is set, so other OpenTelemetry spans in your app are traced too. Off by default, which leaves the global provider to your own OpenTelemetry setup.",
+    )
 
     CONFIDENT_OTEL_URL: Optional[AnyUrl] = Field(
         "https://otel.confident-ai.com",
@@ -1181,7 +1189,9 @@ class Settings(BaseSettings):
         "DEEPEVAL_DISABLE_TIMEOUTS",
         "DEEPEVAL_VERBOSE_MODE",
         "DEEPEVAL_GRPC_LOGGING",
+        "DEEPEVAL_AUTO_TRACE",
         "DEEPEVAL_DISABLE_DOTENV",
+        "DEEPEVAL_GLOBAL_TRACER_PROVIDER",
         "DEEPEVAL_TELEMETRY_OPT_OUT",
         "DEEPEVAL_TELEMETRY_ENABLED",
         "DEEPEVAL_MODEL_THINKING",

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/constants.py

@@ -12,7 +12,6 @@ LOGIN_PROMPT = "\n✨👀 Looking for a place for your LLM test data to live
 
 
 CONFIDENT_TRACE_VERBOSE = "CONFIDENT_TRACE_VERBOSE"
-CONFIDENT_TRACE_FLUSH = "CONFIDENT_TRACE_FLUSH"
 CONFIDENT_TRACE_SAMPLE_RATE = "CONFIDENT_TRACE_SAMPLE_RATE"
 CONFIDENT_TRACE_ENVIRONMENT = "CONFIDENT_TRACE_ENVIRONMENT"
 CONFIDENT_TRACING_ENABLED = "CONFIDENT_TRACING_ENABLED"

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/dataset/dataset.py

@@ -476,9 +476,11 @@ class EvaluationDataset:
             retrieval_context = reconstruct_retrieval_context(
                 json_obj.get(retrieval_context_key_name)
             )
-            tools_called_data = json_obj.get(tools_called_key_name, [])
+            # A key that is present but null (as save_as writes for goldens
+            # without tools) means no tools, same as a missing key.
+            tools_called_data = json_obj.get(tools_called_key_name) or []
             tools_called = [ToolCall(**tool) for tool in tools_called_data]
-            expected_tools_data = json_obj.get(expected_tools_key_name, [])
+            expected_tools_data = json_obj.get(expected_tools_key_name) or []
             expected_tools = [ToolCall(**tool) for tool in expected_tools_data]
             # additional_metadata = json_obj.get(addtional_metadata_key_name)
 

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/dataset/golden.py

@@ -384,6 +384,9 @@ class ConversationalGolden(BaseModel):
             if re.search(pattern, self.user_description) is not None:
                 self.multimodal = True
                 return self
+        if self.context and any(re.search(pattern, c) for c in self.context):
+            self.multimodal = True
+            return self
         if self.turns:
             for turn in self.turns:
                 if re.search(pattern, turn.content) is not None:

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/dataset/utils.py

@@ -32,6 +32,7 @@ def strip_local_expectation_fields(golden_bodies: List[Dict[str, Any]]):
         for field in _LOCAL_ONLY_EXPECTATION_FIELDS:
             expectations.pop(field, None)
 
+
 # RetrievedContextData declares an @model_serializer, so a plain model_dump
 # flattens it and a save/load round-trip loses the source. Serialize each item
 # to a namespaced, parseable marker instead, and reconstruct it on load.

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/errors.py

@@ -67,8 +67,7 @@ class NoMetricsError(DeepEvalError):
     Raised by the ``evals_iterator`` executor when, after iteration completes,
     we can prove that no metrics were declared via:
       - ``evals_iterator(metrics=[...])`` (top-level / trace-level metrics)
-      - ``@observe(metrics=[...])`` or ``@observe(metric_collection=...)``
-        on any span (span-level metrics)
+      - ``@observe(metrics=[...])`` on any span (span-level metrics)
       - ``update_current_trace(metrics=[...])`` inside the traced function
         (trace-level metrics, set at runtime)
 

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/evaluate/evaluate.py

@@ -22,6 +22,7 @@ from deepeval.evaluate.utils import (
     validate_evaluate_inputs,
 )
 from deepeval.evaluate.console_report import EvaluationConsoleReport
+from deepeval.evaluate.local_store import save_results_locally
 from deepeval.dataset import Golden
 from deepeval.prompt import Prompt
 from deepeval.test_case.utils import (
@@ -346,12 +347,18 @@ def evaluate(
 
         if not metrics and not has_expectations:
             # Classifier-only run: classifications are local-only for now, so
-            # there is no test run to finalize or upload.
-            return EvaluationResult(
+            # there is no test run to finalize or upload; only save locally.
+            result = EvaluationResult(
                 test_results=test_results,
                 confident_link=None,
                 test_run_id=None,
             )
+            save_results_locally(
+                result,
+                results_folder=display_config.results_folder,
+                results_subfolder=display_config.results_subfolder,
+            )
+            return result
 
         global_test_run_manager.save_test_run(TEMP_FILE_PATH)
 

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/evaluate/execute/_common.py

@@ -15,7 +15,6 @@ from deepeval.tracing.tracing import (
     trace_manager,
     Trace,
     BaseSpan,
-    LlmSpan,
 )
 from deepeval.errors import MissingTestCaseParamsError
 from deepeval.utils import (
@@ -35,19 +34,9 @@ from deepeval.test_case import (
     LLMTestCase,
     ConversationalTestCase,
 )
-from deepeval.test_run import (
-    global_test_run_manager,
-    TestRunManager,
-)
 from deepeval.evaluate.types import TestResult
 from deepeval.tracing.types import TraceSpanStatus
 from deepeval.config.settings import get_settings
-from deepeval.test_run import TEMP_FILE_PATH
-from deepeval.confident.api import is_confident
-from deepeval.test_run.hyperparameters import (
-    process_hyperparameters,
-    process_prompts,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -116,6 +105,15 @@ def _skip_metrics_for_error(
     return False
 
 
+def _attach_span_tree(
+    test_case: LLMTestCase, trace: Trace, span: Optional[BaseSpan]
+) -> None:
+    """Let metrics that look past their own span (e.g. ArgumentCorrectness)
+    reach the rest of the trace. `span` is None for a trace-level test case."""
+    test_case._trace_root_spans = trace.root_spans
+    test_case._trace_span = span
+
+
 def _trace_error(current_trace: Trace) -> Optional[str]:
     def _first_err(s: BaseSpan) -> Optional[str]:
         if s.status == TraceSpanStatus.ERRORED and s.error:
@@ -317,36 +315,3 @@ def _execute_classifier(
             classifier.label = None
         else:
             raise
-
-
-def log_prompt(
-    llm_span: LlmSpan,
-    test_run_manager: TestRunManager,
-):
-    prompt = llm_span.prompt
-    if prompt is None:
-        return
-
-    span_hyperparameters = {}
-    prompt_hash = prompt.hash if is_confident() else None
-    key = f"{prompt.alias}_{prompt_hash}"
-    span_hyperparameters[key] = prompt
-
-    test_run = test_run_manager.get_test_run()
-    if test_run.prompts is None:
-        test_run.prompts = []
-    if test_run.hyperparameters is None:
-        test_run.hyperparameters = {}
-
-    if key not in test_run.hyperparameters:
-        test_run.hyperparameters.update(
-            process_hyperparameters(span_hyperparameters, False)
-        )
-        existing_prompt_keys = {f"{p.alias}_{p.hash}" for p in test_run.prompts}
-        new_prompts = process_prompts(span_hyperparameters)
-        for new_prompt in new_prompts:
-            new_prompt_key = f"{new_prompt.alias}_{new_prompt.hash}"
-            if new_prompt_key not in existing_prompt_keys:
-                test_run.prompts.append(new_prompt)
-
-    global_test_run_manager.save_test_run(TEMP_FILE_PATH)

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/evaluate/execute/agentic.py

@@ -65,10 +65,10 @@ logger = logging.getLogger(__name__)
 
 
 from deepeval.evaluate.execute._common import (
+    _attach_span_tree,
     _skip_metrics_for_error,
     _trace_error,
     filter_duplicate_results,
-    log_prompt,
 )
 
 
@@ -354,7 +354,6 @@ async def _a_execute_span_test_case(
         trace_api.agent_spans.append(api_span)
     elif isinstance(span, LlmSpan):
         trace_api.llm_spans.append(api_span)
-        log_prompt(span, test_run_manager)
     elif isinstance(span, RetrieverSpan):
         trace_api.retriever_spans.append(api_span)
     elif isinstance(span, ToolSpan):
@@ -416,6 +415,7 @@ async def _a_execute_span_test_case(
         if test_case is None:
             test_case = LLMTestCase(input="None")
         test_case._trace_dict = trace_manager.create_nested_spans_dict(span)
+    _attach_span_tree(test_case, current_trace, span)
 
     for metric in metrics:
         metric.skipped = False
@@ -524,6 +524,7 @@ async def _a_execute_trace_test_case(
         test_case._trace_dict = trace_manager.create_nested_spans_dict(
             trace.root_spans[0]
         )
+    _attach_span_tree(test_case, trace, None)
 
     for metric in metrics:
         metric.skipped = False

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/evaluate/execute/loop.py

@@ -35,6 +35,7 @@ from deepeval.tracing.tracing import (
     ToolSpan,
 )
 from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.otel.setup import start_automatic_tracing
 from deepeval.tracing.api import (
     BaseApiSpan,
 )
@@ -85,6 +86,7 @@ logger = logging.getLogger(__name__)
 
 
 from deepeval.evaluate.execute._common import (
+    _attach_span_tree,
     _await_with_outer_deadline,
     _execute_metric,
     _log_gather_timeout,
@@ -94,7 +96,6 @@ from deepeval.evaluate.execute._common import (
     _snapshot_tasks,
     _trace_error,
     filter_duplicate_results,
-    log_prompt,
 )
 from deepeval.evaluate.execute.agentic import (
     _a_execute_agentic_test_case,
@@ -251,7 +252,6 @@ def execute_agentic_test_cases_from_loop(
                     trace_api.agent_spans.append(api_span)
                 elif isinstance(span, LlmSpan):
                     trace_api.llm_spans.append(api_span)
-                    log_prompt(span, test_run_manager)
                 elif isinstance(span, RetrieverSpan):
                     trace_api.retriever_spans.append(api_span)
                 elif isinstance(span, ToolSpan):
@@ -319,6 +319,7 @@ def execute_agentic_test_cases_from_loop(
                                 advance=count_metrics_in_span_subtree(span),
                             )
                         return
+                _attach_span_tree(llm_test_case, current_trace, span)
 
                 # Preparing metric calculation
                 api_span.metrics_data = []
@@ -411,6 +412,7 @@ def execute_agentic_test_cases_from_loop(
                             current_trace.root_spans[0]
                         )
                     )
+                _attach_span_tree(llm_test_case, current_trace, None)
 
                 if not skip_metrics_for_this_golden:
                     for metric in current_trace.metrics:
@@ -476,6 +478,7 @@ def execute_agentic_test_cases_from_loop(
         ):
             _raise_no_metrics_error()
 
+    stop_automatic_tracing = start_automatic_tracing()
     try:
         if display_config.show_indicator and _use_bar_indicator:
             progress = Progress(
@@ -499,6 +502,7 @@ def execute_agentic_test_cases_from_loop(
     except Exception:
         raise
     finally:
+        stop_automatic_tracing()
         # Atomic exit cleanup: replacing the session resets mode + every
         # per-run collection in a single assignment, so state can't leak
         # into the next run.
@@ -913,6 +917,7 @@ def a_execute_agentic_test_cases_from_loop(
                 )
             )
 
+    stop_automatic_tracing = start_automatic_tracing()
     try:
         if display_config.show_indicator and _use_bar_indicator:
             progress = Progress(
@@ -943,6 +948,7 @@ def a_execute_agentic_test_cases_from_loop(
     except Exception:
         raise
     finally:
+        stop_automatic_tracing()
         # Atomic exit cleanup: replacing the session resets mode + every
         # per-run collection in a single assignment.
         local_trace_manager.eval_session = EvalSession()

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/evaluate/execute/trace_scope.py

@@ -53,10 +53,10 @@ logger = logging.getLogger(__name__)
 
 
 from deepeval.evaluate.execute._common import (
+    _attach_span_tree,
     _execute_metric,
     _skip_metrics_for_error,
     _trace_error,
-    log_prompt,
 )
 
 
@@ -132,6 +132,8 @@ def _assert_test_from_current_trace(
             is_promoted_root = True
         else:
             root_for_dfs = root
+            # A root still parked under the open wrapper points at it too.
+            is_promoted_root = root.parent_uuid is not None
 
     effective_trace_output = (
         current_trace.output
@@ -187,7 +189,6 @@ def _assert_test_from_current_trace(
             trace_api.agent_spans.append(api_span)
         elif isinstance(span, LlmSpan):
             trace_api.llm_spans.append(api_span)
-            log_prompt(span, test_run_manager)
         elif isinstance(span, RetrieverSpan):
             trace_api.retriever_spans.append(api_span)
         elif isinstance(span, ToolSpan):
@@ -239,6 +240,7 @@ def _assert_test_from_current_trace(
                 )
             )
             return
+        _attach_span_tree(llm_test_case, current_trace, span)
 
         api_span.metrics_data = []
         for metric in metrics:
@@ -301,6 +303,7 @@ def _assert_test_from_current_trace(
             llm_test_case_for_trace._trace_dict = (
                 trace_manager.create_nested_spans_dict(root_for_dfs)
             )
+        _attach_span_tree(llm_test_case_for_trace, current_trace, None)
 
         trace_api.metrics_data = []
         for metric in current_trace.metrics:

{deepeval-4.3.2 → deepeval-4.3.4}/deepeval/evaluate/expectations.py

@@ -176,8 +176,7 @@ class _ExpectationEvaluator:
             for condition, verdict in zip(conditions, self.verdicts)
         ]
         return (
-            _REASON_INSTRUCTIONS
-            + f"Score:\n{self.score}\n\n"
+            _REASON_INSTRUCTIONS + f"Score:\n{self.score}\n\n"
             f"Requirements:\n{json.dumps(verdicts, ensure_ascii=False)}\n\n"
             "JSON:\n"
         )