deepeval 4.3.0__tar.gz → 4.3.2__tar.gz

{deepeval-4.3.0 → deepeval-4.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepeval
-Version: 4.3.0
+Version: 4.3.2
 Summary: The LLM Evaluation Framework
 License: Apache-2.0
 License-File: LICENSE.md
@@ -114,18 +114,18 @@ Description-Content-Type: text/markdown
 
 **DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating large-language model systems. It is similar to Pytest but specialized for unit testing LLM apps. DeepEval incorporates the latest research to run evals via metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which use LLM-as-a-judge and other NLP models that run **locally on your machine**.
 
-Whether you're building AI agents, RAG pipelines, or chatbots, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
+Whether you're building AI agents, multi-turn chatbots, voice agents, RAG pipelines, or anything else, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
 
 - **LLM apps end-to-end** as black boxes
-- **Complete agent trajectories** across every decision and action
-- **Individual agent steps** such as LLM calls, tool use, retrieval, and sub-agent handoffs
+- **Complete AI agent trajectories** across every decision and action
+- **Individual AI agent steps** such as LLM calls, tool use, retrieval, and sub-agent handoffs
 
 Use these evaluations to determine the optimal models, prompts, and architecture to improve your AI quality, prevent prompt drifting, or even transition from OpenAI to Claude with confidence.
 
 > Jev is now available in DeepEval, both in existing metrics and the latest [JevEval.](https://deepeval.com/docs/metrics-jev-eval)
 
 > [!IMPORTANT]
-> Want to compare iterations, share evaluation reports, and monitor your AI in production? [Sign up for Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=signup_callout&ref_page=github/readme), the enterprise AI evals and observability platform.
+> If you're already in production, [Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=signup_callout&ref_page=github/readme) surfaces failing traces before anyone has written an eval for them, clusters them into failure modes, and turns them into your next eval.
 >
 > ![Demo GIF](assets/demo.gif)
 
@@ -142,9 +142,9 @@ Use these evaluations to determine the optimal models, prompts, and architecture
     - [DAG](https://deepeval.com/docs/metrics-dag) — DeepEval's graph-based deterministic LLM-as-a-judge metric builder
 
   - <details>
-    <summary><b>Agentic Metrics</b></summary>
+    <summary><b>AI Agent Metrics</b></summary>
 
-    - [Task Completion](https://deepeval.com/docs/metrics-task-completion) — evaluate whether an agent accomplished its goal
+    - [Task Completion](https://deepeval.com/docs/metrics-task-completion) — evaluate whether an AI agent accomplished its goal
     - [Tool Correctness](https://deepeval.com/docs/metrics-tool-correctness) — check if the right tools were called with the right arguments
     - [Goal Accuracy](https://deepeval.com/docs/metrics-goal-accuracy) — measure how accurately the agent achieved the intended goal
     - [Step Efficiency](https://deepeval.com/docs/metrics-step-efficiency) — evaluate whether the agent took unnecessary steps
@@ -155,18 +155,6 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
-  - <details>
-    <summary><b>RAG Metrics</b></summary>
-
-    - [Answer Relevancy](https://deepeval.com/docs/metrics-answer-relevancy) — measure how relevant the RAG pipeline's output is to the input
-    - [Faithfulness](https://deepeval.com/docs/metrics-faithfulness) — evaluate whether the RAG pipeline's output factually aligns with the retrieval context
-    - [Contextual Recall](https://deepeval.com/docs/metrics-contextual-recall) — measure how well the RAG pipeline's retrieval context aligns with the expected output
-    - [Contextual Precision](https://deepeval.com/docs/metrics-contextual-precision) — evaluate whether relevant nodes in the RAG pipeline's retrieval context are ranked higher
-    - [Contextual Relevancy](https://deepeval.com/docs/metrics-contextual-relevancy) — measure the overall relevance of the RAG pipeline's retrieval context to the input
-    - [RAGAS](https://deepeval.com/docs/metrics-ragas) — average of answer relevancy, faithfulness, contextual precision, and contextual recall
-
-    </details>
-
   - <details>
     <summary><b>Multi-Turn Metrics</b></summary>
 
@@ -178,6 +166,29 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
+  - <details>
+    <summary><b>Voice Agent Metrics</b></summary>
+
+    - [Agent Responsiveness](https://deepeval.com/docs/metrics-agent-responsiveness) — check whether the voice agent answered when it was spoken to
+    - [Turn-Taking Naturalness](https://deepeval.com/docs/metrics-turn-taking-naturalness) — measure response latency and interruptions across the call
+    - [Voice Naturalness](https://deepeval.com/docs/metrics-voice-naturalness) — evaluate how natural the voice agent sounds
+    - [Transcription Accuracy](https://deepeval.com/docs/metrics-transcription-accuracy) — check whether the voice agent's speech-to-text heard the caller correctly
+    - [Audio Glitch](https://deepeval.com/docs/metrics-audio-glitch) — detect audio artifacts in the voice agent's speech
+
+    </details>
+
+  - <details>
+    <summary><b>RAG Metrics</b></summary>
+
+    - [Answer Relevancy](https://deepeval.com/docs/metrics-answer-relevancy) — measure how relevant the output is to the input
+    - [Faithfulness](https://deepeval.com/docs/metrics-faithfulness) — evaluate whether the RAG pipeline's output factually aligns with the retrieval context
+    - [Contextual Recall](https://deepeval.com/docs/metrics-contextual-recall) — measure how well the RAG pipeline's retrieval context aligns with the expected output
+    - [Contextual Precision](https://deepeval.com/docs/metrics-contextual-precision) — evaluate whether relevant nodes in the RAG pipeline's retrieval context are ranked higher
+    - [Contextual Relevancy](https://deepeval.com/docs/metrics-contextual-relevancy) — measure the overall relevance of the RAG pipeline's retrieval context to the input
+    - [RAGAS](https://deepeval.com/docs/metrics-ragas) — average of answer relevancy, faithfulness, contextual precision, and contextual recall
+
+    </details>
+
   - <details>
     <summary><b>MCP Metrics</b></summary>
 
@@ -221,7 +232,7 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
 # 🔌 Integrations
 
-DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. For enterprise teams standardizing evals and observability across the organization, **Confident AI** provides a native DeepEval integration.
+DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. The same traces can go to **Confident AI** once you're live, where evals run on every production trace.
 
 ## Frameworks
 
@@ -236,15 +247,16 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 - [Mastra](https://www.deepeval.com/integrations/frameworks/mastra?utm_source=GitHub) — evaluate Mastra agents and workflows with native tracing
 - [OpenAI](https://www.deepeval.com/integrations/frameworks/openai?utm_source=GitHub) — evaluate and trace OpenAI applications via a client wrapper
 - [OpenAI Agents](https://www.deepeval.com/integrations/frameworks/openai-agents?utm_source=GitHub) — evaluate OpenAI Agents end-to-end in under a minute
-- [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate RAG applications built with LlamaIndex
+- [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate AI agents and RAG applications built with LlamaIndex
 
 ## ☁️ Platform + Ecosystem
 
-[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section&ref_page=github/readme) is the enterprise AI evals and observability platform. It gives organizations one consistent standard across product teams, integrates natively with DeepEval, and remains model- and framework-agnostic.
+[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section&ref_page=github/readme) is AI monitoring for applications in production. DeepEval tests what you know to test before you ship; Confident AI finds what you didn't, and it's model- and framework-agnostic.
 
-- Product teams manage datasets, evaluate AI applications before launch, and monitor live traces with online evals and signals in production.
-- Platform teams define one organization-wide quality standard and enforce it through governance and native red teaming.
-- Don't need a UI? Use Confident AI as your persistence layer to run evals, pull datasets, and inspect traces from Claude Code or Cursor through Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
+- Failing traces surface before anyone has written an eval for them, clustered into failure modes, with user sentiment and behavior across traces.
+- Failing traces route to the right annotators and become goldens, so production issues turn into your next eval.
+- For enterprise teams, one system for AI quality: governance as release gates and native red teaming.
+- Prefer to stay in your editor? Inspect failing traces and pull the datasets built from them in Claude Code or Cursor through Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
 
 <p align="center">
   <img src="assets/confident-mcp-architecture.png" alt="Confident AI MCP Architecture" width="500">
@@ -254,7 +266,7 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 
 # 🤖 Vibe-Coder QuickStart
 
-Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your agent, RAG pipeline, or chatbot, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
+Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your AI agent, multi-turn chatbot, voice agent, RAG pipeline, or any other LLM app, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
 
 [Start with the 5-minute vibe-coder guide](https://deepeval.com/docs/vibe-coder-quickstart?utm_source=GitHub).
 
@@ -262,7 +274,7 @@ Want your coding agent to add evals and fix failures for you? Install the DeepEv
 
 # 🚀 Human QuickStart
 
-Let's pretend your LLM application is a RAG based customer support chatbot; here's how DeepEval can help test what you've built.
+Let's pretend your LLM application is a customer support AI agent; here's how DeepEval can help test what you've built.
 
 ## Installation
 
@@ -289,10 +301,10 @@ Follow the instructions in the CLI to create an account, copy your API key, and
 Create a test file:
 
 ```bash
-touch test_chatbot.py
+touch test_ai_agent.py
 ```
 
-Open `test_chatbot.py` and write your first test case to run an **end-to-end** evaluation using DeepEval, which treats your LLM app as a black-box:
+Open `test_ai_agent.py` and write your first test case to run an **end-to-end** evaluation using DeepEval, which treats your LLM app as a black-box:
 
 ```python
 import pytest
@@ -312,7 +324,6 @@ def test_case():
         # Replace this with the actual output from your LLM application
         actual_output="You have 30 days to get a full refund at no extra cost.",
         expected_output="We offer a 30-day full refund at no extra costs.",
-        retrieval_context=["All customers are eligible for a 30 day full refund at no extra costs."]
     )
     assert_test(test_case, [correctness_metric])
 ```
@@ -323,10 +334,10 @@ Set your `OPENAI_API_KEY` as an environment variable (you can also evaluate usin
 export OPENAI_API_KEY="..."
 ```
 
-And finally, run `test_chatbot.py` in the CLI:
+And finally, run `test_ai_agent.py` in the CLI:
 
 ```
-deepeval test run test_chatbot.py
+deepeval test run test_ai_agent.py
 ```
 
 **Congratulations! Your test case should have passed ✅** Let's break down what happened.
@@ -342,7 +353,7 @@ deepeval test run test_chatbot.py
 
 ## Evals With Full Traceability
 
-Use `evals_iterator()` to run the same dataset through your app, whether you instrument it manually or through one of DeepEval's framework integrations. Because tracing captures the ordered sequence of model decisions, tool calls, and intermediate steps, you can run [trajectory-based evaluations](https://deepeval.com/docs/evaluation-trajectory-based-llm-evals?utm_source=GitHub) over the complete path your agent takes.
+Use `evals_iterator()` to run the same dataset through your app, whether you instrument it manually or through one of DeepEval's framework integrations. Because tracing captures the ordered sequence of model decisions, tool calls, and intermediate steps, you can run [trajectory-based evaluations](https://deepeval.com/docs/evaluation-trajectory-based-llm-evals?utm_source=GitHub) over the complete path your AI agent takes.
 
 Here's an example of manual instrumentation:
 
@@ -659,7 +670,7 @@ print(answer_relevancy_metric.score)
 print(answer_relevancy_metric.reason)
 ```
 
-Note that some metrics are for RAG pipelines, while others are for fine-tuning. Make sure to use our docs to pick the right one for your use case.
+Note that some metrics are built for specific use cases, such as AI agents, multi-turn chatbots, voice agents, RAG, and more. Make sure to use our docs to pick the right one for your use case.
 
 ## A Note on Env Variables (.env / .env.local)
 
@@ -674,23 +685,19 @@ cp .env.example .env.local
 
 # DeepEval With Confident AI
 
-[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=cli_login_section&ref_page=github/readme) — the AI observability and evaluation platform for production LLM systems. Trace agent executions, run online evals on live traffic, alert on quality regressions, and let PMs, QA, and domain experts review outputs without writing code. DeepEval traces stream to it with no code changes.
+[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=cli_login_section&ref_page=github/readme) monitors AI applications in production. Evals run on every production trace, failing traces are clustered into failure modes, and each one can be routed to the right annotators — engineers, QA, domain experts or PMs — and turned into a golden for your next eval. The traces your app already produces with DeepEval stream to it with no code changes.
 
-Log in from the CLI to get started:
+Log in from the CLI to connect:
 
 ```bash
 deepeval login
 ```
 
-Then run your tests as usual — results are automatically synced to the platform:
-
-```bash
-deepeval test run test_chatbot.py
-```
+Test runs also appear on the platform once you're logged in.
 
 ![Demo GIF](assets/demo.gif)
 
-Prefer to stay in your IDE? Use DeepEval via [Confident AI's MCP server](https://github.com/confident-ai/confident-mcp-server) as the persistent layer to run evals, pull datasets, and inspect traces without leaving your editor.
+Prefer to stay in your IDE? Inspect failing traces and pull the datasets built from them through [Confident AI's MCP server](https://github.com/confident-ai/confident-mcp-server) without leaving your editor.
 
 <p align="center">
   <img src="assets/confident-mcp-architecture.png" alt="Confident AI MCP Architecture" width="500">

{deepeval-4.3.0 → deepeval-4.3.2}/README.md

@@ -61,18 +61,18 @@
 
 **DeepEval** is a simple-to-use, open-source LLM evaluation framework, for evaluating large-language model systems. It is similar to Pytest but specialized for unit testing LLM apps. DeepEval incorporates the latest research to run evals via metrics such as G-Eval, task completion, answer relevancy, hallucination, etc., which use LLM-as-a-judge and other NLP models that run **locally on your machine**.
 
-Whether you're building AI agents, RAG pipelines, or chatbots, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
+Whether you're building AI agents, multi-turn chatbots, voice agents, RAG pipelines, or anything else, implemented via LangChain or OpenAI, DeepEval has you covered. With it, you can easily evaluate:
 
 - **LLM apps end-to-end** as black boxes
-- **Complete agent trajectories** across every decision and action
-- **Individual agent steps** such as LLM calls, tool use, retrieval, and sub-agent handoffs
+- **Complete AI agent trajectories** across every decision and action
+- **Individual AI agent steps** such as LLM calls, tool use, retrieval, and sub-agent handoffs
 
 Use these evaluations to determine the optimal models, prompts, and architecture to improve your AI quality, prevent prompt drifting, or even transition from OpenAI to Claude with confidence.
 
 > Jev is now available in DeepEval, both in existing metrics and the latest [JevEval.](https://deepeval.com/docs/metrics-jev-eval)
 
 > [!IMPORTANT]
-> Want to compare iterations, share evaluation reports, and monitor your AI in production? [Sign up for Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=signup_callout&ref_page=github/readme), the enterprise AI evals and observability platform.
+> If you're already in production, [Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=signup_callout&ref_page=github/readme) surfaces failing traces before anyone has written an eval for them, clusters them into failure modes, and turns them into your next eval.
 >
 > ![Demo GIF](assets/demo.gif)
 
@@ -89,9 +89,9 @@ Use these evaluations to determine the optimal models, prompts, and architecture
     - [DAG](https://deepeval.com/docs/metrics-dag) — DeepEval's graph-based deterministic LLM-as-a-judge metric builder
 
   - <details>
-    <summary><b>Agentic Metrics</b></summary>
+    <summary><b>AI Agent Metrics</b></summary>
 
-    - [Task Completion](https://deepeval.com/docs/metrics-task-completion) — evaluate whether an agent accomplished its goal
+    - [Task Completion](https://deepeval.com/docs/metrics-task-completion) — evaluate whether an AI agent accomplished its goal
     - [Tool Correctness](https://deepeval.com/docs/metrics-tool-correctness) — check if the right tools were called with the right arguments
     - [Goal Accuracy](https://deepeval.com/docs/metrics-goal-accuracy) — measure how accurately the agent achieved the intended goal
     - [Step Efficiency](https://deepeval.com/docs/metrics-step-efficiency) — evaluate whether the agent took unnecessary steps
@@ -102,18 +102,6 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
-  - <details>
-    <summary><b>RAG Metrics</b></summary>
-
-    - [Answer Relevancy](https://deepeval.com/docs/metrics-answer-relevancy) — measure how relevant the RAG pipeline's output is to the input
-    - [Faithfulness](https://deepeval.com/docs/metrics-faithfulness) — evaluate whether the RAG pipeline's output factually aligns with the retrieval context
-    - [Contextual Recall](https://deepeval.com/docs/metrics-contextual-recall) — measure how well the RAG pipeline's retrieval context aligns with the expected output
-    - [Contextual Precision](https://deepeval.com/docs/metrics-contextual-precision) — evaluate whether relevant nodes in the RAG pipeline's retrieval context are ranked higher
-    - [Contextual Relevancy](https://deepeval.com/docs/metrics-contextual-relevancy) — measure the overall relevance of the RAG pipeline's retrieval context to the input
-    - [RAGAS](https://deepeval.com/docs/metrics-ragas) — average of answer relevancy, faithfulness, contextual precision, and contextual recall
-
-    </details>
-
   - <details>
     <summary><b>Multi-Turn Metrics</b></summary>
 
@@ -125,6 +113,29 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
     </details>
 
+  - <details>
+    <summary><b>Voice Agent Metrics</b></summary>
+
+    - [Agent Responsiveness](https://deepeval.com/docs/metrics-agent-responsiveness) — check whether the voice agent answered when it was spoken to
+    - [Turn-Taking Naturalness](https://deepeval.com/docs/metrics-turn-taking-naturalness) — measure response latency and interruptions across the call
+    - [Voice Naturalness](https://deepeval.com/docs/metrics-voice-naturalness) — evaluate how natural the voice agent sounds
+    - [Transcription Accuracy](https://deepeval.com/docs/metrics-transcription-accuracy) — check whether the voice agent's speech-to-text heard the caller correctly
+    - [Audio Glitch](https://deepeval.com/docs/metrics-audio-glitch) — detect audio artifacts in the voice agent's speech
+
+    </details>
+
+  - <details>
+    <summary><b>RAG Metrics</b></summary>
+
+    - [Answer Relevancy](https://deepeval.com/docs/metrics-answer-relevancy) — measure how relevant the output is to the input
+    - [Faithfulness](https://deepeval.com/docs/metrics-faithfulness) — evaluate whether the RAG pipeline's output factually aligns with the retrieval context
+    - [Contextual Recall](https://deepeval.com/docs/metrics-contextual-recall) — measure how well the RAG pipeline's retrieval context aligns with the expected output
+    - [Contextual Precision](https://deepeval.com/docs/metrics-contextual-precision) — evaluate whether relevant nodes in the RAG pipeline's retrieval context are ranked higher
+    - [Contextual Relevancy](https://deepeval.com/docs/metrics-contextual-relevancy) — measure the overall relevance of the RAG pipeline's retrieval context to the input
+    - [RAGAS](https://deepeval.com/docs/metrics-ragas) — average of answer relevancy, faithfulness, contextual precision, and contextual recall
+
+    </details>
+
   - <details>
     <summary><b>MCP Metrics</b></summary>
 
@@ -168,7 +179,7 @@ Use these evaluations to determine the optimal models, prompts, and architecture
 
 # 🔌 Integrations
 
-DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. For enterprise teams standardizing evals and observability across the organization, **Confident AI** provides a native DeepEval integration.
+DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and more. The same traces can go to **Confident AI** once you're live, where evals run on every production trace.
 
 ## Frameworks
 
@@ -183,15 +194,16 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 - [Mastra](https://www.deepeval.com/integrations/frameworks/mastra?utm_source=GitHub) — evaluate Mastra agents and workflows with native tracing
 - [OpenAI](https://www.deepeval.com/integrations/frameworks/openai?utm_source=GitHub) — evaluate and trace OpenAI applications via a client wrapper
 - [OpenAI Agents](https://www.deepeval.com/integrations/frameworks/openai-agents?utm_source=GitHub) — evaluate OpenAI Agents end-to-end in under a minute
-- [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate RAG applications built with LlamaIndex
+- [LlamaIndex](https://www.deepeval.com/integrations/frameworks/llamaindex?utm_source=GitHub) — evaluate AI agents and RAG applications built with LlamaIndex
 
 ## ☁️ Platform + Ecosystem
 
-[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section&ref_page=github/readme) is the enterprise AI evals and observability platform. It gives organizations one consistent standard across product teams, integrates natively with DeepEval, and remains model- and framework-agnostic.
+[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=platform_section&ref_page=github/readme) is AI monitoring for applications in production. DeepEval tests what you know to test before you ship; Confident AI finds what you didn't, and it's model- and framework-agnostic.
 
-- Product teams manage datasets, evaluate AI applications before launch, and monitor live traces with online evals and signals in production.
-- Platform teams define one organization-wide quality standard and enforce it through governance and native red teaming.
-- Don't need a UI? Use Confident AI as your persistence layer to run evals, pull datasets, and inspect traces from Claude Code or Cursor through Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
+- Failing traces surface before anyone has written an eval for them, clustered into failure modes, with user sentiment and behavior across traces.
+- Failing traces route to the right annotators and become goldens, so production issues turn into your next eval.
+- For enterprise teams, one system for AI quality: governance as release gates and native red teaming.
+- Prefer to stay in your editor? Inspect failing traces and pull the datasets built from them in Claude Code or Cursor through Confident AI's [MCP server](https://github.com/confident-ai/confident-mcp-server).
 
 <p align="center">
   <img src="assets/confident-mcp-architecture.png" alt="Confident AI MCP Architecture" width="500">
@@ -201,7 +213,7 @@ DeepEval plugs into any LLM framework — OpenAI Agents, LangChain, CrewAI, and
 
 # 🤖 Vibe-Coder QuickStart
 
-Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your agent, RAG pipeline, or chatbot, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
+Want your coding agent to add evals and fix failures for you? Install the DeepEval skill, point it at your AI agent, multi-turn chatbot, voice agent, RAG pipeline, or any other LLM app, and ask it to generate a dataset, write the eval suite, run `deepeval test run`, and iterate on the failing metrics.
 
 [Start with the 5-minute vibe-coder guide](https://deepeval.com/docs/vibe-coder-quickstart?utm_source=GitHub).
 
@@ -209,7 +221,7 @@ Want your coding agent to add evals and fix failures for you? Install the DeepEv
 
 # 🚀 Human QuickStart
 
-Let's pretend your LLM application is a RAG based customer support chatbot; here's how DeepEval can help test what you've built.
+Let's pretend your LLM application is a customer support AI agent; here's how DeepEval can help test what you've built.
 
 ## Installation
 
@@ -236,10 +248,10 @@ Follow the instructions in the CLI to create an account, copy your API key, and
 Create a test file:
 
 ```bash
-touch test_chatbot.py
+touch test_ai_agent.py
 ```
 
-Open `test_chatbot.py` and write your first test case to run an **end-to-end** evaluation using DeepEval, which treats your LLM app as a black-box:
+Open `test_ai_agent.py` and write your first test case to run an **end-to-end** evaluation using DeepEval, which treats your LLM app as a black-box:
 
 ```python
 import pytest
@@ -259,7 +271,6 @@ def test_case():
         # Replace this with the actual output from your LLM application
         actual_output="You have 30 days to get a full refund at no extra cost.",
         expected_output="We offer a 30-day full refund at no extra costs.",
-        retrieval_context=["All customers are eligible for a 30 day full refund at no extra costs."]
     )
     assert_test(test_case, [correctness_metric])
 ```
@@ -270,10 +281,10 @@ Set your `OPENAI_API_KEY` as an environment variable (you can also evaluate usin
 export OPENAI_API_KEY="..."
 ```
 
-And finally, run `test_chatbot.py` in the CLI:
+And finally, run `test_ai_agent.py` in the CLI:
 
 ```
-deepeval test run test_chatbot.py
+deepeval test run test_ai_agent.py
 ```
 
 **Congratulations! Your test case should have passed ✅** Let's break down what happened.
@@ -289,7 +300,7 @@ deepeval test run test_chatbot.py
 
 ## Evals With Full Traceability
 
-Use `evals_iterator()` to run the same dataset through your app, whether you instrument it manually or through one of DeepEval's framework integrations. Because tracing captures the ordered sequence of model decisions, tool calls, and intermediate steps, you can run [trajectory-based evaluations](https://deepeval.com/docs/evaluation-trajectory-based-llm-evals?utm_source=GitHub) over the complete path your agent takes.
+Use `evals_iterator()` to run the same dataset through your app, whether you instrument it manually or through one of DeepEval's framework integrations. Because tracing captures the ordered sequence of model decisions, tool calls, and intermediate steps, you can run [trajectory-based evaluations](https://deepeval.com/docs/evaluation-trajectory-based-llm-evals?utm_source=GitHub) over the complete path your AI agent takes.
 
 Here's an example of manual instrumentation:
 
@@ -606,7 +617,7 @@ print(answer_relevancy_metric.score)
 print(answer_relevancy_metric.reason)
 ```
 
-Note that some metrics are for RAG pipelines, while others are for fine-tuning. Make sure to use our docs to pick the right one for your use case.
+Note that some metrics are built for specific use cases, such as AI agents, multi-turn chatbots, voice agents, RAG, and more. Make sure to use our docs to pick the right one for your use case.
 
 ## A Note on Env Variables (.env / .env.local)
 
@@ -621,23 +632,19 @@ cp .env.example .env.local
 
 # DeepEval With Confident AI
 
-[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=cli_login_section&ref_page=github/readme) — the AI observability and evaluation platform for production LLM systems. Trace agent executions, run online evals on live traffic, alert on quality regressions, and let PMs, QA, and domain experts review outputs without writing code. DeepEval traces stream to it with no code changes.
+[Confident AI](https://www.confident-ai.com?utm_source=deepeval&utm_medium=github&utm_content=cli_login_section&ref_page=github/readme) monitors AI applications in production. Evals run on every production trace, failing traces are clustered into failure modes, and each one can be routed to the right annotators — engineers, QA, domain experts or PMs — and turned into a golden for your next eval. The traces your app already produces with DeepEval stream to it with no code changes.
 
-Log in from the CLI to get started:
+Log in from the CLI to connect:
 
 ```bash
 deepeval login
 ```
 
-Then run your tests as usual — results are automatically synced to the platform:
-
-```bash
-deepeval test run test_chatbot.py
-```
+Test runs also appear on the platform once you're logged in.
 
 ![Demo GIF](assets/demo.gif)
 
-Prefer to stay in your IDE? Use DeepEval via [Confident AI's MCP server](https://github.com/confident-ai/confident-mcp-server) as the persistent layer to run evals, pull datasets, and inspect traces without leaving your editor.
+Prefer to stay in your IDE? Inspect failing traces and pull the datasets built from them through [Confident AI's MCP server](https://github.com/confident-ai/confident-mcp-server) without leaving your editor.
 
 <p align="center">
   <img src="assets/confident-mcp-architecture.png" alt="Confident AI MCP Architecture" width="500">

deepeval-4.3.2/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "4.3.2"

{deepeval-4.3.0 → deepeval-4.3.2}/deepeval/cli/auth/command.py

@@ -67,6 +67,7 @@ API_KEY_PATTERN = re.compile(
     r"[A-Za-z0-9+/]+={0,2}$",
     re.IGNORECASE,
 )
+MIN_API_KEY_LENGTH = 20
 
 
 def _prompt_and_persist_region(save: Optional[str]) -> None:
@@ -92,6 +93,15 @@ def _print_api_key_location() -> None:
     )
 
 
+def _get_pasted_api_key_error(api_key: str) -> Optional[str]:
+    if len(api_key) < MIN_API_KEY_LENGTH or any(c.isspace() for c in api_key):
+        return (
+            "That doesn't look like a Confident AI API key. Copy the full "
+            "key from Project Settings > API Keys."
+        )
+    return None
+
+
 def _get_pasted_api_key_warnings(
     api_key: str, configured_region: Optional[str]
 ) -> List[str]:
@@ -132,10 +142,15 @@ def _prompt_paste_api_key() -> str:
         api_key = coerce_blank_to_none(
             typer.prompt("🔐 Enter your project API key", hide_input=True)
         )
-        if api_key:
-            _warn_for_pasted_api_key(api_key)
-            return api_key
-        print("❌ Project API key cannot be empty. Please try again.\n")
+        if not api_key:
+            print("❌ Project API key cannot be empty. Please try again.\n")
+            continue
+        error = _get_pasted_api_key_error(api_key)
+        if error:
+            print(f"❌ {error}\n")
+            continue
+        _warn_for_pasted_api_key(api_key)
+        return api_key
 
 
 def _open_platform_and_prompt_api_key() -> str:
@@ -359,6 +374,9 @@ def login_command(
             if api_key is not None:
                 span.set_method(LoginMethod.API_KEY_FLAG)
                 key = api_key
+                error = _get_pasted_api_key_error(key)
+                if error:
+                    raise AuthFlowError(error)
                 _warn_for_pasted_api_key(key)
             else:
                 key = _resolve_login_key(save, span)
@@ -396,10 +414,16 @@ def login_command(
                 "Follow our quickstart tutorial here: "
                 f"[bold][link={quickstart_url}]{quickstart_url}[/link][/bold]"
             )
+        except (typer.Abort, KeyboardInterrupt):
+            span.set_outcome(LoginOutcome.ABANDONED)
+            print("\nLogin cancelled. You are not logged in.")
         except Exception as e:
             span.set_outcome(LoginOutcome.FAILED)
             print(f"Login failed: {e}")
 
+    if span.outcome is not LoginOutcome.COMPLETED:
+        raise typer.Exit(code=1)
+
 
 def logout_command(
     save: Optional[str] = typer.Option(

{deepeval-4.3.0 → deepeval-4.3.2}/deepeval/cli/main.py

@@ -37,6 +37,11 @@ from deepeval.cli.auth import LOGIN_HELP, login_command, logout_command
 from deepeval.cli.diagnose import diagnose_command
 from deepeval.cli.generate.command import generate_command
 from deepeval.cli.inspect import inspect_command
+from deepeval.cli.setup import (
+    SETUP_CONTEXT_SETTINGS,
+    SETUP_HELP,
+    setup_command,
+)
 from deepeval.cli.test.command import app as test_app
 from deepeval.cli.utils import (
     handle_save_result as _handle_save_result,
@@ -58,6 +63,9 @@ app.command(name="inspect")(inspect_command)
 app.command(name="diagnose")(diagnose_command)
 app.command(name="login", help=LOGIN_HELP)(login_command)
 app.command(name="logout")(logout_command)
+app.command(
+    name="setup", help=SETUP_HELP, context_settings=SETUP_CONTEXT_SETTINGS
+)(setup_command)
 
 
 class Regions(Enum):

deepeval-4.3.2/deepeval/cli/setup/__init__.py

@@ -0,0 +1,11 @@
+from deepeval.cli.setup.command import (
+    SETUP_CONTEXT_SETTINGS,
+    SETUP_HELP,
+    setup_command,
+)
+
+__all__ = [
+    "SETUP_CONTEXT_SETTINGS",
+    "SETUP_HELP",
+    "setup_command",
+]

deepeval-4.3.2/deepeval/cli/setup/command.py

@@ -0,0 +1,36 @@
+"""`deepeval setup` Typer command."""
+
+import typer
+
+from deepeval.cli.setup.launcher import (
+    SetupLauncherError,
+    ensure_binary,
+    run_binary,
+)
+
+SETUP_HELP = (
+    "Set up an evaluation for this project with a coding agent, optionally "
+    "saving results to Confident AI. Downloads the latest setup wizard "
+    "(macOS and Linux), verifies its checksum, caches it, and runs it. Wizard options such as --project-dir and --proj-id are passed "
+    "through; run `deepeval setup -h` to list them."
+)
+
+# Click would otherwise reject the wizard's own flags as unknown options.
+SETUP_CONTEXT_SETTINGS = {
+    "allow_extra_args": True,
+    "ignore_unknown_options": True,
+}
+
+
+def _status(message: str) -> None:
+    typer.echo(message, err=True)
+
+
+def setup_command(ctx: typer.Context) -> None:
+    try:
+        binary = ensure_binary(notify=_status)
+        code = run_binary(binary, list(ctx.args))
+    except SetupLauncherError as error:
+        _status(str(error))
+        raise typer.Exit(1)
+    raise typer.Exit(code)