deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/test_run/test_run.py

@@ -32,6 +32,15 @@ from deepeval.utils import (
 )
 from deepeval.test_run.cache import global_test_run_cache_manager
 from deepeval.constants import CONFIDENT_TEST_CASE_BATCH_SIZE, HIDDEN_DIR
+from deepeval.prompt import (
+    PromptMessage,
+    ModelSettings,
+    PromptInterpolationType,
+    OutputType,
+)
+from rich.panel import Panel
+from rich.columns import Columns
+
 
 TEMP_FILE_PATH = f"{HIDDEN_DIR}/.temp_test_run_data.json"
 LATEST_TEST_RUN_FILE_PATH = f"{HIDDEN_DIR}/.latest_test_run.json"
@@ -71,6 +80,16 @@ class TraceMetricScores(BaseModel):
     base: Dict[str, Dict[str, MetricScores]] = Field(default_factory=dict)
 
 
+class PromptData(BaseModel):
+    alias: Optional[str] = None
+    version: Optional[str] = None
+    text_template: Optional[str] = None
+    messages_template: Optional[List[PromptMessage]] = None
+    model_settings: Optional[ModelSettings] = None
+    output_type: Optional[OutputType] = None
+    interpolation_type: Optional[PromptInterpolationType] = None
+
+
 class MetricsAverageDict:
     def __init__(self):
         self.metric_dict = {}
@@ -123,6 +142,7 @@ class TestRun(BaseModel):
     )
     identifier: Optional[str] = None
     hyperparameters: Optional[Dict[str, Any]] = Field(None)
+    prompts: Optional[List[PromptData]] = Field(None)
     test_passed: Optional[int] = Field(None, alias="testPassed")
     test_failed: Optional[int] = Field(None, alias="testFailed")
     run_duration: float = Field(0.0, alias="runDuration")
@@ -191,65 +211,91 @@ class TestRun(BaseModel):
         valid_scores = 0
 
         def process_metric_data(metric_data: MetricData):
+            """
+            Process and aggregate metric data for overall test metrics.
+
+            Args:
+                metric_data: The metric data to process
+            """
             nonlocal valid_scores
-            name = metric_data.name
+            metric_name = metric_data.name
             score = metric_data.score
             success = metric_data.success
-            # Initialize dict entry if needed.
-            if name not in metrics_dict:
-                metrics_dict[name] = {
+
+            if metric_name not in metrics_dict:
+                metrics_dict[metric_name] = {
                     "scores": [],
                     "passes": 0,
                     "fails": 0,
                     "errors": 0,
                 }
 
+            metric_dict = metrics_dict[metric_name]
+
             if score is None or success is None:
-                metrics_dict[name]["errors"] += 1
+                metric_dict["errors"] += 1
             else:
                 valid_scores += 1
-
-                # Append the score.
-                metrics_dict[name]["scores"].append(score)
-
-                # Increment passes or fails based on the metric_data.success flag.
+                metric_dict["scores"].append(score)
                 if success:
-                    metrics_dict[name]["passes"] += 1
+                    metric_dict["passes"] += 1
                 else:
-                    metrics_dict[name]["fails"] += 1
+                    metric_dict["fails"] += 1
 
         def process_span_metric_data(
-            metric_data: MetricData, type: span_api_type_literals, name: str
+            metric_data: MetricData,
+            span_type: span_api_type_literals,
+            span_name: str,
         ):
+            """
+            Process and aggregate metric data for a specific span.
+
+            Args:
+                metric_data: The metric data to process
+                span_type: The type of span (agent, tool, retriever, llm, base)
+                span_name: The name of the span
+            """
             metric_name = metric_data.name
             score = metric_data.score
             success = metric_data.success
 
-            # Initialize the structure if needed
-            if name not in trace_metrics_dict[type]:
-                trace_metrics_dict[type][name] = {}
+            if span_name not in trace_metrics_dict[span_type]:
+                trace_metrics_dict[span_type][span_name] = {}
 
-            if metric_name not in trace_metrics_dict[type][name]:
-                trace_metrics_dict[type][name][metric_name] = {
+            if metric_name not in trace_metrics_dict[span_type][span_name]:
+                trace_metrics_dict[span_type][span_name][metric_name] = {
                     "scores": [],
                     "passes": 0,
                     "fails": 0,
                     "errors": 0,
                 }
 
+            metric_dict = trace_metrics_dict[span_type][span_name][metric_name]
+
             if score is None or success is None:
-                trace_metrics_dict[type][name][metric_name]["errors"] += 1
+                metric_dict["errors"] += 1
             else:
-                # Append the score
-                trace_metrics_dict[type][name][metric_name]["scores"].append(
-                    score
-                )
-
-                # Increment passes or fails
+                metric_dict["scores"].append(score)
                 if success:
-                    trace_metrics_dict[type][name][metric_name]["passes"] += 1
+                    metric_dict["passes"] += 1
                 else:
-                    trace_metrics_dict[type][name][metric_name]["fails"] += 1
+                    metric_dict["fails"] += 1
+
+        def process_spans(spans, span_type: span_api_type_literals):
+            """
+            Process all metrics for a list of spans of a specific type.
+
+            Args:
+                spans: List of spans to process
+                span_type: The type of spans being processed
+            """
+            for span in spans:
+                if span.metrics_data is not None:
+                    for metric_data in span.metrics_data:
+                        process_metric_data(metric_data)
+                        process_span_metric_data(
+                            metric_data, span_type, span.name
+                        )
 
         # Process non-conversational test cases.
         for test_case in self.test_cases:
@@ -261,45 +307,14 @@ class TestRun(BaseModel):
             if test_case.trace is None:
                 continue
 
-            for span in test_case.trace.agent_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.AGENT.value, span.name
-                        )
-
-            for span in test_case.trace.tool_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.TOOL.value, span.name
-                        )
-
-            for span in test_case.trace.retriever_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.RETRIEVER.value, span.name
-                        )
-
-            for span in test_case.trace.llm_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.LLM.value, span.name
-                        )
-
-            for span in test_case.trace.base_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.BASE.value, span.name
-                        )
+            # Process all span types using the helper function
+            process_spans(test_case.trace.agent_spans, SpanApiType.AGENT.value)
+            process_spans(test_case.trace.tool_spans, SpanApiType.TOOL.value)
+            process_spans(
+                test_case.trace.retriever_spans, SpanApiType.RETRIEVER.value
+            )
+            process_spans(test_case.trace.llm_spans, SpanApiType.LLM.value)
+            process_spans(test_case.trace.base_spans, SpanApiType.BASE.value)
 
         # Process conversational test cases.
         for convo_test_case in self.conversational_test_cases:
@@ -532,105 +547,141 @@ class TestRunManager:
     def clear_test_run(self):
         self.test_run = None
 
-    def display_results_table(
-        self, test_run: TestRun, display: TestRunResultDisplay
-    ):
-        table = Table(title="Test Results")
-        table.add_column("Test case", justify="left")
-        table.add_column("Metric", justify="left")
-        table.add_column("Score", justify="left")
-        table.add_column("Status", justify="left")
-        table.add_column("Overall Success Rate", justify="left")
-
-        for index, test_case in enumerate(test_run.test_cases):
-            if test_case.metrics_data is None:
-                continue
+    @staticmethod
+    def _calculate_success_rate(pass_count: int, fail_count: int) -> str:
+        """Calculate success rate percentage or return error message."""
+        total = pass_count + fail_count
+        if total > 0:
+            return str(round((100 * pass_count) / total, 2))
+        return "Cannot display metrics for component-level evals, please run 'deepeval view' to see results on Confident AI."
+
+    @staticmethod
+    def _get_metric_status(metric_data: MetricData) -> str:
+        """Get formatted status string for a metric."""
+        if metric_data.error:
+            return "[red]ERRORED[/red]"
+        elif metric_data.success:
+            return "[green]PASSED[/green]"
+        return "[red]FAILED[/red]"
+
+    @staticmethod
+    def _format_metric_score(metric_data: MetricData) -> str:
+        """Format metric score with evaluation details."""
+        evaluation_model = metric_data.evaluation_model or "n/a"
+        metric_score = (
+            round(metric_data.score, 2)
+            if metric_data.score is not None
+            else None
+        )
 
-            if (
-                display == TestRunResultDisplay.PASSING
-                and test_case.success is False
-            ):
-                continue
-            elif display == TestRunResultDisplay.FAILING and test_case.success:
-                continue
+        return (
+            f"{metric_score} "
+            f"(threshold={metric_data.threshold}, "
+            f"evaluation model={evaluation_model}, "
+            f"reason={metric_data.reason}, "
+            f"error={metric_data.error})"
+        )
 
-            pass_count = 0
-            fail_count = 0
-            test_case_name = test_case.name
+    @staticmethod
+    def _should_skip_test_case(
+        test_case, display: TestRunResultDisplay
+    ) -> bool:
+        """Determine if test case should be skipped based on display filter."""
+        if display == TestRunResultDisplay.PASSING and not test_case.success:
+            return True
+        elif display == TestRunResultDisplay.FAILING and test_case.success:
+            return True
+        return False
+
+    @staticmethod
+    def _count_metric_results(
+        metrics_data: List[MetricData],
+    ) -> tuple[int, int]:
+        """Count passing and failing metrics."""
+        pass_count = 0
+        fail_count = 0
+        for metric_data in metrics_data:
+            if metric_data.success:
+                pass_count += 1
+            else:
+                fail_count += 1
+        return pass_count, fail_count
 
-            # TODO: recursively iterate through it to calculate pass and fail count
-            if test_case.trace:
-                pass
+    def _add_test_case_header_row(
+        self,
+        table: Table,
+        test_case_name: str,
+        pass_count: int,
+        fail_count: int,
+    ):
+        """Add test case header row with name and success rate."""
+        success_rate = self._calculate_success_rate(pass_count, fail_count)
+        table.add_row(
+            test_case_name,
+            *[""] * 3,
+            f"{success_rate}%",
+        )
 
-            for metric_data in test_case.metrics_data:
-                if metric_data.success:
-                    pass_count += 1
-                else:
-                    fail_count += 1
+    def _add_metric_rows(self, table: Table, metrics_data: List[MetricData]):
+        """Add metric detail rows to the table."""
+        for metric_data in metrics_data:
+            status = self._get_metric_status(metric_data)
+            formatted_score = self._format_metric_score(metric_data)
 
-            success_rate = (
-                round((100 * pass_count) / (pass_count + fail_count), 2)
-                if pass_count + fail_count > 0
-                else "Cannot display metrics for component-level evals, please run 'deepeval view' to see results on Confident AI."
-            )
             table.add_row(
-                test_case_name,
                 "",
+                str(metric_data.name),
+                formatted_score,
+                status,
                 "",
-                "",
-                f"{success_rate}%",
             )
 
-            for metric_data in test_case.metrics_data:
-                if metric_data.error:
-                    status = "[red]ERRORED[/red]"
-                elif metric_data.success:
-                    status = "[green]PASSED[/green]"
-                else:
-                    status = "[red]FAILED[/red]"
+    def _add_separator_row(self, table: Table):
+        """Add empty separator row between test cases."""
+        table.add_row(*[""] * len(table.columns))
 
-                evaluation_model = metric_data.evaluation_model
-                if evaluation_model is None:
-                    evaluation_model = "n/a"
+    def display_results_table(
+        self, test_run: TestRun, display: TestRunResultDisplay
+    ):
+        """Display test results in a formatted table."""
 
-                if metric_data.score is not None:
-                    metric_score = round(metric_data.score, 2)
-                else:
-                    metric_score = None
-
-                table.add_row(
-                    "",
-                    str(metric_data.name),
-                    f"{metric_score} (threshold={metric_data.threshold}, evaluation model={evaluation_model}, reason={metric_data.reason}, error={metric_data.error})",
-                    status,
-                    "",
-                )
+        table = Table(title="Test Results")
+        column_config = dict(justify="left")
+        column_names = [
+            "Test case",
+            "Metric",
+            "Score",
+            "Status",
+            "Overall Success Rate",
+        ]
 
-            if index is not len(self.test_run.test_cases) - 1:
-                table.add_row(
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                )
+        for name in column_names:
+            table.add_column(name, **column_config)
 
+        # Process regular test cases
+        for index, test_case in enumerate(test_run.test_cases):
+            if test_case.metrics_data is None or self._should_skip_test_case(
+                test_case, display
+            ):
+                continue
+            pass_count, fail_count = self._count_metric_results(
+                test_case.metrics_data
+            )
+            self._add_test_case_header_row(
+                table, test_case.name, pass_count, fail_count
+            )
+            self._add_metric_rows(table, test_case.metrics_data)
+
+            if index < len(test_run.test_cases) - 1:
+                self._add_separator_row(table)
+
+        # Process conversational test cases
         for index, conversational_test_case in enumerate(
             test_run.conversational_test_cases
         ):
-            if (
-                display == TestRunResultDisplay.PASSING
-                and conversational_test_case.success is False
-            ):
-                continue
-            elif (
-                display == TestRunResultDisplay.FAILING
-                and conversational_test_case.success
-            ):
+            if self._should_skip_test_case(conversational_test_case, display):
                 continue
 
-            pass_count = 0
-            fail_count = 0
             conversational_test_case_name = conversational_test_case.name
 
             if conversational_test_case.turns:
@@ -691,71 +742,26 @@ class TestRunManager:
                 console.print(
                     f"[dim]No turns recorded for {conversational_test_case_name}.[/dim]"
                 )
-
             if conversational_test_case.metrics_data is not None:
-                for metric_data in conversational_test_case.metrics_data:
-                    if metric_data.success:
-                        pass_count += 1
-                    else:
-                        fail_count += 1
-                table.add_row(
-                    conversational_test_case_name,
-                    "",
-                    "",
-                    "",
-                    f"{round((100*pass_count)/(pass_count+fail_count),2)}%",
+                pass_count, fail_count = self._count_metric_results(
+                    conversational_test_case.metrics_data
                 )
-
-            if conversational_test_case.metrics_data is not None:
-                for metric_data in conversational_test_case.metrics_data:
-                    if metric_data.error:
-                        status = "[red]ERRORED[/red]"
-                    elif metric_data.success:
-                        status = "[green]PASSED[/green]"
-                    else:
-                        status = "[red]FAILED[/red]"
-
-                    evaluation_model = metric_data.evaluation_model
-                    if evaluation_model is None:
-                        evaluation_model = "n/a"
-
-                    if metric_data.score is not None:
-                        metric_score = round(metric_data.score, 2)
-                    else:
-                        metric_score = None
-
-                    table.add_row(
-                        "",
-                        str(metric_data.name),
-                        f"{metric_score} (threshold={metric_data.threshold}, evaluation model={evaluation_model}, reason={metric_data.reason}, error={metric_data.error})",
-                        status,
-                        "",
-                    )
-
-            if index is not len(self.test_run.conversational_test_cases) - 1:
-                table.add_row(
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
+                self._add_test_case_header_row(
+                    table, conversational_test_case.name, pass_count, fail_count
                 )
-
-            if index is not len(self.test_run.test_cases) - 1:
-                table.add_row(
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
+                self._add_metric_rows(
+                    table, conversational_test_case.metrics_data
                 )
 
+            if index < len(test_run.conversational_test_cases) - 1:
+                self._add_separator_row(table)
+
+            if index < len(test_run.test_cases) - 1:
+                self._add_separator_row(table)
+
         table.add_row(
             "[bold red]Note: Use Confident AI with DeepEval to analyze failed test cases for more details[/bold red]",
-            "",
-            "",
-            "",
-            "",
+            *[""] * (len(table.columns) - 1),
         )
         print(table)
 
@@ -799,6 +805,7 @@ class TestRunManager:
             test_run.test_cases = initial_batch
 
         try:
+            test_run.prompts = None
             body = test_run.model_dump(by_alias=True, exclude_none=True)
         except AttributeError:
             # Pydantic version below 2.0
@@ -947,12 +954,28 @@ class TestRunManager:
             global_test_run_cache_manager.disable_write_cache = not bool(
                 get_is_running_deepeval()
             )
-
         global_test_run_cache_manager.wrap_up_cached_test_run()
 
         if display_table:
             self.display_results_table(test_run, display)
 
+        if test_run.hyperparameters is None:
+            console.print(
+                "\n[bold yellow]⚠ WARNING:[/bold yellow] No hyperparameters logged.\n"
+                "» [bold blue][link=https://deepeval.com/docs/evaluation-prompts]Log hyperparameters[/link][/bold blue] to attribute prompts and models to your test runs.\n\n"
+                + "=" * 80
+            )
+        else:
+            if not test_run.prompts:
+                console.print(
+                    "\n[bold yellow]⚠ WARNING:[/bold yellow] No prompts logged.\n"
+                    "» [bold blue][link=https://deepeval.com/docs/evaluation-prompts]Log prompts[/link][/bold blue] to evaluate and optimize your prompt templates and models.\n\n"
+                    + "=" * 80
+                )
+            else:
+                console.print("\n[bold green]✓ Prompts Logged[/bold green]\n")
+                self._render_prompts_panels(prompts=test_run.prompts)
+
         self.save_test_run_locally()
         delete_file_if_exists(self.temp_file_path)
         if is_confident() and self.disable_request is False:
@@ -967,7 +990,7 @@ class TestRunManager:
                 f"» Test Results ({test_run.test_passed + test_run.test_failed} total tests):\n",
                 f"  » Pass Rate: {round((test_run.test_passed / (test_run.test_passed + test_run.test_failed)) * 100, 2)}% | Passed: [bold green]{test_run.test_passed}[/bold green] | Failed: [bold red]{test_run.test_failed}[/bold red]\n\n",
                 "=" * 80,
-                "\n\n» What to share evals with your team, or a place for your test cases to live? ❤️ 🏡\n"
+                "\n\n» Want to share evals with your team, or a place for your test cases to live? ❤️ 🏡\n"
                 "  » Run [bold]'deepeval view'[/bold] to analyze and save testing results on [rgb(106,0,255)]Confident AI[/rgb(106,0,255)].\n\n",
             )
 
@@ -993,5 +1016,68 @@ class TestRunManager:
             pass
         return None
 
+    def _render_prompts_panels(self, prompts: List[PromptData]) -> None:
+
+        def format_string(
+            v, default="[dim]None[/dim]", color: Optional[str] = None
+        ):
+            formatted_string = str(v) if v not in (None, "", []) else default
+            return (
+                f"{formatted_string}"
+                if color is None or v in (None, "", [])
+                else f"[{color}]{formatted_string}[/]"
+            )
+
+        panels = []
+        for prompt in prompts:
+            lines = []
+            p_type = (
+                "messages"
+                if prompt.messages_template
+                else ("text" if prompt.text_template else "—")
+            )
+            if p_type:
+                lines.append(f"type: {format_string(p_type, color='blue')}")
+            if prompt.output_type:
+                lines.append(
+                    f"output_type: {format_string(prompt.output_type, color='blue')}"
+                )
+            if prompt.interpolation_type:
+                lines.append(
+                    f"interpolation_type: {format_string(prompt.interpolation_type, color='blue')}"
+                )
+            if prompt.model_settings:
+                ms = prompt.model_settings
+                settings_lines = [
+                    "Model Settings:",
+                    f"  – provider: {format_string(ms.provider, color='green')}",
+                    f"  – name: {format_string(ms.name, color='green')}",
+                    f"  – temperature: {format_string(ms.temperature, color='green')}",
+                    f"  – max_tokens: {format_string(ms.max_tokens, color='green')}",
+                    f"  – top_p: {format_string(ms.top_p, color='green')}",
+                    f"  – frequency_penalty: {format_string(ms.frequency_penalty, color='green')}",
+                    f"  – presence_penalty: {format_string(ms.presence_penalty, color='green')}",
+                    f"  – stop_sequence: {format_string(ms.stop_sequence, color='green')}",
+                    f"  – reasoning_effort: {format_string(ms.reasoning_effort, color='green')}",
+                    f"  – verbosity: {format_string(ms.verbosity, color='green')}",
+                ]
+                lines.append("")
+                lines.extend(settings_lines)
+            title = f"{format_string(prompt.alias)}"
+            if prompt.version:
+                title += f" (v{prompt.version})"
+            body = "\n".join(lines)
+            panel = Panel(
+                body,
+                title=title,
+                title_align="left",
+                expand=False,
+                padding=(1, 6, 1, 2),
+            )
+            panels.append(panel)
+
+        if panels:
+            console.print(Columns(panels, equal=False, expand=False))
+
 
 global_test_run_manager = TestRunManager()

deepeval/tracing/__init__.py

@@ -4,7 +4,7 @@ from .context import (
     update_retriever_span,
     update_llm_span,
 )
-from .trace_context import trace
+from .trace_context import trace, LlmSpanContext
 from .types import BaseSpan, Trace
 from .tracing import observe, trace_manager
 from .offline_evals import evaluate_thread, evaluate_trace, evaluate_span
@@ -14,6 +14,7 @@ __all__ = [
     "update_current_trace",
     "update_retriever_span",
     "update_llm_span",
+    "LlmSpanContext",
     "BaseSpan",
     "Trace",
     "observe",

deepeval/tracing/api.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union, Literal, Any
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from deepeval.test_case import ToolCall
 
@@ -27,6 +27,8 @@ class PromptApi(BaseModel):
 
 
 class MetricData(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
     name: str
     threshold: float
     success: bool