deepeval 3.6.7__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/test_run/test_run.py

@@ -35,12 +35,10 @@ from deepeval.constants import CONFIDENT_TEST_CASE_BATCH_SIZE, HIDDEN_DIR
 from deepeval.prompt import (
     PromptMessage,
     ModelSettings,
-    OutputType,
     PromptInterpolationType,
     OutputType,
 )
 from rich.panel import Panel
-from rich.text import Text
 from rich.columns import Columns
 
 
@@ -213,65 +211,91 @@ class TestRun(BaseModel):
         valid_scores = 0
 
         def process_metric_data(metric_data: MetricData):
+            """
+            Process and aggregate metric data for overall test metrics.
+
+            Args:
+                metric_data: The metric data to process
+            """
             nonlocal valid_scores
-            name = metric_data.name
+            metric_name = metric_data.name
             score = metric_data.score
             success = metric_data.success
-            # Initialize dict entry if needed.
-            if name not in metrics_dict:
-                metrics_dict[name] = {
+
+            if metric_name not in metrics_dict:
+                metrics_dict[metric_name] = {
                     "scores": [],
                     "passes": 0,
                     "fails": 0,
                     "errors": 0,
                 }
 
+            metric_dict = metrics_dict[metric_name]
+
             if score is None or success is None:
-                metrics_dict[name]["errors"] += 1
+                metric_dict["errors"] += 1
             else:
                 valid_scores += 1
-
-                # Append the score.
-                metrics_dict[name]["scores"].append(score)
-
-                # Increment passes or fails based on the metric_data.success flag.
+                metric_dict["scores"].append(score)
                 if success:
-                    metrics_dict[name]["passes"] += 1
+                    metric_dict["passes"] += 1
                 else:
-                    metrics_dict[name]["fails"] += 1
+                    metric_dict["fails"] += 1
 
         def process_span_metric_data(
-            metric_data: MetricData, type: span_api_type_literals, name: str
+            metric_data: MetricData,
+            span_type: span_api_type_literals,
+            span_name: str,
         ):
+            """
+            Process and aggregate metric data for a specific span.
+
+            Args:
+                metric_data: The metric data to process
+                span_type: The type of span (agent, tool, retriever, llm, base)
+                span_name: The name of the span
+            """
             metric_name = metric_data.name
             score = metric_data.score
             success = metric_data.success
 
-            # Initialize the structure if needed
-            if name not in trace_metrics_dict[type]:
-                trace_metrics_dict[type][name] = {}
+            if span_name not in trace_metrics_dict[span_type]:
+                trace_metrics_dict[span_type][span_name] = {}
 
-            if metric_name not in trace_metrics_dict[type][name]:
-                trace_metrics_dict[type][name][metric_name] = {
+            if metric_name not in trace_metrics_dict[span_type][span_name]:
+                trace_metrics_dict[span_type][span_name][metric_name] = {
                     "scores": [],
                     "passes": 0,
                     "fails": 0,
                     "errors": 0,
                 }
 
+            metric_dict = trace_metrics_dict[span_type][span_name][metric_name]
+
             if score is None or success is None:
-                trace_metrics_dict[type][name][metric_name]["errors"] += 1
+                metric_dict["errors"] += 1
             else:
-                # Append the score
-                trace_metrics_dict[type][name][metric_name]["scores"].append(
-                    score
-                )
-
-                # Increment passes or fails
+                metric_dict["scores"].append(score)
                 if success:
-                    trace_metrics_dict[type][name][metric_name]["passes"] += 1
+                    metric_dict["passes"] += 1
                 else:
-                    trace_metrics_dict[type][name][metric_name]["fails"] += 1
+                    metric_dict["fails"] += 1
+
+        def process_spans(spans, span_type: span_api_type_literals):
+            """
+            Process all metrics for a list of spans of a specific type.
+
+            Args:
+                spans: List of spans to process
+                span_type: The type of spans being processed
+            """
+            for span in spans:
+                if span.metrics_data is not None:
+                    for metric_data in span.metrics_data:
+                        process_metric_data(metric_data)
+                        process_span_metric_data(
+                            metric_data, span_type, span.name
+                        )
 
         # Process non-conversational test cases.
         for test_case in self.test_cases:
@@ -283,45 +307,14 @@ class TestRun(BaseModel):
             if test_case.trace is None:
                 continue
 
-            for span in test_case.trace.agent_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.AGENT.value, span.name
-                        )
-
-            for span in test_case.trace.tool_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.TOOL.value, span.name
-                        )
-
-            for span in test_case.trace.retriever_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.RETRIEVER.value, span.name
-                        )
-
-            for span in test_case.trace.llm_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.LLM.value, span.name
-                        )
-
-            for span in test_case.trace.base_spans:
-                if span.metrics_data is not None:
-                    for metric_data in span.metrics_data:
-                        process_metric_data(metric_data)
-                        process_span_metric_data(
-                            metric_data, SpanApiType.BASE.value, span.name
-                        )
+            # Process all span types using the helper function
+            process_spans(test_case.trace.agent_spans, SpanApiType.AGENT.value)
+            process_spans(test_case.trace.tool_spans, SpanApiType.TOOL.value)
+            process_spans(
+                test_case.trace.retriever_spans, SpanApiType.RETRIEVER.value
+            )
+            process_spans(test_case.trace.llm_spans, SpanApiType.LLM.value)
+            process_spans(test_case.trace.base_spans, SpanApiType.BASE.value)
 
         # Process conversational test cases.
         for convo_test_case in self.conversational_test_cases:
@@ -554,105 +547,141 @@ class TestRunManager:
     def clear_test_run(self):
         self.test_run = None
 
-    def display_results_table(
-        self, test_run: TestRun, display: TestRunResultDisplay
-    ):
-        table = Table(title="Test Results")
-        table.add_column("Test case", justify="left")
-        table.add_column("Metric", justify="left")
-        table.add_column("Score", justify="left")
-        table.add_column("Status", justify="left")
-        table.add_column("Overall Success Rate", justify="left")
-
-        for index, test_case in enumerate(test_run.test_cases):
-            if test_case.metrics_data is None:
-                continue
+    @staticmethod
+    def _calculate_success_rate(pass_count: int, fail_count: int) -> str:
+        """Calculate success rate percentage or return error message."""
+        total = pass_count + fail_count
+        if total > 0:
+            return str(round((100 * pass_count) / total, 2))
+        return "Cannot display metrics for component-level evals, please run 'deepeval view' to see results on Confident AI."
+
+    @staticmethod
+    def _get_metric_status(metric_data: MetricData) -> str:
+        """Get formatted status string for a metric."""
+        if metric_data.error:
+            return "[red]ERRORED[/red]"
+        elif metric_data.success:
+            return "[green]PASSED[/green]"
+        return "[red]FAILED[/red]"
+
+    @staticmethod
+    def _format_metric_score(metric_data: MetricData) -> str:
+        """Format metric score with evaluation details."""
+        evaluation_model = metric_data.evaluation_model or "n/a"
+        metric_score = (
+            round(metric_data.score, 2)
+            if metric_data.score is not None
+            else None
+        )
 
-            if (
-                display == TestRunResultDisplay.PASSING
-                and test_case.success is False
-            ):
-                continue
-            elif display == TestRunResultDisplay.FAILING and test_case.success:
-                continue
+        return (
+            f"{metric_score} "
+            f"(threshold={metric_data.threshold}, "
+            f"evaluation model={evaluation_model}, "
+            f"reason={metric_data.reason}, "
+            f"error={metric_data.error})"
+        )
 
-            pass_count = 0
-            fail_count = 0
-            test_case_name = test_case.name
+    @staticmethod
+    def _should_skip_test_case(
+        test_case, display: TestRunResultDisplay
+    ) -> bool:
+        """Determine if test case should be skipped based on display filter."""
+        if display == TestRunResultDisplay.PASSING and not test_case.success:
+            return True
+        elif display == TestRunResultDisplay.FAILING and test_case.success:
+            return True
+        return False
+
+    @staticmethod
+    def _count_metric_results(
+        metrics_data: List[MetricData],
+    ) -> tuple[int, int]:
+        """Count passing and failing metrics."""
+        pass_count = 0
+        fail_count = 0
+        for metric_data in metrics_data:
+            if metric_data.success:
+                pass_count += 1
+            else:
+                fail_count += 1
+        return pass_count, fail_count
 
-            # TODO: recursively iterate through it to calculate pass and fail count
-            if test_case.trace:
-                pass
+    def _add_test_case_header_row(
+        self,
+        table: Table,
+        test_case_name: str,
+        pass_count: int,
+        fail_count: int,
+    ):
+        """Add test case header row with name and success rate."""
+        success_rate = self._calculate_success_rate(pass_count, fail_count)
+        table.add_row(
+            test_case_name,
+            *[""] * 3,
+            f"{success_rate}%",
+        )
 
-            for metric_data in test_case.metrics_data:
-                if metric_data.success:
-                    pass_count += 1
-                else:
-                    fail_count += 1
+    def _add_metric_rows(self, table: Table, metrics_data: List[MetricData]):
+        """Add metric detail rows to the table."""
+        for metric_data in metrics_data:
+            status = self._get_metric_status(metric_data)
+            formatted_score = self._format_metric_score(metric_data)
 
-            success_rate = (
-                round((100 * pass_count) / (pass_count + fail_count), 2)
-                if pass_count + fail_count > 0
-                else "Cannot display metrics for component-level evals, please run 'deepeval view' to see results on Confident AI."
-            )
             table.add_row(
-                test_case_name,
                 "",
+                str(metric_data.name),
+                formatted_score,
+                status,
                 "",
-                "",
-                f"{success_rate}%",
             )
 
-            for metric_data in test_case.metrics_data:
-                if metric_data.error:
-                    status = "[red]ERRORED[/red]"
-                elif metric_data.success:
-                    status = "[green]PASSED[/green]"
-                else:
-                    status = "[red]FAILED[/red]"
+    def _add_separator_row(self, table: Table):
+        """Add empty separator row between test cases."""
+        table.add_row(*[""] * len(table.columns))
 
-                evaluation_model = metric_data.evaluation_model
-                if evaluation_model is None:
-                    evaluation_model = "n/a"
+    def display_results_table(
+        self, test_run: TestRun, display: TestRunResultDisplay
+    ):
+        """Display test results in a formatted table."""
 
-                if metric_data.score is not None:
-                    metric_score = round(metric_data.score, 2)
-                else:
-                    metric_score = None
-
-                table.add_row(
-                    "",
-                    str(metric_data.name),
-                    f"{metric_score} (threshold={metric_data.threshold}, evaluation model={evaluation_model}, reason={metric_data.reason}, error={metric_data.error})",
-                    status,
-                    "",
-                )
+        table = Table(title="Test Results")
+        column_config = dict(justify="left")
+        column_names = [
+            "Test case",
+            "Metric",
+            "Score",
+            "Status",
+            "Overall Success Rate",
+        ]
 
-            if index is not len(self.test_run.test_cases) - 1:
-                table.add_row(
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
-                )
+        for name in column_names:
+            table.add_column(name, **column_config)
 
+        # Process regular test cases
+        for index, test_case in enumerate(test_run.test_cases):
+            if test_case.metrics_data is None or self._should_skip_test_case(
+                test_case, display
+            ):
+                continue
+            pass_count, fail_count = self._count_metric_results(
+                test_case.metrics_data
+            )
+            self._add_test_case_header_row(
+                table, test_case.name, pass_count, fail_count
+            )
+            self._add_metric_rows(table, test_case.metrics_data)
+
+            if index < len(test_run.test_cases) - 1:
+                self._add_separator_row(table)
+
+        # Process conversational test cases
         for index, conversational_test_case in enumerate(
             test_run.conversational_test_cases
         ):
-            if (
-                display == TestRunResultDisplay.PASSING
-                and conversational_test_case.success is False
-            ):
-                continue
-            elif (
-                display == TestRunResultDisplay.FAILING
-                and conversational_test_case.success
-            ):
+            if self._should_skip_test_case(conversational_test_case, display):
                 continue
 
-            pass_count = 0
-            fail_count = 0
             conversational_test_case_name = conversational_test_case.name
 
             if conversational_test_case.turns:
@@ -713,71 +742,26 @@ class TestRunManager:
                 console.print(
                     f"[dim]No turns recorded for {conversational_test_case_name}.[/dim]"
                 )
-
             if conversational_test_case.metrics_data is not None:
-                for metric_data in conversational_test_case.metrics_data:
-                    if metric_data.success:
-                        pass_count += 1
-                    else:
-                        fail_count += 1
-                table.add_row(
-                    conversational_test_case_name,
-                    "",
-                    "",
-                    "",
-                    f"{round((100*pass_count)/(pass_count+fail_count),2)}%",
+                pass_count, fail_count = self._count_metric_results(
+                    conversational_test_case.metrics_data
                 )
-
-            if conversational_test_case.metrics_data is not None:
-                for metric_data in conversational_test_case.metrics_data:
-                    if metric_data.error:
-                        status = "[red]ERRORED[/red]"
-                    elif metric_data.success:
-                        status = "[green]PASSED[/green]"
-                    else:
-                        status = "[red]FAILED[/red]"
-
-                    evaluation_model = metric_data.evaluation_model
-                    if evaluation_model is None:
-                        evaluation_model = "n/a"
-
-                    if metric_data.score is not None:
-                        metric_score = round(metric_data.score, 2)
-                    else:
-                        metric_score = None
-
-                    table.add_row(
-                        "",
-                        str(metric_data.name),
-                        f"{metric_score} (threshold={metric_data.threshold}, evaluation model={evaluation_model}, reason={metric_data.reason}, error={metric_data.error})",
-                        status,
-                        "",
-                    )
-
-            if index is not len(self.test_run.conversational_test_cases) - 1:
-                table.add_row(
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
+                self._add_test_case_header_row(
+                    table, conversational_test_case.name, pass_count, fail_count
                 )
-
-            if index is not len(self.test_run.test_cases) - 1:
-                table.add_row(
-                    "",
-                    "",
-                    "",
-                    "",
-                    "",
+                self._add_metric_rows(
+                    table, conversational_test_case.metrics_data
                 )
 
+            if index < len(test_run.conversational_test_cases) - 1:
+                self._add_separator_row(table)
+
+            if index < len(test_run.test_cases) - 1:
+                self._add_separator_row(table)
+
         table.add_row(
             "[bold red]Note: Use Confident AI with DeepEval to analyze failed test cases for more details[/bold red]",
-            "",
-            "",
-            "",
-            "",
+            *[""] * (len(table.columns) - 1),
         )
         print(table)
 
@@ -970,7 +954,6 @@ class TestRunManager:
             global_test_run_cache_manager.disable_write_cache = not bool(
                 get_is_running_deepeval()
             )
-
         global_test_run_cache_manager.wrap_up_cached_test_run()
 
         if display_table:

deepeval/tracing/__init__.py

@@ -4,7 +4,7 @@ from .context import (
     update_retriever_span,
     update_llm_span,
 )
-from .trace_context import trace
+from .trace_context import trace, LlmSpanContext
 from .types import BaseSpan, Trace
 from .tracing import observe, trace_manager
 from .offline_evals import evaluate_thread, evaluate_trace, evaluate_span
@@ -14,6 +14,7 @@ __all__ = [
     "update_current_trace",
     "update_retriever_span",
     "update_llm_span",
+    "LlmSpanContext",
     "BaseSpan",
     "Trace",
     "observe",

deepeval/tracing/otel/exporter.py

@@ -254,7 +254,7 @@ class ConfidentSpanExporter(SpanExporter):
 
         base_span = None
         try:
-            base_span = self.__prepare_boilerplate_base_span(span)
+            base_span = self.prepare_boilerplate_base_span(span)
         except Exception:
             pass
 
@@ -453,9 +453,8 @@ class ConfidentSpanExporter(SpanExporter):
         if span_output:
             base_span.output = span_output
 
-    def __prepare_boilerplate_base_span(
-        self, span: ReadableSpan
-    ) -> Optional[BaseSpan]:
+    @staticmethod
+    def prepare_boilerplate_base_span(span: ReadableSpan) -> Optional[BaseSpan]:
 
         ################ Get Span Type ################
         span_type = span.attributes.get("confident.span.type")

deepeval/tracing/otel/utils.py

@@ -11,7 +11,7 @@ from deepeval.tracing import trace_manager, BaseSpan
 from deepeval.tracing.utils import make_json_serializable
 
 
-GEN_AI_OPERATION_NAMES = ["chat", "generate_content", "task_completion"]
+GEN_AI_OPERATION_NAMES = ["chat", "generate_content", "text_completion"]
 
 
 def to_hex_string(id_value: int | bytes, length: int = 32) -> str:
@@ -128,6 +128,10 @@ def check_llm_input_from_gen_ai_attributes(
 
         input = system_instructions + input_messages
 
+        model_parameters = check_model_parameters(span)
+        if model_parameters:
+            input.append(model_parameters)
+
     except Exception:
         pass
     try:
@@ -413,7 +417,7 @@ def post_test_run(traces: List[Trace], test_run_id: Optional[str]):
     # return test_run_manager.post_test_run(test_run) TODO: add after test run with metric collection is implemented
 
 
-def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
+def normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     try:
         raw = span.attributes.get("pydantic_ai.all_messages")
         if not raw:
@@ -438,7 +442,7 @@ def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     except Exception:
         pass
 
-    return None
+    return []
 
 
 def _extract_non_thinking_part_of_last_message(message: dict) -> dict:
@@ -461,7 +465,7 @@ def check_pydantic_ai_agent_input_output(
     output_val: Optional[Any] = None
 
     # Get normalized messages once
-    normalized = _normalize_pydantic_ai_messages(span)
+    normalized = normalize_pydantic_ai_messages(span)
 
     # Input (pydantic_ai.all_messages) - slice up to and including the first 'user' message
     if normalized:
@@ -523,3 +527,18 @@ def check_pydantic_ai_trace_input_output(
         input_val, output_val = check_pydantic_ai_agent_input_output(span)
 
     return input_val, output_val
+
+
+def check_model_parameters(span: ReadableSpan) -> Optional[dict]:
+    try:
+        raw_model_parameters = span.attributes.get("model_request_parameters")
+        if raw_model_parameters and isinstance(raw_model_parameters, str):
+            model_parameters = json.loads(raw_model_parameters)
+            if isinstance(model_parameters, dict):
+                return {
+                    "role": "Model Request Parameters",
+                    "content": model_parameters,
+                }
+    except Exception:
+        pass
+    return None