deepeval 3.6.6__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/test_case/api.py

@@ -0,0 +1,131 @@
+from typing import Union, Optional
+import os
+
+from deepeval.test_run.api import (
+    LLMApiTestCase,
+    ConversationalApiTestCase,
+    TurnApi,
+    TraceApi,
+)
+from deepeval.test_case import (
+    LLMTestCase,
+    ConversationalTestCase,
+    MLLMTestCase,
+    Turn,
+)
+from deepeval.constants import PYTEST_RUN_TEST_NAME
+
+
+def create_api_turn(turn: Turn, index: int) -> TurnApi:
+    return TurnApi(
+        role=turn.role,
+        content=turn.content,
+        user_id=turn.user_id,
+        retrievalContext=turn.retrieval_context,
+        toolsCalled=turn.tools_called,
+        additionalMetadata=turn.additional_metadata,
+        order=index,
+    )
+
+
+def create_api_test_case(
+    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    trace: Optional[TraceApi] = None,
+    index: Optional[int] = None,
+) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
+    if isinstance(test_case, ConversationalTestCase):
+        order = (
+            test_case._dataset_rank
+            if test_case._dataset_rank is not None
+            else index
+        )
+        if test_case.name:
+            name = test_case.name
+        else:
+            name = os.getenv(
+                PYTEST_RUN_TEST_NAME, f"conversational_test_case_{order}"
+            )
+
+        api_test_case = ConversationalApiTestCase(
+            name=name,
+            success=True,
+            metricsData=[],
+            runDuration=0,
+            evaluationCost=None,
+            order=order,
+            scenario=test_case.scenario,
+            expectedOutcome=test_case.expected_outcome,
+            userDescription=test_case.user_description,
+            context=test_case.context,
+            tags=test_case.tags,
+            comments=test_case.comments,
+            additionalMetadata=test_case.additional_metadata,
+        )
+        api_test_case.turns = [
+            create_api_turn(
+                turn=turn,
+                index=index,
+            )
+            for index, turn in enumerate(test_case.turns)
+        ]
+
+        return api_test_case
+    else:
+        order = (
+            test_case._dataset_rank
+            if test_case._dataset_rank is not None
+            else index
+        )
+
+        success = True
+        if test_case.name is not None:
+            name = test_case.name
+        else:
+            name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
+        metrics_data = []
+
+        if isinstance(test_case, LLMTestCase):
+            api_test_case = LLMApiTestCase(
+                name=name,
+                input=test_case.input,
+                actualOutput=test_case.actual_output,
+                expectedOutput=test_case.expected_output,
+                context=test_case.context,
+                retrievalContext=test_case.retrieval_context,
+                toolsCalled=test_case.tools_called,
+                expectedTools=test_case.expected_tools,
+                tokenCost=test_case.token_cost,
+                completionTime=test_case.completion_time,
+                tags=test_case.tags,
+                success=success,
+                metricsData=metrics_data,
+                runDuration=None,
+                evaluationCost=None,
+                order=order,
+                additionalMetadata=test_case.additional_metadata,
+                comments=test_case.comments,
+                trace=trace,
+            )
+        elif isinstance(test_case, MLLMTestCase):
+            api_test_case = LLMApiTestCase(
+                name=name,
+                input="",
+                multimodalInput=test_case.input,
+                multimodalActualOutput=test_case.actual_output,
+                multimodalExpectedOutput=test_case.expected_output,
+                multimodalRetrievalContext=test_case.retrieval_context,
+                multimodalContext=test_case.context,
+                toolsCalled=test_case.tools_called,
+                expectedTools=test_case.expected_tools,
+                tokenCost=test_case.token_cost,
+                completionTime=test_case.completion_time,
+                success=success,
+                metricsData=metrics_data,
+                runDuration=None,
+                evaluationCost=None,
+                order=order,
+                additionalMetadata=test_case.additional_metadata,
+                comments=test_case.comments,
+            )
+        # llm_test_case_lookup_map[instance_id] = api_test_case
+        return api_test_case

deepeval/test_run/__init__.py

@@ -8,6 +8,7 @@ from .test_run import (
     LLMApiTestCase,
     ConversationalApiTestCase,
     TestRunManager,
+    PromptData,
 )
 
 from .hooks import on_test_run_end, invoke_test_run_end_hook

deepeval/test_run/hyperparameters.py

@@ -1,13 +1,15 @@
-from typing import Union, Dict
-
+from typing import Union, Dict, Optional, List
 from deepeval.test_run import global_test_run_manager
 from deepeval.prompt import Prompt
 from deepeval.prompt.api import PromptApi
 from deepeval.test_run.test_run import TEMP_FILE_PATH
+from deepeval.confident.api import is_confident
+from deepeval.test_run.test_run import PromptData
 
 
 def process_hyperparameters(
-    hyperparameters,
+    hyperparameters: Optional[Dict] = None,
+    verbose: bool = True,
 ) -> Union[Dict[str, Union[str, int, float, PromptApi]], None]:
     if hyperparameters is None:
         return None
@@ -16,6 +18,7 @@ def process_hyperparameters(
         raise TypeError("Hyperparameters must be a dictionary or None")
 
     processed_hyperparameters = {}
+    prompts_version_id_map = {}
 
     for key, value in hyperparameters.items():
         if not isinstance(key, str):
@@ -30,14 +33,21 @@ def process_hyperparameters(
             )
 
         if isinstance(value, Prompt):
-            if value._prompt_version_id is not None and value._type is not None:
+            prompt_key = f"{value.alias}_{value.version}"
+            if value._prompt_version_id is not None and value.type is not None:
                 processed_hyperparameters[key] = PromptApi(
                     id=value._prompt_version_id,
-                    type=value._type,
+                    type=value.type,
                 )
-            else:
-                raise ValueError(
-                    f"Cannot log Prompt where template was not pulled from Confident AI. Please import your prompt on Confident AI to continue."
+            elif is_confident():
+                if prompt_key not in prompts_version_id_map:
+                    value.push(_verbose=verbose)
+                    prompts_version_id_map[prompt_key] = (
+                        value._prompt_version_id
+                    )
+                processed_hyperparameters[key] = PromptApi(
+                    id=prompts_version_id_map[prompt_key],
+                    type=value.type,
                 )
         else:
             processed_hyperparameters[key] = str(value)
@@ -64,3 +74,32 @@ def log_hyperparameters(func):
 
     # Return the wrapper function to be used as the decorator
     return wrapper
+
+
+def process_prompts(
+    hyperparameters: Dict[str, Union[str, int, float, Prompt]],
+) -> List[PromptData]:
+    prompts = []
+    if not hyperparameters:
+        return prompts
+    seen_prompts = set()
+    prompt_objects = [
+        value for value in hyperparameters.values() if isinstance(value, Prompt)
+    ]
+    for prompt in prompt_objects:
+        prompt_version = prompt.version if is_confident() else None
+        prompt_key = f"{prompt.alias}_{prompt_version}"
+        if prompt_key in seen_prompts:
+            continue
+        seen_prompts.add(prompt_key)
+        prompt_data = PromptData(
+            alias=prompt.alias,
+            version=prompt_version,
+            text_template=prompt.text_template,
+            messages_template=prompt.messages_template,
+            model_settings=prompt.model_settings,
+            output_type=prompt.output_type,
+            interpolation_type=prompt.interpolation_type,
+        )
+        prompts.append(prompt_data)
+    return prompts

deepeval/test_run/test_run.py

@@ -32,6 +32,17 @@ from deepeval.utils import (
 )
 from deepeval.test_run.cache import global_test_run_cache_manager
 from deepeval.constants import CONFIDENT_TEST_CASE_BATCH_SIZE, HIDDEN_DIR
+from deepeval.prompt import (
+    PromptMessage,
+    ModelSettings,
+    OutputType,
+    PromptInterpolationType,
+    OutputType,
+)
+from rich.panel import Panel
+from rich.text import Text
+from rich.columns import Columns
+
 
 TEMP_FILE_PATH = f"{HIDDEN_DIR}/.temp_test_run_data.json"
 LATEST_TEST_RUN_FILE_PATH = f"{HIDDEN_DIR}/.latest_test_run.json"
@@ -71,6 +82,16 @@ class TraceMetricScores(BaseModel):
     base: Dict[str, Dict[str, MetricScores]] = Field(default_factory=dict)
 
 
+class PromptData(BaseModel):
+    alias: Optional[str] = None
+    version: Optional[str] = None
+    text_template: Optional[str] = None
+    messages_template: Optional[List[PromptMessage]] = None
+    model_settings: Optional[ModelSettings] = None
+    output_type: Optional[OutputType] = None
+    interpolation_type: Optional[PromptInterpolationType] = None
+
+
 class MetricsAverageDict:
     def __init__(self):
         self.metric_dict = {}
@@ -123,6 +144,7 @@ class TestRun(BaseModel):
     )
     identifier: Optional[str] = None
     hyperparameters: Optional[Dict[str, Any]] = Field(None)
+    prompts: Optional[List[PromptData]] = Field(None)
     test_passed: Optional[int] = Field(None, alias="testPassed")
     test_failed: Optional[int] = Field(None, alias="testFailed")
     run_duration: float = Field(0.0, alias="runDuration")
@@ -799,6 +821,7 @@ class TestRunManager:
             test_run.test_cases = initial_batch
 
         try:
+            test_run.prompts = None
             body = test_run.model_dump(by_alias=True, exclude_none=True)
         except AttributeError:
             # Pydantic version below 2.0
@@ -953,6 +976,23 @@ class TestRunManager:
         if display_table:
             self.display_results_table(test_run, display)
 
+        if test_run.hyperparameters is None:
+            console.print(
+                "\n[bold yellow]⚠ WARNING:[/bold yellow] No hyperparameters logged.\n"
+                "» [bold blue][link=https://deepeval.com/docs/evaluation-prompts]Log hyperparameters[/link][/bold blue] to attribute prompts and models to your test runs.\n\n"
+                + "=" * 80
+            )
+        else:
+            if not test_run.prompts:
+                console.print(
+                    "\n[bold yellow]⚠ WARNING:[/bold yellow] No prompts logged.\n"
+                    "» [bold blue][link=https://deepeval.com/docs/evaluation-prompts]Log prompts[/link][/bold blue] to evaluate and optimize your prompt templates and models.\n\n"
+                    + "=" * 80
+                )
+            else:
+                console.print("\n[bold green]✓ Prompts Logged[/bold green]\n")
+                self._render_prompts_panels(prompts=test_run.prompts)
+
         self.save_test_run_locally()
         delete_file_if_exists(self.temp_file_path)
         if is_confident() and self.disable_request is False:
@@ -967,7 +1007,7 @@ class TestRunManager:
                 f"» Test Results ({test_run.test_passed + test_run.test_failed} total tests):\n",
                 f"  » Pass Rate: {round((test_run.test_passed / (test_run.test_passed + test_run.test_failed)) * 100, 2)}% | Passed: [bold green]{test_run.test_passed}[/bold green] | Failed: [bold red]{test_run.test_failed}[/bold red]\n\n",
                 "=" * 80,
-                "\n\n» What to share evals with your team, or a place for your test cases to live? ❤️ 🏡\n"
+                "\n\n» Want to share evals with your team, or a place for your test cases to live? ❤️ 🏡\n"
                 "  » Run [bold]'deepeval view'[/bold] to analyze and save testing results on [rgb(106,0,255)]Confident AI[/rgb(106,0,255)].\n\n",
             )
 
@@ -993,5 +1033,68 @@ class TestRunManager:
             pass
         return None
 
+    def _render_prompts_panels(self, prompts: List[PromptData]) -> None:
+
+        def format_string(
+            v, default="[dim]None[/dim]", color: Optional[str] = None
+        ):
+            formatted_string = str(v) if v not in (None, "", []) else default
+            return (
+                f"{formatted_string}"
+                if color is None or v in (None, "", [])
+                else f"[{color}]{formatted_string}[/]"
+            )
+
+        panels = []
+        for prompt in prompts:
+            lines = []
+            p_type = (
+                "messages"
+                if prompt.messages_template
+                else ("text" if prompt.text_template else "—")
+            )
+            if p_type:
+                lines.append(f"type: {format_string(p_type, color='blue')}")
+            if prompt.output_type:
+                lines.append(
+                    f"output_type: {format_string(prompt.output_type, color='blue')}"
+                )
+            if prompt.interpolation_type:
+                lines.append(
+                    f"interpolation_type: {format_string(prompt.interpolation_type, color='blue')}"
+                )
+            if prompt.model_settings:
+                ms = prompt.model_settings
+                settings_lines = [
+                    "Model Settings:",
+                    f"  – provider: {format_string(ms.provider, color='green')}",
+                    f"  – name: {format_string(ms.name, color='green')}",
+                    f"  – temperature: {format_string(ms.temperature, color='green')}",
+                    f"  – max_tokens: {format_string(ms.max_tokens, color='green')}",
+                    f"  – top_p: {format_string(ms.top_p, color='green')}",
+                    f"  – frequency_penalty: {format_string(ms.frequency_penalty, color='green')}",
+                    f"  – presence_penalty: {format_string(ms.presence_penalty, color='green')}",
+                    f"  – stop_sequence: {format_string(ms.stop_sequence, color='green')}",
+                    f"  – reasoning_effort: {format_string(ms.reasoning_effort, color='green')}",
+                    f"  – verbosity: {format_string(ms.verbosity, color='green')}",
+                ]
+                lines.append("")
+                lines.extend(settings_lines)
+            title = f"{format_string(prompt.alias)}"
+            if prompt.version:
+                title += f" (v{prompt.version})"
+            body = "\n".join(lines)
+            panel = Panel(
+                body,
+                title=title,
+                title_align="left",
+                expand=False,
+                padding=(1, 6, 1, 2),
+            )
+            panels.append(panel)
+
+        if panels:
+            console.print(Columns(panels, equal=False, expand=False))
+
 
 global_test_run_manager = TestRunManager()

deepeval/tracing/api.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union, Literal, Any
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from deepeval.test_case import ToolCall
 
@@ -27,6 +27,8 @@ class PromptApi(BaseModel):
 
 
 class MetricData(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
     name: str
     threshold: float
     success: bool

deepeval/tracing/message_types/__init__.py

@@ -0,0 +1,10 @@
+from .messages import TextMessage, ToolCallMessage
+from .tools import BaseTool, ToolSchema, ToolOutput
+
+__all__ = [
+    "BaseTool",
+    "TextMessage",
+    "ToolCallMessage",
+    "ToolSchema",
+    "ToolOutput",
+]

deepeval/tracing/message_types/base.py

@@ -0,0 +1,6 @@
+from typing import Literal
+from pydantic import BaseModel
+
+
+class BaseMessage(BaseModel):
+    role: Literal["user", "assistant"]

deepeval/tracing/message_types/messages.py

@@ -0,0 +1,14 @@
+from typing import Literal, Dict, Any
+from .base import BaseMessage
+
+
+class TextMessage(BaseMessage):
+    type: Literal["text", "thinking"]
+    content: str
+
+
+class ToolCallMessage(BaseMessage):
+    """This is a message for tool calls in response.choices[0].message.tool_calls"""
+
+    name: str
+    args: Dict[str, Any]

deepeval/tracing/message_types/tools.py

@@ -0,0 +1,18 @@
+from typing import Any, Optional, Dict
+from pydantic import BaseModel
+
+
+class BaseTool(BaseModel):
+    name: str
+    description: Optional[str] = None
+
+
+class ToolSchema(BaseTool):
+    parameters: Dict[str, Any]
+    is_called: Optional[bool] = False
+
+
+class ToolOutput(BaseTool):
+    """Output of the tool function"""
+
+    output: Any

deepeval/tracing/otel/utils.py

@@ -3,7 +3,7 @@ import json
 from typing import List, Optional, Tuple, Any
 from opentelemetry.sdk.trace.export import ReadableSpan
 
-from deepeval.evaluate.utils import create_api_test_case
+from deepeval.test_case.api import create_api_test_case
 from deepeval.test_run.api import LLMApiTestCase
 from deepeval.test_run.test_run import global_test_run_manager
 from deepeval.tracing.types import Trace, LLMTestCase, ToolCall

deepeval/tracing/trace_context.py

@@ -1,14 +1,83 @@
-from .context import current_trace_context
-from .tracing import trace_manager
+from typing import Optional, List, Dict, Any
+from contextvars import ContextVar
 from contextlib import contextmanager
+from dataclasses import dataclass
+
+from .tracing import trace_manager
+from .context import current_trace_context, update_current_trace
+from deepeval.prompt import Prompt
+from deepeval.metrics import BaseMetric
+from deepeval.test_case.llm_test_case import ToolCall
+
+
+@dataclass
+class LlmContext:
+    prompt: Optional[Prompt] = None
+    metrics: Optional[List[BaseMetric]] = None
+    metric_collection: Optional[str] = None
+    expected_output: Optional[str] = None
+    expected_tools: Optional[List[ToolCall]] = None
+    context: Optional[List[str]] = None
+    retrieval_context: Optional[List[str]] = None
+
+
+current_llm_context: ContextVar[Optional[LlmContext]] = ContextVar(
+    "current_llm_context", default=LlmContext()
+)
 
 
 @contextmanager
-def trace():
+def trace(
+    prompt: Optional[Prompt] = None,
+    llm_metrics: Optional[List[BaseMetric]] = None,
+    llm_metric_collection: Optional[str] = None,
+    name: Optional[str] = None,
+    tags: Optional[List[str]] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+    user_id: Optional[str] = None,
+    thread_id: Optional[str] = None,
+    expected_output: Optional[str] = None,
+    expected_tools: Optional[List[ToolCall]] = None,
+    context: Optional[List[str]] = None,
+    retrieval_context: Optional[List[str]] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+):
     current_trace = current_trace_context.get()
 
     if not current_trace:
         current_trace = trace_manager.start_new_trace()
-        current_trace_context.set(current_trace)
+
+    if trace_metrics:
+        current_trace.metrics = trace_metrics
+
+    if trace_metric_collection:
+        current_trace.metric_collection = trace_metric_collection
+
+    current_trace_context.set(current_trace)
+
+    current_llm_context.set(
+        LlmContext(
+            prompt=prompt,
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+            expected_output=expected_output,
+            expected_tools=expected_tools,
+            context=context,
+            retrieval_context=retrieval_context,
+        )
+    )
+
+    # set the current trace attributes
+    if name:
+        update_current_trace(name=name)
+    if tags:
+        update_current_trace(tags=tags)
+    if metadata:
+        update_current_trace(metadata=metadata)
+    if user_id:
+        update_current_trace(user_id=user_id)
+    if thread_id:
+        update_current_trace(thread_id=thread_id)
 
     yield current_trace

deepeval/tracing/tracing.py

@@ -1,5 +1,14 @@
-import os
-from typing import Any, Dict, List, Literal, Optional, Set, Union, Callable
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Union,
+)
 from time import perf_counter
 import threading
 import functools
@@ -20,6 +29,7 @@ from deepeval.constants import (
 )
 from deepeval.confident.api import Api, Endpoints, HttpMethods, is_confident
 from deepeval.metrics import BaseMetric
+from deepeval.test_case.llm_test_case import ToolCall
 from deepeval.tracing.api import (
     BaseApiSpan,
     SpanApiType,
@@ -41,6 +51,7 @@ from deepeval.tracing.types import (
 )
 from deepeval.tracing.utils import (
     Environment,
+    prepare_tool_call_input_parameters,
     replace_self_with_class_name,
     make_json_serializable,
     perf_counter_to_datetime,
@@ -55,6 +66,10 @@ from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.tracing.api import PromptApi
 from deepeval.tracing.trace_test_manager import trace_testing_manager
 
+
+if TYPE_CHECKING:
+    from deepeval.dataset.golden import Golden
+
 EVAL_DUMMY_SPAN_NAME = "evals_iterator"
 
 
@@ -65,6 +80,10 @@ class TraceManager:
         self.active_spans: Dict[str, BaseSpan] = (
             {}
         )  # Map of span_uuid to BaseSpan
+        # Map each trace created during evaluation_loop to the Golden that was active
+        # when it was started. This lets us evaluate traces against the correct golden
+        # since we cannot rely on positional indexing as the order is not guaranteed.
+        self.trace_uuid_to_golden: Dict[str, Golden] = {}
 
         settings = get_settings()
         # Initialize queue and worker thread for trace posting
@@ -86,7 +105,7 @@ class TraceManager:
         )
         validate_environment(self.environment)
 
-        self.sampling_rate = settings.CONFIDENT_SAMPLE_RATE
+        self.sampling_rate = settings.CONFIDENT_TRACE_SAMPLE_RATE
         validate_sampling_rate(self.sampling_rate)
         self.openai_client = None
         self.tracing_enabled = True
@@ -166,6 +185,19 @@ class TraceManager:
         self.traces.append(new_trace)
         if self.evaluation_loop:
             self.traces_to_evaluate_order.append(trace_uuid)
+            # Associate the current Golden with this trace so we can
+            # later evaluate traces against the correct golden, even if more traces
+            # are created than goldens or the order interleaves.
+            try:
+                from deepeval.contextvars import get_current_golden
+
+                current_golden = get_current_golden()
+                if current_golden is not None:
+                    self.trace_uuid_to_golden[trace_uuid] = current_golden
+            except Exception:
+                # not much we can do, but if the golden is not there during evaluation
+                # we will write out a verbose debug log
+                pass
         return new_trace
 
     def end_trace(self, trace_uuid: str):
@@ -861,6 +893,22 @@ class Observer:
         ):
             current_span.prompt = self.prompt
 
+        if not current_span.tools_called:
+            # check any tool span children
+            for child in current_span.children:
+                if isinstance(child, ToolSpan):
+                    current_span.tools_called = current_span.tools_called or []
+                    current_span.tools_called.append(
+                        ToolCall(
+                            name=child.name,
+                            description=child.description,
+                            input_parameters=prepare_tool_call_input_parameters(
+                                child.input
+                            ),
+                            output=child.output,
+                        )
+                    )
+
         trace_manager.remove_span(self.uuid)
         if current_span.parent_uuid:
             parent_span = trace_manager.get_span_by_uuid(

deepeval/tracing/types.py

@@ -3,6 +3,12 @@ from dataclasses import dataclass, field
 from pydantic import BaseModel, Field
 from typing import Any, Dict, List, Optional, Union
 from rich.progress import Progress
+from deepeval.tracing.message_types import (
+    ToolSchema,
+    ToolOutput,
+    TextMessage,
+    ToolCallMessage,
+)
 
 from deepeval.prompt.prompt import Prompt
 from deepeval.test_case.llm_test_case import ToolCall
@@ -88,6 +94,12 @@ class AgentSpan(BaseSpan):
 
 
 class LlmSpan(BaseSpan):
+    input: Optional[
+        Union[Any, List[Union[TextMessage, ToolCallMessage, ToolOutput]]]
+    ] = None
+    output: Optional[Union[Any, List[Union[TextMessage, ToolCallMessage]]]] = (
+        None
+    )
     model: Optional[str] = None
     prompt: Optional[Prompt] = None
     input_token_count: Optional[float] = Field(
@@ -106,6 +118,10 @@ class LlmSpan(BaseSpan):
         None, serialization_alias="tokenTimes"
     )
 
+    # input_tools: Optional[List[ToolSchema]] = Field(None, serialization_alias="inputTools")
+    # invocation_params: Optional[Dict[str, Any]] = Field(None, serialization_alias="invocationParams")
+    # output_metadata: Optional[Dict[str, Any]] = Field(None, serialization_alias="outputMetadata")
+
     # for serializing `prompt`
     model_config = {"arbitrary_types_allowed": True}