deepeval 3.6.5__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/test_case/api.py

@@ -0,0 +1,131 @@
+from typing import Union, Optional
+import os
+
+from deepeval.test_run.api import (
+    LLMApiTestCase,
+    ConversationalApiTestCase,
+    TurnApi,
+    TraceApi,
+)
+from deepeval.test_case import (
+    LLMTestCase,
+    ConversationalTestCase,
+    MLLMTestCase,
+    Turn,
+)
+from deepeval.constants import PYTEST_RUN_TEST_NAME
+
+
+def create_api_turn(turn: Turn, index: int) -> TurnApi:
+    return TurnApi(
+        role=turn.role,
+        content=turn.content,
+        user_id=turn.user_id,
+        retrievalContext=turn.retrieval_context,
+        toolsCalled=turn.tools_called,
+        additionalMetadata=turn.additional_metadata,
+        order=index,
+    )
+
+
+def create_api_test_case(
+    test_case: Union[LLMTestCase, ConversationalTestCase, MLLMTestCase],
+    trace: Optional[TraceApi] = None,
+    index: Optional[int] = None,
+) -> Union[LLMApiTestCase, ConversationalApiTestCase]:
+    if isinstance(test_case, ConversationalTestCase):
+        order = (
+            test_case._dataset_rank
+            if test_case._dataset_rank is not None
+            else index
+        )
+        if test_case.name:
+            name = test_case.name
+        else:
+            name = os.getenv(
+                PYTEST_RUN_TEST_NAME, f"conversational_test_case_{order}"
+            )
+
+        api_test_case = ConversationalApiTestCase(
+            name=name,
+            success=True,
+            metricsData=[],
+            runDuration=0,
+            evaluationCost=None,
+            order=order,
+            scenario=test_case.scenario,
+            expectedOutcome=test_case.expected_outcome,
+            userDescription=test_case.user_description,
+            context=test_case.context,
+            tags=test_case.tags,
+            comments=test_case.comments,
+            additionalMetadata=test_case.additional_metadata,
+        )
+        api_test_case.turns = [
+            create_api_turn(
+                turn=turn,
+                index=index,
+            )
+            for index, turn in enumerate(test_case.turns)
+        ]
+
+        return api_test_case
+    else:
+        order = (
+            test_case._dataset_rank
+            if test_case._dataset_rank is not None
+            else index
+        )
+
+        success = True
+        if test_case.name is not None:
+            name = test_case.name
+        else:
+            name = os.getenv(PYTEST_RUN_TEST_NAME, f"test_case_{order}")
+        metrics_data = []
+
+        if isinstance(test_case, LLMTestCase):
+            api_test_case = LLMApiTestCase(
+                name=name,
+                input=test_case.input,
+                actualOutput=test_case.actual_output,
+                expectedOutput=test_case.expected_output,
+                context=test_case.context,
+                retrievalContext=test_case.retrieval_context,
+                toolsCalled=test_case.tools_called,
+                expectedTools=test_case.expected_tools,
+                tokenCost=test_case.token_cost,
+                completionTime=test_case.completion_time,
+                tags=test_case.tags,
+                success=success,
+                metricsData=metrics_data,
+                runDuration=None,
+                evaluationCost=None,
+                order=order,
+                additionalMetadata=test_case.additional_metadata,
+                comments=test_case.comments,
+                trace=trace,
+            )
+        elif isinstance(test_case, MLLMTestCase):
+            api_test_case = LLMApiTestCase(
+                name=name,
+                input="",
+                multimodalInput=test_case.input,
+                multimodalActualOutput=test_case.actual_output,
+                multimodalExpectedOutput=test_case.expected_output,
+                multimodalRetrievalContext=test_case.retrieval_context,
+                multimodalContext=test_case.context,
+                toolsCalled=test_case.tools_called,
+                expectedTools=test_case.expected_tools,
+                tokenCost=test_case.token_cost,
+                completionTime=test_case.completion_time,
+                success=success,
+                metricsData=metrics_data,
+                runDuration=None,
+                evaluationCost=None,
+                order=order,
+                additionalMetadata=test_case.additional_metadata,
+                comments=test_case.comments,
+            )
+        # llm_test_case_lookup_map[instance_id] = api_test_case
+        return api_test_case

deepeval/test_run/__init__.py

@@ -8,6 +8,7 @@ from .test_run import (
     LLMApiTestCase,
     ConversationalApiTestCase,
     TestRunManager,
+    PromptData,
 )
 
 from .hooks import on_test_run_end, invoke_test_run_end_hook

deepeval/test_run/hyperparameters.py

@@ -1,13 +1,15 @@
-from typing import Union, Dict
-
+from typing import Union, Dict, Optional, List
 from deepeval.test_run import global_test_run_manager
 from deepeval.prompt import Prompt
 from deepeval.prompt.api import PromptApi
 from deepeval.test_run.test_run import TEMP_FILE_PATH
+from deepeval.confident.api import is_confident
+from deepeval.test_run.test_run import PromptData
 
 
 def process_hyperparameters(
-    hyperparameters,
+    hyperparameters: Optional[Dict] = None,
+    verbose: bool = True,
 ) -> Union[Dict[str, Union[str, int, float, PromptApi]], None]:
     if hyperparameters is None:
         return None
@@ -16,6 +18,7 @@ def process_hyperparameters(
         raise TypeError("Hyperparameters must be a dictionary or None")
 
     processed_hyperparameters = {}
+    prompts_version_id_map = {}
 
     for key, value in hyperparameters.items():
         if not isinstance(key, str):
@@ -30,14 +33,21 @@ def process_hyperparameters(
             )
 
         if isinstance(value, Prompt):
-            if value._prompt_version_id is not None and value._type is not None:
+            prompt_key = f"{value.alias}_{value.version}"
+            if value._prompt_version_id is not None and value.type is not None:
                 processed_hyperparameters[key] = PromptApi(
                     id=value._prompt_version_id,
-                    type=value._type,
+                    type=value.type,
                 )
-            else:
-                raise ValueError(
-                    f"Cannot log Prompt where template was not pulled from Confident AI. Please import your prompt on Confident AI to continue."
+            elif is_confident():
+                if prompt_key not in prompts_version_id_map:
+                    value.push(_verbose=verbose)
+                    prompts_version_id_map[prompt_key] = (
+                        value._prompt_version_id
+                    )
+                processed_hyperparameters[key] = PromptApi(
+                    id=prompts_version_id_map[prompt_key],
+                    type=value.type,
                 )
         else:
             processed_hyperparameters[key] = str(value)
@@ -64,3 +74,32 @@ def log_hyperparameters(func):
 
     # Return the wrapper function to be used as the decorator
     return wrapper
+
+
+def process_prompts(
+    hyperparameters: Dict[str, Union[str, int, float, Prompt]],
+) -> List[PromptData]:
+    prompts = []
+    if not hyperparameters:
+        return prompts
+    seen_prompts = set()
+    prompt_objects = [
+        value for value in hyperparameters.values() if isinstance(value, Prompt)
+    ]
+    for prompt in prompt_objects:
+        prompt_version = prompt.version if is_confident() else None
+        prompt_key = f"{prompt.alias}_{prompt_version}"
+        if prompt_key in seen_prompts:
+            continue
+        seen_prompts.add(prompt_key)
+        prompt_data = PromptData(
+            alias=prompt.alias,
+            version=prompt_version,
+            text_template=prompt.text_template,
+            messages_template=prompt.messages_template,
+            model_settings=prompt.model_settings,
+            output_type=prompt.output_type,
+            interpolation_type=prompt.interpolation_type,
+        )
+        prompts.append(prompt_data)
+    return prompts

deepeval/test_run/test_run.py

@@ -32,6 +32,17 @@ from deepeval.utils import (
 )
 from deepeval.test_run.cache import global_test_run_cache_manager
 from deepeval.constants import CONFIDENT_TEST_CASE_BATCH_SIZE, HIDDEN_DIR
+from deepeval.prompt import (
+    PromptMessage,
+    ModelSettings,
+    OutputType,
+    PromptInterpolationType,
+    OutputType,
+)
+from rich.panel import Panel
+from rich.text import Text
+from rich.columns import Columns
+
 
 TEMP_FILE_PATH = f"{HIDDEN_DIR}/.temp_test_run_data.json"
 LATEST_TEST_RUN_FILE_PATH = f"{HIDDEN_DIR}/.latest_test_run.json"
@@ -71,6 +82,16 @@ class TraceMetricScores(BaseModel):
     base: Dict[str, Dict[str, MetricScores]] = Field(default_factory=dict)
 
 
+class PromptData(BaseModel):
+    alias: Optional[str] = None
+    version: Optional[str] = None
+    text_template: Optional[str] = None
+    messages_template: Optional[List[PromptMessage]] = None
+    model_settings: Optional[ModelSettings] = None
+    output_type: Optional[OutputType] = None
+    interpolation_type: Optional[PromptInterpolationType] = None
+
+
 class MetricsAverageDict:
     def __init__(self):
         self.metric_dict = {}
@@ -123,6 +144,7 @@ class TestRun(BaseModel):
     )
     identifier: Optional[str] = None
     hyperparameters: Optional[Dict[str, Any]] = Field(None)
+    prompts: Optional[List[PromptData]] = Field(None)
     test_passed: Optional[int] = Field(None, alias="testPassed")
     test_failed: Optional[int] = Field(None, alias="testFailed")
     run_duration: float = Field(0.0, alias="runDuration")
@@ -799,6 +821,7 @@ class TestRunManager:
             test_run.test_cases = initial_batch
 
         try:
+            test_run.prompts = None
             body = test_run.model_dump(by_alias=True, exclude_none=True)
         except AttributeError:
             # Pydantic version below 2.0
@@ -953,6 +976,23 @@ class TestRunManager:
         if display_table:
             self.display_results_table(test_run, display)
 
+        if test_run.hyperparameters is None:
+            console.print(
+                "\n[bold yellow]⚠ WARNING:[/bold yellow] No hyperparameters logged.\n"
+                "» [bold blue][link=https://deepeval.com/docs/evaluation-prompts]Log hyperparameters[/link][/bold blue] to attribute prompts and models to your test runs.\n\n"
+                + "=" * 80
+            )
+        else:
+            if not test_run.prompts:
+                console.print(
+                    "\n[bold yellow]⚠ WARNING:[/bold yellow] No prompts logged.\n"
+                    "» [bold blue][link=https://deepeval.com/docs/evaluation-prompts]Log prompts[/link][/bold blue] to evaluate and optimize your prompt templates and models.\n\n"
+                    + "=" * 80
+                )
+            else:
+                console.print("\n[bold green]✓ Prompts Logged[/bold green]\n")
+                self._render_prompts_panels(prompts=test_run.prompts)
+
         self.save_test_run_locally()
         delete_file_if_exists(self.temp_file_path)
         if is_confident() and self.disable_request is False:
@@ -967,7 +1007,7 @@ class TestRunManager:
                 f"» Test Results ({test_run.test_passed + test_run.test_failed} total tests):\n",
                 f"  » Pass Rate: {round((test_run.test_passed / (test_run.test_passed + test_run.test_failed)) * 100, 2)}% | Passed: [bold green]{test_run.test_passed}[/bold green] | Failed: [bold red]{test_run.test_failed}[/bold red]\n\n",
                 "=" * 80,
-                "\n\n» What to share evals with your team, or a place for your test cases to live? ❤️ 🏡\n"
+                "\n\n» Want to share evals with your team, or a place for your test cases to live? ❤️ 🏡\n"
                 "  » Run [bold]'deepeval view'[/bold] to analyze and save testing results on [rgb(106,0,255)]Confident AI[/rgb(106,0,255)].\n\n",
             )
 
@@ -993,5 +1033,68 @@ class TestRunManager:
             pass
         return None
 
+    def _render_prompts_panels(self, prompts: List[PromptData]) -> None:
+
+        def format_string(
+            v, default="[dim]None[/dim]", color: Optional[str] = None
+        ):
+            formatted_string = str(v) if v not in (None, "", []) else default
+            return (
+                f"{formatted_string}"
+                if color is None or v in (None, "", [])
+                else f"[{color}]{formatted_string}[/]"
+            )
+
+        panels = []
+        for prompt in prompts:
+            lines = []
+            p_type = (
+                "messages"
+                if prompt.messages_template
+                else ("text" if prompt.text_template else "—")
+            )
+            if p_type:
+                lines.append(f"type: {format_string(p_type, color='blue')}")
+            if prompt.output_type:
+                lines.append(
+                    f"output_type: {format_string(prompt.output_type, color='blue')}"
+                )
+            if prompt.interpolation_type:
+                lines.append(
+                    f"interpolation_type: {format_string(prompt.interpolation_type, color='blue')}"
+                )
+            if prompt.model_settings:
+                ms = prompt.model_settings
+                settings_lines = [
+                    "Model Settings:",
+                    f"  – provider: {format_string(ms.provider, color='green')}",
+                    f"  – name: {format_string(ms.name, color='green')}",
+                    f"  – temperature: {format_string(ms.temperature, color='green')}",
+                    f"  – max_tokens: {format_string(ms.max_tokens, color='green')}",
+                    f"  – top_p: {format_string(ms.top_p, color='green')}",
+                    f"  – frequency_penalty: {format_string(ms.frequency_penalty, color='green')}",
+                    f"  – presence_penalty: {format_string(ms.presence_penalty, color='green')}",
+                    f"  – stop_sequence: {format_string(ms.stop_sequence, color='green')}",
+                    f"  – reasoning_effort: {format_string(ms.reasoning_effort, color='green')}",
+                    f"  – verbosity: {format_string(ms.verbosity, color='green')}",
+                ]
+                lines.append("")
+                lines.extend(settings_lines)
+            title = f"{format_string(prompt.alias)}"
+            if prompt.version:
+                title += f" (v{prompt.version})"
+            body = "\n".join(lines)
+            panel = Panel(
+                body,
+                title=title,
+                title_align="left",
+                expand=False,
+                padding=(1, 6, 1, 2),
+            )
+            panels.append(panel)
+
+        if panels:
+            console.print(Columns(panels, equal=False, expand=False))
+
 
 global_test_run_manager = TestRunManager()

deepeval/tracing/api.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import Dict, List, Optional, Union, Literal, Any
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from deepeval.test_case import ToolCall
 
@@ -27,6 +27,8 @@ class PromptApi(BaseModel):
 
 
 class MetricData(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
     name: str
     threshold: float
     success: bool

deepeval/tracing/message_types/__init__.py

@@ -0,0 +1,10 @@
+from .messages import TextMessage, ToolCallMessage
+from .tools import BaseTool, ToolSchema, ToolOutput
+
+__all__ = [
+    "BaseTool",
+    "TextMessage",
+    "ToolCallMessage",
+    "ToolSchema",
+    "ToolOutput",
+]

deepeval/tracing/message_types/base.py

@@ -0,0 +1,6 @@
+from typing import Literal
+from pydantic import BaseModel
+
+
+class BaseMessage(BaseModel):
+    role: Literal["user", "assistant"]

deepeval/tracing/message_types/messages.py

@@ -0,0 +1,14 @@
+from typing import Literal, Dict, Any
+from .base import BaseMessage
+
+
+class TextMessage(BaseMessage):
+    type: Literal["text", "thinking"]
+    content: str
+
+
+class ToolCallMessage(BaseMessage):
+    """This is a message for tool calls in response.choices[0].message.tool_calls"""
+
+    name: str
+    args: Dict[str, Any]

deepeval/tracing/message_types/tools.py

@@ -0,0 +1,18 @@
+from typing import Any, Optional, Dict
+from pydantic import BaseModel
+
+
+class BaseTool(BaseModel):
+    name: str
+    description: Optional[str] = None
+
+
+class ToolSchema(BaseTool):
+    parameters: Dict[str, Any]
+    is_called: Optional[bool] = False
+
+
+class ToolOutput(BaseTool):
+    """Output of the tool function"""
+
+    output: Any

deepeval/tracing/otel/exporter.py

@@ -90,12 +90,6 @@ class ConfidentSpanExporter(SpanExporter):
         api_key: Optional[str] = None,  # dynamic api key,
         _test_run_id: Optional[str] = None,
     ) -> SpanExportResult:
-        # build forest of spans
-        # for span in spans:
-        #     print("--------------------------------")
-        #     print(span.to_json())
-        #     print("--------------------------------")
-        # return SpanExportResult.SUCCESS
 
         ################ Build Forest of Spans ################
         forest = self._build_span_forest(spans)

deepeval/tracing/otel/utils.py

@@ -3,7 +3,7 @@ import json
 from typing import List, Optional, Tuple, Any
 from opentelemetry.sdk.trace.export import ReadableSpan
 
-from deepeval.evaluate.utils import create_api_test_case
+from deepeval.test_case.api import create_api_test_case
 from deepeval.test_run.api import LLMApiTestCase
 from deepeval.test_run.test_run import global_test_run_manager
 from deepeval.tracing.types import Trace, LLMTestCase, ToolCall
@@ -109,8 +109,24 @@ def check_llm_input_from_gen_ai_attributes(
     input = None
     output = None
     try:
-        input = json.loads(span.attributes.get("gen_ai.input.messages"))
-        input = _flatten_input(input)
+        # check for system instructions
+        system_instructions = []
+        system_instructions_raw = span.attributes.get(
+            "gen_ai.system_instructions"
+        )
+        if system_instructions_raw and isinstance(system_instructions_raw, str):
+            system_instructions_json = json.loads(system_instructions_raw)
+            system_instructions = _flatten_system_instructions(
+                system_instructions_json
+            )
+
+        input_messages = []
+        input_messages_raw = span.attributes.get("gen_ai.input.messages")
+        if input_messages_raw and isinstance(input_messages_raw, str):
+            input_messages_json = json.loads(input_messages_raw)
+            input_messages = _flatten_input(input_messages_json)
+
+        input = system_instructions + input_messages
 
     except Exception:
         pass
@@ -137,6 +153,20 @@ def check_llm_input_from_gen_ai_attributes(
     return input, output
 
 
+def _flatten_system_instructions(system_instructions: list) -> list:
+    if isinstance(system_instructions, list):
+        for system_instruction in system_instructions:
+            if isinstance(system_instruction, dict):
+                role = system_instruction.get("role")
+                if not role:
+                    system_instruction["role"] = "System Instruction"
+        return _flatten_input(system_instructions)
+    elif isinstance(system_instructions, str):
+        return [{"role": "System Instruction", "content": system_instructions}]
+
+    return []
+
+
 def _flatten_input(input: list) -> list:
     if input and isinstance(input, list):
         try:
@@ -411,10 +441,23 @@ def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     return None
 
 
+def _extract_non_thinking_part_of_last_message(message: dict) -> dict:
+
+    if isinstance(message, dict) and message.get("role") == "assistant":
+        parts = message.get("parts")
+        if parts:
+            # Iterate from the last part
+            for part in reversed(parts):
+                if isinstance(part, dict) and part.get("type") == "text":
+                    # Return a modified message with only the text content
+                    return {"role": "assistant", "content": part.get("content")}
+    return None
+
+
 def check_pydantic_ai_agent_input_output(
     span: ReadableSpan,
 ) -> Tuple[Optional[Any], Optional[Any]]:
-    input_val: Optional[Any] = None
+    input_val: list = []
     output_val: Optional[Any] = None
 
     # Get normalized messages once
@@ -445,14 +488,21 @@ def check_pydantic_ai_agent_input_output(
         if span.attributes.get("confident.span.type") == "agent":
             output_val = span.attributes.get("final_result")
             if not output_val and normalized:
-                # Extract the last message if no final_result is available
-                output_val = normalized[-1]
+                output_val = _extract_non_thinking_part_of_last_message(
+                    normalized[-1]
+                )
     except Exception:
         pass
 
+    system_instructions = []
+    system_instruction_raw = span.attributes.get("gen_ai.system_instructions")
+    if system_instruction_raw and isinstance(system_instruction_raw, str):
+        system_instructions = _flatten_system_instructions(
+            json.loads(system_instruction_raw)
+        )
+
     input_val = _flatten_input(input_val)
-    output_val = _flatten_input(output_val)
-    return input_val, output_val
+    return system_instructions + input_val, output_val
 
 
 def check_tool_output(span: ReadableSpan):

deepeval/tracing/trace_context.py

@@ -1,14 +1,83 @@
-from .context import current_trace_context
-from .tracing import trace_manager
+from typing import Optional, List, Dict, Any
+from contextvars import ContextVar
 from contextlib import contextmanager
+from dataclasses import dataclass
+
+from .tracing import trace_manager
+from .context import current_trace_context, update_current_trace
+from deepeval.prompt import Prompt
+from deepeval.metrics import BaseMetric
+from deepeval.test_case.llm_test_case import ToolCall
+
+
+@dataclass
+class LlmContext:
+    prompt: Optional[Prompt] = None
+    metrics: Optional[List[BaseMetric]] = None
+    metric_collection: Optional[str] = None
+    expected_output: Optional[str] = None
+    expected_tools: Optional[List[ToolCall]] = None
+    context: Optional[List[str]] = None
+    retrieval_context: Optional[List[str]] = None
+
+
+current_llm_context: ContextVar[Optional[LlmContext]] = ContextVar(
+    "current_llm_context", default=LlmContext()
+)
 
 
 @contextmanager
-def trace():
+def trace(
+    prompt: Optional[Prompt] = None,
+    llm_metrics: Optional[List[BaseMetric]] = None,
+    llm_metric_collection: Optional[str] = None,
+    name: Optional[str] = None,
+    tags: Optional[List[str]] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+    user_id: Optional[str] = None,
+    thread_id: Optional[str] = None,
+    expected_output: Optional[str] = None,
+    expected_tools: Optional[List[ToolCall]] = None,
+    context: Optional[List[str]] = None,
+    retrieval_context: Optional[List[str]] = None,
+    trace_metric_collection: Optional[str] = None,
+    trace_metrics: Optional[List[BaseMetric]] = None,
+):
     current_trace = current_trace_context.get()
 
     if not current_trace:
         current_trace = trace_manager.start_new_trace()
-        current_trace_context.set(current_trace)
+
+    if trace_metrics:
+        current_trace.metrics = trace_metrics
+
+    if trace_metric_collection:
+        current_trace.metric_collection = trace_metric_collection
+
+    current_trace_context.set(current_trace)
+
+    current_llm_context.set(
+        LlmContext(
+            prompt=prompt,
+            metrics=llm_metrics,
+            metric_collection=llm_metric_collection,
+            expected_output=expected_output,
+            expected_tools=expected_tools,
+            context=context,
+            retrieval_context=retrieval_context,
+        )
+    )
+
+    # set the current trace attributes
+    if name:
+        update_current_trace(name=name)
+    if tags:
+        update_current_trace(tags=tags)
+    if metadata:
+        update_current_trace(metadata=metadata)
+    if user_id:
+        update_current_trace(user_id=user_id)
+    if thread_id:
+        update_current_trace(thread_id=thread_id)
 
     yield current_trace

deepeval/tracing/trace_test_manager.py

@@ -0,0 +1,19 @@
+from typing import Optional, Dict, Any
+import asyncio
+from time import monotonic
+
+
+class TraceTestingManager:
+    test_name: Optional[str] = None
+    test_dict: Optional[Dict[str, Any]] = None
+
+    async def wait_for_test_dict(
+        self, timeout: float = 10.0, poll_interval: float = 0.05
+    ) -> Dict[str, Any]:
+        deadline = monotonic() + timeout
+        while self.test_dict is None and monotonic() < deadline:
+            await asyncio.sleep(poll_interval)
+        return self.test_dict or {}
+
+
+trace_testing_manager = TraceTestingManager()