deepeval 3.6.6__py3-none-any.whl → 3.6.8__py3-none-any.whl

deepeval/openai/utils.py

@@ -1,67 +1,43 @@
-from typing import List
+import json
 import uuid
+from typing import Any, Dict, List, Optional, Iterable
+
+from openai.types.chat.chat_completion_message_param import (
+    ChatCompletionMessageParam,
+)
 
 from deepeval.tracing.types import ToolSpan, TraceSpanStatus
-from deepeval.openai.extractors import InputParameters, OutputParameters
 from deepeval.tracing.context import current_span_context
-from deepeval.test_case import LLMTestCase
-from deepeval.metrics import BaseMetric
-from deepeval.tracing.types import TestCaseMetricPair
-
-openai_test_case_pairs: List[TestCaseMetricPair] = []
-
-
-def set_attr_path(obj, attr_path: str, value):
-    *pre_path, final_attr = attr_path.split(".")
-    for attr in pre_path:
-        obj = getattr(obj, attr, None)
-        if obj is None:
-            return
-    setattr(obj, final_attr, value)
-
-
-def get_attr_path(obj, attr_path: str):
-    for attr in attr_path.split("."):
-        obj = getattr(obj, attr, None)
-        if obj is None:
-            return None
-    return obj
-
-
-def add_test_case(
-    test_case: LLMTestCase,
-    metrics: List[BaseMetric],
-    input_parameters: InputParameters,
-):
-    openai_test_case_pairs.append(
-        TestCaseMetricPair(
-            test_case=test_case,
-            metrics=metrics,
-            hyperparameters=create_hyperparameters_map(input_parameters),
+from deepeval.utils import shorten, len_long
+from deepeval.openai.types import OutputParameters
+
+
+_URL_MAX = 200
+_JSON_MAX = max(
+    len_long(), 400
+)  # <- make this bigger by increasing DEEPEVAL_MAXLEN_LONG above 400
+
+
+def _compact_dump(value: Any) -> str:
+    try:
+        dumped = json.dumps(
+            value, ensure_ascii=False, default=str, separators=(",", ":")
         )
-    )
-
-
-def create_hyperparameters_map(input_parameters: InputParameters):
-    hyperparameters = {"model": input_parameters.model}
-    if input_parameters.instructions:
-        hyperparameters["system_prompt"] = input_parameters.instructions
-    elif input_parameters.messages:
-        system_messages = [
-            m["content"]
-            for m in input_parameters.messages
-            if m["role"] == "system"
-        ]
-        if system_messages:
-            hyperparameters["system_prompt"] = (
-                system_messages[0]
-                if len(system_messages) == 1
-                else str(system_messages)
-            )
-    return hyperparameters
+    except Exception:
+        dumped = repr(value)
+    return shorten(dumped, max_len=_JSON_MAX)
+
+
+def _fmt_url(url: Optional[str]) -> str:
+    if not url:
+        return ""
+    if url.startswith("data:"):
+        return "[data-uri]"
+    return shorten(url, max_len=_URL_MAX)
 
 
 def create_child_tool_spans(output_parameters: OutputParameters):
+
     if output_parameters.tools_called is None:
         return
 
@@ -84,3 +60,176 @@ def create_child_tool_spans(output_parameters: OutputParameters):
             }
         )
         current_span.children.append(tool_span)
+
+
+def stringify_multimodal_content(content: Any) -> str:
+    """
+    Return a short, human-readable summary string for an OpenAI-style multimodal `content` value.
+
+    This is used to populate span summaries, such as `InputParameters.input`. It never raises and
+    never returns huge blobs.
+
+    Notes:
+    - Data URIs are redacted to "[data-uri]".
+    - Output is capped via `deepeval.utils.shorten` (configurable through settings).
+    - Fields that are not explicitly handled are returned as size-capped JSON dumps
+    - This string is for display/summary only, not intended to be parsable.
+
+    Args:
+        content: The value of an OpenAI message `content`, may be a str or list of typed parts,
+                 or any nested structure.
+
+    Returns:
+        A short, readable `str` summary.
+    """
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, (bytes, bytearray)):
+        return f"[bytes:{len(content)}]"
+
+    # list of parts for Chat & Responses
+    if isinstance(content, list):
+        parts: List[str] = []
+        for part in content:
+            s = stringify_multimodal_content(part)
+            if s:
+                parts.append(s)
+        return "\n".join(parts)
+
+    # documented dict shapes (Chat & Responses)
+    if isinstance(content, dict):
+        t = content.get("type")
+
+        # Chat Completions
+        if t == "text":
+            return str(content.get("text", ""))
+        if t == "image_url":
+            image_url = content.get("image_url")
+            if isinstance(image_url, str):
+                url = image_url
+            else:
+                url = (image_url or {}).get("url") or content.get("url")
+            return f"[image:{_fmt_url(url)}]"
+
+        # Responses API variants
+        if t == "input_text":
+            return str(content.get("text", ""))
+        if t == "input_image":
+            image_url = content.get("image_url")
+            if isinstance(image_url, str):
+                url = image_url
+            else:
+                url = (image_url or {}).get("url") or content.get("url")
+            return f"[image:{_fmt_url(url)}]"
+
+        # readability for other input_* types we don't currently handle
+        if t and t.startswith("input_"):
+            return f"[{t}]"
+
+    # unknown dicts and types returned as shortened JSON
+    return _compact_dump(content)
+
+
+def render_messages(
+    messages: Iterable[ChatCompletionMessageParam],
+) -> List[Dict[str, Any]]:
+
+    messages_list = []
+
+    for message in messages:
+        role = message.get("role")
+        content = message.get("content")
+        if role == "assistant" and message.get("tool_calls"):
+            tool_calls = message.get("tool_calls")
+            if isinstance(tool_calls, list):
+                for tool_call in tool_calls:
+                    # Extract type - either "function" or "custom"
+                    tool_type = tool_call.get("type", "function")
+
+                    # Extract name and arguments based on type
+                    if tool_type == "function":
+                        function_data = tool_call.get("function", {})
+                        name = function_data.get("name", "")
+                        arguments = function_data.get("arguments", "")
+                    elif tool_type == "custom":
+                        custom_data = tool_call.get("custom", {})
+                        name = custom_data.get("name", "")
+                        arguments = custom_data.get("input", "")
+                    else:
+                        name = ""
+                        arguments = ""
+
+                    messages_list.append(
+                        {
+                            "id": tool_call.get("id", ""),
+                            "call_id": tool_call.get(
+                                "id", ""
+                            ),  # OpenAI uses 'id', not 'call_id'
+                            "name": name,
+                            "type": tool_type,
+                            "arguments": json.loads(arguments),
+                        }
+                    )
+
+        elif role == "tool":
+            messages_list.append(
+                {
+                    "call_id": message.get("tool_call_id", ""),
+                    "type": role,  # "tool"
+                    "output": message.get("content", {}),
+                }
+            )
+        else:
+            messages_list.append(
+                {
+                    "role": role,
+                    "content": content,
+                }
+            )
+
+    return messages_list
+
+
+def render_response_input(input: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+
+    messages_list = []
+
+    for item in input:
+        type = item.get("type")
+        role = item.get("role")
+
+        if type == "message":
+            messages_list.append(
+                {
+                    "role": role,
+                    "content": item.get("content"),
+                }
+            )
+        else:
+            messages_list.append(item)
+
+    return messages_list
+
+
+def _render_content(content: Dict[str, Any], indent: int = 0) -> str:
+    """
+    Renders a dictionary as a formatted string with indentation for nested structures.
+    """
+    if not content:
+        return ""
+
+    lines = []
+    prefix = "  " * indent
+
+    for key, value in content.items():
+        if isinstance(value, dict):
+            lines.append(f"{prefix}{key}:")
+            lines.append(_render_content(value, indent + 1))
+        elif isinstance(value, list):
+            lines.append(f"{prefix}{key}: {_compact_dump(value)}")
+        else:
+            lines.append(f"{prefix}{key}: {value}")
+
+    return "\n".join(lines)

deepeval/prompt/__init__.py

@@ -1,3 +1,21 @@
 from .prompt import Prompt
+from .api import (
+    PromptMessage,
+    ModelSettings,
+    ModelProvider,
+    Verbosity,
+    ReasoningEffort,
+    OutputType,
+    PromptInterpolationType,
+)
 
-__all__ = ["Prompt"]
+__all__ = [
+    "Prompt",
+    "PromptMessage",
+    "ModelSettings",
+    "ModelProvider",
+    "Verbosity",
+    "ReasoningEffort",
+    "OutputType",
+    "PromptInterpolationType",
+]

deepeval/prompt/api.py

@@ -1,6 +1,119 @@
 from pydantic import BaseModel, Field, AliasChoices
 from enum import Enum
 from typing import List, Optional
+from pydantic import TypeAdapter
+
+###################################
+# Model Settings
+###################################
+
+
+class ReasoningEffort(Enum):
+    MINIMAL = "MINIMAL"
+    LOW = "LOW"
+    MEDIUM = "MEDIUM"
+    HIGH = "HIGH"
+
+
+class Verbosity(Enum):
+    LOW = "LOW"
+    MEDIUM = "MEDIUM"
+    HIGH = "HIGH"
+
+
+class ModelProvider(Enum):
+    OPEN_AI = "OPEN_AI"
+    ANTHROPIC = "ANTHROPIC"
+    GEMINI = "GEMINI"
+    X_AI = "X_AI"
+    DEEPSEEK = "DEEPSEEK"
+    BEDROCK = "BEDROCK"
+
+
+class ModelSettings(BaseModel):
+    provider: Optional[ModelProvider] = None
+    name: Optional[str] = None
+    temperature: Optional[float] = None
+    max_tokens: Optional[int] = Field(
+        default=None,
+        serialization_alias="maxTokens",
+        validation_alias=AliasChoices("max_tokens", "maxTokens"),
+    )
+    top_p: Optional[float] = Field(
+        default=None,
+        serialization_alias="topP",
+        validation_alias=AliasChoices("top_p", "topP"),
+    )
+    frequency_penalty: Optional[float] = Field(
+        default=None,
+        serialization_alias="frequencyPenalty",
+        validation_alias=AliasChoices("frequency_penalty", "frequencyPenalty"),
+    )
+    presence_penalty: Optional[float] = Field(
+        default=None,
+        serialization_alias="presencePenalty",
+        validation_alias=AliasChoices("presence_penalty", "presencePenalty"),
+    )
+    stop_sequence: Optional[List[str]] = Field(
+        default=None,
+        serialization_alias="stopSequence",
+        validation_alias=AliasChoices("stop_sequence", "stopSequence"),
+    )
+    reasoning_effort: Optional[ReasoningEffort] = Field(
+        default=None,
+        serialization_alias="reasoningEffort",
+        validation_alias=AliasChoices("reasoning_effort", "reasoningEffort"),
+    )
+    verbosity: Optional[Verbosity] = Field(
+        default=None,
+        serialization_alias="verbosity",
+        validation_alias=AliasChoices("verbosity", "verbosity"),
+    )
+
+
+###################################
+# Output Settings
+###################################
+
+
+class OutputType(Enum):
+    TEXT = "TEXT"
+    JSON = "JSON"
+    SCHEMA = "SCHEMA"
+
+
+class SchemaDataType(Enum):
+    OBJECT = "OBJECT"
+    STRING = "STRING"
+    FLOAT = "FLOAT"
+    INTEGER = "INTEGER"
+    BOOLEAN = "BOOLEAN"
+    NULL = "NULL"
+
+
+class OutputSchemaField(BaseModel):
+    id: str
+    type: SchemaDataType
+    name: str
+    required: Optional[bool] = False
+    parent_id: Optional[str] = Field(
+        default=None,
+        serialization_alias="parentId",
+        validation_alias=AliasChoices("parent_id", "parentId"),
+    )
+
+    class Config:
+        use_enum_values = True
+
+
+class OutputSchema(BaseModel):
+    fields: Optional[List[OutputSchemaField]] = None
+    name: str
+
+
+###################################
+# Prompt
+###################################
 
 
 class PromptInterpolationType(Enum):
@@ -16,6 +129,9 @@ class PromptMessage(BaseModel):
     content: str
 
 
+PromptMessageList = TypeAdapter(List[PromptMessage])
+
+
 class PromptType(Enum):
     TEXT = "TEXT"
     LIST = "LIST"
@@ -53,6 +169,21 @@ class PromptHttpResponse(BaseModel):
         serialization_alias="interpolationType"
     )
     type: PromptType
+    model_settings: Optional[ModelSettings] = Field(
+        default=None,
+        serialization_alias="modelSettings",
+        validation_alias=AliasChoices("model_settings", "modelSettings"),
+    )
+    output_type: Optional[OutputType] = Field(
+        default=None,
+        serialization_alias="outputType",
+        validation_alias=AliasChoices("output_type", "outputType"),
+    )
+    output_schema: Optional[OutputSchema] = Field(
+        default=None,
+        serialization_alias="outputSchema",
+        validation_alias=AliasChoices("output_schema", "outputSchema"),
+    )
 
 
 class PromptPushRequest(BaseModel):
@@ -62,6 +193,35 @@ class PromptPushRequest(BaseModel):
     interpolation_type: PromptInterpolationType = Field(
         serialization_alias="interpolationType"
     )
+    model_settings: Optional[ModelSettings] = Field(
+        default=None, serialization_alias="modelSettings"
+    )
+    output_schema: Optional[OutputSchema] = Field(
+        default=None, serialization_alias="outputSchema"
+    )
+    output_type: Optional[OutputType] = Field(
+        default=None, serialization_alias="outputType"
+    )
+
+    class Config:
+        use_enum_values = True
+
+
+class PromptUpdateRequest(BaseModel):
+    text: Optional[str] = None
+    messages: Optional[List[PromptMessage]] = None
+    interpolation_type: PromptInterpolationType = Field(
+        serialization_alias="interpolationType"
+    )
+    model_settings: Optional[ModelSettings] = Field(
+        default=None, serialization_alias="modelSettings"
+    )
+    output_schema: Optional[OutputSchema] = Field(
+        default=None, serialization_alias="outputSchema"
+    )
+    output_type: Optional[OutputType] = Field(
+        default=None, serialization_alias="outputType"
+    )
 
     class Config:
         use_enum_values = True