deepeval 3.6.4__py3-none-any.whl → 3.6.6__py3-none-any.whl

deepeval/test_run/test_run.py

@@ -2,9 +2,8 @@ from enum import Enum
 import os
 import json
 from pydantic import BaseModel, Field
-from typing import Any, Optional, List, Dict, Union
+from typing import Any, Optional, List, Dict, Union, Tuple
 import shutil
-import webbrowser
 import sys
 import datetime
 import portalocker
@@ -27,6 +26,9 @@ from deepeval.utils import (
     delete_file_if_exists,
     get_is_running_deepeval,
     open_browser,
+    shorten,
+    format_turn,
+    len_short,
 )
 from deepeval.test_run.cache import global_test_run_cache_manager
 from deepeval.constants import CONFIDENT_TEST_CASE_BATCH_SIZE, HIDDEN_DIR
@@ -546,7 +548,7 @@ class TestRunManager:
 
             if (
                 display == TestRunResultDisplay.PASSING
-                and test_case.success == False
+                and test_case.success is False
             ):
                 continue
             elif display == TestRunResultDisplay.FAILING and test_case.success:
@@ -618,7 +620,7 @@ class TestRunManager:
         ):
             if (
                 display == TestRunResultDisplay.PASSING
-                and conversational_test_case.success == False
+                and conversational_test_case.success is False
             ):
                 continue
             elif (
@@ -631,6 +633,65 @@ class TestRunManager:
             fail_count = 0
             conversational_test_case_name = conversational_test_case.name
 
+            if conversational_test_case.turns:
+                turns_table = Table(
+                    title=f"Conversation - {conversational_test_case_name}",
+                    show_header=True,
+                    header_style="bold",
+                )
+                turns_table.add_column("#", justify="right", width=3)
+                turns_table.add_column("Role", justify="left", width=10)
+
+                # subtract fixed widths + borders and padding.
+                # ~20 as a safe buffer
+                details_max_width = max(
+                    48, min(120, console.width - 3 - 10 - 20)
+                )
+                turns_table.add_column(
+                    "Details",
+                    justify="left",
+                    overflow="fold",
+                    max_width=details_max_width,
+                )
+
+                # truncate when too long
+                tools_max_width = min(60, max(24, console.width // 3))
+                turns_table.add_column(
+                    "Tools",
+                    justify="left",
+                    no_wrap=True,
+                    overflow="ellipsis",
+                    max_width=tools_max_width,
+                )
+
+                sorted_turns = sorted(
+                    conversational_test_case.turns, key=lambda t: t.order
+                )
+
+                for t in sorted_turns:
+                    tools = t.tools_called or []
+                    tool_names = ", ".join(tc.name for tc in tools)
+
+                    # omit order, role and tools since we show them in a separate columns.
+                    details = format_turn(
+                        t,
+                        include_tools_in_header=False,
+                        include_order_role_in_header=False,
+                    )
+
+                    turns_table.add_row(
+                        str(t.order),
+                        t.role,
+                        details,
+                        shorten(tool_names, len_short()),
+                    )
+
+                console.print(turns_table)
+            else:
+                console.print(
+                    f"[dim]No turns recorded for {conversational_test_case_name}.[/dim]"
+                )
+
             if conversational_test_case.metrics_data is not None:
                 for metric_data in conversational_test_case.metrics_data:
                     if metric_data.success:
@@ -698,7 +759,7 @@ class TestRunManager:
         )
         print(table)
 
-    def post_test_run(self, test_run: TestRun) -> Optional[str]:
+    def post_test_run(self, test_run: TestRun) -> Optional[Tuple[str, str]]:
         if (
             len(test_run.test_cases) == 0
             and len(test_run.conversational_test_cases) == 0
@@ -752,6 +813,21 @@ class TestRunManager:
             body=body,
         )
 
+        if not isinstance(data, dict) or "id" not in data:
+            # try to show helpful details
+            detail = None
+            if isinstance(data, dict):
+                detail = (
+                    data.get("detail")
+                    or data.get("message")
+                    or data.get("error")
+                )
+            # fall back to repr for visibility
+            raise RuntimeError(
+                f"Confident API response missing 'id'. "
+                f"detail={detail!r} raw={type(data).__name__}:{repr(data)[:500]}"
+            )
+
         res = TestRunHttpResponse(
             id=data["id"],
         )
@@ -814,7 +890,7 @@ class TestRunManager:
         )
         self.save_final_test_run_link(link)
         open_browser(link)
-        return link
+        return link, res.id
 
     def save_test_run_locally(self):
         local_folder = os.getenv("DEEPEVAL_RESULTS_FOLDER")
@@ -841,7 +917,7 @@ class TestRunManager:
         runDuration: float,
         display_table: bool = True,
         display: Optional[TestRunResultDisplay] = TestRunResultDisplay.ALL,
-    ) -> Optional[str]:
+    ) -> Optional[Tuple[str, str]]:
         test_run = self.get_test_run()
         if test_run is None:
             print("Test Run is empty, please try again.")
@@ -868,8 +944,8 @@ class TestRunManager:
         test_run.sort_test_cases()
 
         if global_test_run_cache_manager.disable_write_cache is None:
-            global_test_run_cache_manager.disable_write_cache = (
-                get_is_running_deepeval() == False
+            global_test_run_cache_manager.disable_write_cache = not bool(
+                get_is_running_deepeval()
             )
 
         global_test_run_cache_manager.wrap_up_cached_test_run()

deepeval/tracing/__init__.py

@@ -4,6 +4,7 @@ from .context import (
     update_retriever_span,
     update_llm_span,
 )
+from .trace_context import trace
 from .types import BaseSpan, Trace
 from .tracing import observe, trace_manager
 from .offline_evals import evaluate_thread, evaluate_trace, evaluate_span
@@ -16,6 +17,7 @@ __all__ = [
     "BaseSpan",
     "Trace",
     "observe",
+    "trace",
     "trace_manager",
     "evaluate_thread",
     "evaluate_trace",

deepeval/tracing/otel/exporter.py

@@ -90,12 +90,6 @@ class ConfidentSpanExporter(SpanExporter):
         api_key: Optional[str] = None,  # dynamic api key,
         _test_run_id: Optional[str] = None,
     ) -> SpanExportResult:
-        # build forest of spans
-        # for span in spans:
-        #     print("--------------------------------")
-        #     print(span.to_json())
-        #     print("--------------------------------")
-        # return SpanExportResult.SUCCESS
 
         ################ Build Forest of Spans ################
         forest = self._build_span_forest(spans)

deepeval/tracing/otel/test_exporter.py

@@ -0,0 +1,35 @@
+from typing import List, Dict, Any, Sequence
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SpanExporter
+from opentelemetry.sdk.trace.export import SpanExportResult
+import json
+from datetime import datetime
+
+
+class TestExporter(SpanExporter):
+    """This exporter is used to test the exporter. It will store the spans in a list of dictionaries."""
+
+    span_json_list: List[Dict[str, Any]] = []
+
+    def export(
+        self, spans: Sequence[ReadableSpan], timeout_millis: int = 30000
+    ) -> SpanExportResult:
+        for span in spans:
+            _span_json = json.loads(span.to_json())
+            self.span_json_list.append(_span_json)
+
+        return SpanExportResult.SUCCESS
+
+    def get_span_json_list(self) -> List[Dict[str, Any]]:
+        return sorted(
+            self.span_json_list,
+            key=lambda x: datetime.fromisoformat(
+                x["start_time"].replace("Z", "+00:00")
+            ),
+        )
+
+    def clear_span_json_list(self):
+        self.span_json_list = []
+
+
+test_exporter = TestExporter()

deepeval/tracing/otel/utils.py

@@ -109,8 +109,24 @@ def check_llm_input_from_gen_ai_attributes(
     input = None
     output = None
     try:
-        input = json.loads(span.attributes.get("gen_ai.input.messages"))
-        input = _flatten_input(input)
+        # check for system instructions
+        system_instructions = []
+        system_instructions_raw = span.attributes.get(
+            "gen_ai.system_instructions"
+        )
+        if system_instructions_raw and isinstance(system_instructions_raw, str):
+            system_instructions_json = json.loads(system_instructions_raw)
+            system_instructions = _flatten_system_instructions(
+                system_instructions_json
+            )
+
+        input_messages = []
+        input_messages_raw = span.attributes.get("gen_ai.input.messages")
+        if input_messages_raw and isinstance(input_messages_raw, str):
+            input_messages_json = json.loads(input_messages_raw)
+            input_messages = _flatten_input(input_messages_json)
+
+        input = system_instructions + input_messages
 
     except Exception:
         pass
@@ -137,6 +153,20 @@ def check_llm_input_from_gen_ai_attributes(
     return input, output
 
 
+def _flatten_system_instructions(system_instructions: list) -> list:
+    if isinstance(system_instructions, list):
+        for system_instruction in system_instructions:
+            if isinstance(system_instruction, dict):
+                role = system_instruction.get("role")
+                if not role:
+                    system_instruction["role"] = "System Instruction"
+        return _flatten_input(system_instructions)
+    elif isinstance(system_instructions, str):
+        return [{"role": "System Instruction", "content": system_instructions}]
+
+    return []
+
+
 def _flatten_input(input: list) -> list:
     if input and isinstance(input, list):
         try:
@@ -411,10 +441,23 @@ def _normalize_pydantic_ai_messages(span: ReadableSpan) -> Optional[list]:
     return None
 
 
+def _extract_non_thinking_part_of_last_message(message: dict) -> dict:
+
+    if isinstance(message, dict) and message.get("role") == "assistant":
+        parts = message.get("parts")
+        if parts:
+            # Iterate from the last part
+            for part in reversed(parts):
+                if isinstance(part, dict) and part.get("type") == "text":
+                    # Return a modified message with only the text content
+                    return {"role": "assistant", "content": part.get("content")}
+    return None
+
+
 def check_pydantic_ai_agent_input_output(
     span: ReadableSpan,
 ) -> Tuple[Optional[Any], Optional[Any]]:
-    input_val: Optional[Any] = None
+    input_val: list = []
     output_val: Optional[Any] = None
 
     # Get normalized messages once
@@ -445,14 +488,21 @@ def check_pydantic_ai_agent_input_output(
         if span.attributes.get("confident.span.type") == "agent":
             output_val = span.attributes.get("final_result")
             if not output_val and normalized:
-                # Extract the last message if no final_result is available
-                output_val = normalized[-1]
+                output_val = _extract_non_thinking_part_of_last_message(
+                    normalized[-1]
+                )
     except Exception:
         pass
 
+    system_instructions = []
+    system_instruction_raw = span.attributes.get("gen_ai.system_instructions")
+    if system_instruction_raw and isinstance(system_instruction_raw, str):
+        system_instructions = _flatten_system_instructions(
+            json.loads(system_instruction_raw)
+        )
+
     input_val = _flatten_input(input_val)
-    output_val = _flatten_input(output_val)
-    return input_val, output_val
+    return system_instructions + input_val, output_val
 
 
 def check_tool_output(span: ReadableSpan):

deepeval/tracing/trace_context.py

@@ -0,0 +1,14 @@
+from .context import current_trace_context
+from .tracing import trace_manager
+from contextlib import contextmanager
+
+
+@contextmanager
+def trace():
+    current_trace = current_trace_context.get()
+
+    if not current_trace:
+        current_trace = trace_manager.start_new_trace()
+        current_trace_context.set(current_trace)
+
+    yield current_trace

deepeval/tracing/trace_test_manager.py

@@ -0,0 +1,19 @@
+from typing import Optional, Dict, Any
+import asyncio
+from time import monotonic
+
+
+class TraceTestingManager:
+    test_name: Optional[str] = None
+    test_dict: Optional[Dict[str, Any]] = None
+
+    async def wait_for_test_dict(
+        self, timeout: float = 10.0, poll_interval: float = 0.05
+    ) -> Dict[str, Any]:
+        deadline = monotonic() + timeout
+        while self.test_dict is None and monotonic() < deadline:
+            await asyncio.sleep(poll_interval)
+        return self.test_dict or {}
+
+
+trace_testing_manager = TraceTestingManager()

deepeval/tracing/tracing.py

@@ -1,3 +1,4 @@
+import os
 from typing import Any, Dict, List, Literal, Optional, Set, Union, Callable
 from time import perf_counter
 import threading
@@ -47,13 +48,12 @@ from deepeval.tracing.utils import (
     tracing_enabled,
     validate_environment,
     validate_sampling_rate,
-    dump_body_to_json_file,
-    get_deepeval_trace_mode,
 )
 from deepeval.utils import dataclass_to_dict
 from deepeval.tracing.context import current_span_context, current_trace_context
 from deepeval.tracing.types import TestCaseMetricPair
 from deepeval.tracing.api import PromptApi
+from deepeval.tracing.trace_test_manager import trace_testing_manager
 
 EVAL_DUMMY_SPAN_NAME = "evals_iterator"
 
@@ -183,13 +183,14 @@ class TraceManager:
             if trace.status == TraceSpanStatus.IN_PROGRESS:
                 trace.status = TraceSpanStatus.SUCCESS
 
-            mode = get_deepeval_trace_mode()
-            if mode == "gen":
+            if trace_testing_manager.test_name:
+                # Trace testing mode is enabled
+                # Instead posting the trace to the queue, it will be stored in this global variable
                 body = self.create_trace_api(trace).model_dump(
                     by_alias=True, exclude_none=True
                 )
-                dump_body_to_json_file(body)
-            # Post the trace to the server before removing it
+                trace_testing_manager.test_dict = make_json_serializable(body)
+            #  Post the trace to the server before removing it
             elif not self.evaluating:
                 self.post_trace(trace)
             else:

deepeval/tracing/utils.py

@@ -1,13 +1,8 @@
 import os
-import inspect
-import json
-import sys
 from datetime import datetime, timezone
 from enum import Enum
 from time import perf_counter
 from collections import deque
-from typing import Any, Dict, Optional
-
 from deepeval.constants import CONFIDENT_TRACING_ENABLED
 
 
@@ -186,84 +181,5 @@ def perf_counter_to_datetime(perf_counter_value: float) -> datetime:
 def replace_self_with_class_name(obj):
     try:
         return f"<{obj.__class__.__name__}>"
-    except Exception:
-        return "<self>"
-
-
-def get_deepeval_trace_mode() -> Optional[str]:
-    deepeval_trace_mode = None
-    try:
-        args = sys.argv
-        for idx, arg in enumerate(args):
-            if isinstance(arg, str) and arg.startswith(
-                "--deepeval-trace-mode="
-            ):
-                deepeval_trace_mode = (
-                    arg.split("=", 1)[1].strip().strip('"').strip("'").lower()
-                )
-                break
-            if arg == "--deepeval-trace-mode" and idx + 1 < len(args):
-                deepeval_trace_mode = (
-                    str(args[idx + 1]).strip().strip('"').strip("'").lower()
-                )
-                break
-    except Exception:
-        deepeval_trace_mode = None
-
-    return deepeval_trace_mode
-
-
-def dump_body_to_json_file(
-    body: Dict[str, Any], file_path: Optional[str] = None
-) -> str:
-    entry_file = None
-    try:
-        cmd0 = sys.argv[0] if sys.argv else None
-        if cmd0 and cmd0.endswith(".py"):
-            entry_file = cmd0
-        else:
-            for frame_info in reversed(inspect.stack()):
-                fp = frame_info.filename
-                if (
-                    fp
-                    and fp.endswith(".py")
-                    and "deepeval/tracing" not in fp
-                    and "site-packages" not in fp
-                ):
-                    entry_file = fp
-                    break
-    except Exception:
-        entry_file = None
-
-    if not entry_file:
-        entry_file = "unknown.py"
-
-    abs_entry = os.path.abspath(entry_file)
-    dir_path = os.path.dirname(abs_entry)
-
-    file_arg = None
-    try:
-        for idx, arg in enumerate(sys.argv):
-            if isinstance(arg, str) and arg.startswith(
-                "--deepeval-trace-file-name="
-            ):
-                file_arg = arg.split("=", 1)[1].strip().strip('"').strip("'")
-                break
-            if arg == "--deepeval-trace-file-name" and idx + 1 < len(sys.argv):
-                file_arg = str(sys.argv[idx + 1]).strip().strip('"').strip("'")
-                break
-    except Exception:
-        file_arg = None
-
-    if file_path:
-        dst_path = os.path.abspath(file_path)
-    elif file_arg:
-        dst_path = os.path.abspath(file_arg)
-    else:
-        base_name = os.path.splitext(os.path.basename(abs_entry))[0]
-        dst_path = os.path.join(dir_path, f"{base_name}.json")
-
-    actual_body = make_json_serializable(body)
-    with open(dst_path, "w", encoding="utf-8") as f:
-        json.dump(actual_body, f, ensure_ascii=False, indent=2, sort_keys=True)
-    return dst_path
+    except:
+        return f"<self>"

deepeval/utils.py

@@ -13,7 +13,7 @@ import math
 
 from contextvars import ContextVar
 from enum import Enum
-from typing import Any, Optional, Dict, List, Union
+from typing import Any, Dict, List, Optional, Protocol, Sequence, Union
 from collections.abc import Iterable
 from dataclasses import asdict, is_dataclass
 from pydantic import BaseModel
@@ -28,6 +28,22 @@ from deepeval.config.utils import (
 )
 
 
+###############
+# Local Types #
+###############
+
+
+class TurnLike(Protocol):
+    order: int
+    role: str
+    content: str
+    user_id: Optional[str]
+    retrieval_context: Optional[Sequence[str]]
+    tools_called: Optional[Sequence[Any]]
+    additional_metadata: Optional[Dict[str, Any]]
+    comments: Optional[str]
+
+
 def get_lcs(seq1, seq2):
     m, n = len(seq1), len(seq2)
     dp = [[0] * (n + 1) for _ in range(m + 1)]
@@ -419,6 +435,138 @@ def is_missing(s: Optional[str]) -> bool:
     return s is None or (isinstance(s, str) and s.strip() == "")
 
 
+def len_tiny() -> int:
+    value = get_settings().DEEPEVAL_MAXLEN_TINY
+    return value if (isinstance(value, int) and value > 0) else 40
+
+
+def len_short() -> int:
+    value = get_settings().DEEPEVAL_MAXLEN_SHORT
+    return value if (isinstance(value, int) and value > 0) else 60
+
+
+def len_medium() -> int:
+    value = get_settings().DEEPEVAL_MAXLEN_MEDIUM
+    return value if (isinstance(value, int) and value > 0) else 120
+
+
+def len_long() -> int:
+    value = get_settings().DEEPEVAL_MAXLEN_LONG
+    return value if (isinstance(value, int) and value > 0) else 240
+
+
+def shorten(
+    text: Optional[object],
+    max_len: Optional[int] = None,
+    suffix: Optional[str] = None,
+) -> str:
+    """
+    Truncate text to max_len characters, appending `suffix` if truncated.
+    - Accepts None and returns "", or any object is returned as str().
+    - Safe when max_len <= len(suffix).
+    """
+    settings = get_settings()
+
+    if max_len is None:
+        max_len = (
+            settings.DEEPEVAL_SHORTEN_DEFAULT_MAXLEN
+            if settings.DEEPEVAL_SHORTEN_DEFAULT_MAXLEN is not None
+            else len_long()
+        )
+    if suffix is None:
+        suffix = (
+            settings.DEEPEVAL_SHORTEN_SUFFIX
+            if settings.DEEPEVAL_SHORTEN_SUFFIX is not None
+            else "..."
+        )
+
+    if text is None:
+        return ""
+    stext = str(text)
+    if max_len <= 0:
+        return ""
+    if len(stext) <= max_len:
+        return stext
+    cut = max_len - len(suffix)
+    if cut <= 0:
+        return suffix[:max_len]
+    return stext[:cut] + suffix
+
+
+def format_turn(
+    turn: TurnLike,
+    *,
+    content_length: Optional[int] = None,
+    max_context_items: Optional[int] = None,
+    context_length: Optional[int] = None,
+    meta_length: Optional[int] = None,
+    include_tools_in_header: bool = True,
+    include_order_role_in_header: bool = True,
+) -> str:
+    """
+    Build a multi-line, human-readable summary for a conversational turn.
+    Safe against missing fields and overly long content.
+    """
+    if content_length is None:
+        content_length = len_long()
+    if max_context_items is None:
+        max_context_items = 2
+    if context_length is None:
+        context_length = len_medium()
+    if meta_length is None:
+        meta_length = len_medium()
+
+    tools = turn.tools_called or []
+    tool_names = ", ".join(getattr(tc, "name", str(tc)) for tc in tools)
+    content = shorten(turn.content, content_length)
+
+    lines = []
+
+    if include_order_role_in_header:
+        header = f"{turn.order:>2}. {turn.role:<9} {content}"
+        if include_tools_in_header and tool_names:
+            header += f"  | tools: {tool_names}"
+        if turn.user_id:
+            header += f"  | user: {shorten(turn.user_id, len_tiny())}"
+        lines.append(header)
+        indent = "      "
+    else:
+        # No order or role prefix in this mode
+        # keep tools out of header as well.
+        first = content
+        if turn.user_id:
+            first += f"  | user: {shorten(turn.user_id, len_tiny())}"
+        lines.append(first)
+        indent = "      "  # ctx and meta indent
+
+    rctx = list(turn.retrieval_context or [])
+    if rctx:
+        show = rctx[:max_context_items]
+        for i, item in enumerate(show):
+            lines.append(f"{indent}↳ ctx[{i}]: {shorten(item, context_length)}")
+        hidden = max(0, len(rctx) - len(show))
+        if hidden:
+            lines.append(f"{indent}↳ ctx: (+{hidden} more)")
+
+    if turn.comments:
+        lines.append(
+            f"{indent}↳ comment: {shorten(str(turn.comments), meta_length)}"
+        )
+
+    meta = turn.additional_metadata or {}
+    if isinstance(meta, dict):
+        for k in list(meta.keys())[:3]:
+            if k in {"user_id", "userId"}:
+                continue
+            v = meta.get(k)
+            if v is not None:
+                lines.append(
+                    f"{indent}↳ meta.{k}: {shorten(str(v), meta_length)}"
+                )
+
+    return "\n".join(lines)
+
+
 ###############################################
 # Source: https://github.com/tingofurro/summac
 ###############################################

{deepeval-3.6.4.dist-info → deepeval-3.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.6.4
+Version: 3.6.6
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0