deepeval 3.5.5__py3-none-any.whl → 3.5.7__py3-none-any.whl

deepeval/_version.py

@@ -1 +1 @@
-__version__: str = "3.5.5"
+__version__: str = "3.5.7"

deepeval/cli/main.py

@@ -28,8 +28,6 @@ import typer
 from enum import Enum
 from pydantic import SecretStr
 from deepeval.key_handler import (
-    KEY_FILE_HANDLER,
-    KeyValues,
     EmbeddingKeyValues,
     ModelKeyValues,
 )
@@ -46,16 +44,9 @@ from deepeval.cli.utils import (
     render_login_message,
     upload_and_open_link,
     PROD,
-    resolve_save_target,
-    save_environ_to_store,
-    unset_environ_in_store,
-    switch_model_provider,
 )
 from deepeval.confident.api import (
-    get_confident_api_key,
     is_confident,
-    set_confident_api_key,
-    CONFIDENT_API_KEY_ENV_VAR,
 )
 
 app = typer.Typer(name="deepeval")
@@ -109,7 +100,7 @@ def set_confident_region_command(
     # Add flag emojis based on region
     flag = "🇺🇸" if region == Regions.US else "🇪🇺"
 
-    setting = get_settings()
+    settings = get_settings()
     with settings.edit(save=save) as edit_ctx:
         settings.CONFIDENT_REGION = region.value
 
@@ -282,23 +273,196 @@ def view():
             upload_and_open_link(_span=span)
 
 
-@app.command(name="enable-grpc-logging")
-def enable_grpc_logging(save: Optional[str] = None):
+@app.command(name="set-debug")
+def set_debug(
+    # Core verbosity
+    log_level: Optional[str] = typer.Option(
+        None,
+        "--log-level",
+        help="Global LOG_LEVEL (DEBUG|INFO|WARNING|ERROR|CRITICAL|NOTSET).",
+    ),
+    verbose: Optional[bool] = typer.Option(
+        None, "--verbose/--no-verbose", help="Toggle DEEPEVAL_VERBOSE_MODE."
+    ),
+    # Retry logging dials
+    retry_before_level: Optional[str] = typer.Option(
+        None,
+        "--retry-before-level",
+        help="Log level before a retry attempt (DEBUG|INFO|WARNING|ERROR|CRITICAL|NOTSET or numeric).",
+    ),
+    retry_after_level: Optional[str] = typer.Option(
+        None,
+        "--retry-after-level",
+        help="Log level after a retry attempt (DEBUG|INFO|WARNING|ERROR|CRITICAL|NOTSET or numeric).",
+    ),
+    # gRPC visibility
+    grpc: Optional[bool] = typer.Option(
+        None, "--grpc/--no-grpc", help="Toggle DEEPEVAL_GRPC_LOGGING."
+    ),
+    grpc_verbosity: Optional[str] = typer.Option(
+        None,
+        "--grpc-verbosity",
+        help="Set GRPC_VERBOSITY (DEBUG|INFO|ERROR|NONE).",
+    ),
+    grpc_trace: Optional[str] = typer.Option(
+        None,
+        "--grpc-trace",
+        help=(
+            "Set GRPC_TRACE to comma-separated tracer names or glob patterns "
+            "(e.g. 'tcp,http,secure_endpoint', '*' for all, 'list_tracers' to print available)."
+        ),
+    ),
+    # Confident tracing
+    trace_verbose: Optional[bool] = typer.Option(
+        None,
+        "--trace-verbose/--no-trace-verbose",
+        help="Enable / disable CONFIDENT_TRACE_VERBOSE.",
+    ),
+    trace_env: Optional[str] = typer.Option(
+        None,
+        "--trace-env",
+        help='Set CONFIDENT_TRACE_ENVIRONMENT ("development", "staging", "production", etc).',
+    ),
+    trace_flush: Optional[bool] = typer.Option(
+        None,
+        "--trace-flush/--no-trace-flush",
+        help="Enable / disable  CONFIDENT_TRACE_FLUSH.",
+    ),
+    # Advanced / potentially surprising
+    error_reporting: Optional[bool] = typer.Option(
+        None,
+        "--error-reporting/--no-error-reporting",
+        help="Enable / disable ERROR_REPORTING.",
+    ),
+    ignore_errors: Optional[bool] = typer.Option(
+        None,
+        "--ignore-errors/--no-ignore-errors",
+        help="Enable / disable IGNORE_DEEPEVAL_ERRORS (not recommended in normal debugging).",
+    ),
+    # Persistence
+    save: Optional[str] = typer.Option(
+        None,
+        "--save",
+        help="Persist CLI parameters as environment variables in a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+):
     """
-    Enable verbose gRPC logging for the current process.
-    Pass --save=dotenv[:path] to persist it (optional).
+    Configure verbose debug behavior for DeepEval.
+
+    This command lets you mix-and-match verbosity flags (global LOG_LEVEL, verbose mode),
+    retry logger levels, gRPC wire logging, and Confident trace toggles. Values apply
+    immediately to the current process and can be persisted to a dotenv file with --save.
+
+    Examples:
+        deepeval set-debug --log-level DEBUG --verbose --grpc --retry-before-level DEBUG --retry-after-level INFO
+        deepeval set-debug --trace-verbose --trace-env staging --save dotenv:.env.local
     """
     settings = get_settings()
     with settings.edit(save=save) as edit_ctx:
-        settings.DEEPEVAL_GRPC_LOGGING = True
+        # Core verbosity
+        if log_level is not None:
+            settings.LOG_LEVEL = log_level
+        if verbose is not None:
+            settings.DEEPEVAL_VERBOSE_MODE = verbose
+
+        # Retry logging
+        if retry_before_level is not None:
+            settings.DEEPEVAL_RETRY_BEFORE_LOG_LEVEL = retry_before_level
+        if retry_after_level is not None:
+            settings.DEEPEVAL_RETRY_AFTER_LOG_LEVEL = retry_after_level
+
+        # gRPC
+        if grpc is not None:
+            settings.DEEPEVAL_GRPC_LOGGING = grpc
+        if grpc_verbosity is not None:
+            settings.GRPC_VERBOSITY = grpc_verbosity
+        if grpc_trace is not None:
+            settings.GRPC_TRACE = grpc_trace
+
+        # Confident tracing
+        if trace_verbose is not None:
+            settings.CONFIDENT_TRACE_VERBOSE = trace_verbose
+        if trace_env is not None:
+            settings.CONFIDENT_TRACE_ENVIRONMENT = trace_env
+        if trace_flush is not None:
+            settings.CONFIDENT_TRACE_FLUSH = trace_flush
+
+        # Advanced
+        if error_reporting is not None:
+            settings.ERROR_REPORTING = error_reporting
+        if ignore_errors is not None:
+            settings.IGNORE_DEEPEVAL_ERRORS = ignore_errors
+
+    handled, path, updated = edit_ctx.result
+
+    if not updated:
+        # no changes were made, so there is nothing to do.
+        return
+
+    if not handled and save is not None:
+        print("Unsupported --save option. Use --save=dotenv[:path].")
+    elif path:
+        print(
+            f"Saved environment variables to {path} (ensure it's git-ignored)."
+        )
+    else:
+        print(
+            "Settings updated for this session. To persist, use --save=dotenv[:path] "
+            "(default .env.local) or set DEEPEVAL_DEFAULT_SAVE=dotenv:.env.local"
+        )
+
+    print(":loud_sound: Debug options updated.")
+
+
+@app.command(name="unset-debug")
+def unset_debug(
+    save: Optional[str] = typer.Option(
+        None,
+        "--save",
+        help="Remove only the debug-related environment variables from a dotenv file. "
+        "Usage: --save=dotenv[:path] (default: .env.local)",
+    ),
+):
+    """
+    Restore default behavior by unsetting debug related variables.
+
+    Behavior:
+    - Resets LOG_LEVEL back to 'info'.
+    - Unsets DEEPEVAL_VERBOSE_MODE, retry log-level overrides, gRPC and Confident trace flags.
+    - If --save is provided (or DEEPEVAL_DEFAULT_SAVE is set), removes these keys from the target dotenv file.
+    """
+    settings = get_settings()
+    with settings.edit(save=save) as edit_ctx:
+        # Back to normal global level
+        settings.LOG_LEVEL = "info"
+        settings.CONFIDENT_TRACE_ENVIRONMENT = "development"
+        settings.CONFIDENT_TRACE_VERBOSE = True
+
+        # Clear optional toggles/overrides
+        settings.DEEPEVAL_VERBOSE_MODE = None
+        settings.DEEPEVAL_RETRY_BEFORE_LOG_LEVEL = None
+        settings.DEEPEVAL_RETRY_AFTER_LOG_LEVEL = None
+
+        settings.DEEPEVAL_GRPC_LOGGING = None
+        settings.GRPC_VERBOSITY = None
+        settings.GRPC_TRACE = None
+
+        settings.CONFIDENT_TRACE_FLUSH = None
+
+        settings.ERROR_REPORTING = None
+        settings.IGNORE_DEEPEVAL_ERRORS = None
 
     handled, path, _ = edit_ctx.result
 
     if not handled and save is not None:
-        # invalid --save format (unsupported)
         print("Unsupported --save option. Use --save=dotenv[:path].")
+    elif path:
+        print(f"Removed debug-related environment variables from {path}.")
     else:
-        print("gRPC logging enabled.")
+        print("Debug settings reverted to defaults for this session.")
+
+    print(":mute: Debug options unset.")
 
 
 #############################################
@@ -1336,7 +1500,7 @@ def set_gemini_model_env(
         )
     else:
         print(
-            f":raising_hands: Congratulations! You're now using Gemini's model for all evals that require an LLM."
+            ":raising_hands: Congratulations! You're now using Gemini's model for all evals that require an LLM."
         )
 
 

deepeval/dataset/dataset.py

@@ -458,6 +458,8 @@ class EvaluationDataset:
         tools_called_col_delimiter: str = ";",
         expected_tools_col_name: Optional[str] = "expected_tools",
         expected_tools_col_delimiter: str = ";",
+        comments_key_name: str = "comments",
+        name_key_name: str = "name",
         source_file_col_name: Optional[str] = None,
         additional_metadata_col_name: Optional[str] = None,
         scenario_col_name: Optional[str] = "scenario",
@@ -526,6 +528,8 @@ class EvaluationDataset:
                 df, expected_tools_col_name, default=""
             )
         ]
+        comments = get_column_data(df, comments_key_name)
+        name = get_column_data(df, name_key_name)
         source_files = get_column_data(df, source_file_col_name)
         additional_metadatas = [
             ast.literal_eval(metadata) if metadata else None
@@ -546,6 +550,8 @@ class EvaluationDataset:
             retrieval_context,
             tools_called,
             expected_tools,
+            comments,
+            name,
             source_file,
             additional_metadata,
             scenario,
@@ -560,6 +566,8 @@ class EvaluationDataset:
             retrieval_contexts,
             tools_called,
             expected_tools,
+            comments,
+            name,
             source_files,
             additional_metadatas,
             scenarios,
@@ -569,7 +577,7 @@ class EvaluationDataset:
         ):
             if scenario:
                 self._multi_turn = True
-                parsed_turns = parse_turns(turns)
+                parsed_turns = parse_turns(turns) if turns else []
                 self.goldens.append(
                     ConversationalGolden(
                         scenario=scenario,
@@ -577,6 +585,8 @@ class EvaluationDataset:
                         expected_outcome=expected_outcome,
                         user_description=user_description,
                         context=context,
+                        comments=comments,
+                        name=name,
                     )
                 )
             else:
@@ -592,6 +602,8 @@ class EvaluationDataset:
                         expected_tools=expected_tools,
                         additional_metadata=additional_metadata,
                         source_file=source_file,
+                        comments=comments,
+                        name=name,
                     )
                 )
 
@@ -605,6 +617,8 @@ class EvaluationDataset:
         retrieval_context_key_name: Optional[str] = "retrieval_context",
         tools_called_key_name: Optional[str] = "tools_called",
         expected_tools_key_name: Optional[str] = "expected_tools",
+        comments_key_name: str = "comments",
+        name_key_name: str = "name",
         source_file_key_name: Optional[str] = "source_file",
         additional_metadata_key_name: Optional[str] = "additional_metadata",
         scenario_key_name: Optional[str] = "scenario",
@@ -628,7 +642,8 @@ class EvaluationDataset:
                 expected_outcome = json_obj.get(expected_outcome_key_name)
                 user_description = json_obj.get(user_description_key_name)
                 context = json_obj.get(context_key_name)
-
+                comments = json_obj.get(comments_key_name)
+                name = json_obj.get(name_key_name)
                 parsed_turns = parse_turns(turns) if turns else []
 
                 self._multi_turn = True
@@ -639,6 +654,8 @@ class EvaluationDataset:
                         expected_outcome=expected_outcome,
                         user_description=user_description,
                         context=context,
+                        comments=comments,
+                        name=name,
                     )
                 )
             else:
@@ -649,6 +666,8 @@ class EvaluationDataset:
                 retrieval_context = json_obj.get(retrieval_context_key_name)
                 tools_called = json_obj.get(tools_called_key_name)
                 expected_tools = json_obj.get(expected_tools_key_name)
+                comments = json_obj.get(comments_key_name)
+                name = json_obj.get(name_key_name)
                 source_file = json_obj.get(source_file_key_name)
                 additional_metadata = json_obj.get(additional_metadata_key_name)
 
@@ -663,6 +682,8 @@ class EvaluationDataset:
                         tools_called=tools_called,
                         expected_tools=expected_tools,
                         additional_metadata=additional_metadata,
+                        comments=comments,
+                        name=name,
                         source_file=source_file,
                     )
                 )
@@ -928,6 +949,8 @@ class EvaluationDataset:
                     expected_outcome=golden.expected_outcome,
                     user_description=golden.user_description,
                     context=golden.context,
+                    name=golden.name,
+                    comments=golden.comments,
                 )
                 for golden in self.goldens
             ]
@@ -939,6 +962,8 @@ class EvaluationDataset:
                     actual_output=golden.actual_output,
                     retrieval_context=golden.retrieval_context,
                     context=golden.context,
+                    name=golden.name,
+                    comments=golden.comments,
                     source_file=golden.source_file,
                 )
                 for golden in self.goldens
@@ -981,6 +1006,8 @@ class EvaluationDataset:
                             "expected_outcome": golden.expected_outcome,
                             "user_description": golden.user_description,
                             "context": golden.context,
+                            "name": golden.name,
+                            "comments": golden.comments,
                         }
                         for golden in goldens
                     ]
@@ -992,6 +1019,8 @@ class EvaluationDataset:
                             "expected_output": golden.expected_output,
                             "retrieval_context": golden.retrieval_context,
                             "context": golden.context,
+                            "name": golden.name,
+                            "comments": golden.comments,
                             "source_file": golden.source_file,
                         }
                         for golden in goldens
@@ -1010,6 +1039,8 @@ class EvaluationDataset:
                             "expected_outcome",
                             "user_description",
                             "context",
+                            "name",
+                            "comments",
                         ]
                     )
                     for golden in goldens:
@@ -1030,6 +1061,8 @@ class EvaluationDataset:
                                 golden.expected_outcome,
                                 golden.user_description,
                                 context,
+                                golden.name,
+                                golden.comments,
                             ]
                         )
                 else:
@@ -1040,6 +1073,8 @@ class EvaluationDataset:
                             "expected_output",
                             "retrieval_context",
                             "context",
+                            "name",
+                            "comments",
                             "source_file",
                         ]
                     )
@@ -1061,6 +1096,8 @@ class EvaluationDataset:
                                 golden.expected_output,
                                 retrieval_context,
                                 context,
+                                golden.name,
+                                golden.comments,
                                 golden.source_file,
                             ]
                         )

deepeval/evaluate/execute.py

@@ -1219,12 +1219,16 @@ async def _a_execute_agentic_test_case(
 
     test_case = LLMTestCase(
         input=golden.input,
-        actual_output=str(trace.output) if trace.output is not None else None,
-        expected_output=trace.expected_output,
-        context=trace.context,
-        retrieval_context=trace.retrieval_context,
-        tools_called=trace.tools_called,
-        expected_tools=trace.expected_tools,
+        actual_output=(
+            str(current_trace.output)
+            if current_trace.output is not None
+            else None
+        ),
+        expected_output=current_trace.expected_output,
+        context=current_trace.context,
+        retrieval_context=current_trace.retrieval_context,
+        tools_called=current_trace.tools_called,
+        expected_tools=current_trace.expected_tools,
         additional_metadata=golden.additional_metadata,
         comments=golden.comments,
         name=golden.name,

deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py

@@ -1,6 +1,6 @@
 """LLM evaluated metric based on the GEval framework: https://arxiv.org/pdf/2303.16634.pdf"""
 
-from typing import Optional, List, Tuple, Union
+from typing import Optional, List, Tuple, Type, Union
 from deepeval.models import DeepEvalBaseMLLM
 from deepeval.metrics import BaseMultimodalMetric
 from deepeval.test_case import (
@@ -10,7 +10,10 @@ from deepeval.test_case import (
 from deepeval.metrics.multimodal_metrics.multimodal_g_eval.template import (
     MultimodalGEvalTemplate,
 )
-from deepeval.metrics.multimodal_metrics.multimodal_g_eval.schema import *
+from deepeval.metrics.multimodal_metrics.multimodal_g_eval.schema import (
+    Steps,
+    ReasonScore,
+)
 from deepeval.utils import get_or_create_event_loop, prettify_list
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.utils import (
@@ -49,6 +52,9 @@ class MultimodalGEval(BaseMultimodalMetric):
         async_mode: bool = True,
         strict_mode: bool = False,
         verbose_mode: bool = False,
+        evaluation_template: Type[
+            MultimodalGEvalTemplate
+        ] = MultimodalGEvalTemplate,
         _include_g_eval_suffix: bool = True,
     ):
         validate_criteria_and_evaluation_steps(criteria, evaluation_steps)
@@ -65,6 +71,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         self.async_mode = async_mode
         self.verbose_mode = verbose_mode
         self._include_g_eval_suffix = _include_g_eval_suffix
+        self.evaluation_template = evaluation_template
 
     def measure(
         self,
@@ -167,7 +174,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         g_eval_params_str = construct_g_eval_params_string(
             self.evaluation_params
         )
-        prompt = MultimodalGEvalTemplate.generate_evaluation_steps(
+        prompt = self.evaluation_template.generate_evaluation_steps(
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
@@ -190,7 +197,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         g_eval_params_str = construct_g_eval_params_string(
             self.evaluation_params
         )
-        prompt = MultimodalGEvalTemplate.generate_evaluation_steps(
+        prompt = self.evaluation_template.generate_evaluation_steps(
             criteria=self.criteria, parameters=g_eval_params_str
         )
         if self.using_native_model:
@@ -218,7 +225,7 @@ class MultimodalGEval(BaseMultimodalMetric):
 
         if not self.strict_mode:
             rubric_str = format_rubrics(self.rubric) if self.rubric else None
-            prompt = MultimodalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=number_evaluation_steps(self.evaluation_steps),
                 test_case_list=test_case_list,
                 parameters=g_eval_params_str,
@@ -227,11 +234,15 @@ class MultimodalGEval(BaseMultimodalMetric):
                 _additional_context=_additional_context,
             )
         else:
-            prompt = MultimodalGEvalTemplate.generate_strict_evaluation_results(
-                evaluation_steps=number_evaluation_steps(self.evaluation_steps),
-                test_case_list=test_case_list,
-                parameters=g_eval_params_str,
-                _additional_context=_additional_context,
+            prompt = (
+                self.evaluation_template.generate_strict_evaluation_results(
+                    evaluation_steps=number_evaluation_steps(
+                        self.evaluation_steps
+                    ),
+                    test_case_list=test_case_list,
+                    parameters=g_eval_params_str,
+                    _additional_context=_additional_context,
+                )
             )
         try:
             # don't use log probabilities for unsupported gpt models
@@ -256,7 +267,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except Exception:
                 return score, reason
         except (
             AttributeError
@@ -289,7 +300,7 @@ class MultimodalGEval(BaseMultimodalMetric):
 
         if not self.strict_mode:
             rubric_str = format_rubrics(self.rubric) if self.rubric else None
-            prompt = MultimodalGEvalTemplate.generate_evaluation_results(
+            prompt = self.evaluation_template.generate_evaluation_results(
                 evaluation_steps=number_evaluation_steps(self.evaluation_steps),
                 test_case_list=test_case_list,
                 parameters=g_eval_params_str,
@@ -298,11 +309,15 @@ class MultimodalGEval(BaseMultimodalMetric):
                 _additional_context=_additional_context,
             )
         else:
-            prompt = MultimodalGEvalTemplate.generate_strict_evaluation_results(
-                evaluation_steps=number_evaluation_steps(self.evaluation_steps),
-                test_case_list=test_case_list,
-                parameters=g_eval_params_str,
-                _additional_context=_additional_context,
+            prompt = (
+                self.evaluation_template.generate_strict_evaluation_results(
+                    evaluation_steps=number_evaluation_steps(
+                        self.evaluation_steps
+                    ),
+                    test_case_list=test_case_list,
+                    parameters=g_eval_params_str,
+                    _additional_context=_additional_context,
+                )
             )
 
         try:
@@ -326,7 +341,7 @@ class MultimodalGEval(BaseMultimodalMetric):
                     score, res
                 )
                 return weighted_summed_score, reason
-            except:
+            except Exception:
                 return score, reason
         except AttributeError:
             # This catches the case where a_generate_raw_response doesn't exist.
@@ -352,7 +367,7 @@ class MultimodalGEval(BaseMultimodalMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except Exception:
                 self.success = False
         return self.success
 

deepeval/openai_agents/agent.py

@@ -164,13 +164,12 @@ class _ObservedModel(Model):
                 ):
 
                     if isinstance(event, ResponseCompletedEvent):
-                        observer.result = (
-                            event.response.output_text
-                        )  # TODO: support other response types
+                        observer.result = make_json_serializable(
+                            event.response.output
+                        )
 
                     yield event
 
-                observer.__exit__(None, None, None)
             except Exception as e:
                 observer.__exit__(type(e), e, e.__traceback__)
                 raise

deepeval/openai_agents/callback_handler.py

@@ -1,9 +1,13 @@
 from deepeval.tracing.tracing import (
     Observer,
     current_span_context,
+    trace_manager,
 )
 from deepeval.openai_agents.extractors import *
 from deepeval.tracing.context import current_trace_context
+from deepeval.tracing.utils import make_json_serializable
+from time import perf_counter
+from deepeval.tracing.types import TraceSpanStatus
 
 try:
     from agents.tracing import Span, Trace, TracingProcessor
@@ -33,14 +37,51 @@ def _check_openai_agents_available():
 class DeepEvalTracingProcessor(TracingProcessor):
     def __init__(self) -> None:
         _check_openai_agents_available()
-        self.root_span_observers: dict[str, Observer] = {}
         self.span_observers: dict[str, Observer] = {}
 
     def on_trace_start(self, trace: "Trace") -> None:
-        pass
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _thread_id = trace_dict.get("group_id")
+        _trace_name = trace_dict.get("workflow_name")
+        _trace_metadata = trace_dict.get("metadata")
+
+        if _thread_id or _trace_metadata:
+            _trace = trace_manager.start_new_trace(trace_uuid=str(_trace_uuid))
+            _trace.thread_id = str(_thread_id)
+            _trace.name = str(_trace_name)
+            _trace.metadata = make_json_serializable(_trace_metadata)
+            current_trace_context.set(_trace)
+
+            trace_manager.add_span(  # adds a dummy root span
+                BaseSpan(
+                    uuid=_trace_uuid,
+                    trace_uuid=_trace_uuid,
+                    parent_uuid=None,
+                    start_time=perf_counter(),
+                    name=_trace_name,
+                    status=TraceSpanStatus.IN_PROGRESS,
+                    children=[],
+                )
+            )
+        else:
+            current_trace = current_trace_context.get()
+            if current_trace:
+                current_trace.name = str(_trace_name)
 
     def on_trace_end(self, trace: "Trace") -> None:
-        pass
+        trace_dict = trace.export()
+        _trace_uuid = trace_dict.get("id")
+        _thread_id = trace_dict.get("group_id")
+        _trace_name = trace_dict.get("workflow_name")
+        _trace_metadata = trace_dict.get("metadata")
+
+        if _thread_id or _trace_metadata:
+            trace_manager.remove_span(
+                _trace_uuid
+            )  # removing the dummy root span
+            trace_manager.end_trace(_trace_uuid)
+            current_trace_context.set(None)
 
     def on_span_start(self, span: "Span") -> None:
         if not span.started_at:

deepeval/openai_agents/runner.py

@@ -109,10 +109,9 @@ class Runner(AgentsRunner):
             metric_collection=metric_collection,
             metrics=metrics,
             func_name="run",
-            function_kwargs={"input": input},
+            function_kwargs={"input": input},  # also set below
         ) as observer:
             update_trace_attributes(
-                input=input,
                 name=name,
                 tags=tags,
                 metadata=metadata,
@@ -123,7 +122,8 @@ class Runner(AgentsRunner):
             )
             current_span = current_span_context.get()
             current_trace = current_trace_context.get()
-            current_trace.input = input
+            if not current_trace.input:
+                current_trace.input = input
             if current_span:
                 current_span.input = input
             res = await super().run(
@@ -138,8 +138,9 @@ class Runner(AgentsRunner):
                 session=session,
                 **kwargs,  # backwards compatibility
             )
+            current_trace_thread_id = current_trace_context.get().thread_id
             _output = None
-            if thread_id:
+            if current_trace_thread_id:
                 _output = res.final_output
             else:
                 _output = str(res)
@@ -170,30 +171,30 @@ class Runner(AgentsRunner):
         **kwargs,
     ) -> RunResult:
         is_agents_available()
-        input_val = input
-
-        update_trace_attributes(
-            input=input_val,
-            name=name,
-            tags=tags,
-            metadata=metadata,
-            thread_id=thread_id,
-            user_id=user_id,
-            metric_collection=metric_collection,
-            metrics=metrics,
-        )
 
         with Observer(
             span_type="custom",
             metric_collection=metric_collection,
             metrics=metrics,
             func_name="run_sync",
-            function_kwargs={"input": input_val},
+            function_kwargs={"input": input},  # also set below
         ) as observer:
+            update_trace_attributes(
+                name=name,
+                tags=tags,
+                metadata=metadata,
+                thread_id=thread_id,
+                user_id=user_id,
+                metric_collection=metric_collection,
+                metrics=metrics,
+            )
+
             current_span = current_span_context.get()
             current_trace = current_trace_context.get()
+            if not current_trace.input:
+                current_trace.input = input
             if current_span:
-                current_span.input = input_val
+                current_span.input = input
             res = super().run_sync(
                 starting_agent,
                 input,
@@ -206,8 +207,9 @@ class Runner(AgentsRunner):
                 session=session,
                 **kwargs,  # backwards compatibility
             )
+            current_trace_thread_id = current_trace_context.get().thread_id
             _output = None
-            if thread_id:
+            if current_trace_thread_id:
                 _output = res.final_output
             else:
                 _output = str(res)
@@ -250,7 +252,6 @@ class Runner(AgentsRunner):
         observer.__enter__()
 
         update_trace_attributes(
-            input=input,
             name=name,
             tags=tags,
             metadata=metadata,
@@ -259,6 +260,9 @@ class Runner(AgentsRunner):
             metric_collection=metric_collection,
             metrics=metrics,
         )
+        current_trace = current_trace_context.get()
+        if not current_trace.input:
+            current_trace.input = input
 
         current_span = current_span_context.get()
         if current_span:

deepeval/scorer/scorer.py

@@ -223,7 +223,7 @@ class Scorer:
         Right now we are using score_one method under the hood. Instead of scoring multiple predictions for faithfulness.
         """
         try:
-            from deepeval.models import SummaCModels
+            from deepeval.models.summac_model import SummaCModels
         except Exception as e:
             print(f"SummaCZS model can not be loaded.\n{e}")
 
@@ -326,7 +326,7 @@ class Scorer:
         from sentence_transformers import util
 
         try:
-            from deepeval.models import (
+            from deepeval.models.answer_relevancy_model import (
                 AnswerRelevancyModel,
                 CrossEncoderAnswerRelevancyModel,
             )

{deepeval-3.5.5.dist-info → deepeval-3.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.5
+Version: 3.5.7
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

{deepeval-3.5.5.dist-info → deepeval-3.5.7.dist-info}/RECORD

@@ -1,5 +1,5 @@
 deepeval/__init__.py,sha256=6fsb813LD_jNhqR-xZnSdE5E-KsBbC3tc4oIg5ZMgTw,2115
-deepeval/_version.py,sha256=CJwAeAyMGnIxrkmBn8fpG6bwbVBsUZaTrtwbstM2LgA,27
+deepeval/_version.py,sha256=5PpKL25tWtYxTPc0_se2v49WDFVCzYaCu8yogWsx_qQ,27
 deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
 deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
 deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
@@ -132,7 +132,7 @@ deepeval/benchmarks/winogrande/template.py,sha256=tDwH8NpNF9x7FbDmQw45XaW1LNqGBV
 deepeval/benchmarks/winogrande/winogrande.py,sha256=_4irJkRPw3c-Ufo-hM4cHpPKUoxozedFQpok9n0csTg,5644
 deepeval/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/cli/dotenv_handler.py,sha256=7PtVjCNUZKAXsVJQxznsLexad7y8x-gQ195xAxmv4gA,2468
-deepeval/cli/main.py,sha256=60FsOU9OGRX49eSSuJkePf8kArSor3-QbeqAZ2bDWCE,51298
+deepeval/cli/main.py,sha256=keY6Ik3h2PLmwFMiNUU8mWEtAGsR8mjzWmYedq0k25w,57296
 deepeval/cli/server.py,sha256=cOm9xiYcPYB9GDeFQw9-Iawf9bNfOqftZs7q7mO_P7I,1979
 deepeval/cli/test.py,sha256=kSIFMRTAfVzBJ4OitwvT829-ylV7UzPMP57P2DePS-Q,5482
 deepeval/cli/types.py,sha256=_7KdthstHNc-JKCWrfpDQCf_j8h9PMxh0qJCHmVXJr0,310
@@ -147,7 +147,7 @@ deepeval/config/utils.py,sha256=gSOVv18Tx1R72GucbdQesbZLFL-Y9EzbS4p7qd2w_xE,3799
 deepeval/constants.py,sha256=Qe-es-WDPJndgBspEQXxddDCVanrAu03YWCpXsUkdo0,1368
 deepeval/dataset/__init__.py,sha256=rcum_VjBXu8eisCdr6sl84BgoZUs3x0tYbB2PnPtHGY,212
 deepeval/dataset/api.py,sha256=ZxkEqAF4nZH_Ys_1f5r9N2LFI_vBcAJxt8eJm7Mplpw,831
-deepeval/dataset/dataset.py,sha256=T2rzGGKeCjIkkhXY0ofnWh13W6gjjdjat9uVHCmhGFI,49493
+deepeval/dataset/dataset.py,sha256=dDWTSPWN8i_mZBOAgZt0r5Id6q6aeDf8jAKxv81mP1o,51113
 deepeval/dataset/golden.py,sha256=T-rTk4Hw1tANx_Iimv977F6Y4QK3s5OIB4PecU5FJDM,2338
 deepeval/dataset/test_run_tracer.py,sha256=5CdpDvhzkEEBRyqWi6egocaxiN6IRS3XfbACxEQZQeM,2544
 deepeval/dataset/types.py,sha256=CWeOIBPK2WdmRUqjFa9gfN-w2da0r8Ilzl3ToDpJQoQ,558
@@ -158,7 +158,7 @@ deepeval/evaluate/api.py,sha256=rkblH0ZFAAdyuF0Ymh7JE1pIJPR9yFuPrn9SQaCEQp4,435
 deepeval/evaluate/compare.py,sha256=tdSJY4E7YJ_zO3dzvpwngZHLiUI2YQcTWJOLI83htsQ,9855
 deepeval/evaluate/configs.py,sha256=QfWjaWNxLsgEe8-5j4PIs5WcSyEckiWt0qdpXSpl57M,928
 deepeval/evaluate/evaluate.py,sha256=NPAJ2iJqJI_RurXKUIC0tft_ozYMIKwZf5iPfmnNhQc,10412
-deepeval/evaluate/execute.py,sha256=45m3w3QSAWVHRNTSqLZcpUI1bA_qRFWIGu292WKTjcA,87953
+deepeval/evaluate/execute.py,sha256=fJLBl45Vf4rA4Pm7k932TG-0BNIvf90klQyurXb-b_4,88057
 deepeval/evaluate/types.py,sha256=IGZ3Xsj0UecPI3JNeTpJaK1gDvlepokfCmHwtItIW9M,831
 deepeval/evaluate/utils.py,sha256=kkliSGzuICeUsXDtlMMPfN95dUKlqarNhfciSffd4gI,23143
 deepeval/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -308,7 +308,7 @@ deepeval/metrics/multimodal_metrics/multimodal_faithfulness/multimodal_faithfuln
 deepeval/metrics/multimodal_metrics/multimodal_faithfulness/schema.py,sha256=b-WtfA7zq4TgQiuqqNEMf7jmohnWBMW4opChHyg49Gc,414
 deepeval/metrics/multimodal_metrics/multimodal_faithfulness/template.py,sha256=9EWRC-Wiyr_UEMPfpuTcX2tvsjPxSRY4n_lClcsK6vw,8389
 deepeval/metrics/multimodal_metrics/multimodal_g_eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py,sha256=gZ7Q4vF12PLGhbHhOUAl9LIFWDOc9-GKhu3ly_LOkQ0,13997
+deepeval/metrics/multimodal_metrics/multimodal_g_eval/multimodal_g_eval.py,sha256=YR2SMmUwVPe8epth2PWtG6UB4vnInBZrTaeeVyF9VHA,14428
 deepeval/metrics/multimodal_metrics/multimodal_g_eval/schema.py,sha256=H_9-iA1BXJwbPKrGEZBqxDO_En4sjXI8_xKSNYc-hnk,167
 deepeval/metrics/multimodal_metrics/multimodal_g_eval/template.py,sha256=6kIC4vTtRxUBCyafjyWLZg5WhVHxsRy-m2Mv7OGbgV0,5235
 deepeval/metrics/multimodal_metrics/multimodal_g_eval/utils.py,sha256=UgY46c1mudFoOglbrrJsXnSrdiJGGRFqpDvrAAZWwV0,2189
@@ -395,11 +395,11 @@ deepeval/openai/extractors.py,sha256=q062nlYKuPVwqfLFYCD1yWv7xHF1U_XrYdAp5ve2l_E
 deepeval/openai/patch.py,sha256=tPDqXaBScBJveM9P5xLT_mVwkubw0bOey-efvdjZIfg,7466
 deepeval/openai/utils.py,sha256=-84VZGUsnzRkYAFWc_DGaGuQTDCUItk0VtUTdjtSxg4,2748
 deepeval/openai_agents/__init__.py,sha256=u-e9laod3LyPfLcI5lr7Yhk8ArfWvlpr-D4_idWIt0A,321
-deepeval/openai_agents/agent.py,sha256=PYOhLELRXfGAP_fje70X3Ovm3WjF24mQYWdwrobwcr4,6173
-deepeval/openai_agents/callback_handler.py,sha256=-tOXJ3SMKqH5u41cB_g7FBjaX5qAuqVAaAv7vQtiBVc,3025
+deepeval/openai_agents/agent.py,sha256=gZcmfqTgrQaJV8g6ChmmdpyArEp6oDIqHSaYPDEd344,6100
+deepeval/openai_agents/callback_handler.py,sha256=jrV2Uv9FjfU1BQQe6V_ltT3QS8ZcalxMbqzJI2vvJXo,4713
 deepeval/openai_agents/extractors.py,sha256=0jZxwgY1NQ3mMxVWPpLcMpKlbj-aYV7rwuzRzG8hdZs,11529
 deepeval/openai_agents/patch.py,sha256=zSmRV5yOReHC6IylhT93SM1nQpmH3sEWfYcJqa_iM84,3684
-deepeval/openai_agents/runner.py,sha256=pRwe6DX6kpXia6btl4TAWlnXpk88MsQfM8yWkGufyk8,10608
+deepeval/openai_agents/runner.py,sha256=U8Kh4jHhDIYVkIIxytcGCKRFHdgxxhpATHd9jnbh1Eg,10999
 deepeval/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/plugins/plugin.py,sha256=_dwsdx4Dg9DbXxK3f7zJY4QWTJQWc7QE1HmIg2Zjjag,1515
 deepeval/progress_context.py,sha256=ZSKpxrE9sdgt9G3REKnVeXAv7GJXHHVGgLynpG1Pudw,3557
@@ -410,7 +410,7 @@ deepeval/prompt/utils.py,sha256=Gk0zj_9BK8MQccs8GmiC8o-YVtkou6ZJEz8kWgW5Mog,1678
 deepeval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 deepeval/red_teaming/README.md,sha256=BY5rAdpp3-sMMToEKwq0Nsd9ivkGDzPE16DeDb8GY7U,154
 deepeval/scorer/__init__.py,sha256=hTvtoV3a4l0dSBjERm-jX7jveTtKZXK0c9JerQo0T_w,27
-deepeval/scorer/scorer.py,sha256=8kp5wXbIlem-8ucyyA6ew_sIeR77HoypW_s4wKmxGUU,18312
+deepeval/scorer/scorer.py,sha256=EmXo1wEMMAL2it8WxNJ4cTqZLCH1ad4BY2VewoX6b10,18348
 deepeval/simulator/__init__.py,sha256=wkyevg9nh46rsVnVrBjY3K5bHlkqjwx4TtrTfyjDCO0,96
 deepeval/simulator/conversation_simulator.py,sha256=Ojng2ZoM31p7GVWEkiT44PE926eEzRoVJP5eRb1yrQI,24262
 deepeval/simulator/schema.py,sha256=16X2-m92plP52YTd-dvECt_-6gsz0U4j7Ut3UdI6gKY,252
@@ -461,8 +461,8 @@ deepeval/tracing/tracing.py,sha256=b-0T3W6lAEOEGhODx0e-yIwBkm5V46EDNAWS9lcWkD0,4
 deepeval/tracing/types.py,sha256=l_utWKerNlE5H3mOKpeUJLsvpP3cMyjH7HRANNgTmSQ,5306
 deepeval/tracing/utils.py,sha256=w_kdhuyBCygllnbqLpDdKJqpJo42t3ZMlGhNicV2A8c,6467
 deepeval/utils.py,sha256=r8tV_NYJSi6ib-oQw6cLw3L7ZSe4KIJVJc1ng6-kDX4,17179
-deepeval-3.5.5.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
-deepeval-3.5.5.dist-info/METADATA,sha256=Js_9nOjXPh0YQOokcbPvquIW7lBRQMphrLHTYZ8-pAE,18721
-deepeval-3.5.5.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
-deepeval-3.5.5.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
-deepeval-3.5.5.dist-info/RECORD,,
+deepeval-3.5.7.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
+deepeval-3.5.7.dist-info/METADATA,sha256=NldO1OinDSv_gGUP-kkFk1zpMGXKTceoYMtF92XGbgs,18721
+deepeval-3.5.7.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
+deepeval-3.5.7.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
+deepeval-3.5.7.dist-info/RECORD,,