deepeval 3.8.2__py3-none-any.whl → 3.8.4__py3-none-any.whl

deepeval/integrations/crewai/wrapper.py

@@ -17,8 +17,9 @@ def wrap_crew_kickoff():
             func_name="kickoff",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_kickoff(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -36,8 +37,9 @@ def wrap_crew_kickoff_for_each():
             func_name="kickoff_for_each",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_kickoff_for_each(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -55,8 +57,9 @@ def wrap_crew_kickoff_async():
             func_name="kickoff_async",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = await original_kickoff_async(self, *args, **kwargs)
+            observer.result = str(result) if result else None
 
         return result
 
@@ -74,33 +77,61 @@ def wrap_crew_kickoff_for_each_async():
             func_name="kickoff_for_each_async",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = await original_kickoff_for_each_async(
                 self, *args, **kwargs
             )
+            observer.result = str(result) if result else None
 
         return result
 
     Crew.kickoff_for_each_async = wrapper
 
 
-def wrap_llm_call():
-    original_llm_call = LLM.call
+def wrap_crew_akickoff():
+    if not hasattr(Crew, "akickoff"):
+        return
 
-    @wraps(original_llm_call)
-    def wrapper(self, *args, **kwargs):
+    original_akickoff = Crew.akickoff
+
+    @wraps(original_akickoff)
+    async def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="akickoff",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ) as observer:
+            result = await original_akickoff(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+
+        return result
+
+    Crew.akickoff = wrapper
+
+
+def wrap_crew_akickoff_for_each():
+    if not hasattr(Crew, "akickoff_for_each"):
+        return
+
+    original_akickoff_for_each = Crew.akickoff_for_each
+
+    @wraps(original_akickoff_for_each)
+    async def wrapper(self, *args, **kwargs):
         metric_collection, metrics = _check_metrics_and_metric_collection(self)
         with Observer(
-            span_type="llm",
-            func_name="call",
-            observe_kwargs={"model": "temp_model"},
+            span_type="crew",
+            func_name="akickoff_for_each",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
-            result = original_llm_call(self, *args, **kwargs)
+        ) as observer:
+            result = await original_akickoff_for_each(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+
         return result
 
-    LLM.call = wrapper
+    Crew.akickoff_for_each = wrapper
 
 
 def wrap_agent_execute_task():
@@ -114,13 +145,36 @@ def wrap_agent_execute_task():
             func_name="execute_task",
             metric_collection=metric_collection,
             metrics=metrics,
-        ):
+        ) as observer:
             result = original_execute_task(self, *args, **kwargs)
+            observer.result = str(result) if result else None
         return result
 
     Agent.execute_task = wrapper
 
 
+def wrap_agent_aexecute_task():
+    if not hasattr(Agent, "aexecute_task"):
+        return
+
+    original_aexecute_task = Agent.aexecute_task
+
+    @wraps(original_aexecute_task)
+    async def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="agent",
+            func_name="aexecute_task",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ) as observer:
+            result = await original_aexecute_task(self, *args, **kwargs)
+            observer.result = str(result) if result else None
+        return result
+
+    Agent.aexecute_task = wrapper
+
+
 def _check_metrics_and_metric_collection(obj: Any):
     metric_collection = getattr(obj, "_metric_collection", None)
     metrics = getattr(obj, "_metrics", None)

deepeval/integrations/langchain/callback.py

@@ -84,6 +84,7 @@ class CallbackHandler(BaseCallbackHandler):
         user_id: Optional[str] = None,
         metrics: Optional[List[BaseMetric]] = None,
         metric_collection: Optional[str] = None,
+        test_case_id: Optional[str] = None,
     ):
         is_langchain_installed()
         with capture_tracing_integration("langchain.callback.CallbackHandler"):
@@ -108,6 +109,7 @@ class CallbackHandler(BaseCallbackHandler):
                 "metadata": metadata,
                 "thread_id": thread_id,
                 "user_id": user_id,
+                "test_case_id": test_case_id,
             }
             self._trace_init_fields: Dict[str, Any] = dict(
                 self._original_init_fields
@@ -200,6 +202,8 @@ class CallbackHandler(BaseCallbackHandler):
                 trace.thread_id = fields["thread_id"]
             if fields.get("user_id") is not None:
                 trace.user_id = fields["user_id"]
+            if fields.get("test_case_id") is not None:
+                trace.test_case_id = fields["test_case_id"]
             # prevent re-applying on every callback
             self._trace_init_fields = {}
 

deepeval/integrations/llama_index/handler.py

@@ -21,6 +21,7 @@ from deepeval.tracing.types import (
 from deepeval.tracing.trace_context import (
     current_llm_context,
     current_agent_context,
+    current_trace_context,
 )
 from deepeval.test_case import ToolCall
 from deepeval.tracing.utils import make_json_serializable
@@ -40,7 +41,10 @@ try:
         LLMChatStartEvent,
         LLMChatEndEvent,
     )
-    from llama_index_instrumentation.dispatcher import Dispatcher
+    from llama_index.core.instrumentation import Dispatcher
+    from llama_index.core.instrumentation.events.retrieval import (
+        RetrievalEndEvent,
+    )
     from deepeval.integrations.llama_index.utils import (
         parse_id,
         prepare_input_llm_test_case_params,
@@ -82,15 +86,23 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 input_messages.append({"role": role, "content": content})
 
             llm_span_context = current_llm_context.get()
-            # create the span
+
+            parent_span = trace_manager.get_span_by_uuid(event.span_id)
+            if parent_span:
+                trace_uuid = parent_span.trace_uuid
+            else:
+                current_trace = current_trace_context.get()
+                if current_trace:
+                    trace_uuid = current_trace.uuid
+                else:
+                    trace_uuid = trace_manager.start_new_trace().uuid
+
             llm_span = LlmSpan(
                 name="ConfidentLLMSpan",
                 uuid=str(uuid.uuid4()),
                 status=TraceSpanStatus.IN_PROGRESS,
                 children=[],
-                trace_uuid=trace_manager.get_span_by_uuid(
-                    event.span_id
-                ).trace_uuid,
+                trace_uuid=trace_uuid,
                 parent_uuid=event.span_id,
                 start_time=perf_counter(),
                 model=getattr(event, "model_dict", {}).get(
@@ -128,6 +140,13 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     trace_manager.remove_span(llm_span.uuid)
                     del self.open_ai_astream_to_llm_span_map[event.span_id]
 
+        if isinstance(event, RetrievalEndEvent):
+            span = trace_manager.get_span_by_uuid(event.span_id)
+            if span:
+                span.retrieval_context = [
+                    node.node.get_content() for node in event.nodes
+                ]
+
     def new_span(
         self,
         id_: str,
@@ -139,18 +158,30 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
     ) -> Optional[LlamaIndexBaseSpan]:
         class_name, method_name = parse_id(id_)
 
-        # check if it is a root span
-        if parent_span_id is None:
-            trace_uuid = trace_manager.start_new_trace().uuid
-        elif class_name == "Workflow" and method_name == "run":
-            trace_uuid = trace_manager.start_new_trace().uuid
-            parent_span_id = None  # since workflow is the root span, we need to set the parent span id to None
+        current_trace = current_trace_context.get()
+        trace_uuid = None
+
+        if parent_span_id is None or (
+            class_name == "Workflow" and method_name == "run"
+        ):
+            if current_trace:
+                trace_uuid = current_trace.uuid
+            else:
+                trace_uuid = trace_manager.start_new_trace().uuid
+
+            if class_name == "Workflow" and method_name == "run":
+                parent_span_id = None
+
         elif trace_manager.get_span_by_uuid(parent_span_id):
             trace_uuid = trace_manager.get_span_by_uuid(
                 parent_span_id
             ).trace_uuid
+
         else:
-            trace_uuid = trace_manager.start_new_trace().uuid
+            if current_trace:
+                trace_uuid = current_trace.uuid
+            else:
+                trace_uuid = trace_manager.start_new_trace().uuid
 
         self.root_span_trace_id_map[id_] = trace_uuid
 
@@ -195,7 +226,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                     else None
                 ),
             )
-        elif method_name == "acall":
+        elif method_name in ["acall", "call_tool", "acall_tool"]:
             span = ToolSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -206,7 +237,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 input=bound_args.arguments,
                 name="Tool",
             )
-        # prepare input test case params for the span
+
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments
         )
@@ -215,6 +246,22 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         return span
 
+    def _get_output_value(self, result: Any) -> Any:
+        """Helper to ensure AgentChatResponse and similar objects are serialized as dicts."""
+        if hasattr(result, "response") and hasattr(result, "sources"):
+            if hasattr(result, "model_dump"):
+                return result.model_dump()
+            if hasattr(result, "to_dict"):
+                return result.to_dict()
+            return {"response": result.response, "sources": result.sources}
+
+        if hasattr(result, "response"):
+            if hasattr(result, "model_dump"):
+                return result.model_dump()
+            return {"response": result.response}
+
+        return result
+
     def prepare_to_exit_span(
         self,
         id_: str,
@@ -229,7 +276,8 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
             return None
 
         class_name, method_name = parse_id(id_)
-        if method_name == "call_tool":
+
+        if method_name in ["call_tool", "acall_tool"]:
             output_json = make_json_serializable(result)
             if output_json and isinstance(output_json, dict):
                 if base_span.tools_called is None:
@@ -243,7 +291,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 )
         base_span.end_time = perf_counter()
         base_span.status = TraceSpanStatus.SUCCESS
-        base_span.output = result
+        base_span.output = self._get_output_value(result)
 
         if isinstance(base_span, ToolSpan):
             result_json = make_json_serializable(result)
@@ -265,7 +313,8 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         if base_span.parent_uuid is None:
             trace_manager.end_trace(base_span.trace_uuid)
-            self.root_span_trace_id_map.pop(base_span.uuid)
+            if base_span.uuid in self.root_span_trace_id_map:
+                self.root_span_trace_id_map.pop(base_span.uuid)
 
         return base_span
 
@@ -282,13 +331,12 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
             return None
 
         base_span.end_time = perf_counter()
-        base_span.status = (
-            TraceSpanStatus.SUCCESS
-        )  # find a way to add error and handle the span without the parent id
+        base_span.status = TraceSpanStatus.SUCCESS
 
         if base_span.parent_uuid is None:
             trace_manager.end_trace(base_span.trace_uuid)
-            self.root_span_trace_id_map.pop(base_span.uuid)
+            if base_span.uuid in self.root_span_trace_id_map:
+                self.root_span_trace_id_map.pop(base_span.uuid)
 
         return base_span
 

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -36,7 +36,10 @@ try:
         SpanProcessor as _SpanProcessor,
         TracerProvider,
     )
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace.export import (
+        BatchSpanProcessor,
+        SimpleSpanProcessor,
+    )
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
@@ -172,7 +175,9 @@ class ConfidentInstrumentationSettings(InstrumentationSettings):
         trace_provider.add_span_processor(span_interceptor)
 
         if is_test_mode:
-            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+            trace_provider.add_span_processor(
+                SimpleSpanProcessor(ConfidentSpanExporter())
+            )
         else:
             trace_provider.add_span_processor(
                 BatchSpanProcessor(
@@ -345,7 +350,6 @@ class SpanInterceptor(SpanProcessor):
                 trace.status = TraceSpanStatus.SUCCESS
                 trace.end_time = perf_counter()
                 trace_manager.traces_to_evaluate.append(trace)
-                test_exporter.clear_span_json_list()
 
     def _add_agent_span(self, span, name):
         span.set_attribute("confident.span.type", "agent")

deepeval/prompt/__init__.py

@@ -7,6 +7,7 @@ from .api import (
     ReasoningEffort,
     OutputType,
     PromptInterpolationType,
+    Tool,
 )
 
 __all__ = [
@@ -18,4 +19,5 @@ __all__ = [
     "ReasoningEffort",
     "OutputType",
     "PromptInterpolationType",
+    "Tool",
 ]

deepeval/prompt/api.py

@@ -1,6 +1,14 @@
-from pydantic import BaseModel, Field, AliasChoices, ConfigDict
+from pydantic import (
+    BaseModel,
+    Field,
+    AliasChoices,
+    ConfigDict,
+    model_validator,
+    model_serializer,
+)
 from enum import Enum
-from typing import List, Optional
+import uuid
+from typing import List, Optional, Dict, Any, Union, Type
 from pydantic import TypeAdapter
 
 from deepeval.utils import make_model_config
@@ -33,6 +41,12 @@ class ModelProvider(Enum):
     OPENROUTER = "OPENROUTER"
 
 
+class ToolMode(str, Enum):
+    ALLOW_ADDITIONAL = "ALLOW_ADDITIONAL"
+    NO_ADDITIONAL = "NO_ADDITIONAL"
+    STRICT = "STRICT"
+
+
 class ModelSettings(BaseModel):
     provider: Optional[ModelProvider] = None
     name: Optional[str] = None
@@ -100,6 +114,7 @@ class OutputSchemaField(BaseModel):
     id: str
     type: SchemaDataType
     name: str
+    description: Optional[str] = None
     required: Optional[bool] = False
     parent_id: Optional[str] = Field(
         default=None,
@@ -109,8 +124,36 @@ class OutputSchemaField(BaseModel):
 
 
 class OutputSchema(BaseModel):
+    id: Optional[str] = None
     fields: Optional[List[OutputSchemaField]] = None
+    name: Optional[str] = None
+
+
+class Tool(BaseModel):
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
     name: str
+    description: Optional[str] = None
+    mode: ToolMode
+    structured_schema: Optional[Union[Type[BaseModel], OutputSchema]] = Field(
+        serialization_alias="structuredSchema",
+        validation_alias=AliasChoices("structured_schema", "structuredSchema"),
+    )
+
+    @model_validator(mode="after")
+    def update_schema(self):
+        if not isinstance(self.structured_schema, OutputSchema):
+            from deepeval.prompt.utils import construct_output_schema
+
+            self.structured_schema = construct_output_schema(
+                self.structured_schema
+            )
+        return self
+
+    @property
+    def input_schema(self) -> Dict[str, Any]:
+        from deepeval.prompt.utils import output_schema_to_json_schema
+
+        return output_schema_to_json_schema(self.structured_schema)
 
 
 ###################################
@@ -186,6 +229,7 @@ class PromptHttpResponse(BaseModel):
         serialization_alias="outputSchema",
         validation_alias=AliasChoices("output_schema", "outputSchema"),
     )
+    tools: Optional[List[Tool]] = None
 
 
 class PromptPushRequest(BaseModel):
@@ -196,6 +240,7 @@ class PromptPushRequest(BaseModel):
     alias: str
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
+    tools: Optional[List[Tool]] = None
     interpolation_type: PromptInterpolationType = Field(
         serialization_alias="interpolationType"
     )
@@ -215,6 +260,7 @@ class PromptUpdateRequest(BaseModel):
 
     text: Optional[str] = None
     messages: Optional[List[PromptMessage]] = None
+    tools: Optional[List[Tool]] = None
     interpolation_type: PromptInterpolationType = Field(
         serialization_alias="interpolationType"
     )

deepeval/prompt/prompt.py

@@ -25,6 +25,7 @@ from deepeval.prompt.api import (
     ModelSettings,
     OutputSchema,
     OutputType,
+    Tool,
 )
 from deepeval.prompt.utils import (
     interpolate_text,
@@ -101,6 +102,7 @@ class CachedPrompt(BaseModel):
     model_settings: Optional[ModelSettings]
     output_type: Optional[OutputType]
     output_schema: Optional[OutputSchema]
+    tools: Optional[List[Tool]] = None
 
 
 class Prompt:
@@ -131,6 +133,7 @@ class Prompt:
             interpolation_type or PromptInterpolationType.FSTRING
         )
         self.confident_api_key = confident_api_key
+        self.tools: Optional[List[Tool]] = None
 
         self._version = None
         self._prompt_version_id: Optional[str] = None
@@ -308,6 +311,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[OutputSchema] = None,
+        tools: Optional[List[Tool]] = None,
     ):
         if portalocker is None or not self.alias:
             return
@@ -354,6 +358,7 @@ class Prompt:
                     "model_settings": model_settings,
                     "output_type": output_type,
                     "output_schema": output_schema,
+                    "tools": tools,
                 }
 
                 if cache_key == VERSION_CACHE_KEY:
@@ -415,6 +420,7 @@ class Prompt:
             self.output_schema = construct_base_model(
                 cached_prompt.output_schema
             )
+            self.tools = cached_prompt.tools
 
         end_time = time.perf_counter()
         time_taken = format(end_time - start_time, ".2f")
@@ -494,6 +500,7 @@ class Prompt:
                         self.output_schema = construct_base_model(
                             cached_prompt.output_schema
                         )
+                        self.tools = cached_prompt.tools
                     return
             except Exception:
                 pass
@@ -547,6 +554,7 @@ class Prompt:
                     model_settings=data.get("modelSettings", None),
                     output_type=data.get("outputType", None),
                     output_schema=data.get("outputSchema", None),
+                    tools=data.get("tools", None),
                 )
             except Exception:
                 if fallback_to_cache:
@@ -573,6 +581,7 @@ class Prompt:
                 self.output_schema = construct_base_model(
                     response.output_schema
                 )
+                self.tools = response.tools
 
             end_time = time.perf_counter()
             time_taken = format(end_time - start_time, ".2f")
@@ -594,6 +603,7 @@ class Prompt:
                     model_settings=response.model_settings,
                     output_type=response.output_type,
                     output_schema=response.output_schema,
+                    tools=response.tools,
                 )
 
     def push(
@@ -606,6 +616,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Tool]] = None,
         _verbose: Optional[bool] = True,
     ):
         if self.alias is None:
@@ -628,6 +639,7 @@ class Prompt:
             output_type=output_type or self.output_type,
             output_schema=construct_output_schema(output_schema)
             or construct_output_schema(self.output_schema),
+            tools=tools or self.tools,
         )
         try:
             body = body.model_dump(
@@ -655,6 +667,7 @@ class Prompt:
             self.model_settings = model_settings or self.model_settings
             self.output_type = output_type or self.output_type
             self.output_schema = output_schema or self.output_schema
+            self.tools = tools or self.tools
             self.type = PromptType.TEXT if text_template else PromptType.LIST
             if _verbose:
                 console = Console()
@@ -674,6 +687,7 @@ class Prompt:
         model_settings: Optional[ModelSettings] = None,
         output_type: Optional[OutputType] = None,
         output_schema: Optional[Type[BaseModel]] = None,
+        tools: Optional[List[Tool]] = None,
     ):
         if self.alias is None:
             raise ValueError(
@@ -687,6 +701,7 @@ class Prompt:
             model_settings=model_settings,
             output_type=output_type,
             output_schema=construct_output_schema(output_schema),
+            tools=tools,
         )
         try:
             body = body.model_dump(
@@ -712,6 +727,7 @@ class Prompt:
             self.model_settings = model_settings
             self.output_type = output_type
             self.output_schema = output_schema
+            self.tools = tools
             self.type = PromptType.TEXT if text else PromptType.LIST
             console = Console()
             console.print("✅ Prompt successfully updated on Confident AI!")
@@ -796,6 +812,10 @@ class Prompt:
                     messages=data.get("messages", None),
                     type=data["type"],
                     interpolation_type=data["interpolationType"],
+                    model_settings=data.get("modelSettings", None),
+                    output_type=data.get("outputType", None),
+                    output_schema=data.get("outputSchema", None),
+                    tools=data.get("tools", None),
                 )
 
                 # Update the cache with fresh data from server
@@ -808,6 +828,10 @@ class Prompt:
                     prompt_version_id=response.id,
                     type=response.type,
                     interpolation_type=response.interpolation_type,
+                    model_settings=response.model_settings,
+                    output_type=response.output_type,
+                    output_schema=response.output_schema,
+                    tools=response.tools,
                 )
 
                 # Update in-memory properties with fresh data (thread-safe)
@@ -819,6 +843,12 @@ class Prompt:
                     self._prompt_version_id = response.id
                     self.type = response.type
                     self.interpolation_type = response.interpolation_type
+                    self.model_settings = response.model_settings
+                    self.output_type = response.output_type
+                    self.output_schema = construct_base_model(
+                        response.output_schema
+                    )
+                    self.tools = response.tools
 
             except Exception:
                 pass