deepeval 3.5.8__tar.gz → 3.5.9__tar.gz

{deepeval-3.5.8 → deepeval-3.5.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepeval
-Version: 3.5.8
+Version: 3.5.9
 Summary: The LLM Evaluation Framework
 Home-page: https://github.com/confident-ai/deepeval
 License: Apache-2.0

deepeval-3.5.9/deepeval/_version.py

@@ -0,0 +1 @@
+__version__: str = "3.5.9"

deepeval-3.5.9/deepeval/integrations/pydantic_ai/__init__.py

@@ -0,0 +1,5 @@
+from .agent import DeepEvalPydanticAIAgent as Agent
+from .instrumentator import ConfidentInstrumentationSettings
+from .otel import instrument_pydantic_ai
+
+__all__ = ["ConfidentInstrumentationSettings"]

deepeval-3.5.9/deepeval/integrations/pydantic_ai/agent.py

@@ -0,0 +1,21 @@
+import warnings
+
+try:
+    from pydantic_ai.agent import Agent
+
+    is_pydantic_ai_installed = True
+except:
+    is_pydantic_ai_installed = False
+
+
+class DeepEvalPydanticAIAgent(Agent):
+
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "instrument_pydantic_ai is deprecated and will be removed in a future version. "
+            "Please use the new ConfidentInstrumentationSettings instead. Docs: https://www.confident-ai.com/docs/integrations/third-party/pydantic-ai",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        super().__init__(*args, **kwargs)

deepeval-3.5.9/deepeval/integrations/pydantic_ai/instrumentator.py

@@ -0,0 +1,196 @@
+import json
+import os
+from typing import Literal, Optional, List
+
+try:
+    from pydantic_ai.models.instrumented import InstrumentationSettings
+    from opentelemetry.sdk.trace import SpanProcessor, TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+        OTLPSpanExporter,
+    )
+
+    dependency_installed = True
+except:
+    dependency_installed = False
+
+
+def is_dependency_installed():
+    if not dependency_installed:
+        raise ImportError(
+            "Dependencies are not installed. Please install it with `pip install pydantic-ai opentelemetry-sdk opentelemetry-exporter-otlp-proto-http`."
+        )
+    return True
+
+
+from deepeval.confident.api import get_confident_api_key
+from deepeval.prompt import Prompt
+
+# OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
+OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
+
+
+class SpanInterceptor(SpanProcessor):
+    def __init__(self, settings_instance):
+        # Keep a reference to the settings instance instead of copying values
+        self.settings: ConfidentInstrumentationSettings = settings_instance
+
+    def on_start(self, span, parent_context):
+
+        # set trace attributes
+        if self.settings.thread_id:
+            span.set_attribute(
+                "confident.trace.thread_id", self.settings.thread_id
+            )
+        if self.settings.user_id:
+            span.set_attribute("confident.trace.user_id", self.settings.user_id)
+        if self.settings.metadata:
+            span.set_attribute(
+                "confident.trace.metadata", json.dumps(self.settings.metadata)
+            )
+        if self.settings.tags:
+            span.set_attribute("confident.trace.tags", self.settings.tags)
+        if self.settings.metric_collection:
+            span.set_attribute(
+                "confident.trace.metric_collection",
+                self.settings.metric_collection,
+            )
+        if self.settings.environment:
+            span.set_attribute(
+                "confident.trace.environment", self.settings.environment
+            )
+        if self.settings.name:
+            span.set_attribute("confident.trace.name", self.settings.name)
+        if self.settings.confident_prompt:
+            span.set_attribute(
+                "confident.span.prompt",
+                json.dumps(
+                    {
+                        "alias": self.settings.confident_prompt.alias,
+                        "version": self.settings.confident_prompt.version,
+                    }
+                ),
+            )
+
+        # set trace metric collection
+        if self.settings.trace_metric_collection:
+            span.set_attribute(
+                "confident.trace.metric_collection",
+                self.settings.trace_metric_collection,
+            )
+
+        # set agent name and metric collection
+        if span.attributes.get("agent_name"):
+            span.set_attribute("confident.span.type", "agent")
+            span.set_attribute(
+                "confident.span.name", span.attributes.get("agent_name")
+            )
+            if self.settings.agent_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    self.settings.agent_metric_collection,
+                )
+
+        # set llm metric collection
+        if span.attributes.get("gen_ai.operation.name") in [
+            "chat",
+            "generate_content",
+            "text_completion",
+        ]:
+            if self.settings.llm_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    self.settings.llm_metric_collection,
+                )
+
+        # set tool metric collection
+        tool_name = span.attributes.get("gen_ai.tool.name")
+        if tool_name:
+            tool_metric_collection = (
+                self.settings.tool_metric_collection_map.get(tool_name)
+            )
+            if tool_metric_collection:
+                span.set_attribute(
+                    "confident.span.metric_collection",
+                    str(tool_metric_collection),
+                )
+
+    def on_end(self, span):
+        pass
+
+
+class ConfidentInstrumentationSettings(InstrumentationSettings):
+
+    name: Optional[str] = None
+    thread_id: Optional[str] = None
+    user_id: Optional[str] = None
+    metadata: Optional[dict] = None
+    tags: Optional[List[str]] = None
+    environment: Literal["production", "staging", "development", "testing"] = (
+        None
+    )
+    metric_collection: Optional[str] = None
+    confident_prompt: Optional[Prompt] = None
+    llm_metric_collection: Optional[str] = None
+    agent_metric_collection: Optional[str] = None
+    tool_metric_collection_map: dict = {}
+    trace_metric_collection: Optional[str] = None
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        name: Optional[str] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metadata: Optional[dict] = None,
+        tags: Optional[List[str]] = None,
+        metric_collection: Optional[str] = None,
+        confident_prompt: Optional[Prompt] = None,
+        llm_metric_collection: Optional[str] = None,
+        agent_metric_collection: Optional[str] = None,
+        tool_metric_collection_map: dict = {},
+        trace_metric_collection: Optional[str] = None,
+    ):
+        is_dependency_installed()
+
+        _environment = os.getenv("CONFIDENT_TRACE_ENVIRONMENT", "development")
+        if _environment and _environment in [
+            "production",
+            "staging",
+            "development",
+            "testing",
+        ]:
+            self.environment = _environment
+
+        self.tool_metric_collection_map = tool_metric_collection_map
+        self.name = name
+        self.thread_id = thread_id
+        self.user_id = user_id
+        self.metadata = metadata
+        self.tags = tags
+        self.metric_collection = metric_collection
+        self.confident_prompt = confident_prompt
+        self.llm_metric_collection = llm_metric_collection
+        self.agent_metric_collection = agent_metric_collection
+        self.trace_metric_collection = trace_metric_collection
+
+        if not api_key:
+            api_key = get_confident_api_key()
+            if not api_key:
+                raise ValueError("CONFIDENT_API_KEY is not set")
+
+        trace_provider = TracerProvider()
+
+        # Pass the entire settings instance instead of individual values
+        span_interceptor = SpanInterceptor(self)
+        trace_provider.add_span_processor(span_interceptor)
+
+        trace_provider.add_span_processor(
+            BatchSpanProcessor(
+                OTLPSpanExporter(
+                    endpoint=OTLP_ENDPOINT,
+                    headers={"x-confident-api-key": api_key},
+                )
+            )
+        )
+        super().__init__(tracer_provider=trace_provider)

{deepeval-3.5.8 → deepeval-3.5.9}/deepeval/integrations/pydantic_ai/otel.py

@@ -1,7 +1,6 @@
+import warnings
 from typing import Optional
-import deepeval
 from deepeval.telemetry import capture_tracing_integration
-from deepeval.confident.api import get_confident_api_key
 
 try:
     from opentelemetry import trace
@@ -28,6 +27,13 @@ OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
 
 
 def instrument_pydantic_ai(api_key: Optional[str] = None):
+    warnings.warn(
+        "instrument_pydantic_ai is deprecated and will be removed in a future version. "
+        "Please use the new ConfidentInstrumentationSettings instead. Docs: https://www.confident-ai.com/docs/integrations/third-party/pydantic-ai",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
     with capture_tracing_integration("pydantic_ai"):
         is_opentelemetry_available()