deepeval 3.6.6__py3-none-any.whl → 3.6.7__py3-none-any.whl

deepeval/integrations/crewai/subs.py

@@ -0,0 +1,51 @@
+from typing import List, Optional, Type, TypeVar
+from pydantic import PrivateAttr
+
+from deepeval.metrics.base_metric import BaseMetric
+
+try:
+    from crewai import Crew, Agent, LLM
+
+    is_crewai_installed = True
+except ImportError:
+    is_crewai_installed = False
+
+
+def is_crewai_installed():
+    if not is_crewai_installed:
+        raise ImportError(
+            "CrewAI is not installed. Please install it with `pip install crewai`."
+        )
+
+
+T = TypeVar("T")
+
+
+def create_deepeval_class(base_class: Type[T], class_name: str) -> Type[T]:
+    """Factory function to create DeepEval-enabled CrewAI classes"""
+
+    class DeepEvalClass(base_class):
+        _metric_collection: Optional[str] = PrivateAttr(default=None)
+        _metrics: Optional[List[BaseMetric]] = PrivateAttr(default=None)
+
+        def __init__(
+            self,
+            *args,
+            metrics: Optional[List[BaseMetric]] = None,
+            metric_collection: Optional[str] = None,
+            **kwargs
+        ):
+            is_crewai_installed()
+            super().__init__(*args, **kwargs)
+            self._metric_collection = metric_collection
+            self._metrics = metrics
+
+    DeepEvalClass.__name__ = class_name
+    DeepEvalClass.__qualname__ = class_name
+    return DeepEvalClass
+
+
+# Create the classes
+DeepEvalCrew = create_deepeval_class(Crew, "DeepEvalCrew")
+DeepEvalAgent = create_deepeval_class(Agent, "DeepEvalAgent")
+DeepEvalLLM = create_deepeval_class(LLM, "DeepEvalLLM")

deepeval/integrations/crewai/wrapper.py

@@ -3,6 +3,7 @@ from crewai.crew import Crew
 from crewai.agent import Agent
 from functools import wraps
 from deepeval.tracing.tracing import Observer
+from typing import Any
 
 
 def wrap_crew_kickoff():
@@ -10,7 +11,13 @@ def wrap_crew_kickoff():
 
     @wraps(original_kickoff)
     def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = original_kickoff(self, *args, **kwargs)
 
         return result
@@ -23,7 +30,13 @@ def wrap_crew_kickoff_for_each():
 
     @wraps(original_kickoff_for_each)
     def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff_for_each"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff_for_each",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = original_kickoff_for_each(self, *args, **kwargs)
 
         return result
@@ -36,7 +49,13 @@ def wrap_crew_kickoff_async():
 
     @wraps(original_kickoff_async)
     async def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff_async"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff_async",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = await original_kickoff_async(self, *args, **kwargs)
 
         return result
@@ -49,7 +68,13 @@ def wrap_crew_kickoff_for_each_async():
 
     @wraps(original_kickoff_for_each_async)
     async def wrapper(self, *args, **kwargs):
-        with Observer(span_type="crew", func_name="kickoff_for_each_async"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="crew",
+            func_name="kickoff_for_each_async",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = await original_kickoff_for_each_async(
                 self, *args, **kwargs
             )
@@ -64,10 +89,13 @@ def wrap_llm_call():
 
     @wraps(original_llm_call)
     def wrapper(self, *args, **kwargs):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
         with Observer(
             span_type="llm",
             func_name="call",
             observe_kwargs={"model": "temp_model"},
+            metric_collection=metric_collection,
+            metrics=metrics,
         ):
             result = original_llm_call(self, *args, **kwargs)
         return result
@@ -80,8 +108,20 @@ def wrap_agent_execute_task():
 
     @wraps(original_execute_task)
     def wrapper(self, *args, **kwargs):
-        with Observer(span_type="agent", func_name="execute_task"):
+        metric_collection, metrics = _check_metrics_and_metric_collection(self)
+        with Observer(
+            span_type="agent",
+            func_name="execute_task",
+            metric_collection=metric_collection,
+            metrics=metrics,
+        ):
             result = original_execute_task(self, *args, **kwargs)
         return result
 
     Agent.execute_task = wrapper
+
+
+def _check_metrics_and_metric_collection(obj: Any):
+    metric_collection = getattr(obj, "_metric_collection", None)
+    metrics = getattr(obj, "_metrics", None)
+    return metric_collection, metrics

deepeval/metrics/answer_relevancy/answer_relevancy.py

@@ -16,6 +16,7 @@ from deepeval.models import DeepEvalBaseLLM
 from deepeval.metrics.answer_relevancy.template import AnswerRelevancyTemplate
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.answer_relevancy.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class AnswerRelevancyMetric(BaseMetric):
@@ -50,8 +51,8 @@ class AnswerRelevancyMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -65,6 +66,7 @@ class AnswerRelevancyMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -85,6 +87,10 @@ class AnswerRelevancyMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
 
             return self.score
 
@@ -93,8 +99,8 @@ class AnswerRelevancyMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -121,7 +127,10 @@ class AnswerRelevancyMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str) -> str:

deepeval/metrics/api.py

@@ -0,0 +1,281 @@
+from typing import Optional, Set, Any, Dict, List, Union
+import threading
+import asyncio
+import queue
+import atexit
+from time import perf_counter
+from enum import Enum
+from pydantic import Field
+from rich.console import Console
+
+from deepeval.confident.api import Api, HttpMethods, Endpoints, is_confident
+from deepeval.constants import (
+    CONFIDENT_METRIC_LOGGING_FLUSH,
+    CONFIDENT_METRIC_LOGGING_VERBOSE,
+)
+from deepeval.metrics.base_metric import BaseConversationalMetric, BaseMetric
+from deepeval.test_case.conversational_test_case import ConversationalTestCase
+from deepeval.test_case.llm_test_case import LLMTestCase
+from deepeval.test_case.api import create_api_test_case
+from deepeval.test_run.api import LLMApiTestCase, ConversationalApiTestCase
+from deepeval.tracing.api import MetricData
+from deepeval.config.settings import get_settings
+
+
+class MetricWorkerStatus(Enum):
+    SUCCESS = "success"
+    FAILURE = "failure"
+    WARNING = "warning"
+
+
+class ApiMetricData(MetricData):
+    llm_test_case: Optional[LLMApiTestCase] = Field(None, alias="llmTestCase")
+    conversational_test_case: Optional[ConversationalApiTestCase] = Field(
+        None, alias="conversationalTestCase"
+    )
+
+
+class MetricDataManager:
+    """Manager for posting metric data asynchronously in background thread."""
+
+    def __init__(self):
+        settings = get_settings()
+        # Initialize queue and worker thread for metric posting
+        self._metric_queue = queue.Queue()
+        self._worker_thread = None
+        self._min_interval = 0.2  # Minimum time between API calls (seconds)
+        self._last_post_time = 0
+        self._in_flight_tasks: Set[asyncio.Task[Any]] = set()
+        self._flush_enabled = bool(settings.CONFIDENT_METRIC_LOGGING_FLUSH)
+        self._daemon = not self._flush_enabled
+        self._thread_lock = threading.Lock()
+        self.metric_logging_enabled = bool(
+            settings.CONFIDENT_METRIC_LOGGING_ENABLED
+        )
+
+        # Register an exit handler to warn about unprocessed metrics
+        atexit.register(self._warn_on_exit)
+
+    def post_metric_if_enabled(
+        self,
+        metric: Union[BaseMetric, BaseConversationalMetric],
+        test_case: Optional[Union[LLMTestCase, ConversationalTestCase]] = None,
+    ):
+        """Post metric data asynchronously in a background thread."""
+        if not self.metric_logging_enabled or not is_confident():
+            return
+
+        from deepeval.evaluate.utils import create_metric_data
+
+        metric_data = create_metric_data(metric)
+        api_metric_data = ApiMetricData(
+            **metric_data.model_dump(by_alias=True, exclude_none=True)
+        )
+
+        if isinstance(test_case, LLMTestCase):
+            api_metric_data.llm_test_case = create_api_test_case(test_case)
+        elif isinstance(test_case, ConversationalTestCase):
+            api_metric_data.conversational_test_case = create_api_test_case(
+                test_case
+            )
+
+        self._ensure_worker_thread_running()
+        self._metric_queue.put(api_metric_data)
+
+    def _warn_on_exit(self):
+        """Warn if there are unprocessed metrics on exit."""
+        queue_size = self._metric_queue.qsize()
+        in_flight = len(self._in_flight_tasks)
+        remaining_tasks = queue_size + in_flight
+
+        if not self._flush_enabled and remaining_tasks > 0:
+            self._print_metric_data_status(
+                metric_worker_status=MetricWorkerStatus.WARNING,
+                message=f"Exiting with {queue_size + in_flight} abandoned metric(s).",
+                description=f"Set {CONFIDENT_METRIC_LOGGING_FLUSH}=1 as an environment variable to flush remaining metrics to Confident AI.",
+            )
+
+    def _ensure_worker_thread_running(self):
+        """Ensure the background worker thread is running."""
+        with self._thread_lock:
+            if (
+                self._worker_thread is None
+                or not self._worker_thread.is_alive()
+            ):
+                self._worker_thread = threading.Thread(
+                    target=self._process_metric_queue,
+                    daemon=self._daemon,
+                )
+                self._worker_thread.start()
+
+    def _print_metric_data_status(
+        self,
+        metric_worker_status: MetricWorkerStatus,
+        message: str,
+        description: Optional[str] = None,
+    ):
+        """Print metric data worker status messages."""
+        if getattr(get_settings(), CONFIDENT_METRIC_LOGGING_VERBOSE, False):
+            console = Console()
+            message_prefix = "[dim][Confident AI Metric Data Log][/dim]"
+            if metric_worker_status == MetricWorkerStatus.SUCCESS:
+                message = f"[green]{message}[/green]"
+            elif metric_worker_status == MetricWorkerStatus.FAILURE:
+                message = f"[red]{message}[/red]"
+            elif metric_worker_status == MetricWorkerStatus.WARNING:
+                message = f"[yellow]{message}[/yellow]"
+
+            if bool(CONFIDENT_METRIC_LOGGING_VERBOSE):
+                if description:
+                    message += f": {description}"
+
+                console.print(
+                    message_prefix,
+                    message,
+                    f"\nTo disable dev logging, set {CONFIDENT_METRIC_LOGGING_VERBOSE}=0 as an environment variable.",
+                )
+
+    def _process_metric_queue(self):
+        """Worker thread function that processes the metric queue."""
+        import threading
+
+        main_thr = threading.main_thread()
+
+        # Create a new event loop
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # Buffer for payloads that need to be sent after main exits
+        remaining_metric_request_bodies: List[Dict[str, Any]] = []
+
+        async def _a_send_metric(metric_data: ApiMetricData):
+            nonlocal remaining_metric_request_bodies
+            try:
+                # Build API object & payload
+                try:
+                    body = metric_data.model_dump(
+                        by_alias=True,
+                        exclude_none=True,
+                    )
+                except AttributeError:
+                    # Pydantic version below 2.0
+                    body = metric_data.dict(by_alias=True, exclude_none=True)
+
+                # If the main thread is still alive, send now
+                if main_thr.is_alive():
+                    api = Api()
+                    _, _ = await api.a_send_request(
+                        method=HttpMethods.POST,
+                        endpoint=Endpoints.METRIC_DATA_ENDPOINT,
+                        body=body,
+                    )
+                    queue_size = self._metric_queue.qsize()
+                    in_flight = len(self._in_flight_tasks)
+                    status = f"({queue_size} metric{'s' if queue_size!=1 else ''} remaining in queue, {in_flight} in flight)"
+                    self._print_metric_data_status(
+                        metric_worker_status=MetricWorkerStatus.SUCCESS,
+                        message=f"Successfully posted metric data {status}",
+                    )
+                elif self._flush_enabled:
+                    # Main thread gone → to be flushed
+                    remaining_metric_request_bodies.append(body)
+
+            except Exception as e:
+                queue_size = self._metric_queue.qsize()
+                in_flight = len(self._in_flight_tasks)
+                status = f"({queue_size} metric{'s' if queue_size!=1 else ''} remaining in queue, {in_flight} in flight)"
+                self._print_metric_data_status(
+                    metric_worker_status=MetricWorkerStatus.FAILURE,
+                    message=f"Error posting metric data {status}",
+                    description=str(e),
+                )
+            finally:
+                task = asyncio.current_task()
+                if task:
+                    self._in_flight_tasks.discard(task)
+
+        async def async_worker():
+            # Continue while user code is running or work remains
+            while (
+                main_thr.is_alive()
+                or not self._metric_queue.empty()
+                or self._in_flight_tasks
+            ):
+                try:
+                    metric_data = self._metric_queue.get(
+                        block=True, timeout=1.0
+                    )
+
+                    # Rate-limit
+                    now = perf_counter()
+                    elapsed = now - self._last_post_time
+                    if elapsed < self._min_interval:
+                        await asyncio.sleep(self._min_interval - elapsed)
+                    self._last_post_time = perf_counter()
+
+                    # Schedule async send
+                    task = asyncio.create_task(_a_send_metric(metric_data))
+                    self._in_flight_tasks.add(task)
+                    self._metric_queue.task_done()
+
+                except queue.Empty:
+                    await asyncio.sleep(0.1)
+                    continue
+                except Exception as e:
+                    self._print_metric_data_status(
+                        message="Error in metric worker",
+                        metric_worker_status=MetricWorkerStatus.FAILURE,
+                        description=str(e),
+                    )
+                    await asyncio.sleep(1.0)
+
+        try:
+            loop.run_until_complete(async_worker())
+        finally:
+            # Drain any pending tasks
+            pending = asyncio.all_tasks(loop=loop)
+            if pending:
+                loop.run_until_complete(
+                    asyncio.gather(*pending, return_exceptions=True)
+                )
+            self._flush_metrics(remaining_metric_request_bodies)
+            loop.run_until_complete(loop.shutdown_asyncgens())
+            loop.close()
+
+    def _flush_metrics(
+        self, remaining_metric_request_bodies: List[Dict[str, Any]]
+    ):
+        """Flush remaining metrics synchronously."""
+        if not remaining_metric_request_bodies:
+            return
+
+        self._print_metric_data_status(
+            MetricWorkerStatus.WARNING,
+            message=f"Flushing {len(remaining_metric_request_bodies)} remaining metric(s)",
+        )
+
+        for body in remaining_metric_request_bodies:
+            try:
+                api = Api()
+                _, link = api.send_request(
+                    method=HttpMethods.POST,
+                    endpoint=Endpoints.METRIC_DATA_ENDPOINT,
+                    body=body,
+                )
+                qs = self._metric_queue.qsize()
+                self._print_metric_data_status(
+                    metric_worker_status=MetricWorkerStatus.SUCCESS,
+                    message=f"Successfully posted metric data ({qs} metrics remaining in queue, 1 in flight)",
+                    description=link,
+                )
+            except Exception as e:
+                qs = self._metric_queue.qsize()
+                self._print_metric_data_status(
+                    metric_worker_status=MetricWorkerStatus.FAILURE,
+                    message="Error flushing remaining metric(s)",
+                    description=str(e),
+                )
+
+
+# Global metric manager instance
+metric_data_manager = MetricDataManager()

deepeval/metrics/argument_correctness/argument_correctness.py

@@ -19,6 +19,7 @@ from deepeval.metrics.argument_correctness.template import (
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.argument_correctness.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ArgumentCorrectnessMetric(BaseMetric):
@@ -53,6 +54,7 @@ class ArgumentCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -68,6 +70,7 @@ class ArgumentCorrectnessMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -91,7 +94,10 @@ class ArgumentCorrectnessMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -99,6 +105,7 @@ class ArgumentCorrectnessMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -130,7 +137,10 @@ class ArgumentCorrectnessMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str) -> str:

deepeval/metrics/bias/bias.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Type, Union
 
 from deepeval.metrics import BaseMetric
+from deepeval.metrics.api import metric_data_manager
 from deepeval.test_case import (
     LLMTestCase,
     LLMTestCaseParams,
@@ -48,8 +49,8 @@ class BiasMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -63,6 +64,7 @@ class BiasMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -81,7 +83,10 @@ class BiasMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -89,8 +94,8 @@ class BiasMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -116,6 +121,10 @@ class BiasMetric(BaseMetric):
                 ],
             )
 
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self) -> str:

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -18,6 +18,7 @@ from deepeval.metrics.contextual_precision.template import (
 )
 from deepeval.metrics.indicator import metric_progress_indicator
 from deepeval.metrics.contextual_precision.schema import *
+from deepeval.metrics.api import metric_data_manager
 
 
 class ContextualPrecisionMetric(BaseMetric):
@@ -53,8 +54,8 @@ class ContextualPrecisionMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
-
         check_llm_test_case_params(test_case, self._required_params, self)
 
         self.evaluation_cost = 0 if self.using_native_model else None
@@ -68,6 +69,7 @@ class ContextualPrecisionMetric(BaseMetric):
                         test_case,
                         _show_indicator=False,
                         _in_component=_in_component,
+                        _log_metric_to_confident=_log_metric_to_confident,
                     )
                 )
             else:
@@ -88,7 +90,10 @@ class ContextualPrecisionMetric(BaseMetric):
                         f"Score: {self.score}\nReason: {self.reason}",
                     ],
                 )
-
+                if _log_metric_to_confident:
+                    metric_data_manager.post_metric_if_enabled(
+                        self, test_case=test_case
+                    )
             return self.score
 
     async def a_measure(
@@ -96,6 +101,7 @@ class ContextualPrecisionMetric(BaseMetric):
         test_case: LLMTestCase,
         _show_indicator: bool = True,
         _in_component: bool = False,
+        _log_metric_to_confident: bool = True,
     ) -> float:
 
         check_llm_test_case_params(test_case, self._required_params, self)
@@ -124,7 +130,10 @@ class ContextualPrecisionMetric(BaseMetric):
                     f"Score: {self.score}\nReason: {self.reason}",
                 ],
             )
-
+            if _log_metric_to_confident:
+                metric_data_manager.post_metric_if_enabled(
+                    self, test_case=test_case
+                )
             return self.score
 
     async def _a_generate_reason(self, input: str):