deepeval 3.6.7__py3-none-any.whl → 3.6.9__py3-none-any.whl

deepeval/evaluate/types.py

@@ -10,6 +10,7 @@ from deepeval.test_case import MLLMImage
 class TestResult:
     """Returned from run_test"""
 
+    __test__ = False
     name: str
     success: bool
     metrics_data: Union[List[MetricData], None]

deepeval/evaluate/utils.py

@@ -5,8 +5,6 @@ import os
 import time
 
 from deepeval.utils import format_turn
-from deepeval.test_case.conversational_test_case import Turn
-from deepeval.test_run.api import TurnApi
 from deepeval.test_run.test_run import TestRunResultDisplay
 from deepeval.dataset import Golden
 from deepeval.metrics import (
@@ -481,6 +479,18 @@ def count_metrics_in_trace(trace: Trace) -> int:
     return sum(count_metrics_recursive(span) for span in trace.root_spans)
 
 
+def count_total_metrics_for_trace(trace: Trace) -> int:
+    """Span subtree metrics + trace-level metrics."""
+    return count_metrics_in_trace(trace=trace) + len(trace.metrics or [])
+
+
+def count_metrics_in_span_subtree(span: BaseSpan) -> int:
+    total = len(span.metrics or [])
+    for c in span.children or []:
+        total += count_metrics_in_span_subtree(c)
+    return total
+
+
 def extract_trace_test_results(trace_api: TraceApi) -> List[TestResult]:
     test_results: List[TestResult] = []
     # extract trace result
@@ -523,7 +533,7 @@ def extract_span_test_results(span_api: BaseApiSpan) -> List[TestResult]:
         test_results.append(
             TestResult(
                 name=span_api.name,
-                success=span_api.status == "SUCCESS",
+                success=span_api.status == TraceSpanApiStatus.SUCCESS,
                 metrics_data=span_api.metrics_data,
                 input=span_api.input,
                 actual_output=span_api.output,

deepeval/integrations/crewai/__init__.py

@@ -4,5 +4,6 @@ from .subs import (
     DeepEvalAgent as Agent,
     DeepEvalLLM as LLM,
 )
+from .tool import tool
 
-__all__ = ["instrument_crewai", "Crew", "Agent", "LLM"]
+__all__ = ["instrument_crewai", "Crew", "Agent", "LLM", "tool"]

deepeval/integrations/crewai/tool.py

@@ -0,0 +1,71 @@
+import functools
+from typing import Callable
+from crewai.tools import tool as crewai_tool
+
+from deepeval.tracing.context import current_span_context
+from deepeval.tracing.types import ToolSpan
+
+
+def tool(*args, metric=None, metric_collection=None, **kwargs) -> Callable:
+    """
+    Simple wrapper around crewai.tools.tool that:
+      - prints the original function's input and output
+      - accepts additional parameters: metric and metric_collection (unused, for compatibility)
+      - remains backward compatible with CrewAI's decorator usage patterns
+    """
+    crewai_kwargs = kwargs
+
+    # Case 1: @tool (function passed directly)
+    if len(args) == 1 and callable(args[0]):
+        f = args[0]
+        tool_name = f.__name__
+
+        @functools.wraps(f)
+        def wrapped(*f_args, **f_kwargs):
+            current_span = current_span_context.get()
+            if current_span and isinstance(current_span, ToolSpan):
+                current_span.metric_collection = metric_collection
+                current_span.metrics = metric
+            result = f(*f_args, **f_kwargs)
+            return result
+
+        return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+
+    # Case 2: @tool("name")
+    if len(args) == 1 and isinstance(args[0], str):
+        tool_name = args[0]
+
+        def _decorator(f: Callable) -> Callable:
+            @functools.wraps(f)
+            def wrapped(*f_args, **f_kwargs):
+                current_span = current_span_context.get()
+                if current_span and isinstance(current_span, ToolSpan):
+                    current_span.metric_collection = metric_collection
+                    current_span.metrics = metric
+                result = f(*f_args, **f_kwargs)
+                return result
+
+            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+
+        return _decorator
+
+    # Case 3: @tool(result_as_answer=True, ...) — kwargs only
+    if len(args) == 0:
+
+        def _decorator(f: Callable) -> Callable:
+            tool_name = f.__name__
+
+            @functools.wraps(f)
+            def wrapped(*f_args, **f_kwargs):
+                current_span = current_span_context.get()
+                if current_span and isinstance(current_span, ToolSpan):
+                    current_span.metric_collection = metric_collection
+                    current_span.metrics = metric
+                result = f(*f_args, **f_kwargs)
+                return result
+
+            return crewai_tool(tool_name, **crewai_kwargs)(wrapped)
+
+        return _decorator
+
+    raise ValueError("Invalid arguments")

deepeval/integrations/llama_index/__init__.py

@@ -1,10 +1,6 @@
 from .handler import instrument_llama_index
-from .agent.patched import FunctionAgent, ReActAgent, CodeActAgent
 
 
 __all__ = [
     "instrument_llama_index",
-    "FunctionAgent",
-    "ReActAgent",
-    "CodeActAgent",
 ]

deepeval/integrations/llama_index/handler.py

@@ -5,6 +5,10 @@ import uuid
 from deepeval.telemetry import capture_tracing_integration
 from deepeval.tracing import trace_manager
 from deepeval.tracing.types import AgentSpan, BaseSpan, LlmSpan, TraceSpanStatus
+from deepeval.tracing.trace_context import (
+    current_llm_context,
+    current_agent_context,
+)
 
 try:
     from llama_index.core.instrumentation.events.base import BaseEvent
@@ -22,11 +26,6 @@ try:
         LLMChatEndEvent,
     )
     from llama_index_instrumentation.dispatcher import Dispatcher
-    from deepeval.integrations.llama_index.agent.patched import (
-        FunctionAgent as PatchedFunctionAgent,
-        ReActAgent as PatchedReActAgent,
-        CodeActAgent as PatchedCodeActAgent,
-    )
     from deepeval.integrations.llama_index.utils import (
         parse_id,
         prepare_input_llm_test_case_params,
@@ -67,6 +66,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 ).strip()
                 input_messages.append({"role": role, "content": content})
 
+            llm_span_context = current_llm_context.get()
             # create the span
             llm_span = LlmSpan(
                 name="ConfidentLLMSpan",
@@ -83,6 +83,12 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 ),  # check the model name not coming in this option
                 input=input_messages,
                 output="",
+                metrics=llm_span_context.metrics if llm_span_context else None,
+                metric_collection=(
+                    llm_span_context.metric_collection
+                    if llm_span_context
+                    else None
+                ),
             )
             trace_manager.add_span(llm_span)
             trace_manager.add_span_to_trace(llm_span)
@@ -144,6 +150,7 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
 
         # conditions to qualify as agent start run span
         if method_name == "run":
+            agent_span_context = current_agent_context.get()
             span = AgentSpan(
                 uuid=id_,
                 status=TraceSpanStatus.IN_PROGRESS,
@@ -153,24 +160,16 @@ class LLamaIndexHandler(BaseEventHandler, BaseSpanHandler):
                 start_time=perf_counter(),
                 name="Agent",  # TODO: decide the name of the span
                 input=bound_args.arguments,
+                metrics=(
+                    agent_span_context.metrics if agent_span_context else None
+                ),
+                metric_collection=(
+                    agent_span_context.metric_collection
+                    if agent_span_context
+                    else None
+                ),
             )
 
-            # check if the instance is a PatchedFunctionAgent
-            if isinstance(instance, PatchedFunctionAgent):
-                span.name = "FunctionAgent"
-                span.metric_collection = instance.metric_collection
-                span.metrics = instance.metrics
-
-            if isinstance(instance, PatchedReActAgent):
-                span.name = "ReActAgent"
-                span.metric_collection = instance.metric_collection
-                span.metrics = instance.metrics
-
-            if isinstance(instance, PatchedCodeActAgent):
-                span.name = "CodeActAgent"
-                span.metric_collection = instance.metric_collection
-                span.metrics = instance.metrics
-
         # prepare input test case params for the span
         prepare_input_llm_test_case_params(
             class_name, method_name, span, bound_args.arguments

deepeval/integrations/pydantic_ai/instrumentator.py

@@ -1,14 +1,19 @@
 import json
 import logging
 import os
+from time import perf_counter
 from typing import Literal, Optional, List
 
 from deepeval.config.settings import get_settings
 from deepeval.confident.api import get_confident_api_key
+from deepeval.metrics.base_metric import BaseMetric
 from deepeval.prompt import Prompt
 from deepeval.tracing.context import current_trace_context
 from deepeval.tracing.types import Trace
 from deepeval.tracing.otel.utils import to_hex_string
+from deepeval.tracing.tracing import trace_manager
+from deepeval.tracing.otel.utils import normalize_pydantic_ai_messages
+from deepeval.tracing.otel.exporter import ConfidentSpanExporter
 
 
 logger = logging.getLogger(__name__)
@@ -21,6 +26,7 @@ try:
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
         OTLPSpanExporter,
     )
+    from opentelemetry.sdk.trace import ReadableSpan
 
     dependency_installed = True
 except ImportError as e:
@@ -48,24 +54,96 @@ def is_dependency_installed():
     return True
 
 
+from deepeval.tracing.types import AgentSpan
 from deepeval.confident.api import get_confident_api_key
 from deepeval.prompt import Prompt
 from deepeval.tracing.otel.test_exporter import test_exporter
 from deepeval.tracing.context import current_trace_context
 from deepeval.tracing.types import Trace
 from deepeval.tracing.otel.utils import to_hex_string
+from deepeval.tracing.types import TraceSpanStatus, ToolCall
+from deepeval.tracing.perf_epoch_bridge import init_clock_bridge
 
 # OTLP_ENDPOINT = "http://127.0.0.1:4318/v1/traces"
 OTLP_ENDPOINT = "https://otel.confident-ai.com/v1/traces"
+init_clock_bridge()  # initialize clock bridge for perf_counter() to epoch_nanos conversion
+
+
+class ConfidentInstrumentationSettings(InstrumentationSettings):
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        name: Optional[str] = None,
+        thread_id: Optional[str] = None,
+        user_id: Optional[str] = None,
+        metadata: Optional[dict] = None,
+        tags: Optional[List[str]] = None,
+        metric_collection: Optional[str] = None,
+        confident_prompt: Optional[Prompt] = None,
+        llm_metric_collection: Optional[str] = None,
+        agent_metric_collection: Optional[str] = None,
+        tool_metric_collection_map: Optional[dict] = None,
+        trace_metric_collection: Optional[str] = None,
+        is_test_mode: Optional[bool] = False,
+        agent_metrics: Optional[List[BaseMetric]] = None,
+    ):
+        is_dependency_installed()
+
+        _environment = os.getenv("CONFIDENT_TRACE_ENVIRONMENT", "development")
+        if _environment and _environment in [
+            "production",
+            "staging",
+            "development",
+            "testing",
+        ]:
+            self.environment = _environment
+
+        self.tool_metric_collection_map = tool_metric_collection_map or {}
+        self.name = name
+        self.thread_id = thread_id
+        self.user_id = user_id
+        self.metadata = metadata
+        self.tags = tags
+        self.metric_collection = metric_collection
+        self.confident_prompt = confident_prompt
+        self.llm_metric_collection = llm_metric_collection
+        self.agent_metric_collection = agent_metric_collection
+        self.trace_metric_collection = trace_metric_collection
+        self.is_test_mode = is_test_mode
+        self.agent_metrics = agent_metrics
+
+        if not api_key:
+            api_key = get_confident_api_key()
+            if not api_key:
+                raise ValueError("CONFIDENT_API_KEY is not set")
+
+        trace_provider = TracerProvider()
+
+        # Pass the entire settings instance instead of individual values
+        span_interceptor = SpanInterceptor(self)
+        trace_provider.add_span_processor(span_interceptor)
+
+        if is_test_mode:
+            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
+        else:
+            trace_provider.add_span_processor(
+                BatchSpanProcessor(
+                    OTLPSpanExporter(
+                        endpoint=OTLP_ENDPOINT,
+                        headers={"x-confident-api-key": api_key},
+                    )
+                )
+            )
+        super().__init__(tracer_provider=trace_provider)
 
 
 class SpanInterceptor(SpanProcessor):
-    def __init__(self, settings_instance):
+    def __init__(self, settings_instance: ConfidentInstrumentationSettings):
         # Keep a reference to the settings instance instead of copying values
-        self.settings: ConfidentInstrumentationSettings = settings_instance
+        self.settings = settings_instance
 
     def on_start(self, span, parent_context):
-
         # set trace uuid
         _current_trace_context = current_trace_context.get()
         if _current_trace_context and isinstance(_current_trace_context, Trace):
@@ -151,85 +229,56 @@ class SpanInterceptor(SpanProcessor):
                 )
 
     def on_end(self, span):
-        pass
-
+        if self.settings.is_test_mode:
+            if span.attributes.get("confident.span.type") == "agent":
 
-class ConfidentInstrumentationSettings(InstrumentationSettings):
-
-    name: Optional[str] = None
-    thread_id: Optional[str] = None
-    user_id: Optional[str] = None
-    metadata: Optional[dict] = None
-    tags: Optional[List[str]] = None
-    environment: Literal["production", "staging", "development", "testing"] = (
-        None
-    )
-    metric_collection: Optional[str] = None
-    confident_prompt: Optional[Prompt] = None
-    llm_metric_collection: Optional[str] = None
-    agent_metric_collection: Optional[str] = None
-    tool_metric_collection_map: dict = {}
-    trace_metric_collection: Optional[str] = None
+                def create_agent_span_for_evaluation(
+                    span: ReadableSpan,
+                ) -> AgentSpan:
 
-    def __init__(
-        self,
-        api_key: Optional[str] = None,
-        name: Optional[str] = None,
-        thread_id: Optional[str] = None,
-        user_id: Optional[str] = None,
-        metadata: Optional[dict] = None,
-        tags: Optional[List[str]] = None,
-        metric_collection: Optional[str] = None,
-        confident_prompt: Optional[Prompt] = None,
-        llm_metric_collection: Optional[str] = None,
-        agent_metric_collection: Optional[str] = None,
-        tool_metric_collection_map: Optional[dict] = None,
-        trace_metric_collection: Optional[str] = None,
-        is_test_mode: Optional[bool] = False,
-    ):
-        is_dependency_installed()
+                    agent_span = (
+                        ConfidentSpanExporter.prepare_boilerplate_base_span(
+                            span
+                        )
+                    )
 
-        _environment = os.getenv("CONFIDENT_TRACE_ENVIRONMENT", "development")
-        if _environment and _environment in [
-            "production",
-            "staging",
-            "development",
-            "testing",
-        ]:
-            self.environment = _environment
+                    # tools called
+                    normalized_messages = normalize_pydantic_ai_messages(span)
+                    tools_called = []
 
-        self.tool_metric_collection_map = tool_metric_collection_map or {}
-        self.name = name
-        self.thread_id = thread_id
-        self.user_id = user_id
-        self.metadata = metadata
-        self.tags = tags
-        self.metric_collection = metric_collection
-        self.confident_prompt = confident_prompt
-        self.llm_metric_collection = llm_metric_collection
-        self.agent_metric_collection = agent_metric_collection
-        self.trace_metric_collection = trace_metric_collection
+                    for message in normalized_messages:
+                        for part in message.get("parts", []):
+                            if part.get("type") == "tool_call":
+                                name = part.get("name")
+                                try:
+                                    input_parameters = json.loads(
+                                        part.get("arguments")
+                                    )
+                                except Exception:
+                                    input_parameters = {}
 
-        if not api_key:
-            api_key = get_confident_api_key()
-            if not api_key:
-                raise ValueError("CONFIDENT_API_KEY is not set")
+                                tools_called.append(
+                                    ToolCall(
+                                        name=name,
+                                        input_parameters=input_parameters,
+                                    )
+                                )
 
-        trace_provider = TracerProvider()
+                    # agent_span.tools_called = tools_called
+                    return agent_span
 
-        # Pass the entire settings instance instead of individual values
-        span_interceptor = SpanInterceptor(self)
-        trace_provider.add_span_processor(span_interceptor)
+                agent_span = create_agent_span_for_evaluation(span)
+                agent_span.metrics = self.settings.agent_metrics
 
-        if is_test_mode:
-            trace_provider.add_span_processor(BatchSpanProcessor(test_exporter))
-        else:
-            trace_provider.add_span_processor(
-                BatchSpanProcessor(
-                    OTLPSpanExporter(
-                        endpoint=OTLP_ENDPOINT,
-                        headers={"x-confident-api-key": api_key},
+                # create a trace for evaluation
+                trace = trace_manager.get_trace_by_uuid(agent_span.trace_uuid)
+                if not trace:
+                    trace = trace_manager.start_new_trace(
+                        trace_uuid=agent_span.trace_uuid
                     )
-                )
-            )
-        super().__init__(tracer_provider=trace_provider)
+
+                trace.root_spans.append(agent_span)
+                trace.status = TraceSpanStatus.SUCCESS
+                trace.end_time = perf_counter()
+                trace_manager.traces_to_evaluate.append(trace)
+                test_exporter.clear_span_json_list()

deepeval/metrics/__init__.py

@@ -27,6 +27,12 @@ from .tool_correctness.tool_correctness import ToolCorrectnessMetric
 from .json_correctness.json_correctness import JsonCorrectnessMetric
 from .prompt_alignment.prompt_alignment import PromptAlignmentMetric
 from .task_completion.task_completion import TaskCompletionMetric
+from .topic_adherence.topic_adherence import TopicAdherenceMetric
+from .step_efficiency.step_efficiency import StepEfficiencyMetric
+from .plan_adherence.plan_adherence import PlanAdherenceMetric
+from .plan_quality.plan_quality import PlanQualityMetric
+from .tool_use.tool_use import ToolUseMetric
+from .goal_accuracy.goal_accuracy import GoalAccuracyMetric
 from .argument_correctness.argument_correctness import ArgumentCorrectnessMetric
 from .mcp.mcp_task_completion import MCPTaskCompletionMetric
 from .mcp.multi_turn_mcp_use_metric import MultiTurnMCPUseMetric
@@ -98,6 +104,13 @@ __all__ = [
     "TaskCompletionMetric",
     "ArgumentCorrectnessMetric",
     "KnowledgeRetentionMetric",
+    # Agentic metrics
+    "TopicAdherenceMetric",
+    "StepEfficiencyMetric",
+    "PlanAdherenceMetric",
+    "PlanQualityMetric",
+    "ToolUseMetric",
+    "GoalAccuracyMetric",
     # Conversational metrics
     "TurnRelevancyMetric",
     "ConversationCompletenessMetric",

deepeval/metrics/base_metric.py

@@ -27,6 +27,7 @@ class BaseMetric:
     evaluation_cost: Optional[float] = None
     verbose_logs: Optional[str] = None
     skipped = False
+    requires_trace: bool = False
     model = Optional[DeepEvalBaseLLM]
     using_native_model = Optional[bool]
 

deepeval/metrics/contextual_precision/contextual_precision.py

@@ -17,7 +17,7 @@ from deepeval.metrics.contextual_precision.template import (
     ContextualPrecisionTemplate,
 )
 from deepeval.metrics.indicator import metric_progress_indicator
-from deepeval.metrics.contextual_precision.schema import *
+import deepeval.metrics.contextual_precision.schema as cpschema
 from deepeval.metrics.api import metric_data_manager
 
 
@@ -73,7 +73,7 @@ class ContextualPrecisionMetric(BaseMetric):
                     )
                 )
             else:
-                self.verdicts: List[ContextualPrecisionVerdict] = (
+                self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                     self._generate_verdicts(
                         test_case.input,
                         test_case.expected_output,
@@ -113,7 +113,7 @@ class ContextualPrecisionMetric(BaseMetric):
             _show_indicator=_show_indicator,
             _in_component=_in_component,
         ):
-            self.verdicts: List[ContextualPrecisionVerdict] = (
+            self.verdicts: List[cpschema.ContextualPrecisionVerdict] = (
                 await self._a_generate_verdicts(
                     test_case.input,
                     test_case.expected_output,
@@ -141,7 +141,7 @@ class ContextualPrecisionMetric(BaseMetric):
             return None
 
         retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            {"verdict": verdict.verdict, "reason": verdict.reason}
             for verdict in self.verdicts
         ]
         prompt = self.evaluation_template.generate_reason(
@@ -152,15 +152,15 @@ class ContextualPrecisionMetric(BaseMetric):
 
         if self.using_native_model:
             res, cost = await self.model.a_generate(
-                prompt, schema=ContextualPrecisionScoreReason
+                prompt, schema=cpschema.ContextualPrecisionScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: ContextualPrecisionScoreReason = (
+                res: cpschema.ContextualPrecisionScoreReason = (
                     await self.model.a_generate(
-                        prompt, schema=ContextualPrecisionScoreReason
+                        prompt, schema=cpschema.ContextualPrecisionScoreReason
                     )
                 )
                 return res.reason
@@ -174,7 +174,7 @@ class ContextualPrecisionMetric(BaseMetric):
             return None
 
         retrieval_contexts_verdicts = [
-            {"verdict": verdict.verdict, "reasons": verdict.reason}
+            {"verdict": verdict.verdict, "reason": verdict.reason}
             for verdict in self.verdicts
         ]
         prompt = self.evaluation_template.generate_reason(
@@ -185,14 +185,16 @@ class ContextualPrecisionMetric(BaseMetric):
 
         if self.using_native_model:
             res, cost = self.model.generate(
-                prompt, schema=ContextualPrecisionScoreReason
+                prompt, schema=cpschema.ContextualPrecisionScoreReason
             )
             self.evaluation_cost += cost
             return res.reason
         else:
             try:
-                res: ContextualPrecisionScoreReason = self.model.generate(
-                    prompt, schema=ContextualPrecisionScoreReason
+                res: cpschema.ContextualPrecisionScoreReason = (
+                    self.model.generate(
+                        prompt, schema=cpschema.ContextualPrecisionScoreReason
+                    )
                 )
                 return res.reason
             except TypeError:
@@ -202,21 +204,23 @@ class ContextualPrecisionMetric(BaseMetric):
 
     async def _a_generate_verdicts(
         self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
+    ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = await self.model.a_generate(prompt, schema=Verdicts)
+            res, cost = await self.model.a_generate(
+                prompt, schema=cpschema.Verdicts
+            )
             self.evaluation_cost += cost
             verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
-                res: Verdicts = await self.model.a_generate(
-                    prompt, schema=Verdicts
+                res: cpschema.Verdicts = await self.model.a_generate(
+                    prompt, schema=cpschema.Verdicts
                 )
                 verdicts = [item for item in res.verdicts]
                 return verdicts
@@ -224,34 +228,36 @@ class ContextualPrecisionMetric(BaseMetric):
                 res = await self.model.a_generate(prompt)
                 data = trimAndLoadJson(res, self)
                 verdicts = [
-                    ContextualPrecisionVerdict(**item)
+                    cpschema.ContextualPrecisionVerdict(**item)
                     for item in data["verdicts"]
                 ]
                 return verdicts
 
     def _generate_verdicts(
         self, input: str, expected_output: str, retrieval_context: List[str]
-    ) -> List[ContextualPrecisionVerdict]:
+    ) -> List[cpschema.ContextualPrecisionVerdict]:
         prompt = self.evaluation_template.generate_verdicts(
             input=input,
             expected_output=expected_output,
             retrieval_context=retrieval_context,
         )
         if self.using_native_model:
-            res, cost = self.model.generate(prompt, schema=Verdicts)
+            res, cost = self.model.generate(prompt, schema=cpschema.Verdicts)
             self.evaluation_cost += cost
             verdicts = [item for item in res.verdicts]
             return verdicts
         else:
             try:
-                res: Verdicts = self.model.generate(prompt, schema=Verdicts)
+                res: cpschema.Verdicts = self.model.generate(
+                    prompt, schema=cpschema.Verdicts
+                )
                 verdicts = [item for item in res.verdicts]
                 return verdicts
             except TypeError:
                 res = self.model.generate(prompt)
                 data = trimAndLoadJson(res, self)
                 verdicts = [
-                    ContextualPrecisionVerdict(**item)
+                    cpschema.ContextualPrecisionVerdict(**item)
                     for item in data["verdicts"]
                 ]
                 return verdicts
@@ -288,7 +294,7 @@ class ContextualPrecisionMetric(BaseMetric):
         else:
             try:
                 self.success = self.score >= self.threshold
-            except:
+            except TypeError:
                 self.success = False
         return self.success