agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/metrics/agents/types.py

@@ -0,0 +1,160 @@
+"""
+Types for Agent Evaluation.
+
+These types support trajectory-based evaluation of AI agent performance,
+including multi-step analysis and tool usage tracking.
+"""
+
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, Field
+
+from ...types import BaseMetricInput
+
+
+class ToolCall(BaseModel):
+    """Represents a tool/function call made by an agent."""
+
+    name: str = Field(..., description="Name of the tool called")
+    arguments: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Arguments passed to the tool"
+    )
+    result: Optional[Any] = Field(
+        default=None,
+        description="Result returned from the tool"
+    )
+    success: bool = Field(
+        default=True,
+        description="Whether the tool call succeeded"
+    )
+    error: Optional[str] = Field(
+        default=None,
+        description="Error message if the call failed"
+    )
+
+
+class AgentStep(BaseModel):
+    """Represents a single step in an agent's trajectory."""
+
+    step_number: int = Field(..., description="Step number in the trajectory (1-indexed)")
+    thought: Optional[str] = Field(
+        default=None,
+        description="Agent's reasoning/thought for this step"
+    )
+    action: Optional[str] = Field(
+        default=None,
+        description="Action taken by the agent"
+    )
+    tool_calls: List[ToolCall] = Field(
+        default_factory=list,
+        description="Tool calls made in this step"
+    )
+    observation: Optional[str] = Field(
+        default=None,
+        description="Observation from the environment after the action"
+    )
+    is_final: bool = Field(
+        default=False,
+        description="Whether this is the final step"
+    )
+    timestamp_ms: Optional[int] = Field(
+        default=None,
+        description="Timestamp of the step in milliseconds"
+    )
+
+
+class TaskDefinition(BaseModel):
+    """Definition of the task the agent should accomplish."""
+
+    description: str = Field(..., description="Natural language description of the task")
+    expected_outcome: Optional[str] = Field(
+        default=None,
+        description="Expected outcome or result"
+    )
+    required_tools: Optional[List[str]] = Field(
+        default=None,
+        description="List of tools required to complete the task"
+    )
+    max_steps: Optional[int] = Field(
+        default=None,
+        description="Maximum allowed steps"
+    )
+    success_criteria: Optional[List[str]] = Field(
+        default=None,
+        description="Criteria for determining task success"
+    )
+
+
+class ExpectedStep(BaseModel):
+    """Expected step in the optimal trajectory."""
+
+    description: str = Field(..., description="Description of the expected action")
+    required_tools: Optional[List[str]] = Field(
+        default=None,
+        description="Tools that should be used in this step"
+    )
+    key_arguments: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="Key arguments that should be present"
+    )
+
+
+class AgentTrajectoryInput(BaseMetricInput):
+    """
+    Input for agent trajectory evaluation metrics.
+
+    Evaluates multi-step agent execution including:
+    - Task completion
+    - Step efficiency
+    - Tool usage accuracy
+    - Trajectory quality
+    """
+
+    # The agent's execution trajectory
+    trajectory: List[AgentStep] = Field(
+        ...,
+        description="List of steps taken by the agent"
+    )
+
+    # The task definition
+    task: TaskDefinition = Field(
+        ...,
+        description="Definition of the task to accomplish"
+    )
+
+    # Optional: expected trajectory for comparison
+    expected_trajectory: Optional[List[ExpectedStep]] = Field(
+        default=None,
+        description="Expected/optimal trajectory for comparison"
+    )
+
+    # Optional: final result produced by the agent
+    final_result: Optional[Any] = Field(
+        default=None,
+        description="Final output/result from the agent"
+    )
+
+    # Optional: ground truth result for comparison
+    expected_result: Optional[Any] = Field(
+        default=None,
+        description="Expected/ground truth result"
+    )
+
+    # Available tools for the task
+    available_tools: Optional[List[str]] = Field(
+        default=None,
+        description="List of available tools the agent can use"
+    )
+
+
+class TrajectoryAnalysis(BaseModel):
+    """Detailed analysis of an agent trajectory."""
+
+    total_steps: int = Field(..., description="Total number of steps taken")
+    successful_tool_calls: int = Field(..., description="Number of successful tool calls")
+    failed_tool_calls: int = Field(..., description="Number of failed tool calls")
+    unique_tools_used: List[str] = Field(default_factory=list, description="List of unique tools used")
+    unnecessary_steps: int = Field(default=0, description="Number of unnecessary/redundant steps")
+    backtracking_count: int = Field(default=0, description="Number of times agent backtracked")
+    task_completed: bool = Field(..., description="Whether the task was completed")
+    efficiency_score: float = Field(..., description="Efficiency score (0-1)")

fi/evals/metrics/base_llm_metric.py

@@ -0,0 +1,111 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional, Type, TypeVar
+from pydantic import BaseModel, ValidationError
+import re
+import json
+
+from .base_metric import BaseMetric
+from .heuristics.json_metrics import ContainsJson
+from ..types import TextMetricInput
+from .llm_as_judges.types import BaseLLMJudgeInput
+from ..llm.base_llm_provider import LLMProvider
+from ..llm.providers.litellm import LiteLLMProvider
+
+LLMJudgeInputType = TypeVar("LLMJudgeInputType", bound=BaseLLMJudgeInput)
+
+
+class BaseLLMJudgeMetric(BaseMetric[LLMJudgeInputType], ABC):
+    """
+    The definitive, Pydantic-driven base class for LLM-as-a-judge metrics.
+    It orchestrates prompt creation, calling a provider for a structured response,
+    and robustly parsing the output into a validated Pydantic model.
+    """
+
+    def __init__(
+        self,
+        provider: LLMProvider,
+        config: Optional[Dict[str, Any]] = None,
+        **litellm_kwargs,
+    ):
+        super().__init__(config)
+        self.provider = provider
+        self.model = self.config.get("model", "gpt-4o")
+        self.provider_kwargs = {**litellm_kwargs}
+
+    @property
+    @abstractmethod
+    def output_pydantic_model(self) -> Type[BaseModel]:
+        """The Pydantic model defining the structure of the judge's output."""
+        pass
+
+    @abstractmethod
+    def _create_prompt_messages(
+        self, inputs: LLMJudgeInputType
+    ) -> List[Dict[str, Any]]:
+        """Creates the full prompt message list for the API call."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def _normalize_score(self, parsed_output: BaseModel) -> Dict[str, Any]:
+        """
+        Takes the validated Pydantic output object and converts it into the
+        final metric result (a dict with 'output' and 'reason' keys).
+        """
+        raise NotImplementedError
+
+    def _parse_response_with_fallback(self, response_text: str) -> BaseModel:
+        """
+        Attempts to parse the LLM's response string into the target Pydantic model.
+        It first tries a direct parse, and if that fails, it uses a regex-based
+        fallback to extract a JSON object and tries again.
+        """
+        try:
+            return self.output_pydantic_model.model_validate_json(response_text)
+        except (ValidationError, json.JSONDecodeError):
+            # Fallback attempt: extract JSON from messy text and retry
+            # use our metric to check if it contains json
+            if (
+                ContainsJson().compute_one(
+                    inputs=TextMetricInput(response=response_text)
+                )["output"]
+                == 1.0
+            ):
+                match = re.search(r"\{.*\}|\[.*\]", response_text, re.DOTALL)
+                cleaned_json_str = match.group(0)
+                try:
+                    return self.output_pydantic_model.model_validate_json(
+                        cleaned_json_str
+                    )
+                except (ValidationError, json.JSONDecodeError) as final_e:
+                    raise ValueError(
+                        "Failed to validate the extracted JSON."
+                    ) from final_e
+            else:
+                raise ValueError("Failed to find JSON in the response.")
+
+    def compute_one(self, inputs: LLMJudgeInputType) -> Dict[str, Any]:
+        messages = self._create_prompt_messages(inputs)
+
+        response_format = self.output_pydantic_model
+        # response schema check
+        if isinstance(self.provider, LiteLLMProvider):
+            if not self.provider._supports_schema(self.model):
+                response_format = {"type": "json_object"}
+        try:
+            response_text = self.provider.get_completion(
+                model=self.model,
+                messages=messages,
+                response_format=response_format,
+                **self.provider_kwargs,
+            )
+        except Exception as e:
+            return {"output": 0.0, "reason": f"LLM provider failed: {e}"}
+
+        try:
+            parsed_output = self._parse_response_with_fallback(response_text)
+            return self._normalize_score(parsed_output)
+        except Exception as e:
+            return {
+                "output": 0.0,
+                "reason": f"Failed to parse or validate LLM response. Raw response: '{response_text}'. Error: {e}",
+            }

fi/evals/metrics/base_metric.py

@@ -0,0 +1,138 @@
+from abc import ABC, abstractmethod
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    Iterator,
+    List,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+)
+from pydantic import ValidationError
+from typing_extensions import get_original_bases
+import time
+
+from ..types import BatchRunResult, EvalResult, BaseMetricInput
+
+BaseMetricInputType = TypeVar("BaseMetricInputType", bound=BaseMetricInput)
+
+
+class BaseMetric(Generic[BaseMetricInputType], ABC):
+    """The abstract base class for all metric evaluations.
+
+    It provides a unified structure, centralizes batch processing,
+    and handles standardized error logging and timing. Subclasses are only
+    required to implement the logic for a single computation.
+    """
+
+    supports_llm_judge: bool = False
+    judge_description: str = ""
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
+        self.config = config or {}
+
+        # magic code to get the input metric basemodel to use for input validation.
+        # This traverses the MRO to find the generic BaseMetric specialization.
+        self.input_model: Type[BaseMetricInputType]
+        for cls in self.__class__.__mro__:
+            for base in get_original_bases(cls):
+                if hasattr(base, "__origin__") and base.__origin__ is BaseMetric:
+                    type_args = get_args(base)
+                    if type_args:
+                        self.input_model = type_args[0]
+                        break
+            if hasattr(self, "input_model"):
+                break
+
+        if not hasattr(self, "input_model"):
+            raise TypeError(
+                f"Could not determine input model for {self.__class__.__name__}. "
+                "Ensure it inherits from BaseMetric with a specific input type, "
+                "e.g., class MyMetric(BaseMetric[TextMetricInput]):"
+            )
+
+    @property
+    @abstractmethod
+    def metric_name(self) -> str:
+        """The official name of the metric"""
+        pass
+
+    def _validate_and_yield_inputs(
+        self,
+        inputs: List[Any],
+    ) -> Iterator[Union[BaseMetricInputType, Exception]]:
+        """
+        A generator that validates each item from an input list.
+        """
+        for i, item in enumerate(inputs):
+            try:
+                if isinstance(item, dict):
+                    # Validate and yield the dictionary as a Pydantic model
+                    yield self.input_model.model_validate(item)
+                elif isinstance(item, self.input_model):
+                    # The item is already a valid Pydantic model, yield it directly
+                    yield item
+                else:
+                    # The item is an unsupported type
+                    raise TypeError(
+                        f"Item at index {i} has an invalid type '{type(item).__name__}'. "
+                        f"Expected a 'dict' or a '{self.input_model.__name__}' instance."
+                    )
+            except (ValidationError, TypeError) as e:
+                # If validation fails for any reason, yield the exception
+                yield e
+
+    @abstractmethod
+    def compute_one(self, inputs: BaseMetricInputType) -> Dict[str, Any]:
+        """
+        Computes the metric for a single, strongly-typed input.
+        Pydantic validates the data before this method is ever called.
+
+        Returns:
+            A dictionary containing at least an 'output' key with the score.
+            Can optionally include a 'reason' key for more details.
+        """
+        raise NotImplementedError
+
+    def evaluate(
+        self, inputs: Union[List[BaseMetricInputType], List[Dict[str, Any]]]
+    ) -> BatchRunResult:
+        """
+        Evaluates the metric over a batch of strongly-typed inputs.
+        """
+        if not inputs:
+            return BatchRunResult(eval_results=[])
+
+        eval_results: List[Optional[EvalResult]] = []
+
+        # Use the generator to process each item one by one
+        for prepared_item in self._validate_and_yield_inputs(inputs):
+            start_time = time.perf_counter()
+            output = 0.0
+            reason = ""
+
+            # Check if the prepared_item is an exception from the validation step
+            if isinstance(prepared_item, Exception):
+                reason = f"Input validation failed: {prepared_item}"
+            else:
+                # If validation succeeded, run the actual metric computation
+                result_dict = self.compute_one(prepared_item)
+                output = result_dict.get("output")
+                reason = result_dict.get("reason")
+
+            end_time = time.perf_counter()
+            runtime_ms = int((end_time - start_time) * 1000)
+
+            eval_result = EvalResult(
+                name=self.metric_name,
+                output=output,
+                reason=reason,
+                runtime=runtime_ms,
+                output_type="score",  # Default, can be customized if needed
+            )
+            eval_results.append(eval_result)
+
+        return BatchRunResult(eval_results=eval_results)