agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/metrics/heuristics/string_metrics.py

@@ -0,0 +1,391 @@
+import re
+from typing import Any, Dict, List, Optional
+
+from ..base_metric import BaseMetric
+from ...types import TextMetricInput
+import requests
+
+
+class Regex(BaseMetric[TextMetricInput]):
+    """Checks if a regex pattern is found in the response text."""
+
+    @property
+    def metric_name(self) -> str:
+        return "regex"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(config)
+        self.pattern = self.config.get("pattern")
+        if not self.pattern:
+            raise ValueError("Regex metric requires a 'pattern' in its config.")
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        match = re.search(self.pattern, inputs.response)
+        if match:
+            return {
+                "output": 1.0,  # Using 1.0 for success (True)
+                "reason": f"Regex pattern '{self.pattern}' found in response.",
+            }
+        else:
+            return {
+                "output": 0.0,  # Using 0.0 for failure (False)
+                "reason": f"Regex pattern '{self.pattern}' not found in response.",
+            }
+
+
+class Contains(BaseMetric[TextMetricInput]):
+    @property
+    def metric_name(self) -> str:
+        return "contains"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.keyword = self.config.get("keyword")
+        self.case_sensitive = self.config.get("case_sensitive", False)
+        if not self.keyword:
+            raise ValueError("Contains metric requires a 'keyword' config.")
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        text, kw = (
+            (inputs.response, self.keyword)
+            if self.case_sensitive
+            else (inputs.response.lower(), self.keyword.lower())
+        )
+        is_present = kw in text
+        return {
+            "output": 1.0 if is_present else 0.0,
+            "reason": f"Keyword '{self.keyword}' found"
+            if is_present
+            else f"Keyword '{self.keyword}' not found",
+        }
+
+
+class _BaseContainsKeywords(BaseMetric[TextMetricInput]):
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.keywords = self.config.get("keywords")
+        self.case_sensitive = self.config.get("case_sensitive", False)
+        if not self.keywords or not isinstance(self.keywords, list):
+            raise ValueError(
+                f"{self.metric_name} metric requires a 'keywords' list in config."
+            )
+
+    def _get_found_keywords(self, text: str) -> List[str]:
+        text_to_check = text if self.case_sensitive else text.lower()
+        found = [
+            kw
+            for kw in self.keywords
+            if (kw if self.case_sensitive else kw.lower()) in text_to_check
+        ]
+        return found
+
+
+class ContainsAll(_BaseContainsKeywords):
+    @property
+    def metric_name(self) -> str:
+        return "contains_all"
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        found = self._get_found_keywords(inputs.response)
+        if len(found) == len(self.keywords):
+            return {
+                "output": 1.0,
+                "reason": f"All {len(self.keywords)} keywords found.",
+            }
+        missing = [kw for kw in self.keywords if kw not in found]
+        return {"output": 0.0, "reason": f"Missing keywords: {', '.join(missing)}"}
+
+
+class ContainsAny(_BaseContainsKeywords):
+    """Checks if the response text contains any of the provided keywords."""
+
+    @property
+    def metric_name(self) -> str:
+        return "contains_any"
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        found_keywords = self._get_found_keywords(inputs.response)
+        if found_keywords:
+            return {
+                "output": 1.0,
+                "reason": f"Found keywords: {', '.join(found_keywords)}",
+            }
+        return {"output": 0.0, "reason": "No keywords found in response."}
+
+
+class ContainsNone(_BaseContainsKeywords):
+    """Checks if the response text contains none of the provided keywords."""
+
+    @property
+    def metric_name(self) -> str:
+        return "contains_none"
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        found_keywords = self._get_found_keywords(inputs.response)
+        if not found_keywords:
+            return {"output": 1.0, "reason": "No forbidden keywords found."}
+        return {
+            "output": 0.0,
+            "reason": f"Found forbidden keywords: {', '.join(found_keywords)}",
+        }
+
+
+def _standardize_url(url: str) -> str:
+    if url.startswith("http://") or url.startswith("https://"):
+        return url
+    return f"http://{url}"
+
+
+class OneLine(BaseMetric[TextMetricInput]):
+    """Checks if the text is a single line."""
+
+    @property
+    def metric_name(self) -> str:
+        return "one_line"
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        is_one_line = "\n" not in inputs.response.strip()
+        return {
+            "output": 1.0 if is_one_line else 0.0,
+            "reason": "Response is a single line."
+            if is_one_line
+            else "Response contains multiple lines.",
+        }
+
+
+class ContainsEmail(Regex):
+    """Checks if the text contains an email address."""
+
+    @property
+    def metric_name(self) -> str:
+        return "contains_email"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        # Pass the specific regex pattern to the parent Regex class
+        super().__init__({"pattern": r"[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+"})
+
+
+class IsEmail(Regex):
+    """Checks if the entire text is a valid email address."""
+
+    @property
+    def metric_name(self) -> str:
+        return "is_email"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(
+            {"pattern": r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"}
+        )
+
+
+class ContainsLink(Regex):
+    """Checks if the text contains a link."""
+
+    @property
+    def metric_name(self) -> str:
+        return "contains_link"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__({"pattern": r"(?!.*@)(?:https?://)?(?:www\.)?\S+\.\S+"})
+
+
+class ContainsValidLink(BaseMetric[TextMetricInput]):
+    """Checks if the text contains a link that returns a 2xx status code."""
+
+    @property
+    def metric_name(self) -> str:
+        return "contains_valid_link"
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        pattern = r"(?!.*@)(?:https?://)?(?:www\.)?\S+\.\S+"
+        match = re.search(pattern, inputs.response)
+        if not match:
+            return {"output": 0.0, "reason": "No link found in response."}
+
+        url = _standardize_url(match.group(0))
+        try:
+            response = requests.head(url, timeout=5)
+            if 200 <= response.status_code < 300:
+                return {
+                    "output": 1.0,
+                    "reason": f"Valid link '{url}' found (Status: {response.status_code})",
+                }
+            else:
+                return {
+                    "output": 0.0,
+                    "reason": f"Invalid link '{url}' found (Status: {response.status_code})",
+                }
+        except requests.RequestException as e:
+            return {
+                "output": 0.0,
+                "reason": f"Unreachable link '{url}' found. Error: {e}",
+            }
+
+
+class Equals(BaseMetric[TextMetricInput]):
+    """Checks if the response text exactly matches the expected text."""
+
+    @property
+    def metric_name(self) -> str:
+        return "equals"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.case_sensitive = self.config.get("case_sensitive", False)
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        if inputs.expected_response is None:
+            raise ValueError("Equals metric requires 'expected_response' to be provided.")
+        if not isinstance(inputs.expected_response, str):
+            raise TypeError("Equals metric requires 'expected_response' to be a string.")
+        resp, expected = (
+            (inputs.response, inputs.expected_response)
+            if self.case_sensitive
+            else (inputs.response.lower(), inputs.expected_response.lower())
+        )
+        return {
+            "output": 1.0 if resp == expected else 0.0,
+            "reason": "Response matches expected text."
+            if resp == expected
+            else "Response does not match.",
+        }
+
+
+class StartsWith(BaseMetric[TextMetricInput]):
+    """Checks if the response text starts with the expected text."""
+
+    @property
+    def metric_name(self) -> str:
+        return "starts_with"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.case_sensitive = self.config.get("case_sensitive", False)
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        if inputs.expected_response is None:
+            raise ValueError("StartsWith metric requires 'expected_response' to be provided.")
+        if not isinstance(inputs.expected_response, str):
+            raise TypeError("StartsWith requires 'expected_response' to be a string.")
+        resp, prefix = (
+            (inputs.response, inputs.expected_response)
+            if self.case_sensitive
+            else (inputs.response.lower(), inputs.expected_response.lower())
+        )
+        starts = resp.startswith(prefix)
+        return {
+            "output": 1.0 if starts else 0.0,
+            "reason": f"Response starts with '{inputs.expected_response}'."
+            if starts
+            else f"Response does not start with '{inputs.expected_response}'.",
+        }
+
+
+class EndsWith(BaseMetric[TextMetricInput]):
+    """Checks if the response text ends with the expected text."""
+
+    @property
+    def metric_name(self) -> str:
+        return "ends_with"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.case_sensitive = self.config.get("case_sensitive", False)
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        if inputs.expected_response is None:
+            raise ValueError("EndsWith metric requires 'expected_response' to be provided.")
+        if not isinstance(inputs.expected_response, str):
+            raise TypeError("EndsWith requires 'expected_response' to be a string.")
+        resp, suffix = (
+            (inputs.response, inputs.expected_response)
+            if self.case_sensitive
+            else (inputs.response.lower(), inputs.expected_response.lower())
+        )
+        ends = resp.endswith(suffix)
+        return {
+            "output": 1.0 if ends else 0.0,
+            "reason": f"Response ends with '{inputs.expected_response}'."
+            if ends
+            else f"Response does not end with '{inputs.expected_response}'.",
+        }
+
+
+# --- Length Metrics ---
+
+
+class LengthLessThan(BaseMetric[TextMetricInput]):
+    """Checks if text length is less than a max_length."""
+
+    @property
+    def metric_name(self) -> str:
+        return "length_less_than"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.max_length = self.config.get("max_length")
+        if not isinstance(self.max_length, int):
+            raise ValueError(
+                "LengthLessThan metric requires an integer 'max_length' config."
+            )
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        is_less = len(inputs.response) < self.max_length
+        return {
+            "output": 1.0 if is_less else 0.0,
+            "reason": f"Length {len(inputs.response)} < {self.max_length}"
+            if is_less
+            else f"Length {len(inputs.response)} >= {self.max_length}",
+        }
+
+
+class LengthGreaterThan(BaseMetric[TextMetricInput]):
+    """Checks if text length is greater than a min_length."""
+
+    @property
+    def metric_name(self) -> str:
+        return "length_greater_than"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.min_length = self.config.get("min_length")
+        if not isinstance(self.min_length, int):
+            raise ValueError(
+                "LengthGreaterThan metric requires an integer 'min_length' config."
+            )
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        is_greater = len(inputs.response) > self.min_length
+        return {
+            "output": 1.0 if is_greater else 0.0,
+            "reason": f"Length {len(inputs.response)} > {self.min_length}"
+            if is_greater
+            else f"Length {len(inputs.response)} <= {self.min_length}",
+        }
+
+
+class LengthBetween(BaseMetric[TextMetricInput]):
+    """Checks if text length is between a min_length and max_length."""
+
+    @property
+    def metric_name(self) -> str:
+        return "length_between"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.min_length = self.config.get("min_length")
+        self.max_length = self.config.get("max_length")
+        if not isinstance(self.min_length, int) or not isinstance(self.max_length, int):
+            raise ValueError(
+                "LengthBetween requires integer 'min_length' and 'max_length' configs."
+            )
+
+    def compute_one(self, inputs: TextMetricInput) -> Dict[str, Any]:
+        length = len(inputs.response)
+        is_between = self.min_length <= length <= self.max_length
+        reason = (
+            f"Length {length} is between [{self.min_length}, {self.max_length}]"
+            if is_between
+            else f"Length {length} is not between [{self.min_length}, {self.max_length}]"
+        )
+        return {"output": 1.0 if is_between else 0.0, "reason": reason}

fi/evals/metrics/llm_as_judges/__init__.py

@@ -0,0 +1,17 @@
+from .custom_judge.metric import CustomLLMJudge
+from .types import (
+    CustomInput,
+    BaseLLMJudgeInput,
+    LLMFewShotExample,
+    LLMMessage,
+    DefaultJudgeOutput,
+)
+
+__all__ = [
+    "CustomLLMJudge",
+    "CustomInput",
+    "BaseLLMJudgeInput",
+    "LLMFewShotExample",
+    "LLMMessage",
+    "DefaultJudgeOutput",
+]

fi/evals/metrics/llm_as_judges/custom_judge/metric.py

@@ -0,0 +1,112 @@
+import json
+from typing import Any, Dict, List, Type
+from pydantic import BaseModel
+from jinja2 import Environment, BaseLoader
+
+from ...base_llm_metric import BaseLLMJudgeMetric
+from ..types import CustomInput, DefaultJudgeOutput
+from ....llm.base_llm_provider import LLMProvider
+from .prompts import DEFAULT_USER_PROMPT_TEMPLATE
+
+
+class CustomLLMJudge(BaseLLMJudgeMetric[CustomInput]):
+    """
+    A smart, user-configurable LLM-as-a-judge metric that prioritizes ease of use.
+
+    For the most common use cases, the user only needs to provide their
+    grading criteria. The judge provides sensible defaults for the prompt
+    template and output format, which can be optionally overridden for
+    advanced customization.
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return self.config.get("name", "custom_llm_judge")
+
+    def __init__(self, provider: LLMProvider, config: Dict[str, Any], **litellm_kwargs):
+        # The ONLY required key is now 'grading_criteria'
+        if "grading_criteria" not in config:
+            raise ValueError(
+                "CustomLLMJudge config must contain a 'grading_criteria' key."
+            )
+
+        super().__init__(provider, config, **litellm_kwargs)
+
+        # Explicitly set the input model, as this class is generic
+        self.input_model = CustomInput
+
+        # Smartly decide which Pydantic model to uset
+        self._output_model = DefaultJudgeOutput
+
+    @property
+    def output_pydantic_model(self) -> Type[BaseModel]:
+        return self._output_model
+
+    # Keys whose values are media URLs that should be sent as content parts
+    _IMAGE_KEYS = {"image_url", "input_image_url", "output_image_url", "image"}
+    _AUDIO_KEYS = {"audio_url", "input_audio_url", "audio"}
+
+    def _create_prompt_messages(self, inputs: CustomInput) -> List[Dict[str, Any]]:
+        jinja_env = Environment(loader=BaseLoader())
+        jinja_env.filters["tojson"] = json.dumps
+
+        # Use the user-provided template if it exists, otherwise use the default
+        template_str = self.config.get(
+            "user_prompt_template", DEFAULT_USER_PROMPT_TEMPLATE
+        )
+        template = jinja_env.from_string(template_str)
+
+        render_context = {
+            "grading_criteria": self.config["grading_criteria"],
+            "few_shot_examples": self.config.get("few_shot_examples", []),
+            "task_input": inputs.model_dump(),
+        }
+
+        user_prompt = template.render(render_context)
+
+        system_prompt = self.config.get(
+            "system_prompt",
+            "You are an expert AI evaluator. Follow the user's instructions and output format precisely.",
+        )
+
+        # Detect multimodal inputs and build content parts
+        user_content = self._build_content(user_prompt, inputs.model_dump())
+
+        return [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_content},
+        ]
+
+    def _build_content(self, text: str, input_data: Dict[str, Any]):
+        """Build message content — plain string or list of content parts with media."""
+        media_parts = []
+
+        for key, value in input_data.items():
+            if not value or not isinstance(value, str):
+                continue
+            if key in self._IMAGE_KEYS:
+                media_parts.append({
+                    "type": "image_url",
+                    "image_url": {"url": value},
+                })
+            elif key in self._AUDIO_KEYS:
+                # LiteLLM translates image_url type to the correct provider
+                # format for audio URLs too (Gemini, OpenAI, etc.)
+                media_parts.append({
+                    "type": "image_url",
+                    "image_url": {"url": value},
+                })
+
+        if not media_parts:
+            return text
+
+        return [{"type": "text", "text": text}] + media_parts
+
+    def _normalize_score(self, parsed_output: BaseModel) -> Dict[str, Any]:
+        """Normalizes the score from the validated Pydantic output."""
+        output_dict = parsed_output.model_dump()
+
+        # Prioritize finding a "score" field, as it's our default
+        score_val = output_dict.get("score", 1.0)
+
+        return {"output": float(score_val), "reason": json.dumps(output_dict, indent=2)}

fi/evals/metrics/llm_as_judges/custom_judge/prompts.py

@@ -0,0 +1,26 @@
+DEFAULT_USER_PROMPT_TEMPLATE = """
+### GRADING CRITERIA ###
+{{ grading_criteria }}
+
+### OUTPUT FORMAT ###
+You MUST return a valid JSON object with two keys: "score" (a float between 0.0 and 1.0) and "reason" (a brief explanation of your score).
+
+{% if few_shot_examples %}
+### EXAMPLES ###
+{% for example in few_shot_examples -%}
+---
+INPUT:
+{{ example.inputs | tojson(indent=2) }}
+
+EXPECTED JUDGEMENT:
+{{ example.output }}
+---
+{% endfor %}
+{% endif %}
+
+### TASK ###
+Based on the grading criteria, please evaluate the following input.
+
+### INPUT ###
+{{ task_input | tojson(indent=2) }}
+"""

fi/evals/metrics/llm_as_judges/types.py

@@ -0,0 +1,48 @@
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel, Field
+from ...types import BaseMetricInput
+
+
+class BaseLLMJudgeInput(BaseMetricInput):
+    pass
+
+
+class LLMFewShotExample(BaseModel):
+    inputs: Dict[str, Any] = Field(
+        ..., description="A dictionary representing an input model"
+    )
+    output: str = Field(
+        ...,
+        description="The ideal JSON string the judge LLM should produce for these inputs.",
+    )
+
+
+class CustomInput(BaseLLMJudgeInput):
+    """A flexible input model for the CustomLLMJudge that allows any field."""
+
+    class Config:
+        extra = "allow"
+
+
+class DefaultJudgeOutput(BaseModel):
+    """The default output format for a custom judge."""
+
+    score: float = Field(
+        ...,
+        ge=0.0,
+        le=1.0,
+        description="The normalized evaluation score from 0.0 to 1.0.",
+    )
+    reason: str = Field(..., description="A brief explanation of the score.")
+
+
+class LLMMessage(BaseModel):
+    role: str
+    content: str
+    name: Optional[str]
+    function_call: Optional[str]
+    tool_call_id: Optional[str]
+
+
+class ConversationInput(BaseLLMJudgeInput):
+    messages: List[LLMMessage]