agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/metrics/rag/retrieval/noise_sensitivity.py

@@ -0,0 +1,163 @@
+"""
+Noise Sensitivity Metric.
+
+Measures RAG system robustness to irrelevant/noisy context
+mixed with relevant context.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from ...base_metric import BaseMetric
+from ..types import NoiseSensitivityInput
+from ..utils import (
+    extract_claims,
+    check_claim_supported,
+    compute_semantic_similarity,
+)
+
+
+class NoiseSensitivity(BaseMetric[NoiseSensitivityInput]):
+    """
+    Measures RAG system robustness to irrelevant context.
+
+    Compares response quality when noise is added to the context.
+    Lower sensitivity (higher score) means more robust system.
+
+    Requires both clean and noisy response variants to compare.
+
+    Approach:
+    1. Calculate faithfulness of clean response (relevant contexts only)
+    2. Calculate faithfulness of noisy response (relevant + irrelevant contexts)
+    3. Sensitivity = degradation when noise added
+    4. Robustness = 1 - sensitivity (returned as output)
+
+    Score: 0.0 (very sensitive to noise) to 1.0 (robust to noise)
+
+    Example:
+        >>> noise_sens = NoiseSensitivity()
+        >>> result = noise_sens.evaluate([{
+        ...     "query": "What is the capital of France?",
+        ...     "response_clean": "The capital of France is Paris.",
+        ...     "response_noisy": "The capital of France is Paris, a beautiful city.",
+        ...     "relevant_contexts": ["Paris is the capital of France."],
+        ...     "irrelevant_contexts": ["The Eiffel Tower is 330 meters tall."],
+        ... }])
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "noise_sensitivity"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.faithfulness_threshold = self.config.get("faithfulness_threshold", 0.5)
+
+    def compute_one(self, inputs: NoiseSensitivityInput) -> Dict[str, Any]:
+        # Validate inputs
+        if not inputs.relevant_contexts:
+            return {
+                "output": 0.0,
+                "reason": "No relevant contexts provided",
+            }
+
+        # Calculate faithfulness of clean response
+        clean_faithfulness = self._calculate_faithfulness(
+            inputs.response_clean, inputs.relevant_contexts
+        )
+
+        # Calculate faithfulness of noisy response
+        all_contexts = inputs.relevant_contexts + inputs.irrelevant_contexts
+        noisy_faithfulness = self._calculate_faithfulness(
+            inputs.response_noisy, all_contexts
+        )
+
+        # Calculate semantic similarity between responses
+        response_similarity = compute_semantic_similarity(
+            inputs.response_clean, inputs.response_noisy
+        )
+
+        # Calculate sensitivity metrics
+        # Sensitivity = degradation when noise added
+        faithfulness_degradation = max(0, clean_faithfulness - noisy_faithfulness)
+
+        # Also check if noisy response introduced irrelevant info
+        irrelevant_contamination = self._check_contamination(
+            inputs.response_noisy,
+            inputs.irrelevant_contexts,
+            inputs.relevant_contexts
+        )
+
+        # Combined sensitivity score
+        sensitivity = (
+            0.5 * faithfulness_degradation +
+            0.3 * irrelevant_contamination +
+            0.2 * (1.0 - response_similarity)
+        )
+
+        # Convert to robustness score (1 - sensitivity)
+        robustness = max(0.0, 1.0 - sensitivity)
+
+        return {
+            "output": round(robustness, 4),
+            "reason": f"Robustness: {robustness:.2f} (clean={clean_faithfulness:.2f}, noisy={noisy_faithfulness:.2f})",
+            "clean_faithfulness": round(clean_faithfulness, 4),
+            "noisy_faithfulness": round(noisy_faithfulness, 4),
+            "faithfulness_degradation": round(faithfulness_degradation, 4),
+            "irrelevant_contamination": round(irrelevant_contamination, 4),
+            "response_similarity": round(response_similarity, 4),
+            "sensitivity": round(sensitivity, 4),
+        }
+
+    def _calculate_faithfulness(
+        self, response: str, contexts: List[str]
+    ) -> float:
+        """Calculate faithfulness score for a response given contexts."""
+        claims = extract_claims(response)
+
+        if not claims:
+            return 1.0  # No claims to verify
+
+        supported = 0
+        for claim in claims:
+            is_supported, score, _ = check_claim_supported(
+                claim, contexts, self.faithfulness_threshold
+            )
+            if is_supported:
+                supported += 1
+
+        return supported / len(claims)
+
+    def _check_contamination(
+        self,
+        response: str,
+        irrelevant_contexts: List[str],
+        relevant_contexts: List[str]
+    ) -> float:
+        """
+        Check if response contains information from irrelevant contexts
+        that is NOT in relevant contexts.
+        """
+        if not irrelevant_contexts:
+            return 0.0
+
+        claims = extract_claims(response)
+        if not claims:
+            return 0.0
+
+        contaminated_claims = 0
+
+        for claim in claims:
+            # Check if claim is supported by irrelevant contexts
+            irrel_supported, irrel_score, _ = check_claim_supported(
+                claim, irrelevant_contexts, 0.4
+            )
+
+            # But NOT supported by relevant contexts
+            rel_supported, rel_score, _ = check_claim_supported(
+                claim, relevant_contexts, 0.4
+            )
+
+            if irrel_supported and not rel_supported:
+                contaminated_claims += 1
+
+        return contaminated_claims / len(claims)

fi/evals/metrics/rag/retrieval/ranking.py

@@ -0,0 +1,261 @@
+"""
+Ranking Metrics for RAG Retrieval.
+
+Provides NDCG (Normalized Discounted Cumulative Gain) and
+MRR (Mean Reciprocal Rank) metrics.
+"""
+
+import math
+from typing import Any, Dict, List, Optional
+
+from ...base_metric import BaseMetric
+from ..types import RAGRankingInput
+
+
+class NDCG(BaseMetric[RAGRankingInput]):
+    """
+    Normalized Discounted Cumulative Gain for ranked retrieval.
+
+    Accounts for graded relevance (not just binary) and position.
+    Higher scores for relevant items appearing early in ranking.
+
+    Formula:
+        DCG@k = Σ (2^rel_i - 1) / log2(i + 2)
+        NDCG@k = DCG@k / IDCG@k
+
+    Where IDCG is the ideal DCG (perfect ranking).
+
+    Score: 0.0 (worst ranking) to 1.0 (perfect ranking)
+
+    Example:
+        >>> ndcg = NDCG(config={"k": 5})
+        >>> result = ndcg.evaluate([{
+        ...     "query": "machine learning",
+        ...     "contexts": ["ML intro", "Unrelated", "ML advanced"],
+        ...     "relevance_scores": [1.0, 0.0, 0.8]
+        ... }])
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "ndcg"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.k = self.config.get("k", None)  # None = use all
+
+    def compute_one(self, inputs: RAGRankingInput) -> Dict[str, Any]:
+        scores = inputs.relevance_scores
+
+        if not scores:
+            return {
+                "output": 0.0,
+                "reason": "No relevance scores provided",
+            }
+
+        k = self.k if self.k is not None else len(scores)
+        k = min(k, len(scores))
+
+        # DCG@k
+        dcg = self._compute_dcg(scores[:k])
+
+        # Ideal DCG@k (perfect ranking - sorted descending)
+        ideal_scores = sorted(scores, reverse=True)
+        idcg = self._compute_dcg(ideal_scores[:k])
+
+        if idcg == 0:
+            # All scores are 0
+            return {
+                "output": 0.0,
+                "reason": "All relevance scores are 0",
+                "dcg": 0.0,
+                "idcg": 0.0,
+            }
+
+        ndcg = dcg / idcg
+
+        return {
+            "output": round(ndcg, 4),
+            "reason": f"NDCG@{k}={ndcg:.3f}, DCG={dcg:.3f}, IDCG={idcg:.3f}",
+            "k": k,
+            "dcg": round(dcg, 4),
+            "idcg": round(idcg, 4),
+            "relevance_scores": scores[:k],
+        }
+
+    def _compute_dcg(self, scores: List[float]) -> float:
+        """Compute Discounted Cumulative Gain."""
+        dcg = 0.0
+        for i, score in enumerate(scores):
+            # Position is 1-indexed in the formula
+            dcg += (2**score - 1) / math.log2(i + 2)
+        return dcg
+
+
+class MRR(BaseMetric[RAGRankingInput]):
+    """
+    Mean Reciprocal Rank - how quickly the first relevant result appears.
+
+    Best for single-answer retrieval tasks where you only need
+    one correct result.
+
+    Formula:
+        MRR = 1 / rank_of_first_relevant
+
+    Score: 0.0 (no relevant results) to 1.0 (first result is relevant)
+
+    Example:
+        >>> mrr = MRR(config={"relevance_threshold": 0.5})
+        >>> result = mrr.evaluate([{
+        ...     "query": "capital of France",
+        ...     "contexts": ["Unrelated", "Paris is capital", "More info"],
+        ...     "relevance_scores": [0.1, 0.9, 0.3]
+        ... }])
+        >>> print(result.eval_results[0].output)  # 0.5 (found at position 2)
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "mrr"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.relevance_threshold = self.config.get("relevance_threshold", 0.5)
+
+    def compute_one(self, inputs: RAGRankingInput) -> Dict[str, Any]:
+        scores = inputs.relevance_scores
+
+        if not scores:
+            return {
+                "output": 0.0,
+                "reason": "No relevance scores provided",
+            }
+
+        # Find first relevant result
+        for i, score in enumerate(scores, 1):
+            if score >= self.relevance_threshold:
+                reciprocal_rank = 1.0 / i
+                return {
+                    "output": round(reciprocal_rank, 4),
+                    "reason": f"First relevant result at position {i}",
+                    "first_relevant_position": i,
+                    "first_relevant_score": round(score, 4),
+                    "threshold": self.relevance_threshold,
+                }
+
+        return {
+            "output": 0.0,
+            "reason": f"No results above relevance threshold ({self.relevance_threshold})",
+            "first_relevant_position": None,
+            "threshold": self.relevance_threshold,
+        }
+
+
+class PrecisionAtK(BaseMetric[RAGRankingInput]):
+    """
+    Precision@K - fraction of top-K results that are relevant.
+
+    Simple metric for evaluating retrieval quality at a fixed cutoff.
+
+    Formula:
+        Precision@K = |relevant in top K| / K
+
+    Score: 0.0 (no relevant in top K) to 1.0 (all top K are relevant)
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "precision_at_k"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.k = self.config.get("k", 5)
+        self.relevance_threshold = self.config.get("relevance_threshold", 0.5)
+
+    def compute_one(self, inputs: RAGRankingInput) -> Dict[str, Any]:
+        scores = inputs.relevance_scores
+
+        if not scores:
+            return {
+                "output": 0.0,
+                "reason": "No relevance scores provided",
+            }
+
+        k = min(self.k, len(scores))
+        top_k_scores = scores[:k]
+
+        relevant_count = sum(
+            1 for score in top_k_scores if score >= self.relevance_threshold
+        )
+
+        precision = relevant_count / k
+
+        return {
+            "output": round(precision, 4),
+            "reason": f"Precision@{k}={precision:.3f}, {relevant_count}/{k} relevant",
+            "k": k,
+            "relevant_count": relevant_count,
+            "threshold": self.relevance_threshold,
+        }
+
+
+class RecallAtK(BaseMetric[RAGRankingInput]):
+    """
+    Recall@K - fraction of all relevant results that appear in top K.
+
+    Measures how many of the relevant items were retrieved.
+
+    Formula:
+        Recall@K = |relevant in top K| / |total relevant|
+
+    Score: 0.0 (no relevant recalled) to 1.0 (all relevant in top K)
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "recall_at_k"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.k = self.config.get("k", 5)
+        self.relevance_threshold = self.config.get("relevance_threshold", 0.5)
+
+    def compute_one(self, inputs: RAGRankingInput) -> Dict[str, Any]:
+        scores = inputs.relevance_scores
+
+        if not scores:
+            return {
+                "output": 0.0,
+                "reason": "No relevance scores provided",
+            }
+
+        # Count total relevant
+        total_relevant = sum(
+            1 for score in scores if score >= self.relevance_threshold
+        )
+
+        if total_relevant == 0:
+            return {
+                "output": 0.0,
+                "reason": "No relevant items to recall",
+                "k": self.k,
+                "total_relevant": 0,
+            }
+
+        k = min(self.k, len(scores))
+        top_k_scores = scores[:k]
+
+        recalled = sum(
+            1 for score in top_k_scores if score >= self.relevance_threshold
+        )
+
+        recall = recalled / total_relevant
+
+        return {
+            "output": round(recall, 4),
+            "reason": f"Recall@{k}={recall:.3f}, {recalled}/{total_relevant} recalled",
+            "k": k,
+            "recalled": recalled,
+            "total_relevant": total_relevant,
+            "threshold": self.relevance_threshold,
+        }

fi/evals/metrics/rag/types.py

@@ -0,0 +1,100 @@
+"""
+RAG Evaluation Input/Output Types.
+
+Defines strongly-typed inputs for all RAG metrics.
+"""
+
+from typing import List, Optional
+from pydantic import Field
+
+from ...types import BaseMetricInput
+
+
+class RAGInput(BaseMetricInput):
+    """Standard RAG evaluation input for generation metrics."""
+
+    query: str = Field(..., description="The user query/question")
+    response: str = Field(..., description="The generated response")
+    contexts: List[str] = Field(..., description="Retrieved context passages")
+    reference: Optional[str] = Field(None, description="Ground truth answer")
+
+
+class RAGRetrievalInput(BaseMetricInput):
+    """Input for retrieval-only metrics (recall, precision, entity recall)."""
+
+    query: str = Field(..., description="The user query")
+    contexts: List[str] = Field(..., description="Retrieved context passages")
+    reference: str = Field(..., description="Ground truth/reference text")
+    relevance_labels: Optional[List[int]] = Field(
+        None, description="Binary relevance labels (1=relevant, 0=irrelevant)"
+    )
+
+
+class RAGRankingInput(BaseMetricInput):
+    """Input for ranking metrics (NDCG, MRR)."""
+
+    query: str = Field(..., description="The user query")
+    contexts: List[str] = Field(..., description="Retrieved contexts in ranked order")
+    relevance_scores: List[float] = Field(
+        ..., description="Relevance scores/grades for each context (0-1 or graded)"
+    )
+
+
+class NoiseSensitivityInput(BaseMetricInput):
+    """Input for noise sensitivity evaluation."""
+
+    query: str = Field(..., description="The user query")
+    response_clean: str = Field(
+        ..., description="Response generated with only relevant contexts"
+    )
+    response_noisy: str = Field(
+        ..., description="Response generated with relevant + irrelevant contexts"
+    )
+    relevant_contexts: List[str] = Field(..., description="Relevant context passages")
+    irrelevant_contexts: List[str] = Field(
+        ..., description="Irrelevant/noisy context passages"
+    )
+    reference: Optional[str] = Field(None, description="Ground truth answer")
+
+
+class RAGMultiHopInput(BaseMetricInput):
+    """Input for multi-hop reasoning evaluation."""
+
+    query: str = Field(..., description="Complex query requiring multiple hops")
+    response: str = Field(..., description="Generated response")
+    contexts: List[str] = Field(..., description="Retrieved context passages")
+    hop_chain: Optional[List[str]] = Field(
+        None, description="Expected reasoning chain across contexts"
+    )
+    reference: Optional[str] = Field(None, description="Ground truth answer")
+
+
+class SourceAttributionInput(BaseMetricInput):
+    """Input for source attribution evaluation."""
+
+    response: str = Field(..., description="Generated response with citations")
+    contexts: List[str] = Field(..., description="Source documents")
+    citation_format: str = Field(
+        "bracketed", description="Citation format: bracketed [1], inline, footnote"
+    )
+    require_citations: bool = Field(
+        True, description="Whether citations are required for all claims"
+    )
+
+
+class ContextUtilizationInput(BaseMetricInput):
+    """Input for context utilization evaluation."""
+
+    query: str = Field(..., description="The user query")
+    response: str = Field(..., description="The generated response")
+    contexts: List[str] = Field(..., description="Retrieved context passages")
+
+
+class AnswerRelevancyInput(BaseMetricInput):
+    """Input for answer relevancy evaluation."""
+
+    query: str = Field(..., description="The user query")
+    response: str = Field(..., description="The generated response")
+    contexts: Optional[List[str]] = Field(
+        None, description="Retrieved context passages (optional)"
+    )

fi/evals/metrics/rag/utils/__init__.py

@@ -0,0 +1,62 @@
+"""
+RAG Evaluation Utilities.
+
+Provides common functions for NLI, claim extraction,
+entity extraction, and semantic similarity.
+"""
+
+from .nli import (
+    NLILabel,
+    check_entailment,
+    check_entailment_heuristic,
+    check_claim_supported,
+    check_attribution,
+    check_contradiction,
+)
+from .claims import (
+    split_into_sentences,
+    extract_claims,
+    extract_key_phrases,
+    extract_atomic_claims,
+)
+from .entities import (
+    extract_entities,
+    extract_entities_heuristic,
+    entities_match,
+    normalize_entity,
+)
+from .similarity import (
+    compute_text_similarity,
+    compute_word_overlap,
+    compute_semantic_similarity,
+    compute_ngram_overlap,
+    extract_keywords,
+    normalize_text,
+)
+
+__all__ = [
+    # NLI
+    "NLILabel",
+    "check_entailment",
+    "check_entailment_heuristic",
+    "check_claim_supported",
+    "check_attribution",
+    "check_contradiction",
+    # Claims
+    "split_into_sentences",
+    "extract_claims",
+    "extract_key_phrases",
+    "extract_atomic_claims",
+    # Entities
+    "extract_entities",
+    "extract_entities_heuristic",
+    "entities_match",
+    "normalize_entity",
+    # Similarity
+    "compute_text_similarity",
+    "compute_word_overlap",
+    "compute_semantic_similarity",
+    "compute_ngram_overlap",
+    "extract_keywords",
+    "normalize_text",
+]