agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/metrics/rag/rag_score.py

@@ -0,0 +1,277 @@
+"""
+Comprehensive RAG Score Metric.
+
+Combines multiple RAG metrics into a single comprehensive score.
+"""
+
+from typing import Any, Dict, Optional
+
+from ..base_metric import BaseMetric
+from .types import RAGInput, RAGRetrievalInput
+from .retrieval import ContextRecall, ContextPrecision
+from .generation import AnswerRelevancy, ContextUtilization, RAGFaithfulness
+
+
+class RAGScore(BaseMetric[RAGInput]):
+    """
+    Comprehensive RAG evaluation combining all metrics.
+
+    Evaluates both retrieval and generation components in a single call.
+
+    Default weights (configurable):
+    - Retrieval (40%): Context Recall, Context Precision
+    - Generation (40%): Faithfulness, Answer Relevancy
+    - Advanced (20%): Context Utilization
+
+    Score: 0.0 (poor RAG performance) to 1.0 (excellent RAG performance)
+
+    Example:
+        >>> rag_score = RAGScore()
+        >>> result = rag_score.evaluate([{
+        ...     "query": "What is the capital of France?",
+        ...     "response": "The capital of France is Paris.",
+        ...     "contexts": ["Paris is the capital and largest city of France."],
+        ...     "reference": "Paris is the capital of France."
+        ... }])
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "rag_score"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.weights = self.config.get("weights", {
+            "retrieval": 0.4,
+            "generation": 0.4,
+            "advanced": 0.2,
+        })
+        self.include_details = self.config.get("include_details", True)
+
+    def compute_one(self, inputs: RAGInput) -> Dict[str, Any]:
+        scores = {}
+        details = {}
+
+        # Prepare retrieval input — only if reference is available
+        reference = inputs.reference
+        has_reference = bool(reference and reference.strip())
+
+        if has_reference:
+            retrieval_input = RAGRetrievalInput(
+                query=inputs.query,
+                contexts=inputs.contexts,
+                reference=reference,
+            )
+
+            recall_metric = ContextRecall()
+            recall_result = recall_metric.compute_one(retrieval_input)
+            scores["context_recall"] = recall_result["output"]
+            if self.include_details:
+                details["context_recall"] = recall_result.get("reason", "")
+
+            precision_metric = ContextPrecision()
+            precision_result = precision_metric.compute_one(retrieval_input)
+            scores["context_precision"] = precision_result["output"]
+            if self.include_details:
+                details["context_precision"] = precision_result.get("reason", "")
+        else:
+            scores["context_recall"] = 0.0
+            scores["context_precision"] = 0.0
+            if self.include_details:
+                details["context_recall"] = "No reference provided — skipped"
+                details["context_precision"] = "No reference provided — skipped"
+
+        retrieval_score = (scores["context_recall"] + scores["context_precision"]) / 2
+
+        # Generation metrics
+        from .types import AnswerRelevancyInput, ContextUtilizationInput
+
+        relevancy_input = AnswerRelevancyInput(
+            query=inputs.query,
+            response=inputs.response,
+            contexts=inputs.contexts,
+        )
+        relevancy_metric = AnswerRelevancy()
+        relevancy_result = relevancy_metric.compute_one(relevancy_input)
+        scores["answer_relevancy"] = relevancy_result["output"]
+        if self.include_details:
+            details["answer_relevancy"] = relevancy_result.get("reason", "")
+
+        faithfulness_metric = RAGFaithfulness()
+        faithfulness_result = faithfulness_metric.compute_one(inputs)
+        scores["faithfulness"] = faithfulness_result["output"]
+        if self.include_details:
+            details["faithfulness"] = faithfulness_result.get("reason", "")
+
+        generation_score = (scores["answer_relevancy"] + scores["faithfulness"]) / 2
+
+        # Advanced metrics
+        utilization_input = ContextUtilizationInput(
+            query=inputs.query,
+            response=inputs.response,
+            contexts=inputs.contexts,
+        )
+        utilization_metric = ContextUtilization()
+        utilization_result = utilization_metric.compute_one(utilization_input)
+        scores["context_utilization"] = utilization_result["output"]
+        if self.include_details:
+            details["context_utilization"] = utilization_result.get("reason", "")
+
+        advanced_score = scores["context_utilization"]
+
+        # Weighted combination
+        final_score = (
+            self.weights["retrieval"] * retrieval_score +
+            self.weights["generation"] * generation_score +
+            self.weights["advanced"] * advanced_score
+        )
+
+        result = {
+            "output": round(final_score, 4),
+            "reason": f"Retrieval={retrieval_score:.2f}, Generation={generation_score:.2f}, Advanced={advanced_score:.2f}",
+            "retrieval_score": round(retrieval_score, 4),
+            "generation_score": round(generation_score, 4),
+            "advanced_score": round(advanced_score, 4),
+            "component_scores": {k: round(v, 4) for k, v in scores.items()},
+        }
+
+        if self.include_details:
+            result["component_details"] = details
+
+        return result
+
+
+class RAGScoreDetailed(BaseMetric[RAGInput]):
+    """
+    Detailed RAG evaluation with all available metrics.
+
+    Runs all RAG metrics and provides comprehensive analysis.
+    More expensive but gives complete picture.
+
+    Components:
+    - Retrieval: Recall, Precision, Entity Recall
+    - Generation: Faithfulness, Relevancy, Groundedness
+    - Advanced: Context Utilization, Multi-hop (if applicable)
+
+    Score: 0.0 to 1.0
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "rag_score_detailed"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.retrieval_weight = self.config.get("retrieval_weight", 0.35)
+        self.generation_weight = self.config.get("generation_weight", 0.35)
+        self.advanced_weight = self.config.get("advanced_weight", 0.30)
+
+    def compute_one(self, inputs: RAGInput) -> Dict[str, Any]:
+        from .retrieval import ContextRecall, ContextPrecision, ContextEntityRecall
+        from .generation import AnswerRelevancy, ContextUtilization, Groundedness, RAGFaithfulness
+        from .types import AnswerRelevancyInput, ContextUtilizationInput
+
+        all_scores = {}
+        all_details = {}
+
+        # Prepare retrieval input — only if reference is available
+        reference = inputs.reference
+        has_reference = bool(reference and reference.strip())
+
+        if has_reference:
+            retrieval_input = RAGRetrievalInput(
+                query=inputs.query,
+                contexts=inputs.contexts,
+                reference=reference,
+            )
+
+            # === RETRIEVAL METRICS ===
+            recall = ContextRecall().compute_one(retrieval_input)
+            all_scores["context_recall"] = recall["output"]
+            all_details["context_recall"] = recall
+
+            precision = ContextPrecision().compute_one(retrieval_input)
+            all_scores["context_precision"] = precision["output"]
+            all_details["context_precision"] = precision
+
+            entity_recall = ContextEntityRecall().compute_one(retrieval_input)
+            all_scores["context_entity_recall"] = entity_recall["output"]
+            all_details["context_entity_recall"] = entity_recall
+        else:
+            for key in ("context_recall", "context_precision", "context_entity_recall"):
+                all_scores[key] = 0.0
+                all_details[key] = {"output": 0.0, "reason": "No reference provided — skipped"}
+
+        retrieval_avg = (
+            all_scores["context_recall"] +
+            all_scores["context_precision"] +
+            all_scores["context_entity_recall"]
+        ) / 3
+
+        # === GENERATION METRICS ===
+        # Faithfulness
+        faithfulness = RAGFaithfulness().compute_one(inputs)
+        all_scores["faithfulness"] = faithfulness["output"]
+        all_details["faithfulness"] = faithfulness
+
+        # Answer Relevancy
+        relevancy_input = AnswerRelevancyInput(
+            query=inputs.query,
+            response=inputs.response,
+            contexts=inputs.contexts,
+        )
+        relevancy = AnswerRelevancy().compute_one(relevancy_input)
+        all_scores["answer_relevancy"] = relevancy["output"]
+        all_details["answer_relevancy"] = relevancy
+
+        # Groundedness
+        groundedness = Groundedness().compute_one(inputs)
+        all_scores["groundedness"] = groundedness["output"]
+        all_details["groundedness"] = groundedness
+
+        generation_avg = (
+            all_scores["faithfulness"] +
+            all_scores["answer_relevancy"] +
+            all_scores["groundedness"]
+        ) / 3
+
+        # === ADVANCED METRICS ===
+        # Context Utilization
+        util_input = ContextUtilizationInput(
+            query=inputs.query,
+            response=inputs.response,
+            contexts=inputs.contexts,
+        )
+        utilization = ContextUtilization().compute_one(util_input)
+        all_scores["context_utilization"] = utilization["output"]
+        all_details["context_utilization"] = utilization
+
+        advanced_avg = all_scores["context_utilization"]
+
+        # === FINAL SCORE ===
+        final_score = (
+            self.retrieval_weight * retrieval_avg +
+            self.generation_weight * generation_avg +
+            self.advanced_weight * advanced_avg
+        )
+
+        # Determine quality level
+        if final_score >= 0.8:
+            quality = "excellent"
+        elif final_score >= 0.6:
+            quality = "good"
+        elif final_score >= 0.4:
+            quality = "fair"
+        else:
+            quality = "poor"
+
+        return {
+            "output": round(final_score, 4),
+            "reason": f"Quality: {quality} - Retrieval={retrieval_avg:.2f}, Generation={generation_avg:.2f}, Advanced={advanced_avg:.2f}",
+            "quality_level": quality,
+            "retrieval_average": round(retrieval_avg, 4),
+            "generation_average": round(generation_avg, 4),
+            "advanced_average": round(advanced_avg, 4),
+            "all_scores": {k: round(v, 4) for k, v in all_scores.items()},
+            "details": all_details,
+        }

fi/evals/metrics/rag/retrieval/__init__.py

@@ -0,0 +1,20 @@
+"""
+RAG Retrieval Metrics.
+
+Metrics for evaluating the retrieval component of RAG systems.
+"""
+
+from .context_recall import ContextRecall
+from .context_precision import ContextPrecision
+from .context_entity_recall import ContextEntityRecall
+from .noise_sensitivity import NoiseSensitivity
+from .ranking import NDCG, MRR
+
+__all__ = [
+    "ContextRecall",
+    "ContextPrecision",
+    "ContextEntityRecall",
+    "NoiseSensitivity",
+    "NDCG",
+    "MRR",
+]

fi/evals/metrics/rag/retrieval/context_entity_recall.py

@@ -0,0 +1,124 @@
+"""
+Context Entity Recall Metric.
+
+Measures entity-level retrieval coverage - what fraction of
+entities from the reference appear in retrieved contexts.
+"""
+
+from typing import Any, Dict, Optional, Set
+
+from ...base_metric import BaseMetric
+from ..types import RAGRetrievalInput
+from ..utils import extract_entities, entities_match
+
+
+class ContextEntityRecall(BaseMetric[RAGRetrievalInput]):
+    """
+    Measures entity-level retrieval completeness.
+
+    Uses NER to extract entities from reference and contexts,
+    then calculates what fraction of reference entities appear
+    in the retrieved contexts.
+
+    Critical for fact-based use cases where specific entities
+    (names, dates, locations, amounts) must not be missed.
+
+    Formula:
+        Context Entity Recall = |Entities in Context ∩ Entities in Reference| / |Entities in Reference|
+
+    Score: 0.0 (no entities recalled) to 1.0 (all entities recalled)
+
+    Example:
+        >>> entity_recall = ContextEntityRecall()
+        >>> result = entity_recall.evaluate([{
+        ...     "query": "When did Einstein win the Nobel Prize?",
+        ...     "contexts": ["Albert Einstein received the Nobel Prize in Physics in 1921."],
+        ...     "reference": "Albert Einstein won the Nobel Prize in 1921."
+        ... }])
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "context_entity_recall"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.entity_types = self.config.get("entity_types", None)  # None = all types
+        self.fuzzy_match = self.config.get("fuzzy_match", True)
+        self.case_sensitive = self.config.get("case_sensitive", False)
+
+    def compute_one(self, inputs: RAGRetrievalInput) -> Dict[str, Any]:
+        # Handle empty inputs
+        if not inputs.reference or not inputs.reference.strip():
+            return {
+                "output": 0.0,
+                "reason": "No reference provided — cannot measure entity recall",
+            }
+
+        # Extract entities from reference
+        reference_entities = extract_entities(
+            inputs.reference,
+            entity_types=self.entity_types,
+        )
+
+        if not reference_entities:
+            return {
+                "output": 0.0,
+                "reason": "No entities found in reference",
+                "reference_entities": [],
+            }
+
+        if not inputs.contexts:
+            return {
+                "output": 0.0,
+                "reason": "No contexts provided - cannot recall any entities",
+                "reference_entities": list(reference_entities),
+                "recalled_entities": [],
+                "missing_entities": list(reference_entities),
+            }
+
+        # Extract entities from all contexts
+        context_entities: Set[str] = set()
+        for ctx in inputs.contexts:
+            ctx_ents = extract_entities(ctx, entity_types=self.entity_types)
+            context_entities.update(ctx_ents)
+
+        # Calculate entity overlap
+        recalled = set()
+        missing = set()
+
+        for ref_entity in reference_entities:
+            found = False
+
+            if self.fuzzy_match:
+                # Check for fuzzy matches
+                for ctx_entity in context_entities:
+                    if entities_match(ref_entity, ctx_entity):
+                        found = True
+                        break
+            else:
+                # Exact match (case-insensitive by default)
+                if self.case_sensitive:
+                    found = ref_entity in context_entities
+                else:
+                    found = ref_entity.lower() in {e.lower() for e in context_entities}
+
+            if found:
+                recalled.add(ref_entity)
+            else:
+                missing.add(ref_entity)
+
+        # Calculate recall
+        recall = len(recalled) / len(reference_entities)
+
+        return {
+            "output": round(recall, 4),
+            "reason": f"{len(recalled)}/{len(reference_entities)} entities recalled",
+            "total_reference_entities": len(reference_entities),
+            "recalled_count": len(recalled),
+            "missing_count": len(missing),
+            "reference_entities": list(reference_entities),
+            "recalled_entities": list(recalled),
+            "missing_entities": list(missing),
+            "context_entities": list(context_entities)[:20],  # Limit for readability
+        }

fi/evals/metrics/rag/retrieval/context_precision.py

@@ -0,0 +1,158 @@
+"""
+Context Precision Metric.
+
+Measures retrieval ranking quality - whether relevant contexts
+appear before irrelevant ones.
+"""
+
+from typing import Any, Dict, Optional
+
+from ...base_metric import BaseMetric
+from ..types import RAGRetrievalInput
+from ..utils import (
+    compute_semantic_similarity,
+    compute_word_overlap,
+    check_entailment,
+    NLILabel,
+)
+
+
+class ContextPrecision(BaseMetric[RAGRetrievalInput]):
+    """
+    Measures retrieval ranking quality.
+
+    Evaluates whether relevant contexts are ranked higher than
+    irrelevant ones. A single irrelevant chunk at position 1
+    significantly reduces the score vs. the same chunk at position 5.
+
+    Formula (Average Precision style):
+        Context Precision@K = Σ(Precision@k × v_k) / |Relevant items in top K|
+
+        Where:
+        - Precision@k = TP@k / (TP@k + FP@k)
+        - v_k ∈ {0,1} indicates relevance at rank k
+
+    Score: 0.0 (poor ranking) to 1.0 (perfect ranking)
+
+    Example:
+        >>> precision = ContextPrecision()
+        >>> result = precision.evaluate([{
+        ...     "query": "What is machine learning?",
+        ...     "contexts": [
+        ...         "Machine learning is a branch of AI.",  # Relevant
+        ...         "The weather is nice today.",  # Irrelevant
+        ...     ],
+        ...     "reference": "Machine learning is an AI technique."
+        ... }])
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "context_precision"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.relevance_threshold = self.config.get("relevance_threshold", 0.3)
+        self.use_reference = self.config.get("use_reference", True)
+
+    def compute_one(self, inputs: RAGRetrievalInput) -> Dict[str, Any]:
+        if not inputs.contexts:
+            return {
+                "output": 0.0,
+                "reason": "No contexts to evaluate",
+            }
+
+        # Determine relevance of each context
+        relevance = []
+        relevance_details = []
+
+        for i, ctx in enumerate(inputs.contexts):
+            if inputs.relevance_labels is not None and i < len(inputs.relevance_labels):
+                # Use provided labels
+                is_relevant = inputs.relevance_labels[i] == 1
+                relevance_score = 1.0 if is_relevant else 0.0
+            else:
+                # Compute relevance
+                relevance_score = self._compute_relevance(
+                    ctx, inputs.reference, inputs.query
+                )
+                is_relevant = relevance_score >= self.relevance_threshold
+
+            relevance.append(1 if is_relevant else 0)
+            relevance_details.append({
+                "position": i + 1,
+                "relevant": is_relevant,
+                "score": round(relevance_score, 3),
+                "context_preview": ctx[:80] + "..." if len(ctx) > 80 else ctx,
+            })
+
+        # Calculate Average Precision
+        precision_sum = 0.0
+        relevant_count = 0
+
+        for k, is_rel in enumerate(relevance, 1):
+            if is_rel:
+                relevant_count += 1
+                precision_at_k = relevant_count / k
+                precision_sum += precision_at_k
+
+        total_relevant = sum(relevance)
+        if total_relevant == 0:
+            return {
+                "output": 0.0,
+                "reason": "No relevant contexts found",
+                "relevance_by_position": relevance,
+                "details": relevance_details,
+            }
+
+        avg_precision = precision_sum / total_relevant
+
+        return {
+            "output": round(avg_precision, 4),
+            "reason": f"AP={avg_precision:.3f}, {total_relevant}/{len(relevance)} contexts relevant",
+            "total_contexts": len(relevance),
+            "relevant_contexts": total_relevant,
+            "relevance_by_position": relevance,
+            "details": relevance_details,
+        }
+
+    def _compute_relevance(
+        self, context: str, reference: str, query: str
+    ) -> float:
+        """
+        Compute relevance score for a context.
+
+        Combines multiple signals:
+        - Semantic similarity to reference
+        - Word overlap with reference
+        - Query relevance (if reference unavailable)
+        """
+        scores = []
+
+        if reference:
+            # Semantic similarity to reference
+            sem_sim = compute_semantic_similarity(context, reference)
+            scores.append(sem_sim)
+
+            # Word overlap with reference
+            word_overlap = compute_word_overlap(context, reference)
+            scores.append(word_overlap)
+
+            # NLI-based check
+            label, nli_score = check_entailment(context, reference)
+            if label == NLILabel.ENTAILMENT:
+                scores.append(nli_score)
+            elif label == NLILabel.CONTRADICTION:
+                scores.append(0.0)
+            else:
+                scores.append(nli_score * 0.5)
+
+        # Query relevance as fallback or additional signal
+        if query:
+            query_sim = compute_semantic_similarity(context, query)
+            scores.append(query_sim * 0.8)  # Slightly lower weight
+
+        if not scores:
+            return 0.0
+
+        return sum(scores) / len(scores)

fi/evals/metrics/rag/retrieval/context_recall.py

@@ -0,0 +1,106 @@
+"""
+Context Recall Metric.
+
+Measures retrieval completeness - how much of the ground truth
+information is covered by retrieved contexts.
+"""
+
+from typing import Any, Dict, Optional
+
+from ...base_metric import BaseMetric
+from ..types import RAGRetrievalInput
+from ..utils import split_into_sentences, check_attribution
+
+
+class ContextRecall(BaseMetric[RAGRetrievalInput]):
+    """
+    Measures retrieval completeness.
+
+    Evaluates whether retrieved contexts contain all information
+    required to produce the ground truth answer.
+
+    Uses NLI-based attribution to check if reference sentences
+    can be inferred from the retrieved contexts.
+
+    Formula (Ragas-style):
+        Context Recall = |GT sentences attributable to context| / |GT sentences|
+
+    Score: 0.0 (no coverage) to 1.0 (complete coverage)
+
+    Example:
+        >>> recall = ContextRecall()
+        >>> result = recall.evaluate([{
+        ...     "query": "What is the capital of France?",
+        ...     "contexts": ["Paris is the capital of France."],
+        ...     "reference": "The capital of France is Paris."
+        ... }])
+        >>> print(result.eval_results[0].output)  # Close to 1.0
+    """
+
+    @property
+    def metric_name(self) -> str:
+        return "context_recall"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self.attribution_threshold = self.config.get("attribution_threshold", 0.5)
+        self.min_sentence_words = self.config.get("min_sentence_words", 3)
+
+    def compute_one(self, inputs: RAGRetrievalInput) -> Dict[str, Any]:
+        # Handle empty inputs
+        if not inputs.reference or not inputs.reference.strip():
+            return {
+                "output": 0.0,
+                "reason": "No reference provided — cannot measure recall",
+            }
+
+        if not inputs.contexts:
+            return {
+                "output": 0.0,
+                "reason": "No contexts provided - cannot recall any information",
+            }
+
+        # Split reference into sentences
+        reference_sentences = split_into_sentences(inputs.reference)
+
+        # Filter very short sentences
+        reference_sentences = [
+            s for s in reference_sentences
+            if len(s.split()) >= self.min_sentence_words
+        ]
+
+        if not reference_sentences:
+            return {
+                "output": 0.0,
+                "reason": "No verifiable sentences in reference",
+            }
+
+        # For each sentence, check attribution to any context
+        attributed = 0
+        attribution_details = []
+
+        for sentence in reference_sentences:
+            is_attributed, best_context, score = check_attribution(
+                sentence, inputs.contexts, self.attribution_threshold
+            )
+
+            if is_attributed:
+                attributed += 1
+
+            attribution_details.append({
+                "sentence": sentence[:100] + "..." if len(sentence) > 100 else sentence,
+                "attributed": is_attributed,
+                "score": round(score, 3),
+                "matched_context": best_context[:80] + "..." if best_context and len(best_context) > 80 else best_context,
+            })
+
+        # Calculate recall
+        recall = attributed / len(reference_sentences)
+
+        return {
+            "output": round(recall, 4),
+            "reason": f"{attributed}/{len(reference_sentences)} reference sentences found in context",
+            "total_sentences": len(reference_sentences),
+            "attributed_sentences": attributed,
+            "details": attribution_details,
+        }