agent-learning-kit 0.1.0__py3-none-any.whl

fi/evals/metrics/rag/utils/claims.py

@@ -0,0 +1,189 @@
+"""
+Claim extraction utilities for RAG evaluation.
+
+Provides functions to split text into sentences and extract
+verifiable claims from responses.
+"""
+
+import re
+from typing import List, Set
+
+
+def split_into_sentences(text: str) -> List[str]:
+    """
+    Split text into sentences.
+
+    Uses regex-based sentence boundary detection.
+    Handles common abbreviations and edge cases.
+
+    Args:
+        text: The text to split
+
+    Returns:
+        List of sentence strings
+    """
+    if not text or not text.strip():
+        return []
+
+    # Handle common abbreviations
+    text = re.sub(r"\b(Mr|Mrs|Ms|Dr|Prof|Sr|Jr|etc|vs|i\.e|e\.g)\.", r"\1<DOT>", text)
+
+    # Split on sentence boundaries
+    sentences = re.split(r"(?<=[.!?])\s+", text.strip())
+
+    # Restore abbreviations
+    sentences = [s.replace("<DOT>", ".").strip() for s in sentences if s.strip()]
+
+    return sentences
+
+
+def extract_claims(text: str, min_words: int = 3) -> List[str]:
+    """
+    Extract verifiable claims from text.
+
+    Filters out questions, meta-statements, and very short sentences.
+
+    Args:
+        text: The text to extract claims from
+        min_words: Minimum number of words for a valid claim
+
+    Returns:
+        List of claim strings
+    """
+    sentences = split_into_sentences(text)
+    claims = []
+
+    for sentence in sentences:
+        # Skip very short sentences
+        if len(sentence.split()) < min_words:
+            continue
+
+        # Skip questions
+        if sentence.strip().endswith("?"):
+            continue
+
+        # Skip meta-statements and hedging
+        meta_patterns = [
+            r"^(I think|I believe|In my opinion|It seems|Perhaps|Maybe)",
+            r"^(Here is|Here are|The following|Below is|Above is)",
+            r"^(Note:|Important:|Warning:|Disclaimer:|Please note)",
+            r"^(Let me|I will|I can|I would|I should)",
+            r"^(This is|That is|These are|Those are)\s+(a|an|the)\s+(question|query|request)",
+        ]
+        is_meta = any(re.match(p, sentence, re.IGNORECASE) for p in meta_patterns)
+        if is_meta:
+            continue
+
+        # Skip acknowledgments and greetings
+        ack_patterns = [
+            r"^(Thank you|Thanks|Sure|Of course|Certainly|Absolutely)",
+            r"^(Hello|Hi|Hey|Good morning|Good afternoon|Good evening)",
+        ]
+        is_ack = any(re.match(p, sentence, re.IGNORECASE) for p in ack_patterns)
+        if is_ack:
+            continue
+
+        claims.append(sentence)
+
+    return claims
+
+
+def extract_key_phrases(text: str, max_phrases: int = 20) -> List[str]:
+    """
+    Extract key phrases from text.
+
+    Uses heuristics to identify important noun phrases and
+    subject-verb-object patterns.
+
+    Args:
+        text: The text to extract phrases from
+        max_phrases: Maximum number of phrases to return
+
+    Returns:
+        List of key phrase strings
+    """
+    phrases = set()
+
+    # Extract capitalized phrases (proper nouns)
+    proper_nouns = re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b", text)
+    phrases.update(proper_nouns)
+
+    # Extract quoted phrases
+    quoted = re.findall(r'"([^"]+)"', text)
+    phrases.update(quoted)
+
+    # Extract phrases with numbers
+    numbered = re.findall(
+        r"\b\d+(?:\.\d+)?(?:\s+(?:percent|%|million|billion|thousand|hundred|dollars?|euros?|pounds?|years?|months?|days?|hours?|minutes?|seconds?))\b",
+        text,
+        re.IGNORECASE,
+    )
+    phrases.update(numbered)
+
+    # Extract noun phrases using simple patterns
+    # Pattern: adjective(s) + noun(s)
+    noun_phrases = re.findall(
+        r"\b(?:the\s+)?(?:[A-Za-z]+\s+){0,2}[A-Za-z]+(?:tion|ment|ness|ity|ence|ance|ing|ed)\b",
+        text,
+    )
+    phrases.update(noun_phrases[:10])
+
+    # Filter and limit
+    filtered = [p for p in phrases if len(p) > 2 and len(p.split()) <= 5]
+
+    return filtered[:max_phrases]
+
+
+def extract_atomic_claims(text: str) -> List[str]:
+    """
+    Extract atomic (indivisible) claims from text.
+
+    Breaks down compound sentences into simpler claims.
+
+    Args:
+        text: The text to extract atomic claims from
+
+    Returns:
+        List of atomic claim strings
+    """
+    claims = extract_claims(text)
+    atomic_claims = []
+
+    for claim in claims:
+        # Split compound sentences on conjunctions
+        parts = re.split(r"\b(?:and|but|however|although|while|whereas)\b", claim)
+
+        for part in parts:
+            part = part.strip()
+            # Clean up leading/trailing punctuation
+            part = re.sub(r"^[,;:\s]+|[,;:\s]+$", "", part)
+
+            if len(part.split()) >= 3:
+                atomic_claims.append(part)
+
+    return atomic_claims
+
+
+def get_claim_entities(claim: str) -> Set[str]:
+    """
+    Extract entity-like tokens from a claim.
+
+    Simple extraction without requiring NER models.
+
+    Args:
+        claim: The claim text
+
+    Returns:
+        Set of entity-like strings
+    """
+    entities = set()
+
+    # Capitalized words (likely proper nouns)
+    caps = re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b", claim)
+    entities.update(caps)
+
+    # Numbers with context
+    numbers = re.findall(r"\b\d+(?:\.\d+)?(?:\s+\w+)?\b", claim)
+    entities.update(numbers)
+
+    return entities

fi/evals/metrics/rag/utils/entities.py

@@ -0,0 +1,244 @@
+"""
+Entity extraction utilities for RAG evaluation.
+
+Provides functions to extract named entities from text.
+Supports spaCy NER with heuristic fallback.
+"""
+
+import re
+from typing import List, Optional, Set
+
+# Optional spaCy import
+_SPACY_AVAILABLE = False
+_nlp = None
+
+try:
+    import spacy
+
+    _SPACY_AVAILABLE = True
+except ImportError:
+    spacy = None
+
+
+def _get_spacy_model():
+    """Lazy-load spaCy model."""
+    global _nlp
+    if _nlp is None and _SPACY_AVAILABLE:
+        try:
+            _nlp = spacy.load("en_core_web_sm")
+        except OSError:
+            # Model not downloaded, use fallback
+            _nlp = False
+    return _nlp
+
+
+def extract_entities_spacy(
+    text: str, entity_types: Optional[List[str]] = None
+) -> Set[str]:
+    """
+    Extract entities using spaCy NER.
+
+    Args:
+        text: The text to extract entities from
+        entity_types: Optional list of entity types to filter by
+
+    Returns:
+        Set of entity strings
+    """
+    nlp = _get_spacy_model()
+    if not nlp:
+        return extract_entities_heuristic(text)
+
+    try:
+        doc = nlp(text)
+        entities = set()
+
+        # Default entity types of interest
+        default_types = {
+            "PERSON",
+            "ORG",
+            "GPE",
+            "LOC",
+            "DATE",
+            "TIME",
+            "MONEY",
+            "QUANTITY",
+            "PRODUCT",
+            "EVENT",
+            "WORK_OF_ART",
+            "LAW",
+            "LANGUAGE",
+            "PERCENT",
+            "CARDINAL",
+            "ORDINAL",
+        }
+        target_types = set(entity_types) if entity_types else default_types
+
+        for ent in doc.ents:
+            if ent.label_ in target_types:
+                entities.add(ent.text)
+
+        return entities
+    except Exception:
+        return extract_entities_heuristic(text)
+
+
+def extract_entities_heuristic(text: str) -> Set[str]:
+    """
+    Fallback entity extraction using heuristics.
+
+    Extracts:
+    - Capitalized phrases (likely proper nouns)
+    - Numbers with units
+    - Dates in common formats
+    - Monetary values
+
+    Args:
+        text: The text to extract entities from
+
+    Returns:
+        Set of entity strings
+    """
+    entities = set()
+
+    # Capitalized words/phrases (proper nouns)
+    # Match sequences of capitalized words
+    proper_nouns = re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b", text)
+    # Filter out sentence starters by checking position
+    for noun in proper_nouns:
+        # Check if it appears mid-sentence (not after . or at start)
+        pattern = rf"[a-z,;:]\s+{re.escape(noun)}\b"
+        if re.search(pattern, text) or text.strip().startswith(noun):
+            entities.add(noun)
+
+    # Dates - various formats
+    date_patterns = [
+        r"\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b",  # 01/15/2024
+        r"\b(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}\b",
+        r"\b\d{1,2}(?:st|nd|rd|th)?\s+(?:January|February|March|April|May|June|July|August|September|October|November|December),?\s+\d{4}\b",
+        r"\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\.?\s+\d{1,2},?\s+\d{4}\b",
+        r"\b\d{4}\b",  # Years
+    ]
+    for pattern in date_patterns:
+        matches = re.findall(pattern, text, re.IGNORECASE)
+        entities.update(matches)
+
+    # Monetary values
+    money_patterns = [
+        r"\$[\d,]+(?:\.\d{2})?(?:\s*(?:million|billion|trillion))?",
+        r"[\d,]+(?:\.\d{2})?\s*(?:dollars?|USD|euros?|EUR|pounds?|GBP)",
+        r"\d+(?:\.\d+)?\s*(?:million|billion|trillion)\s*(?:dollars?|euros?|pounds?)?",
+    ]
+    for pattern in money_patterns:
+        matches = re.findall(pattern, text, re.IGNORECASE)
+        entities.update(matches)
+
+    # Percentages
+    percentages = re.findall(r"\b\d+(?:\.\d+)?%", text)
+    entities.update(percentages)
+
+    # Quantities with units
+    quantities = re.findall(
+        r"\b\d+(?:\.\d+)?\s*(?:kg|km|miles?|meters?|feet|inches|pounds?|ounces?|liters?|gallons?|hours?|minutes?|seconds?|days?|weeks?|months?|years?)\b",
+        text,
+        re.IGNORECASE,
+    )
+    entities.update(quantities)
+
+    # Email addresses
+    emails = re.findall(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", text)
+    entities.update(emails)
+
+    # URLs
+    urls = re.findall(r"https?://[^\s]+", text)
+    entities.update(urls)
+
+    return entities
+
+
+def extract_entities(
+    text: str,
+    entity_types: Optional[List[str]] = None,
+    use_llm: bool = False,
+) -> Set[str]:
+    """
+    Main entity extraction function.
+
+    Tries spaCy first, falls back to heuristics.
+
+    Args:
+        text: The text to extract entities from
+        entity_types: Optional list of entity types to filter by
+        use_llm: Whether to use LLM for extraction (not implemented)
+
+    Returns:
+        Set of entity strings
+    """
+    if use_llm:
+        # LLM-based extraction could be added here
+        pass
+
+    if _SPACY_AVAILABLE:
+        return extract_entities_spacy(text, entity_types)
+
+    return extract_entities_heuristic(text)
+
+
+def normalize_entity(entity: str) -> str:
+    """
+    Normalize an entity for comparison.
+
+    Handles case, whitespace, and common variations.
+
+    Args:
+        entity: The entity string to normalize
+
+    Returns:
+        Normalized entity string
+    """
+    # Lowercase
+    normalized = entity.lower().strip()
+
+    # Normalize whitespace
+    normalized = re.sub(r"\s+", " ", normalized)
+
+    # Remove common prefixes/suffixes
+    normalized = re.sub(r"^(the|a|an)\s+", "", normalized)
+
+    return normalized
+
+
+def entities_match(entity1: str, entity2: str, threshold: float = 0.8) -> bool:
+    """
+    Check if two entities match.
+
+    Handles exact matches and fuzzy matching for variations.
+
+    Args:
+        entity1: First entity
+        entity2: Second entity
+        threshold: Similarity threshold for fuzzy matching
+
+    Returns:
+        True if entities match
+    """
+    norm1 = normalize_entity(entity1)
+    norm2 = normalize_entity(entity2)
+
+    # Exact match
+    if norm1 == norm2:
+        return True
+
+    # Substring match
+    if norm1 in norm2 or norm2 in norm1:
+        return True
+
+    # Simple character overlap for fuzzy matching
+    if len(norm1) > 3 and len(norm2) > 3:
+        chars1 = set(norm1)
+        chars2 = set(norm2)
+        overlap = len(chars1 & chars2) / max(len(chars1), len(chars2))
+        if overlap >= threshold:
+            return True
+
+    return False

fi/evals/metrics/rag/utils/nli.py

@@ -0,0 +1,92 @@
+"""
+Natural Language Inference utilities for RAG evaluation.
+
+Provides entailment checking between claims and context.
+Reuses the shared NLI pipeline from the hallucination module to avoid
+loading the model twice. Falls back to word-overlap heuristic when
+transformers is not installed.
+"""
+
+from typing import Tuple, List, Optional
+
+from ...hallucination.nli import (
+    NLILabel,
+    check_entailment,
+    check_entailment_heuristic,
+    check_contradiction,
+)
+
+# Re-export for backwards compatibility
+__all__ = [
+    "NLILabel",
+    "check_entailment",
+    "check_entailment_heuristic",
+    "check_claim_supported",
+    "check_attribution",
+    "check_contradiction",
+]
+
+
+def check_claim_supported(
+    claim: str, contexts: List[str], threshold: float = 0.5
+) -> Tuple[bool, float, Optional[str]]:
+    """
+    Check if a claim is supported by any of the contexts.
+
+    Args:
+        claim: The claim to verify
+        contexts: List of context passages
+        threshold: Minimum score to consider claim supported
+
+    Returns:
+        Tuple of (is_supported, best_score, best_supporting_context)
+    """
+    best_score = 0.0
+    best_context = None
+
+    for ctx in contexts:
+        label, score = check_entailment(ctx, claim)
+
+        if label == NLILabel.ENTAILMENT and score > best_score:
+            best_score = score
+            best_context = ctx[:200] + "..." if len(ctx) > 200 else ctx
+
+    is_supported = best_score >= threshold
+
+    return is_supported, best_score, best_context
+
+
+def check_attribution(
+    sentence: str, contexts: List[str], threshold: float = 0.5
+) -> Tuple[bool, Optional[str], float]:
+    """
+    Check if a sentence can be attributed to any context.
+
+    Args:
+        sentence: The sentence to check attribution for
+        contexts: List of context passages
+        threshold: Minimum score to consider attributed
+
+    Returns:
+        Tuple of (is_attributed, best_context, best_score)
+    """
+    best_score = 0.0
+    best_context = None
+
+    for ctx in contexts:
+        label, score = check_entailment(ctx, sentence)
+
+        if label == NLILabel.ENTAILMENT:
+            if score > best_score:
+                best_score = score
+                best_context = ctx[:200] + "..." if len(ctx) > 200 else ctx
+        elif label == NLILabel.NEUTRAL and score > 0.5:
+            # High-confidence neutral as partial attribution
+            adjusted_score = score * 0.7
+            if adjusted_score > best_score:
+                best_score = adjusted_score
+                best_context = ctx[:200] + "..." if len(ctx) > 200 else ctx
+
+    is_attributed = best_score >= threshold
+
+    return is_attributed, best_context, best_score