crprotocol 2.0.0__py3-none-any.whl

crp/provenance/attribution_scorer.py

@@ -0,0 +1,252 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Attribution Scorer — map claims to envelope facts (§7.14.3).
+
+For each factual claim in the LLM output, scores how likely it came from
+each envelope fact vs. parametric knowledge.  Uses two signals:
+
+  1. **Semantic similarity**: When available, uses dense sentence-transformer
+     embeddings (all-MiniLM-L6-v2, 384-dim) for genuine semantic comparison.
+     Falls back to hash-projected bag-of-words when the model is unavailable.
+  2. **Lexical overlap**: Token-level Jaccard overlap — fast, complements
+     semantic similarity for surface-level attribution.
+
+The composite score is a weighted blend:
+    composite = semantic_weight × semantic + lexical_weight × lexical
+
+Claims with no fact scoring above ``similarity_threshold`` are flagged
+as PARAMETRIC (likely from model training data).  Claims with a top score
+between ``mixed_threshold`` and ``similarity_threshold`` are MIXED.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+import re
+from collections.abc import Sequence
+from typing import Any
+
+from crp.envelope.packer import PackedFact
+
+from ._embeddings import cosine_similarity as _dense_cosine
+from ._embeddings import encode_texts
+from ._types import (
+    AttributionType,
+    ClaimAttribution,
+    ClaimType,
+    FactScore,
+    ProvenanceConfig,
+)
+from .claim_detector import DetectedClaim
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Internal: lightweight text vectorisation (mirrors crp.envelope.decomposer)
+# ---------------------------------------------------------------------------
+
+_STOP = frozenset(
+    [
+        "a", "an", "the", "and", "or", "but", "in", "on", "at", "to",
+        "for", "of", "is", "it", "its", "are", "was", "were", "be",
+        "been", "being", "have", "has", "had", "do", "does", "did",
+        "will", "would", "shall", "should", "may", "might", "can",
+        "could", "this", "that", "these", "those", "i", "we", "you",
+        "he", "she", "they", "me", "us", "him", "her", "them", "my",
+        "our", "your", "his", "their", "with", "from", "by", "as",
+        "not", "no",
+    ]
+)
+
+_EMBED_DIM = 256
+
+
+def _tokenize(text: str) -> list[str]:
+    """Lower-case word tokeniser (alphanum only, stopwords removed)."""
+    return [w for w in re.findall(r"[a-z0-9]+", text.lower()) if w not in _STOP]
+
+
+def _bag_vector(tokens: Sequence[str], dim: int = _EMBED_DIM) -> list[float]:
+    """Sparse bag-of-words vector with hash projection, L2-normalised."""
+    vec = [0.0] * dim
+    for tok in tokens:
+        idx = int(hashlib.sha256(tok.encode()).hexdigest(), 16) % dim
+        vec[idx] += 1.0
+    norm = math.sqrt(sum(v * v for v in vec)) or 1.0
+    return [v / norm for v in vec]
+
+
+def _cosine(a: Sequence[float], b: Sequence[float]) -> float:
+    """Cosine similarity between two equal-length vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = math.sqrt(sum(x * x for x in a)) or 1e-12
+    norm_b = math.sqrt(sum(x * x for x in b)) or 1e-12
+    return dot / (norm_a * norm_b)
+
+
+def _lexical_overlap(tokens_a: list[str], tokens_b: list[str]) -> float:
+    """Token-level Jaccard overlap (|intersection| / |union|)."""
+    if not tokens_a or not tokens_b:
+        return 0.0
+    set_a = set(tokens_a)
+    set_b = set(tokens_b)
+    intersection = set_a & set_b
+    union = set_a | set_b
+    return len(intersection) / len(union) if union else 0.0
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def score_claim_against_facts(
+    claim: DetectedClaim,
+    packed_facts: list[PackedFact],
+    *,
+    config: ProvenanceConfig | None = None,
+    _embedder_override: Any = None,
+) -> ClaimAttribution:
+    """Score a single claim against all envelope facts.
+
+    Attempts to use dense sentence-transformer embeddings for semantic
+    similarity.  Falls back to hash-projected bag-of-words if the
+    embedding model is unavailable.
+
+    Args:
+        claim: Detected claim from claim_detector.
+        packed_facts: Facts that were packed into the envelope.
+        config: Scoring configuration (thresholds, weights).
+        _embedder_override: Injected embedding model for testing.
+
+    Returns:
+        ClaimAttribution with ranked fact scores and attribution type.
+    """
+    cfg = config or ProvenanceConfig()
+
+    claim_tokens = _tokenize(claim.text)
+
+    # --- Try dense embeddings first ---
+    texts_to_encode = [claim.text] + [pf.text for pf in packed_facts]
+    dense_embeddings = encode_texts(texts_to_encode, _model_override=_embedder_override)
+    use_dense = dense_embeddings is not None and len(dense_embeddings) == len(texts_to_encode)
+
+    if use_dense:
+        claim_emb = dense_embeddings[0]
+        fact_embs = dense_embeddings[1:]
+    else:
+        # Fallback: hash-projected bag-of-words
+        claim_vec = _bag_vector(claim_tokens)
+
+    fact_scores: list[FactScore] = []
+
+    for i, pf in enumerate(packed_facts):
+        fact_tokens = _tokenize(pf.text)
+
+        if use_dense:
+            semantic = max(_dense_cosine(claim_emb, fact_embs[i]), 0.0)
+        else:
+            fact_vec = _bag_vector(fact_tokens)
+            semantic = max(_cosine(claim_vec, fact_vec), 0.0)
+
+        lexical = _lexical_overlap(claim_tokens, fact_tokens)
+
+        composite = (
+            cfg.semantic_weight * semantic
+            + cfg.lexical_weight * lexical
+        )
+
+        fact_scores.append(FactScore(
+            fact_id=pf.fact_id,
+            fact_text_preview=pf.text[:cfg.fact_preview_length],
+            semantic_similarity=round(semantic, 4),
+            lexical_overlap=round(lexical, 4),
+            composite_score=round(composite, 4),
+            fact_source_window="",   # Filled by provenance chain builder
+            fact_extraction_stage=0,  # Filled by provenance chain builder
+        ))
+
+    # Sort by composite score descending
+    fact_scores.sort(key=lambda fs: fs.composite_score, reverse=True)
+
+    # Determine attribution type from top score
+    top = fact_scores[0].composite_score if fact_scores else 0.0
+
+    if top >= cfg.similarity_threshold:
+        attr_type = AttributionType.CONTEXT_GROUNDED
+    elif top >= cfg.mixed_threshold:
+        attr_type = AttributionType.MIXED
+    elif top > 0.0:
+        attr_type = AttributionType.PARAMETRIC
+    else:
+        attr_type = AttributionType.UNCERTAIN
+
+    # Confidence: distance from thresholds → 0.5–0.95 range
+    if attr_type == AttributionType.CONTEXT_GROUNDED:
+        confidence = min(0.60 + (top - cfg.similarity_threshold), 0.95)
+    elif attr_type == AttributionType.MIXED:
+        confidence = 0.40 + (top - cfg.mixed_threshold) * 0.5
+    elif attr_type == AttributionType.PARAMETRIC:
+        confidence = 0.50 + (cfg.mixed_threshold - top) * 0.3
+    else:
+        confidence = 0.20
+
+    return ClaimAttribution(
+        claim_text=claim.text,
+        claim_index=claim.index,
+        claim_type=claim.claim_type,
+        attributed_facts=fact_scores[:5],  # Top-5 facts per claim
+        top_score=round(top, 4),
+        attribution_type=attr_type,
+        confidence=round(min(max(confidence, 0.0), 1.0), 4),
+    )
+
+
+def score_all_claims(
+    claims: list[DetectedClaim],
+    packed_facts: list[PackedFact],
+    *,
+    config: ProvenanceConfig | None = None,
+    _embedder_override: Any = None,
+) -> list[ClaimAttribution]:
+    """Score all claims against all envelope facts.
+
+    Only FACTUAL_CLAIM and HEDGE types are scored against facts.
+    Other types (OPINION, PROCEDURAL, CONNECTIVE) are returned with
+    empty attribution (they don't require source grounding).
+
+    Args:
+        claims: All detected claims from claim_detector.
+        packed_facts: Facts that were packed into the envelope.
+        config: Scoring configuration.
+        _embedder_override: Injected embedding model for testing.
+
+    Returns:
+        List of ClaimAttribution objects in claim order.
+    """
+    cfg = config or ProvenanceConfig()
+    attributions: list[ClaimAttribution] = []
+
+    for claim in claims:
+        if claim.claim_type in (ClaimType.FACTUAL_CLAIM, ClaimType.HEDGE):
+            attr = score_claim_against_facts(
+                claim, packed_facts, config=cfg,
+                _embedder_override=_embedder_override,
+            )
+        else:
+            # Non-factual claims don't need source attribution
+            attr = ClaimAttribution(
+                claim_text=claim.text,
+                claim_index=claim.index,
+                claim_type=claim.claim_type,
+                attributed_facts=[],
+                top_score=0.0,
+                attribution_type=AttributionType.UNCERTAIN,
+                confidence=0.0,
+            )
+        attributions.append(attr)
+
+    return attributions

crp/provenance/claim_detector.py

@@ -0,0 +1,229 @@
+# Copyright © 2025 Constantinos Vidiniotis. All rights reserved.
+# Licensed under Elastic License 2.0 — see LICENSE.md for details.
+"""Claim Detector — segment LLM output into attributable claims (§7.14.3).
+
+Splits model output into individual sentences/claims and classifies each as:
+  - FACTUAL_CLAIM: verifiable factual assertion (requires attribution)
+  - OPINION: subjective view or judgment
+  - PROCEDURAL: action or instruction
+  - HEDGE: qualified/uncertain statement
+  - CONNECTIVE: structural/transitional text
+
+Uses rule-based heuristics — no ML model required for classification, keeping
+overhead under 5ms for typical outputs.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from ._types import ClaimType
+
+
+# ---------------------------------------------------------------------------
+# Sentence boundary detection
+# ---------------------------------------------------------------------------
+
+# Regex-based sentence splitter: split on sentence-ending punctuation
+# followed by whitespace or end-of-string, but avoid splitting on
+# common abbreviations (e.g., Mr., Dr., U.S., etc.) and decimals.
+_ABBREV = r"(?<![A-Z][a-z])(?<!Mr)(?<!Mrs)(?<!Ms)(?<!Dr)(?<!Prof)(?<!Jr)(?<!Sr)(?<!Inc)(?<!Ltd)(?<!Corp)(?<!vs)(?<!etc)(?<!e\.g)(?<!i\.e)"
+_SENTENCE_SPLIT = re.compile(
+    _ABBREV + r"[.!?]+(?:\s+|$)"
+    r"|(?:\n\s*\n)"  # Double newline = paragraph boundary
+)
+
+# ---------------------------------------------------------------------------
+# Claim classification patterns
+# ---------------------------------------------------------------------------
+
+# Hedge indicators
+_HEDGE_PATTERNS = re.compile(
+    r"\b("
+    r"may\s|might\s|could\s|possibly|perhaps|likely|unlikely"
+    r"|it\s+is\s+possible|it\s+appears|it\s+seems"
+    r"|suggest(?:s|ing)?\s|indicat(?:e|es|ing)\s"
+    r"|approximately|roughly|about\s+\d"
+    r"|tend(?:s)?\s+to|potential(?:ly)?"
+    r")\b",
+    re.IGNORECASE,
+)
+
+# Opinion indicators
+_OPINION_PATTERNS = re.compile(
+    r"\b("
+    r"I\s+(?:think|believe|feel|consider|recommend|suggest)"
+    r"|in\s+my\s+(?:opinion|view|experience)"
+    r"|(?:good|bad|best|worst|excellent|poor|great|terrible)"
+    r"|(?:should|ought|must)\s"
+    r"|(?:important(?:ly)?|crucial(?:ly)?|essential(?:ly)?)"
+    r"|unfortunately|fortunately|hopefully"
+    r"|(?:prefer|favorite|ideal)"
+    r")\b",
+    re.IGNORECASE,
+)
+
+# Procedural indicators
+_PROCEDURAL_PATTERNS = re.compile(
+    r"\b("
+    r"(?:click|press|open|close|select|choose|enter|type|run|execute|install)"
+    r"|(?:step\s+\d|first,?\s|then,?\s|next,?\s|finally,?\s)"
+    r"|(?:navigate\s+to|go\s+to|switch\s+to)"
+    r"|(?:make\s+sure|ensure\s+that|verify\s+that)"
+    r"|(?:to\s+do\s+this|follow\s+these|here's\s+how)"
+    r")\b",
+    re.IGNORECASE,
+)
+
+# Connective indicators — very short or purely structural
+_CONNECTIVE_PATTERNS = re.compile(
+    r"^("
+    r"(?:furthermore|moreover|additionally|however|nevertheless|therefore)"
+    r"|(?:in\s+conclusion|to\s+summarize|in\s+summary|overall)"
+    r"|(?:for\s+example|for\s+instance|such\s+as|e\.g\.|i\.e\.)"
+    r"|(?:on\s+the\s+other\s+hand|in\s+contrast|conversely)"
+    r"|(?:as\s+mentioned|as\s+noted|as\s+discussed)"
+    r")(?:\s*[,:]?\s*$)",
+    re.IGNORECASE,
+)
+
+# Factual indicators — numbers, dates, named entities, specific claims
+_FACTUAL_PATTERNS = re.compile(
+    r"("
+    r"\d+(?:\.\d+)?%"                     # Percentages
+    r"|\$\d"                               # Dollar amounts
+    r"|\d{4}"                              # Years
+    r"|(?:according\s+to|based\s+on)"      # Citation-like
+    r"|(?:was|were|is|are|has|have)\s+\w+"  # State assertions
+    r"|\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+"  # Proper nouns (2+ words)
+    r")",
+)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class DetectedClaim:
+    """A single detected claim with its classification."""
+
+    text: str = ""
+    index: int = 0              # Position in the output (0-based sentence index)
+    claim_type: ClaimType = ClaimType.CONNECTIVE
+    type_confidence: float = 0.0  # Confidence in the classification
+
+
+def split_into_sentences(text: str) -> list[str]:
+    """Split text into sentences.
+
+    Uses regex-based sentence boundary detection with abbreviation
+    handling and paragraph boundary support.
+
+    Returns:
+        List of sentence strings, stripped of leading/trailing whitespace.
+    """
+    if not text or not text.strip():
+        return []
+
+    # First, split on paragraph boundaries (double newlines)
+    paragraphs = re.split(r"\n\s*\n", text.strip())
+
+    sentences: list[str] = []
+    for para in paragraphs:
+        para = para.strip()
+        if not para:
+            continue
+
+        # Split paragraph into sentences on terminal punctuation
+        parts = re.split(r"(?<=[.!?])\s+", para)
+        for part in parts:
+            part = part.strip()
+            if part:
+                sentences.append(part)
+
+    return sentences
+
+
+def classify_claim(text: str) -> tuple[ClaimType, float]:
+    """Classify a single sentence/claim by type.
+
+    Returns:
+        Tuple of (ClaimType, confidence_score).
+        confidence_score is 0.0-1.0 indicating classification confidence.
+    """
+    stripped = text.strip()
+
+    # Very short fragments are connective
+    if len(stripped) < 15:
+        return ClaimType.CONNECTIVE, 0.90
+
+    # Check connective patterns first (structural text)
+    if _CONNECTIVE_PATTERNS.search(stripped) and len(stripped) < 80:
+        return ClaimType.CONNECTIVE, 0.85
+
+    # Check hedge patterns
+    hedge_matches = len(_HEDGE_PATTERNS.findall(stripped))
+    opinion_matches = len(_OPINION_PATTERNS.findall(stripped))
+    procedural_matches = len(_PROCEDURAL_PATTERNS.findall(stripped))
+    factual_matches = len(_FACTUAL_PATTERNS.findall(stripped))
+
+    # Score each type
+    scores = {
+        ClaimType.HEDGE: hedge_matches * 0.40,
+        ClaimType.OPINION: opinion_matches * 0.35,
+        ClaimType.PROCEDURAL: procedural_matches * 0.35,
+        ClaimType.FACTUAL_CLAIM: factual_matches * 0.30,
+    }
+
+    # If no strong signals, default based on length and structure
+    max_type = max(scores, key=scores.get)  # type: ignore[arg-type]
+    max_score = scores[max_type]
+
+    if max_score < 0.20:
+        # No strong signal — sentences with assertions are likely factual
+        if re.search(r"\b(?:is|are|was|were|has|have|had)\b", stripped):
+            return ClaimType.FACTUAL_CLAIM, 0.50
+        return ClaimType.CONNECTIVE, 0.40
+
+    # Map raw score to confidence (0.5 - 0.95 range)
+    confidence = min(0.50 + max_score, 0.95)
+    return max_type, confidence
+
+
+def detect_claims(
+    text: str,
+    *,
+    min_length: int = 10,
+    max_claims: int = 50,
+) -> list[DetectedClaim]:
+    """Detect and classify all claims in LLM output text.
+
+    Args:
+        text: Raw LLM output text.
+        min_length: Minimum character length for a claim (shorter → skipped).
+        max_claims: Maximum number of claims to return (safety limit).
+
+    Returns:
+        List of DetectedClaim objects, ordered by position in text.
+    """
+    sentences = split_into_sentences(text)
+
+    claims: list[DetectedClaim] = []
+    for idx, sentence in enumerate(sentences):
+        if len(sentence.strip()) < min_length:
+            continue
+        if len(claims) >= max_claims:
+            break
+
+        claim_type, confidence = classify_claim(sentence)
+        claims.append(DetectedClaim(
+            text=sentence.strip(),
+            index=idx,
+            claim_type=claim_type,
+            type_confidence=confidence,
+        ))
+
+    return claims