repeatability-evals 0.1.0__py3-none-any.whl

repeatability_evals/similarity/embedding.py

@@ -0,0 +1,56 @@
+"""Embedding-based semantic similarity. Bring your own embedding function: no dependencies."""
+from __future__ import annotations
+
+import math
+from typing import Any, Callable, Dict, List, Sequence
+
+from .base import Context, Similarity
+
+EmbedFn = Callable[[List[str]], Sequence[Sequence[float]]]
+
+
+class EmbeddingCosine(Similarity):
+    """Cosine similarity of text embeddings.
+
+    ``embed`` takes a list of strings and returns one vector each, which is the shape of
+    ``SentenceTransformer.encode`` and of most embedding API clients, e.g.::
+
+        from sentence_transformers import SentenceTransformer
+        model = SentenceTransformer("all-MiniLM-L6-v2")
+        sim = EmbeddingCosine(lambda texts: model.encode(texts).tolist())
+
+    ``rescale="clip"`` maps negative cosines to 0 (the usual convention for text
+    embeddings); ``"affine"`` maps [-1, 1] onto [0, 1]. Embeddings are cached per text
+    because the same item is compared against several others.
+    """
+
+    name = "embedding_cosine"
+
+    def __init__(self, embed: EmbedFn, *, rescale: str = "clip", max_cache: int = 20000) -> None:
+        if rescale not in ("clip", "affine"):
+            raise ValueError("rescale must be 'clip' or 'affine'")
+        self._embed, self._rescale, self._max_cache = embed, rescale, max_cache
+        self._cache: Dict[str, Sequence[float]] = {}
+
+    def _vectors(self, texts: List[str]) -> List[Sequence[float]]:
+        missing = [t for t in dict.fromkeys(texts) if t not in self._cache]
+        if missing:
+            if len(self._cache) + len(missing) > self._max_cache:
+                self._cache.clear()
+            vecs = self._embed(missing)
+            if len(vecs) != len(missing):
+                raise ValueError("embed() must return one vector per input text")
+            self._cache.update(zip(missing, vecs))
+        return [self._cache[t] for t in texts]
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        ta, tb = str(a), str(b)
+        if ta == tb:
+            return 1.0
+        va, vb = self._vectors([ta, tb])
+        na = math.sqrt(sum(x * x for x in va))
+        nb = math.sqrt(sum(x * x for x in vb))
+        if na == 0 or nb == 0:
+            return 0.0
+        cos = sum(x * y for x, y in zip(va, vb)) / (na * nb)
+        return max(0.0, cos) if self._rescale == "clip" else (cos + 1) / 2

repeatability_evals/similarity/judge.py

@@ -0,0 +1,185 @@
+"""LLM-as-judge semantic equivalence. Model-agnostic: you supply the completion function."""
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Awaitable, Callable, Dict, Optional, Sequence, Tuple
+
+from ..prompts import DEFAULT_DIMENSIONS, SEMANTIC_EQUIVALENCE_PROMPT, Dimension, build_rubric_prompt
+from ..types import SimilarityResult
+from .base import Context, Similarity
+
+CompleteFn = Callable[[str], str]
+ACompleteFn = Callable[[str], Awaitable[str]]
+
+
+class JudgeParseError(ValueError):
+    """The judge's reply contained no usable score."""
+
+
+def parse_judge_output(text: str, scale: Tuple[float, float]) -> Tuple[float, Optional[str]]:
+    """Extract (score, reason) from a judge reply: JSON first, then 'score: N', then last number."""
+    lo, hi = scale
+    match = re.search(r"\{.*\}", text, re.S)
+    if match:
+        try:
+            obj = json.loads(match.group(0))
+            if isinstance(obj, dict) and "score" in obj:
+                return float(obj["score"]), obj.get("reason")
+        except (ValueError, TypeError):
+            pass
+    m = re.search(r"(?i)\b(?:score|rating)\b\D{0,12}(-?\d+(?:\.\d+)?)", text)
+    if m:
+        return float(m.group(1)), None
+    nums = [float(x) for x in re.findall(r"(?<![\d.])\d+(?:\.\d+)?(?![\d.])", text)]
+    nums = [x for x in nums if lo <= x <= hi]
+    if nums:
+        return nums[-1], None
+    raise JudgeParseError(f"no score found in judge reply: {text[:200]!r}")
+
+
+class LLMJudge(Similarity):
+    """Semantic equivalence of two items, scored by an LLM on a 1-5 rubric.
+
+    ``llm`` is any ``str -> str`` completion function (and/or ``allm``, its async twin), so
+    this works with any provider or local model. The score is normalised to [0, 1]; the
+    raw rating and the judge's reason are kept on the ``SimilarityResult``. Put the query
+    both items answer in ``context={"query": ...}`` and the prompt will include it.
+
+    ``symmetric=True`` judges both (a, b) and (b, a) and averages them, removing position
+    bias at twice the cost.
+    """
+
+    name = "llm_judge"
+
+    def __init__(
+        self,
+        llm: Optional[CompleteFn] = None,
+        allm: Optional[ACompleteFn] = None,
+        *,
+        prompt: str = SEMANTIC_EQUIVALENCE_PROMPT,
+        scale: Tuple[float, float] = (1.0, 5.0),
+        symmetric: bool = False,
+    ) -> None:
+        if llm is None and allm is None:
+            raise ValueError("provide llm and/or allm")
+        if scale[1] <= scale[0]:
+            raise ValueError("scale must be (low, high) with high > low")
+        self._llm, self._allm = llm, allm
+        self.prompt, self.scale, self.symmetric = prompt, scale, symmetric
+
+    def _render(self, a: Any, b: Any, context: Context) -> str:
+        query = str((context or {}).get("query", "") or "(not provided)")
+        return (
+            self.prompt.replace("{query}", query)
+            .replace("{item_a}", str(a))
+            .replace("{item_b}", str(b))
+        )
+
+    def _parse(self, reply: str) -> Tuple[float, Optional[str]]:
+        raw, reason = parse_judge_output(reply, self.scale)
+        return min(max(raw, self.scale[0]), self.scale[1]), reason
+
+    def _finish(self, parsed: list) -> SimilarityResult:
+        lo, hi = self.scale
+        raw = sum(p[0] for p in parsed) / len(parsed)
+        reason = next((p[1] for p in parsed if p[1]), None)
+        return SimilarityResult(score=(raw - lo) / (hi - lo), raw=raw, reason=reason)
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> SimilarityResult:
+        if self._llm is None:
+            raise RuntimeError("this judge only has an async llm; use aevaluate()/ascore()")
+        orders = [(a, b), (b, a)] if self.symmetric else [(a, b)]
+        return self._finish([self._parse(self._llm(self._render(x, y, context))) for x, y in orders])
+
+    async def acompute(self, a: Any, b: Any, *, context: Context = None) -> SimilarityResult:
+        if self._allm is None:
+            return await super().acompute(a, b, context=context)  # type: ignore[return-value]
+        orders = [(a, b), (b, a)] if self.symmetric else [(a, b)]
+        parsed = [self._parse(await self._allm(self._render(x, y, context))) for x, y in orders]
+        return self._finish(parsed)
+
+
+def parse_rubric_output(
+    text: str, names: Sequence[str], scale: Tuple[float, float]
+) -> Tuple[Dict[str, float], Optional[str]]:
+    """Extract ({dimension: score}, reason) from a rubric judge reply (JSON with a "scores" object)."""
+    match = re.search(r"\{.*\}", text, re.S)
+    if not match:
+        raise JudgeParseError(f"no JSON found in judge reply: {text[:200]!r}")
+    try:
+        obj = json.loads(match.group(0))
+        scores = obj["scores"]
+        out = {n: min(max(float(scores[n]), scale[0]), scale[1]) for n in names}
+    except (ValueError, TypeError, KeyError) as e:
+        raise JudgeParseError(f"missing or invalid dimension scores ({e!r}): {text[:200]!r}") from e
+    return out, obj.get("reason")
+
+
+class RubricJudge(Similarity):
+    """Multi-dimension LLM judge: one call scores several aspects of agreement (1-5 each).
+
+    Default dimensions: ``core_facts``, ``specifics``, ``supporting_detail``, ``stance``,
+    ``format`` (see ``repeatability_evals.prompts.DEFAULT_DIMENSIONS``). The similarity score is
+    their weighted mean, normalised to [0, 1]; the per-dimension scores are kept on the result
+    (``metadata["dimensions"]``) and averaged per dimension in ``PairwiseConsistency`` details, so
+    you can see *what* varies between runs (facts? format?) and not just *whether* they differ.
+
+    Pass your own ``dimensions`` (any number, any weights) to adapt it, e.g. for agent traces.
+    Weights are normalised to sum to 1. ``symmetric=True`` judges both orders and averages.
+    """
+
+    name = "rubric_judge"
+
+    def __init__(
+        self,
+        llm: Optional[CompleteFn] = None,
+        allm: Optional[ACompleteFn] = None,
+        *,
+        dimensions: Sequence[Dimension] = DEFAULT_DIMENSIONS,
+        prompt: Optional[str] = None,
+        scale: Tuple[float, float] = (1.0, 5.0),
+        symmetric: bool = False,
+    ) -> None:
+        if llm is None and allm is None:
+            raise ValueError("provide llm and/or allm")
+        if scale[1] <= scale[0]:
+            raise ValueError("scale must be (low, high) with high > low")
+        if not dimensions or any(d.weight <= 0 for d in dimensions):
+            raise ValueError("need at least one dimension, each with weight > 0")
+        if len({d.name for d in dimensions}) != len(dimensions):
+            raise ValueError("dimension names must be unique")
+        self._llm, self._allm = llm, allm
+        self.dimensions = tuple(dimensions)
+        self.prompt = prompt or build_rubric_prompt(self.dimensions)
+        self.scale, self.symmetric = scale, symmetric
+        self._names = [d.name for d in self.dimensions]
+        total = sum(d.weight for d in self.dimensions)
+        self._weights = {d.name: d.weight / total for d in self.dimensions}
+
+    _render = LLMJudge._render
+
+    def _parse(self, reply: str) -> Tuple[Dict[str, float], Optional[str]]:
+        return parse_rubric_output(reply, self._names, self.scale)
+
+    def _finish(self, parsed: list) -> SimilarityResult:
+        lo, hi = self.scale
+        raw_dims = {n: sum(p[0][n] for p in parsed) / len(parsed) for n in self._names}
+        norm = {n: (v - lo) / (hi - lo) for n, v in raw_dims.items()}
+        score = sum(self._weights[n] * norm[n] for n in self._names)
+        raw = lo + score * (hi - lo)
+        reason = next((p[1] for p in parsed if p[1]), None)
+        return SimilarityResult(score, raw=raw, reason=reason, metadata={"dimensions": norm})
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> SimilarityResult:
+        if self._llm is None:
+            raise RuntimeError("this judge only has an async llm; use aevaluate()/ascore()")
+        orders = [(a, b), (b, a)] if self.symmetric else [(a, b)]
+        return self._finish([self._parse(self._llm(self._render(x, y, context))) for x, y in orders])
+
+    async def acompute(self, a: Any, b: Any, *, context: Context = None) -> SimilarityResult:
+        if self._allm is None:
+            return await super().acompute(a, b, context=context)  # type: ignore[return-value]
+        orders = [(a, b), (b, a)] if self.symmetric else [(a, b)]
+        parsed = [self._parse(await self._allm(self._render(x, y, context))) for x, y in orders]
+        return self._finish(parsed)

repeatability_evals/similarity/ranking.py

@@ -0,0 +1,46 @@
+"""Similarity for *ordered* lists, where position matters (ranked retrieval, tool sequences)."""
+from __future__ import annotations
+
+from typing import Any, Hashable, List, Sequence
+
+from .base import Context, Similarity
+
+
+def _unique(seq: Sequence[Hashable]) -> List[Hashable]:
+    return list(dict.fromkeys(seq))
+
+
+class RankBiasedOverlap(Similarity):
+    """Rank-biased overlap (Webber et al., 2010), extrapolated form on the common depth.
+
+    Top ranks count more than lower ones; ``p`` sets how top-heavy (0.9 means roughly the
+    top ~10 ranks carry most of the weight). Identical rankings score 1.0, disjoint 0.0.
+    Lists are truncated to the shorter length; duplicates keep their first position.
+    """
+
+    name = "rank_biased_overlap"
+
+    def __init__(self, p: float = 0.9) -> None:
+        if not 0.0 < p < 1.0:
+            raise ValueError("p must be in (0, 1)")
+        self.p = p
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        la, lb = _unique(list(a)), _unique(list(b))
+        if not la and not lb:
+            return 1.0
+        k = min(len(la), len(lb))
+        if k == 0:
+            return 0.0
+        p, seen_a, seen_b = self.p, set(), set()
+        overlap, weighted = 0, 0.0
+        for d in range(1, k + 1):
+            x, y = la[d - 1], lb[d - 1]
+            if x == y:
+                overlap += 1
+            else:
+                overlap += (x in seen_b) + (y in seen_a)
+            seen_a.add(x)
+            seen_b.add(y)
+            weighted += (overlap / d) * p**d
+        return (overlap / k) * p**k + ((1 - p) / p) * weighted

repeatability_evals/similarity/sets.py

@@ -0,0 +1,53 @@
+"""Set-overlap similarities over collections of hashable items (IDs, labels, URLs...)."""
+from __future__ import annotations
+
+from typing import Any, Hashable, Iterable, Set
+
+from .base import Context, Similarity
+
+
+def to_set(x: Iterable[Hashable]) -> Set[Hashable]:
+    if isinstance(x, (str, bytes)):
+        raise TypeError(
+            "set similarities compare collections; got a bare string. "
+            "Wrap it (e.g. [text]) or use a text similarity such as NgramJaccard."
+        )
+    return set(x)
+
+
+class JaccardSimilarity(Similarity):
+    """|A & B| / |A | B|. Two empty collections count as identical (1.0)."""
+
+    name = "jaccard"
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        sa, sb = to_set(a), to_set(b)
+        if not sa and not sb:
+            return 1.0
+        return len(sa & sb) / len(sa | sb)
+
+
+class DiceSimilarity(Similarity):
+    """2|A & B| / (|A| + |B|): like Jaccard but more forgiving of partial overlap."""
+
+    name = "dice"
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        sa, sb = to_set(a), to_set(b)
+        if not sa and not sb:
+            return 1.0
+        return 2 * len(sa & sb) / (len(sa) + len(sb))
+
+
+class OverlapCoefficient(Similarity):
+    """|A & B| / min(|A|, |B|): 1.0 when one collection is contained in the other."""
+
+    name = "overlap_coefficient"
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        sa, sb = to_set(a), to_set(b)
+        if not sa and not sb:
+            return 1.0
+        if not sa or not sb:
+            return 0.0
+        return len(sa & sb) / min(len(sa), len(sb))

repeatability_evals/similarity/text.py

@@ -0,0 +1,114 @@
+"""Lexical text similarities (no model calls)."""
+from __future__ import annotations
+
+import difflib
+import re
+from typing import Any, Callable, List, Optional, Sequence, Set, Tuple, Union
+
+from .base import Context, Similarity
+
+_TOKEN_RE = re.compile(r"\w+", re.UNICODE)
+
+
+def tokenize(text: str) -> List[str]:
+    """Lower-cased word tokens (Unicode-aware, punctuation dropped)."""
+    return _TOKEN_RE.findall(text.lower())
+
+
+def default_normalize(text: str) -> str:
+    """Case-fold and collapse whitespace."""
+    return " ".join(text.casefold().split())
+
+
+def ngram_set(text: str, n: int = 3) -> Set[Tuple[str, ...]]:
+    """Word n-grams of ``text``. Text shorter than ``n`` yields one short gram."""
+    if n < 1:
+        raise ValueError("n must be >= 1")
+    tokens = tokenize(text)
+    if not tokens:
+        return set()
+    if len(tokens) < n:
+        return {tuple(tokens)}
+    return {tuple(tokens[k : k + n]) for k in range(len(tokens) - n + 1)}
+
+
+class ExactMatch(Similarity):
+    """1.0 if the normalised strings are equal, else 0.0."""
+
+    name = "exact_match"
+
+    def __init__(self, normalize: Optional[Callable[[str], str]] = default_normalize) -> None:
+        self._normalize = normalize or (lambda s: s)
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        return 1.0 if self._normalize(str(a)) == self._normalize(str(b)) else 0.0
+
+
+class NgramJaccard(Similarity):
+    """Jaccard over word n-grams.
+
+    Each item is a string or a sequence of strings (e.g. retrieved chunks); a sequence is
+    pooled into one n-gram set. That makes it robust to re-chunking: when chunk IDs
+    change but the underlying text does not, the score stays high.
+    """
+
+    name = "ngram_jaccard"
+
+    def __init__(self, n: int = 3) -> None:
+        if n < 1:
+            raise ValueError("n must be >= 1")
+        self.n = n
+
+    def _grams(self, item: Union[str, Sequence[str]]) -> Set[Tuple[str, ...]]:
+        texts = [item] if isinstance(item, str) else list(item)
+        grams: Set[Tuple[str, ...]] = set()
+        for t in texts:
+            grams |= ngram_set(t, self.n)
+        return grams
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        ga, gb = self._grams(a), self._grams(b)
+        if not ga and not gb:
+            return 1.0
+        return len(ga & gb) / len(ga | gb)
+
+
+class RougeL(Similarity):
+    """ROUGE-L F-measure: longest common token subsequence, so word order matters."""
+
+    name = "rouge_l"
+
+    @staticmethod
+    def _lcs(x: List[str], y: List[str]) -> int:
+        if not x or not y:
+            return 0
+        prev = [0] * (len(y) + 1)
+        for xi in x:
+            cur = [0]
+            for j, yj in enumerate(y, 1):
+                cur.append(prev[j - 1] + 1 if xi == yj else max(prev[j], cur[j - 1]))
+            prev = cur
+        return prev[-1]
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        ta, tb = tokenize(str(a)), tokenize(str(b))
+        if not ta and not tb:
+            return 1.0
+        lcs = self._lcs(ta, tb)
+        if lcs == 0:
+            return 0.0
+        p, r = lcs / len(tb), lcs / len(ta)
+        return 2 * p * r / (p + r)
+
+
+class EditSimilarity(Similarity):
+    """Character-level similarity ratio (``difflib``); also works on lists of tokens."""
+
+    name = "edit_similarity"
+
+    def compute(self, a: Any, b: Any, *, context: Context = None) -> float:
+        if isinstance(a, str):
+            a, b = default_normalize(a), default_normalize(str(b))
+        if len(a) == 0 and len(b) == 0:
+            return 1.0
+        return difflib.SequenceMatcher(None, a, b, autojunk=False).ratio()

repeatability_evals/stats.py

@@ -0,0 +1,27 @@
+"""Small statistics helpers (standard library only)."""
+from __future__ import annotations
+
+import random
+from typing import Sequence, Tuple
+
+
+def mean(values: Sequence[float]) -> float:
+    if not values:
+        raise ValueError("mean() of an empty sequence")
+    return sum(values) / len(values)
+
+
+def bootstrap_ci(
+    values: Sequence[float], n_boot: int = 2000, alpha: float = 0.05, seed: int = 0
+) -> Tuple[float, float]:
+    """Percentile bootstrap confidence interval for the mean of ``values``."""
+    if not values:
+        raise ValueError("bootstrap_ci() of an empty sequence")
+    if len(values) == 1:
+        return (values[0], values[0])
+    rng = random.Random(seed)
+    n = len(values)
+    means = sorted(sum(values[rng.randrange(n)] for _ in range(n)) / n for _ in range(n_boot))
+    lo = means[int((alpha / 2) * n_boot)]
+    hi = means[min(n_boot - 1, int((1 - alpha / 2) * n_boot))]
+    return (lo, hi)

repeatability_evals/types.py

@@ -0,0 +1,43 @@
+"""Result and data types shared across the package."""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+Pair = Tuple[int, int]
+
+
+@dataclass(frozen=True)
+class SimilarityResult:
+    """Outcome of comparing two items. ``score`` is always in [0, 1] (1 = identical)."""
+
+    score: float
+    raw: Optional[float] = None  # the un-normalised value, e.g. a 1-5 judge rating
+    reason: Optional[str] = None  # free-text rationale, e.g. from an LLM judge
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class PairScore:
+    i: int
+    j: int
+    score: float
+    raw: Optional[float] = None
+    reason: Optional[str] = None
+    dimensions: Optional[Dict[str, float]] = None  # per-rubric-dimension scores in [0, 1]
+
+
+@dataclass
+class ConsistencyResult:
+    """Consistency of one group of N items (e.g. N runs of the same input)."""
+
+    metric: str
+    score: float  # headline value in [0, 1]; 1 = fully consistent
+    min_score: float  # the weakest compared pair (the "worst-case" view)
+    n_items: int
+    n_pairs: int  # pairs actually compared (== number of similarity calls)
+    pairs: List[PairScore] = field(default_factory=list)
+    details: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)

repeatability_evals-0.1.0.dist-info/METADATA

@@ -0,0 +1,147 @@
+Metadata-Version: 2.5
+Name: repeatability-evals
+Version: 0.1.0
+Summary: Consistency evals for non-deterministic systems: LLM generation, agentic retrieval, agent workflows.
+Project-URL: Homepage, https://github.com/krishna-gogineni-765/repeatability-evals
+Project-URL: Issues, https://github.com/krishna-gogineni-765/repeatability-evals/issues
+Author: Krishna Gogineni
+License: MIT
+License-File: LICENSE
+Keywords: agents,consistency,evaluation,llm,llm-as-judge,non-determinism,rag,reliability,repeatability,retrieval
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# repeatability-evals
+
+**Consistency evals for non-deterministic systems: LLM generation, agentic retrieval, agent workflows.**
+
+Composable tools for measuring **how consistent an LLM system is with itself**: the same
+input run several times, or an input and its paraphrases, and how much the outputs agree.
+
+Most eval libraries score one run for quality. A system can average 90% on quality and still
+give a customer a different answer every time they ask. This package measures that spread.
+
+It is deliberately **unaware of what you are comparing**. It has no notion of "answers",
+"retrieval" or "agents": it compares *items*. You choose the items and the similarity:
+
+| You have N of ...                    | Use a similarity such as                         |
+|--------------------------------------|--------------------------------------------------|
+| final answers (free text)            | `LLMJudge`, `EmbeddingCosine`, `RougeL`          |
+| retrieved chunk IDs                  | `JaccardSimilarity`                              |
+| retrieved chunk text                 | `NgramJaccard` (robust to re-chunking)           |
+| ranked result lists                  | `RankBiasedOverlap`                              |
+| agent tool-call traces               | `EditSimilarity` / `ExactMatch` on a trace string|
+| anything else                        | subclass `Similarity`, or pass a plain function  |
+
+Consistency is **not** correctness: a system that is consistently wrong scores 1.0. Pair this
+with an accuracy metric.
+
+## Install
+
+```bash
+pip install repeatability-evals   # pure standard library, no required dependencies
+```
+
+## Quickstart
+
+```python
+from repeatability_evals import (
+    PairwiseConsistency, JaccardSimilarity, NgramJaccard, LLMJudge, evaluate_groups,
+)
+
+# 1. Retrieval: did the 5 runs fetch the same chunks?
+ids = PairwiseConsistency(JaccardSimilarity())
+print(ids.evaluate([["a", "b", "c"], ["a", "b", "d"], ["a", "b", "c"]]).score)
+
+# 2. Same evidence, different chunk boundaries?
+text = PairwiseConsistency(NgramJaccard(n=3))
+print(text.evaluate([["backup jobs run nightly"], ["backup jobs", "run nightly"]]).score)
+
+# 3. Answers: an LLM judge scores semantic equivalence. Bring any `str -> str` function.
+def complete(prompt: str) -> str: ...            # your model call
+answers = PairwiseConsistency(LLMJudge(complete), agreement_threshold=0.75)
+r = answers.evaluate(["Retention is 30 days.", "30 days.", "Retention is 90 days."],
+                     context={"query": "How long are backups retained?"})
+print(r.score, r.details["agreement_rate"], [p.reason for p in r.pairs])
+
+# 4. A whole dataset: one group per question, with a bootstrap confidence interval.
+report = evaluate_groups({"q1": runs_q1, "q2": runs_q2}, ids)
+print(report.score, report.ci, report.worst(3))
+```
+
+## Design
+
+Three small pieces, each replaceable:
+
+* **`Similarity`**: compare two items, return a score in `[0, 1]` (optionally with a raw value
+  and a rationale). Built in: set overlap (`Jaccard`, `Dice`, `OverlapCoefficient`), text
+  (`ExactMatch`, `NgramJaccard`, `RougeL`, `EditSimilarity`), ranking (`RankBiasedOverlap`),
+  embeddings (`EmbeddingCosine`, bring your own embed function), and `RubricJudge` and `LLMJudge`.
+* **`PairSampler`**: which pairs of the N items to compare (see below).
+* **`ConsistencyMetric`**: combines them. `PairwiseConsistency` averages similarity over
+  sampled pairs; `ClusterConsistency` groups items into "same answer" clusters and reports the
+  largest cluster's share plus the entropy of the partition; `TotalAgreement` is the
+  all-N-identical rate.
+
+Everything has an async twin (`aevaluate`, `aevaluate_groups`) with bounded concurrency, which
+matters when the similarity is an LLM call.
+
+### Sampling instead of all pairs
+
+All pairs cost N(N-1)/2 comparisons: 10 for 5 runs, 45 for 10. The default `CycleSampler`
+compares a random cycle through the runs, so N pairs, and every run is compared at least twice.
+It is seeded and reproducible. `AllPairs()` gives the exact value; `RandomPairs(k)` and
+`AnchorPairs()` (every run against one reference) are also available. `benchmarks/sampling_error.py`
+measures the error this introduces on synthetic data so you can see the trade-off for your N.
+
+### The LLM judges
+
+`RubricJudge` is the recommended judge for free-text outputs. One call scores five dimensions
+of agreement (1-5 each): `core_facts`, `specifics` (names, numbers, dates), `supporting_detail`,
+`stance` (answer vs hedge vs refuse) and `format`. The headline score is their weighted mean
+(core facts count most), and `PairwiseConsistency` reports the mean per dimension in
+`details["dimensions"]`, so you can see *what* varies between runs: a system whose facts are
+stable but whose formatting drifts looks very different from one that changes its answer.
+Pass your own `Dimension`s to add, remove or reweight (e.g. for agent traces).
+
+```python
+from repeatability_evals import RubricJudge, PairwiseConsistency
+m = PairwiseConsistency(RubricJudge(complete))
+r = m.evaluate(runs, context={"query": question})
+print(r.score, r.details["dimensions"])   # {'core_facts': 0.95, 'format': 0.4, ...}
+```
+
+`LLMJudge` is the single-score version (one 1-5 semantic-equivalence rating). Both take any completion function, so it works with any provider or a local model. The
+default rubrics (1-5, normalised to `[0, 1]`) are in `repeatability_evals.prompts`. `symmetric=True`
+judges both orderings to cancel position bias at twice the cost. **Validate any judge against
+human ratings on your own data before relying on it**; agreement depends on the domain and the
+judge model.
+
+## Writing your own similarity
+
+```python
+from repeatability_evals import Similarity, PairwiseConsistency
+
+class SameFirstSentence(Similarity):
+    name = "same_first_sentence"
+    def compute(self, a, b, *, context=None):
+        return float(a.split(".")[0] == b.split(".")[0])
+
+PairwiseConsistency(SameFirstSentence())
+```
+
+or just `PairwiseConsistency(lambda a, b: ...)`.
+
+## Status
+
+Early (0.1). The API may change before 1.0.
+
+## License
+
+MIT