repeatability-evals 0.1.0__py3-none-any.whl

repeatability_evals/__init__.py

@@ -0,0 +1,38 @@
+"""repeatability-evals: consistency evals for non-deterministic LLM systems."""
+from .dataset import DatasetResult, aevaluate_groups, evaluate_groups
+from .metrics import ClusterConsistency, ConsistencyMetric, PairwiseConsistency, TotalAgreement
+from .prompts import DEFAULT_DIMENSIONS, SEMANTIC_EQUIVALENCE_PROMPT, Dimension, build_rubric_prompt
+from .sampling import AllPairs, AnchorPairs, CycleSampler, PairSampler, RandomPairs, cycle_pairs
+from .similarity import (
+    DiceSimilarity,
+    EditSimilarity,
+    EmbeddingCosine,
+    ExactMatch,
+    FunctionSimilarity,
+    JaccardSimilarity,
+    JudgeParseError,
+    LLMJudge,
+    NgramJaccard,
+    OverlapCoefficient,
+    RankBiasedOverlap,
+    RougeL,
+    RubricJudge,
+    Similarity,
+    as_similarity,
+)
+from .stats import bootstrap_ci
+from .types import ConsistencyResult, PairScore, SimilarityResult
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ConsistencyMetric", "PairwiseConsistency", "ClusterConsistency", "TotalAgreement",
+    "evaluate_groups", "aevaluate_groups", "DatasetResult", "bootstrap_ci",
+    "Similarity", "FunctionSimilarity", "as_similarity",
+    "JaccardSimilarity", "DiceSimilarity", "OverlapCoefficient",
+    "ExactMatch", "NgramJaccard", "RougeL", "EditSimilarity",
+    "RankBiasedOverlap", "EmbeddingCosine", "LLMJudge", "RubricJudge", "JudgeParseError",
+    "Dimension", "DEFAULT_DIMENSIONS", "build_rubric_prompt",
+    "PairSampler", "AllPairs", "CycleSampler", "RandomPairs", "AnchorPairs", "cycle_pairs",
+    "ConsistencyResult", "PairScore", "SimilarityResult", "SEMANTIC_EQUIVALENCE_PROMPT",
+]

repeatability_evals/dataset.py

@@ -0,0 +1,80 @@
+"""Run a metric over many groups (one group = the N runs of one input) and summarise."""
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass, field
+from typing import Any, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple
+
+from .metrics import ConsistencyMetric
+from .stats import bootstrap_ci, mean
+from .types import ConsistencyResult
+
+
+@dataclass
+class DatasetResult:
+    metric: str
+    score: float  # mean of per-group scores
+    n_groups: int
+    ci: Optional[Tuple[float, float]] = None  # bootstrap CI of ``score``
+    per_group: Dict[Hashable, ConsistencyResult] = field(default_factory=dict)
+
+    def worst(self, k: int = 5) -> List[Tuple[Hashable, ConsistencyResult]]:
+        """The ``k`` least consistent groups: where to look first when debugging."""
+        return sorted(self.per_group.items(), key=lambda kv: kv[1].score)[:k]
+
+    def to_records(self) -> List[Dict[str, Any]]:
+        return [
+            {"group": key, "score": r.score, "min_score": r.min_score,
+             "n_items": r.n_items, "n_pairs": r.n_pairs, **{f"d_{k}": v for k, v in r.details.items()
+                                                             if isinstance(v, (int, float, str, bool))}}
+            for key, r in self.per_group.items()
+        ]
+
+
+def _summarise(
+    metric: ConsistencyMetric, results: Dict[Hashable, ConsistencyResult],
+    ci: bool, n_boot: int, seed: int,
+) -> DatasetResult:
+    if not results:
+        raise ValueError("no groups to evaluate")
+    scores = [r.score for r in results.values()]
+    interval = bootstrap_ci(scores, n_boot=n_boot, seed=seed) if ci else None
+    return DatasetResult(metric.name, mean(scores), len(results), interval, results)
+
+
+def evaluate_groups(
+    groups: Mapping[Hashable, Sequence[Any]],
+    metric: ConsistencyMetric,
+    *,
+    contexts: Optional[Mapping[Hashable, Mapping[str, Any]]] = None,
+    ci: bool = True,
+    n_boot: int = 2000,
+    seed: int = 0,
+) -> DatasetResult:
+    """``groups`` maps a group key (e.g. a question ID) to its N items; ``contexts`` optionally
+    maps the same keys to per-group context such as ``{"query": "..."}`` for an LLM judge."""
+    results = {
+        key: metric.evaluate(items, context=(contexts or {}).get(key))
+        for key, items in groups.items()
+    }
+    return _summarise(metric, results, ci, n_boot, seed)
+
+
+async def aevaluate_groups(
+    groups: Mapping[Hashable, Sequence[Any]],
+    metric: ConsistencyMetric,
+    *,
+    contexts: Optional[Mapping[Hashable, Mapping[str, Any]]] = None,
+    concurrency: int = 4,
+    ci: bool = True,
+    n_boot: int = 2000,
+    seed: int = 0,
+) -> DatasetResult:
+    sem = asyncio.Semaphore(concurrency)
+
+    async def one(key: Hashable, items: Sequence[Any]) -> Tuple[Hashable, ConsistencyResult]:
+        async with sem:
+            return key, await metric.aevaluate(items, context=(contexts or {}).get(key))
+
+    pairs = await asyncio.gather(*(one(k, v) for k, v in groups.items()))
+    return _summarise(metric, dict(pairs), ci, n_boot, seed)

repeatability_evals/metrics.py

@@ -0,0 +1,223 @@
+"""Consistency metrics: turn a Similarity (or an equality test) into a score for N items.
+
+A metric knows nothing about *what* the items are. Feed it the N final answers of N runs,
+N sets of retrieved chunk IDs, N tool-call traces, N paraphrase-conditioned outputs... and
+pick the Similarity that suits them.
+"""
+from __future__ import annotations
+
+import asyncio
+import math
+from abc import ABC, abstractmethod
+from typing import Any, Callable, List, Optional, Sequence
+
+from .sampling import CycleSampler, PairSampler
+from .similarity.base import Context, Similarity, as_similarity
+from .similarity.text import default_normalize
+from .stats import mean
+from .types import ConsistencyResult, PairScore, SimilarityResult
+
+
+class ConsistencyMetric(ABC):
+    """Score how consistent a group of N items is. Higher is more consistent (1.0 = fully)."""
+
+    name: str = "consistency"
+
+    @abstractmethod
+    def evaluate(self, items: Sequence[Any], *, context: Context = None) -> ConsistencyResult: ...
+
+    async def aevaluate(
+        self, items: Sequence[Any], *, context: Context = None
+    ) -> ConsistencyResult:
+        return await asyncio.to_thread(self.evaluate, items, context=context)
+
+
+def _degenerate(name: str, n_items: int) -> ConsistencyResult:
+    # Nothing to disagree with: zero or one item is trivially consistent.
+    return ConsistencyResult(name, 1.0, 1.0, n_items, 0, [], {"degenerate": True})
+
+
+def _pair_score(i: int, j: int, r: SimilarityResult) -> PairScore:
+    return PairScore(i, j, r.score, r.raw, r.reason, r.metadata.get("dimensions"))
+
+
+class PairwiseConsistency(ConsistencyMetric):
+    """Average similarity over a sampled set of item pairs.
+
+    Args:
+        similarity: a ``Similarity`` (or any ``fn(a, b) -> float``).
+        sampler: which pairs to compare. Default ``CycleSampler()``: N pairs, every item
+            covered, deterministic. Use ``AllPairs()`` for the exact all-pairs value.
+        preprocess: applied to each item before comparing, e.g. ``lambda r: r["chunk_ids"]``.
+        agreement_threshold: if set, ``details["agreement_rate"]`` reports the share of
+            compared pairs scoring at least this (on the normalised [0, 1] scale).
+        aggregate: how to combine pair scores into ``score`` (default: mean).
+        concurrency: max simultaneous similarity calls in ``aevaluate``.
+    """
+
+    def __init__(
+        self,
+        similarity: Any,
+        *,
+        sampler: Optional[PairSampler] = None,
+        preprocess: Optional[Callable[[Any], Any]] = None,
+        agreement_threshold: Optional[float] = None,
+        aggregate: Optional[Callable[[Sequence[float]], float]] = None,
+        name: Optional[str] = None,
+        concurrency: int = 8,
+    ) -> None:
+        self.similarity: Similarity = as_similarity(similarity)
+        self.sampler = sampler or CycleSampler()
+        self.preprocess = preprocess
+        self.agreement_threshold = agreement_threshold
+        self.aggregate = aggregate or mean
+        self.name = name or f"pairwise_{self.similarity.name}"
+        self.concurrency = concurrency
+
+    def _prepare(self, items: Sequence[Any]) -> List[Any]:
+        return [self.preprocess(x) for x in items] if self.preprocess else list(items)
+
+    def _finish(self, n_items: int, pairs: List[PairScore]) -> ConsistencyResult:
+        scores = [p.score for p in pairs]
+        details = {"similarity": self.similarity.name, "sampler": type(self.sampler).__name__}
+        if self.agreement_threshold is not None:
+            details["agreement_rate"] = sum(s >= self.agreement_threshold for s in scores) / len(scores)
+        dims = [p.dimensions for p in pairs if p.dimensions]
+        if dims:  # rubric judges: mean score per dimension, so you can see *what* varies
+            details["dimensions"] = {k: mean([d[k] for d in dims]) for k in dims[0]}
+        return ConsistencyResult(
+            self.name, self.aggregate(scores), min(scores), n_items, len(pairs), pairs, details
+        )
+
+    def evaluate(self, items: Sequence[Any], *, context: Context = None) -> ConsistencyResult:
+        prepared = self._prepare(items)
+        pairs = self.sampler.sample(len(prepared))
+        if not pairs:
+            return _degenerate(self.name, len(prepared))
+        scored = [
+            _pair_score(i, j, self.similarity.score(prepared[i], prepared[j], context=context))
+            for i, j in pairs
+        ]
+        return self._finish(len(prepared), scored)
+
+    async def aevaluate(
+        self, items: Sequence[Any], *, context: Context = None
+    ) -> ConsistencyResult:
+        prepared = self._prepare(items)
+        pairs = self.sampler.sample(len(prepared))
+        if not pairs:
+            return _degenerate(self.name, len(prepared))
+        sem = asyncio.Semaphore(self.concurrency)
+
+        async def one(i: int, j: int) -> PairScore:
+            async with sem:
+                r = await self.similarity.ascore(prepared[i], prepared[j], context=context)
+            return _pair_score(i, j, r)
+
+        scored = await asyncio.gather(*(one(i, j) for i, j in pairs))
+        return self._finish(len(prepared), list(scored))
+
+
+class ClusterConsistency(ConsistencyMetric):
+    """Group items into 'same answer' clusters; the score is the largest cluster's share.
+
+    Greedy: each item joins the first cluster whose first member it matches
+    (``similarity >= threshold``), else starts a new one. Order-dependent by construction;
+    it never compares more than items x clusters pairs. ``details`` reports the cluster
+    sizes and the normalised entropy of the partition (the discrete form of semantic
+    entropy: 0 = one cluster, 1 = every item alone).
+    """
+
+    def __init__(
+        self, similarity: Any, *, threshold: float = 0.75, name: Optional[str] = None,
+        preprocess: Optional[Callable[[Any], Any]] = None, concurrency: int = 8,
+    ) -> None:
+        if not 0.0 <= threshold <= 1.0:
+            raise ValueError("threshold must be in [0, 1]")
+        self.similarity: Similarity = as_similarity(similarity)
+        self.threshold, self.preprocess, self.concurrency = threshold, preprocess, concurrency
+        self.name = name or f"cluster_{self.similarity.name}"
+
+    def _prepare(self, items: Sequence[Any]) -> List[Any]:
+        return [self.preprocess(x) for x in items] if self.preprocess else list(items)
+
+    def _result(self, n: int, members: List[List[int]], pairs: List[PairScore]) -> ConsistencyResult:
+        sizes = sorted((len(m) for m in members), reverse=True)
+        probs = [s / n for s in sizes]
+        entropy = -sum(p * math.log(p) for p in probs)
+        norm = entropy / math.log(n) if n > 1 else 0.0
+        details = {
+            "similarity": self.similarity.name, "threshold": self.threshold,
+            "cluster_sizes": sizes, "n_clusters": len(sizes), "normalized_entropy": norm,
+            "clusters": members,
+        }
+        scores = [p.score for p in pairs]
+        return ConsistencyResult(
+            self.name, sizes[0] / n, min(scores) if scores else 1.0, n, len(pairs), pairs, details
+        )
+
+    def evaluate(self, items: Sequence[Any], *, context: Context = None) -> ConsistencyResult:
+        prepared = self._prepare(items)
+        if len(prepared) < 2:
+            return _degenerate(self.name, len(prepared))
+        members: List[List[int]] = []
+        pairs: List[PairScore] = []
+        for idx, item in enumerate(prepared):
+            for cluster in members:
+                r = self.similarity.score(prepared[cluster[0]], item, context=context)
+                pairs.append(_pair_score(cluster[0], idx, r))
+                if r.score >= self.threshold:
+                    cluster.append(idx)
+                    break
+            else:
+                members.append([idx])
+        return self._result(len(prepared), members, pairs)
+
+    async def aevaluate(
+        self, items: Sequence[Any], *, context: Context = None
+    ) -> ConsistencyResult:
+        prepared = self._prepare(items)
+        if len(prepared) < 2:
+            return _degenerate(self.name, len(prepared))
+        sem = asyncio.Semaphore(self.concurrency)
+        members: List[List[int]] = []
+        pairs: List[PairScore] = []
+
+        async def cmp(rep: int, idx: int) -> PairScore:
+            async with sem:
+                r = await self.similarity.ascore(prepared[rep], prepared[idx], context=context)
+            return _pair_score(rep, idx, r)
+
+        for idx in range(len(prepared)):
+            # compare against every current cluster at once, then take the first match
+            results = await asyncio.gather(*(cmp(c[0], idx) for c in members))
+            pairs.extend(results)
+            hit = next((k for k, p in enumerate(results) if p.score >= self.threshold), None)
+            if hit is None:
+                members.append([idx])
+            else:
+                members[hit].append(idx)
+        return self._result(len(prepared), members, pairs)
+
+
+class TotalAgreement(ConsistencyMetric):
+    """1.0 if *all* items are identical after normalisation, else 0.0.
+
+    The mean over a dataset is the total agreement rate (TAR@N of Atil et al., 2024).
+    Pass ``normalize=str`` for raw-output agreement, or an answer extractor for parsed
+    agreement.
+    """
+
+    name = "total_agreement"
+
+    def __init__(self, normalize: Callable[[Any], Any] = default_normalize) -> None:
+        self.normalize = normalize
+
+    def evaluate(self, items: Sequence[Any], *, context: Context = None) -> ConsistencyResult:
+        if len(items) < 2:
+            return _degenerate(self.name, len(items))
+        distinct = {self.normalize(x) for x in items}
+        score = 1.0 if len(distinct) == 1 else 0.0
+        return ConsistencyResult(
+            self.name, score, score, len(items), 0, [], {"n_distinct": len(distinct)}
+        )

repeatability_evals/prompts.py

@@ -0,0 +1,120 @@
+"""Default prompt for the LLM judge. Placeholders: {query}, {item_a}, {item_b}."""
+
+SEMANTIC_EQUIVALENCE_PROMPT = """\
+You are comparing two responses that were produced independently for the same input. \
+Decide how closely they agree in meaning.
+
+Input (may be empty):
+{query}
+
+Response A:
+{item_a}
+
+Response B:
+{item_b}
+
+Rate their semantic equivalence on a 1-5 scale:
+5 = Identical meaning: same core facts, no meaningful differences
+4 = Very similar: only minor phrasing or presentational differences
+3 = Partially equivalent: some differences in detail or completeness
+2 = Mostly different: notable factual or informational differences
+1 = Completely different information: little to no overlap
+
+Guidelines:
+- Judge meaning, not wording, length, formatting, or the order of points.
+- Do not judge which response is better or more correct. Only judge whether they agree; \
+two equally wrong responses that say the same thing score 5.
+- Extra detail in one response lowers the score only when it adds or changes substantive \
+information that the other lacks or contradicts.
+
+Reply with JSON only: {"reason": "<one or two sentences>", "score": <integer 1-5>}
+"""
+
+
+# --------------------------------------------------------------------------------------
+# Multi-dimension rubric. One LLM call scores each dimension 1-5; ``RubricJudge`` combines
+# them with the weights below. Edit, drop or add dimensions by passing your own ``Dimension``s.
+# --------------------------------------------------------------------------------------
+from dataclasses import dataclass  # noqa: E402
+
+
+@dataclass(frozen=True)
+class Dimension:
+    name: str
+    description: str  # what 5 vs 1 means; shown to the judge verbatim
+    weight: float = 1.0
+
+
+DEFAULT_DIMENSIONS = (
+    Dimension(
+        "core_facts",
+        "Do both responses make the same central claims, conclusions or answer? "
+        "5 = same core answer; 3 = same topic but the main conclusion is partly different or "
+        "one omits part of it; 1 = different or contradictory core answers.",
+        0.40,
+    ),
+    Dimension(
+        "specifics",
+        "Do the concrete specifics agree: names, numbers, dates, versions, identifiers, "
+        "quantities, units, cited sources? "
+        "5 = all shared specifics match; 3 = some differ or are missing; "
+        "1 = specifics conflict. If neither response contains specifics, score 5.",
+        0.25,
+    ),
+    Dimension(
+        "supporting_detail",
+        "Do the non-core details agree: explanations, examples, caveats, background, extra "
+        "context? 5 = same supporting information; 3 = noticeable additions or omissions; "
+        "1 = largely different supporting material. Differences here matter less than core facts.",
+        0.15,
+    ),
+    Dimension(
+        "stance",
+        "Do the responses take the same stance and certainty: direct answer vs hedge vs refusal "
+        "vs 'I don't know', the same recommendation or next step, similar confidence? "
+        "5 = same stance; 3 = same direction but different confidence or caveats; "
+        "1 = opposite stance (e.g. one answers, one declines).",
+        0.10,
+    ),
+    Dimension(
+        "format",
+        "Do the responses have the same form: structure (list vs prose vs table), length, "
+        "ordering, level of verbosity? 5 = essentially the same form; 3 = noticeably different "
+        "structure or length; 1 = completely different form. Ignore wording that doesn't change "
+        "structure.",
+        0.10,
+    ),
+)
+
+RUBRIC_PROMPT_TEMPLATE = """\
+You are comparing two responses that were produced independently for the same input. \
+Judge how closely they agree on each dimension below.
+
+Input (may be empty):
+{query}
+
+Response A:
+{item_a}
+
+Response B:
+{item_b}
+
+Score each dimension from 1 (completely different) to 5 (identical):
+{rubric}
+
+Guidelines:
+- Judge agreement between the two responses only. Do not judge which one is better or more \
+correct; two equally wrong responses that say the same thing agree.
+- Score each dimension independently. A formatting difference must not lower core_facts, and \
+vice versa.
+- Judge meaning, not surface wording.
+
+Reply with JSON only: \
+{"reason": "<one or two sentences on the main differences>", "scores": {%s}}
+"""
+
+
+def build_rubric_prompt(dimensions=DEFAULT_DIMENSIONS) -> str:
+    rubric = "\n".join(f"- {d.name}: {d.description}" for d in dimensions)
+    keys = ", ".join(f'"{d.name}": <integer 1-5>' for d in dimensions)
+    return RUBRIC_PROMPT_TEMPLATE.replace("{rubric}", rubric).replace("%s", keys)

repeatability_evals/sampling.py

@@ -0,0 +1,102 @@
+"""Choosing which pairs of items to compare.
+
+Comparing all N-choose-2 pairs is quadratic; with an LLM judge that gets expensive. The
+samplers here pick a subset deterministically (seeded) so results are reproducible.
+"""
+from __future__ import annotations
+
+import itertools
+import random
+from dataclasses import dataclass
+from typing import List, Optional, Protocol
+
+from .types import Pair
+
+
+class PairSampler(Protocol):
+    def sample(self, n_items: int) -> List[Pair]:
+        """Return distinct (i, j) pairs with i < j, drawn from range(n_items)."""
+        ...
+
+
+def _norm(a: int, b: int) -> Pair:
+    return (a, b) if a < b else (b, a)
+
+
+def cycle_pairs(n_items: int, budget: Optional[int] = None, seed: int = 0) -> List[Pair]:
+    """A random Hamiltonian cycle over the items, topped up with random extra pairs.
+
+    Every item appears in at least two sampled pairs when ``budget >= n_items``, so no
+    item can hide from the comparison. ``budget`` defaults to ``n_items``. When the
+    budget is smaller than the cycle, alternate cycle edges (a matching) are used first
+    so as many items as possible are still covered. When it reaches N-choose-2 you get
+    every pair.
+    """
+    if n_items < 2:
+        return []
+    all_pairs = list(itertools.combinations(range(n_items), 2))
+    budget = n_items if budget is None else budget
+    if budget < 1:
+        raise ValueError("budget must be >= 1")
+    if budget >= len(all_pairs):
+        return all_pairs
+
+    rng = random.Random(seed)
+    order = list(range(n_items))
+    rng.shuffle(order)  # random cycle: no positional bias
+    ring = list(dict.fromkeys(_norm(order[k], order[(k + 1) % n_items]) for k in range(n_items)))
+    ring = ring[0::2] + ring[1::2]  # even edges form a matching -> best coverage if truncated
+    chosen = ring[:budget]
+    if len(chosen) < budget:
+        taken = set(chosen)
+        rest = [p for p in all_pairs if p not in taken]
+        rng.shuffle(rest)
+        chosen += rest[: budget - len(chosen)]
+    return sorted(chosen)
+
+
+@dataclass(frozen=True)
+class AllPairs:
+    """Exact: every pair. Cost grows quadratically."""
+
+    def sample(self, n_items: int) -> List[Pair]:
+        return list(itertools.combinations(range(n_items), 2))
+
+
+@dataclass(frozen=True)
+class CycleSampler:
+    """Default. ``budget`` pairs (default: N) with coverage guaranteed; see ``cycle_pairs``."""
+
+    budget: Optional[int] = None
+    seed: int = 0
+
+    def sample(self, n_items: int) -> List[Pair]:
+        return cycle_pairs(n_items, self.budget, self.seed)
+
+
+@dataclass(frozen=True)
+class RandomPairs:
+    """``budget`` distinct pairs drawn uniformly, with no coverage guarantee."""
+
+    budget: int
+    seed: int = 0
+
+    def sample(self, n_items: int) -> List[Pair]:
+        all_pairs = list(itertools.combinations(range(n_items), 2))
+        if self.budget >= len(all_pairs):
+            return all_pairs
+        return sorted(random.Random(self.seed).sample(all_pairs, self.budget))
+
+
+@dataclass(frozen=True)
+class AnchorPairs:
+    """Compare every item to one anchor (e.g. a reference or baseline run): N-1 pairs."""
+
+    anchor: int = 0
+
+    def sample(self, n_items: int) -> List[Pair]:
+        if n_items < 2:
+            return []
+        if not 0 <= self.anchor < n_items:
+            raise IndexError(f"anchor {self.anchor} out of range for {n_items} items")
+        return [_norm(self.anchor, k) for k in range(n_items) if k != self.anchor]

repeatability_evals/similarity/__init__.py

@@ -0,0 +1,14 @@
+from .base import FunctionSimilarity, Similarity, as_similarity
+from .embedding import EmbeddingCosine
+from .judge import JudgeParseError, LLMJudge, RubricJudge, parse_judge_output, parse_rubric_output
+from .ranking import RankBiasedOverlap
+from .sets import DiceSimilarity, JaccardSimilarity, OverlapCoefficient
+from .text import EditSimilarity, ExactMatch, NgramJaccard, RougeL, ngram_set, tokenize
+
+__all__ = [
+    "Similarity", "FunctionSimilarity", "as_similarity",
+    "JaccardSimilarity", "DiceSimilarity", "OverlapCoefficient",
+    "ExactMatch", "NgramJaccard", "RougeL", "EditSimilarity", "ngram_set", "tokenize",
+    "RankBiasedOverlap", "EmbeddingCosine",
+    "LLMJudge", "RubricJudge", "JudgeParseError", "parse_judge_output", "parse_rubric_output",
+]

repeatability_evals/similarity/base.py

@@ -0,0 +1,85 @@
+"""The one abstraction everything else is built on: comparing two items."""
+from __future__ import annotations
+
+import asyncio
+import math
+from abc import ABC, abstractmethod
+from typing import Any, Awaitable, Callable, Mapping, Optional, Union
+
+from ..types import SimilarityResult
+
+Context = Optional[Mapping[str, Any]]
+_EPS = 1e-9
+
+
+def coerce_result(value: Union[float, int, SimilarityResult], source: str) -> SimilarityResult:
+    """Accept a bare number or a SimilarityResult; enforce the [0, 1] contract."""
+    result = value if isinstance(value, SimilarityResult) else SimilarityResult(score=float(value))
+    s = result.score
+    if math.isnan(s) or s < -_EPS or s > 1 + _EPS:
+        raise ValueError(f"{source} returned {s!r}; similarity scores must be in [0, 1]")
+    if s != min(max(s, 0.0), 1.0):
+        result = SimilarityResult(min(max(s, 0.0), 1.0), result.raw, result.reason, result.metadata)
+    return result
+
+
+class Similarity(ABC):
+    """Compare two items and return how alike they are, in [0, 1] (1 = identical).
+
+    Subclass and implement ``compute``. Return a float, or a ``SimilarityResult`` to
+    attach a raw value or a rationale. Items can be anything (strings, sets of IDs,
+    lists of chunks, tool-call traces); ``context`` carries side information such as the
+    query both items answer. Override ``acompute`` if your similarity does async I/O.
+    """
+
+    name: str = "similarity"
+
+    @abstractmethod
+    def compute(
+        self, a: Any, b: Any, *, context: Context = None
+    ) -> Union[float, SimilarityResult]: ...
+
+    async def acompute(
+        self, a: Any, b: Any, *, context: Context = None
+    ) -> Union[float, SimilarityResult]:
+        return await asyncio.to_thread(self.compute, a, b, context=context)
+
+    # public entry points -------------------------------------------------
+    def score(self, a: Any, b: Any, *, context: Context = None) -> SimilarityResult:
+        return coerce_result(self.compute(a, b, context=context), self.name)
+
+    async def ascore(self, a: Any, b: Any, *, context: Context = None) -> SimilarityResult:
+        return coerce_result(await self.acompute(a, b, context=context), self.name)
+
+    def __call__(self, a: Any, b: Any, *, context: Context = None) -> float:
+        return self.score(a, b, context=context).score
+
+
+class FunctionSimilarity(Similarity):
+    """Wrap a plain ``fn(a, b) -> float`` (and optionally an async twin) as a Similarity."""
+
+    def __init__(
+        self,
+        fn: Callable[..., Union[float, SimilarityResult]],
+        *,
+        afn: Optional[Callable[..., Awaitable[Union[float, SimilarityResult]]]] = None,
+        name: Optional[str] = None,
+    ) -> None:
+        self._fn, self._afn = fn, afn
+        self.name = name or getattr(fn, "__name__", "function_similarity")
+
+    def compute(self, a: Any, b: Any, *, context: Context = None):
+        return self._fn(a, b)
+
+    async def acompute(self, a: Any, b: Any, *, context: Context = None):
+        if self._afn is not None:
+            return await self._afn(a, b)
+        return await super().acompute(a, b, context=context)
+
+
+def as_similarity(obj: Any) -> Similarity:
+    if isinstance(obj, Similarity):
+        return obj
+    if callable(obj):
+        return FunctionSimilarity(obj)
+    raise TypeError(f"expected a Similarity or a callable, got {type(obj).__name__}")