upheld 0.1.0__py3-none-any.whl

upheld/__init__.py

@@ -0,0 +1,28 @@
+from upheld.auditor import Auditor, audit
+from upheld.models import AuditReport, AuditRequest, CitedSpan, Source, Verdict
+from upheld.pipeline import run_audit
+from upheld.providers import Provider, from_anthropic, from_gemini, from_openai, from_perplexity, from_response
+from upheld.verify import EnsembleVerifier, GraniteJudgeVerifier, LettuceVerifier, LexicalVerifier, MiniCheckVerifier, make_verifier
+
+__all__ = [
+    "Auditor",
+    "EnsembleVerifier",
+    "GraniteJudgeVerifier",
+    "MiniCheckVerifier",
+    "AuditReport",
+    "AuditRequest",
+    "CitedSpan",
+    "LettuceVerifier",
+    "LexicalVerifier",
+    "Provider",
+    "Source",
+    "Verdict",
+    "audit",
+    "from_anthropic",
+    "from_gemini",
+    "from_openai",
+    "from_perplexity",
+    "from_response",
+    "make_verifier",
+    "run_audit",
+]

upheld/api.py

@@ -0,0 +1,69 @@
+import asyncio
+import os
+from contextlib import asynccontextmanager, nullcontext
+from importlib.metadata import version
+from pathlib import Path
+from typing import Annotated, Any
+
+from fastapi import Body, FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from upheld.auditor import to_request
+from upheld.evidence import make_embedder, retrieval_name
+from upheld.models import AuditReport, AuditRequest
+from upheld.pipeline import run_audit
+from upheld.providers import Provider
+from upheld.verify import make_verifier
+
+_state: dict = {}
+
+# The built website (web/dist). Served at / when present, so `upheld serve` hosts the
+# landing page and the playground next to the API.
+SITE_DIR = Path(os.environ.get("UPHELD_SITE_DIR", Path(__file__).resolve().parents[2] / "web" / "dist"))
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Load the models once at startup, not per request.
+    _state["verifier"] = make_verifier(os.environ.get("UPHELD_VERIFIER", "ensemble"))
+    _state["embedder"] = make_embedder(os.environ.get("UPHELD_RETRIEVAL", "auto"))
+    # A public deployment can cap concurrent audits so requests queue instead of
+    # competing for one model on a small machine.
+    limit = int(os.environ.get("UPHELD_MAX_CONCURRENT", "0"))
+    _state["slots"] = asyncio.Semaphore(limit) if limit > 0 else None
+    yield
+    _state.clear()
+
+
+app = FastAPI(title="upheld", version=version("upheld"), lifespan=lifespan)
+
+# Origins allowed to call the API from a browser, comma-separated, e.g. a static
+# deploy of the website on another domain.
+if origins := [o.strip() for o in os.environ.get("UPHELD_CORS_ORIGINS", "").split(",") if o.strip()]:
+    app.add_middleware(CORSMiddleware, allow_origins=origins, allow_methods=["GET", "POST"], allow_headers=["Content-Type"])
+
+
+@app.get("/health")
+async def health() -> dict[str, str]:
+    return {"status": "ok", "verifier": _state["verifier"].name, "retrieval": retrieval_name(_state["embedder"])}
+
+
+@app.post("/audit")
+async def audit(req: AuditRequest) -> AuditReport:
+    async with _state["slots"] or nullcontext():
+        return await run_audit(req, _state["verifier"], embedder=_state["embedder"])
+
+
+@app.post("/audit/{provider}")
+async def audit_response(provider: Provider, response: Annotated[dict[str, Any], Body()], question: str | None = None) -> AuditReport:
+    """Audit a provider's API response as it came back. auto tells the provider from the response's shape."""
+    try:
+        req = to_request(response, question=question, provider=provider)
+    except ValueError as e:
+        raise HTTPException(422, str(e)) from e
+    return await audit(req)
+
+
+if SITE_DIR.is_dir():
+    app.mount("/", StaticFiles(directory=SITE_DIR, html=True), name="site")

upheld/auditor.py

@@ -0,0 +1,124 @@
+"""Audit from Python in one call: an answer with its sources, or a provider's whole response."""
+
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from functools import lru_cache
+from typing import Any, Coroutine, TypeVar
+
+from upheld.evidence import make_embedder
+from upheld.models import AuditReport, AuditRequest, Source
+from upheld.pipeline import run_audit
+from upheld.providers import Provider, from_response
+from upheld.verify import Verifier, make_verifier
+
+T = TypeVar("T")
+
+SourcesArg = list[Source | dict | str] | None
+
+
+class Auditor:
+    """A verifier and retrieval model, loaded once and reused for every audit.
+
+    The default ensemble takes a few seconds and about 4 GB of memory to load,
+    so make one Auditor and keep it. `audit` blocks; `audit_async` is for code
+    that already runs an event loop, such as a web server.
+    """
+
+    def __init__(
+        self,
+        verifier: str | Verifier = "ensemble",
+        *,
+        threshold: float | None = None,
+        retrieval: str = "auto",
+        device: str | None = None,
+    ):
+        self.verifier = make_verifier(verifier, threshold, device) if isinstance(verifier, str) else verifier
+        self.embedder = make_embedder(retrieval)
+
+    def audit(
+        self,
+        answer: Any,
+        sources: SourcesArg = None,
+        *,
+        question: str | None = None,
+        provider: Provider | str = Provider.AUTO,
+        fetch: bool = True,
+        allow_private: bool = False,
+    ) -> AuditReport:
+        return _run(self.audit_async(answer, sources, question=question, provider=provider, fetch=fetch, allow_private=allow_private))
+
+    async def audit_async(
+        self,
+        answer: Any,
+        sources: SourcesArg = None,
+        *,
+        question: str | None = None,
+        provider: Provider | str = Provider.AUTO,
+        fetch: bool = True,
+        allow_private: bool = False,
+    ) -> AuditReport:
+        req = to_request(answer, sources, question=question, provider=provider)
+        return await run_audit(req, self.verifier, embedder=self.embedder, fetch=fetch, allow_private=allow_private)
+
+
+def audit(
+    answer: Any,
+    sources: SourcesArg = None,
+    *,
+    question: str | None = None,
+    verifier: str = "ensemble",
+    threshold: float | None = None,
+    retrieval: str = "auto",
+    provider: Provider | str = Provider.AUTO,
+    fetch: bool = True,
+    allow_private: bool = False,
+) -> AuditReport:
+    """Audit one answer and return its report.
+
+    `answer` is the answer's text with [1]-style markers or inline links, an
+    AuditRequest, or a provider's response (see upheld.providers). `sources`
+    go with text: each a Source, a dict, or a string, which is a URL if it
+    starts with http:// or https:// and the source's text otherwise, numbered
+    from 1. The models load on the first call and stay loaded for the next
+    call with the same verifier, threshold and retrieval.
+    """
+    auditor = _auditor(verifier, threshold, retrieval)
+    return auditor.audit(answer, sources, question=question, provider=provider, fetch=fetch, allow_private=allow_private)
+
+
+@lru_cache(maxsize=1)
+def _auditor(verifier: str, threshold: float | None, retrieval: str) -> Auditor:
+    return Auditor(verifier, threshold=threshold, retrieval=retrieval)
+
+
+def to_request(answer: Any, sources: SourcesArg = None, *, question: str | None = None, provider: Provider | str = Provider.AUTO) -> AuditRequest:
+    """Turn what `audit` accepts into an AuditRequest."""
+    if isinstance(answer, str):
+        return AuditRequest(answer=answer, question=question, sources=[_source(s, n) for n, s in enumerate(sources or [], 1)])
+    if sources:
+        raise ValueError("sources go with an answer's text; a request or a provider response carries its own")
+    if isinstance(answer, AuditRequest):
+        answer = answer.model_dump()
+    if isinstance(answer, dict) and "answer" in answer and provider == Provider.AUTO:
+        return AuditRequest.model_validate({**answer, "question": question} if question else answer)
+    return from_response(answer, provider, question=question)
+
+
+def _source(source: Source | dict | str, n: int) -> Source:
+    if isinstance(source, Source):
+        return source
+    if isinstance(source, dict):
+        return Source.model_validate({"id": str(n), **source})
+    if source.startswith(("http://", "https://")):
+        return Source(id=str(n), url=source)
+    return Source(id=str(n), text=source)
+
+
+def _run(coro: Coroutine[Any, Any, T]) -> T:
+    """Run a coroutine to completion, also from inside a running event loop (a Jupyter notebook, say)."""
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        return pool.submit(asyncio.run, coro).result()

upheld/cli.py

@@ -0,0 +1,173 @@
+import asyncio
+import json
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from upheld.auditor import to_request
+from upheld.evals import load_examples, metrics_at, score_examples
+from upheld.evidence import make_embedder
+from upheld.models import AuditReport, AuditRequest, Verdict
+from upheld.pipeline import run_audit
+from upheld.providers import Provider
+from upheld.verify import make_verifier
+
+app = typer.Typer(no_args_is_help=True, help="Check whether an AI answer's citations actually support its claims.")
+console = Console()
+
+VERDICT_STYLE = {
+    Verdict.SUPPORTED: "green",
+    Verdict.UNSUPPORTED: "red",
+    Verdict.WRONG_ENTITY: "magenta",
+    Verdict.SOURCE_MISSING: "red bold",
+    Verdict.UNVERIFIABLE: "yellow",
+    Verdict.UNCITED: "dim",
+}
+
+VerifierOpt = Annotated[
+    str,
+    typer.Option(
+        help="ensemble (MiniCheck, word overlap and LettuceDetect combined), ensemble+judge (adds the Granite Guardian 3.3 8B judge: most accurate, slower, needs the judge extra), minicheck (MiniCheck-Flan-T5), lettuce (LettuceDetect), lexical (word overlap, no model), or judge (the Granite judge alone)."
+    ),
+]
+ThresholdOpt = Annotated[float | None, typer.Option(help="Support score below this counts as unsupported.")]
+RetrievalOpt = Annotated[
+    str, typer.Option(help="How to find evidence in each source: hybrid (BM25 + embeddings), bm25, or auto (hybrid if installed).")
+]
+
+
+@app.command()
+def audit(
+    path: Annotated[
+        Path,
+        typer.Argument(help="Answer as .json (an AuditRequest, or a provider's response), or markdown/text with a reference list. Use - for stdin."),
+    ],
+    question: Annotated[str | None, typer.Option(help="The question the answer responds to.")] = None,
+    provider: Annotated[
+        Provider,
+        typer.Option(
+            "--from",
+            help="Whose API response the JSON is: openai, anthropic, perplexity or gemini. auto tells them apart by shape, and reads JSON with an `answer` as an AuditRequest.",
+        ),
+    ] = Provider.AUTO,
+    verifier: VerifierOpt = "ensemble",
+    threshold: ThresholdOpt = None,
+    retrieval: RetrievalOpt = "auto",
+    fetch: Annotated[bool, typer.Option(help="Fetch cited URLs. Off means only inline source text is used.")] = True,
+    allow_private: Annotated[bool, typer.Option(help="Allow fetching localhost and private-network URLs.")] = False,
+    as_json: Annotated[bool, typer.Option("--json", help="Print the full report as JSON.")] = False,
+) -> None:
+    """Audit one answer."""
+    text = sys.stdin.read() if str(path) == "-" else path.read_text()
+    try:
+        if path.suffix == ".json" or provider != Provider.AUTO or (str(path) == "-" and text.lstrip().startswith("{")):
+            req = to_request(json.loads(text), question=question, provider=provider)
+        else:
+            req = AuditRequest(answer=text, question=question)
+    except ValueError as e:  # bad JSON, or not a response upheld can read
+        console.print(f"[red]error:[/] {e}")
+        raise typer.Exit(1) from e
+    report = asyncio.run(
+        run_audit(
+            req, make_verifier(verifier, threshold), embedder=make_embedder(retrieval), fetch=fetch, allow_private=allow_private
+        )
+    )
+    if as_json:
+        print(report.model_dump_json(indent=2))
+    else:
+        render(report)
+
+
+@app.command("eval")
+def evaluate(
+    path: Annotated[Path, typer.Argument(help="Labeled JSONL: claim, evidence, label (supported|unsupported).")],
+    verifier: VerifierOpt = "ensemble",
+    threshold: ThresholdOpt = None,
+) -> None:
+    """Measure a verifier against human labels, with a threshold sweep."""
+    examples = load_examples(path)
+    v = make_verifier(verifier, threshold)
+    scores = score_examples(examples, v)
+
+    table = Table(title=f"{v.name} on {path.name} (n={len(examples)}, positive class = unsupported)")
+    for col in ("threshold", "precision", "recall", "F1", "macro-F1", "accuracy", "flagged"):
+        table.add_column(col, justify="right")
+    for t in sorted({0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, v.threshold}):
+        m = metrics_at(t, scores, examples)
+        style = "bold" if t == v.threshold else None
+        table.add_row(
+            f"{t:.2f}",
+            f"{m.precision:.3f}",
+            f"{m.recall:.3f}",
+            f"{m.f1:.3f}",
+            f"{m.macro_f1:.3f}",
+            f"{m.accuracy:.3f}",
+            f"{m.flagged_rate:.1%}",
+            style=style,
+        )
+    console.print(table)
+
+
+@app.command()
+def serve(
+    host: str = "127.0.0.1",
+    port: int = 8000,
+    verifier: VerifierOpt = "ensemble",
+    retrieval: RetrievalOpt = "auto",
+) -> None:
+    """Run the HTTP API, plus the website and playground when web/dist is built."""
+    import os
+
+    try:
+        import fastapi  # noqa: F401
+        import uvicorn
+    except ImportError as e:
+        console.print(f'[red]error:[/] upheld serve needs the api extra ({e.name} is missing): pip install "upheld\\[api]"')
+        raise typer.Exit(1) from e
+
+    os.environ["UPHELD_VERIFIER"] = verifier
+    os.environ["UPHELD_RETRIEVAL"] = retrieval
+    uvicorn.run("upheld.api:app", host=host, port=port)
+
+
+def render(report: AuditReport) -> None:
+    s = report.summary
+    console.print(f"[bold]verifier[/] {report.verifier} · [bold]retrieval[/] {report.retrieval}")
+    console.print(
+        f"[bold]{s.supported_claims}/{s.cited_claims}[/] cited claims supported · "
+        f"{s.citations_checked} citations checked · {s.claim_verdicts.get('uncited', 0)} uncited claims\n"
+    )
+
+    claims = Table(show_lines=True, expand=True)
+    claims.add_column("#", justify="right", width=3)
+    claims.add_column("verdict", width=14)
+    claims.add_column("claim", ratio=3)
+    claims.add_column("details", ratio=2)
+    for c in report.claims:
+        details = []
+        for check in c.checks:
+            score = f" {check.support_score:.2f}" if check.support_score is not None else ""
+            line = f"[{check.source_id}] [{VERDICT_STYLE[check.verdict]}]{check.verdict.value}[/]{score}"
+            if check.missing_entities:
+                line += f" · not in source: {', '.join(check.missing_entities)}"
+            if check.unsupported_spans:
+                line += " · flagged: " + "; ".join(f'"{sp.text.strip()}"' for sp in check.unsupported_spans)
+            if check.note:
+                line += f" · {check.note}"
+            details.append(line)
+        if c.note:
+            details.append(f"[dim]{c.note}[/]")
+        claims.add_row(str(c.index), f"[{VERDICT_STYLE[c.verdict]}]{c.verdict.value}[/]", c.text, "\n".join(details))
+    console.print(claims)
+
+    sources = Table(title="sources", expand=True)
+    for col in ("id", "status", "url / note"):
+        sources.add_column(col)
+    for src in report.sources:
+        resolved = f"→ {src.resolved_url}" if src.resolved_url else None
+        sources.add_row(src.id, src.status.value, " · ".join(x for x in (src.url, resolved, src.note) if x))
+    console.print(sources)

upheld/entities.py

@@ -0,0 +1,68 @@
+"""Catch "deceptive grounding": evidence that supports the claim's shape but is about a different entity.
+
+Support verifiers judge whether a passage backs a statement; they can pass a
+claim about drug X when the passage describes drug Y. This check is cheap and
+high precision: it flags named entities from the claim that appear nowhere in
+the cited source.
+"""
+
+import re
+
+from wordfreq import zipf_frequency
+
+COMMON_ZIPF = 4.0  # roughly: appears at least 10 times per million words
+
+_NAME = r"[A-Z][\w'’-]*(?:\s+\d+\b)?"  # "Ozempic", "GPT-4o", "STEP 1", "Windows 11"
+_CAPITALIZED_RUN = re.compile(rf"\b{_NAME}(?:\s+(?:(?:of|de|the|and|&)\s+)?{_NAME})*")
+_CONNECTORS = frozenset("of de the and &".split())
+_NOT_ENTITIES = frozenset(
+    "The A An In On At By For From To Of And But Or If As It Its This That These Those There Their They He She We You I "
+    "However Also According Although While When Where Which Who What Why How After Before During Since Despite "
+    "Without With Under Over Between Among Through Unlike Like Following Because Unless Until Though Thus Therefore "
+    "Both Each Every Many Most Some Several Other Another All Any Such Not No Only Even Then Meanwhile Instead Here Today "
+    "January February March April May June July August September October November December "
+    "Monday Tuesday Wednesday Thursday Friday Saturday Sunday".split()
+)
+
+
+_EDGE_PUNCT = re.compile(r"^['‘’\"“”]+|(?:['’]s)?['‘’\"“”]*$")  # quotes and possessive 's: "Charles II's" -> "Charles II"
+
+
+def named_entities(claim: str) -> list[str]:
+    found: list[str] = []
+    # Where the sentence's first word starts, after any bullet, quote or markdown.
+    first_word = next((i for i, ch in enumerate(claim) if ch.isalnum()), 0)
+    for m in _CAPITALIZED_RUN.finditer(claim):
+        words = [_EDGE_PUNCT.sub("", w) for w in m.group().split()]
+        words = [w for w in words if w]
+        lead = 0
+        while words and (words[0] in _NOT_ENTITIES or words[0] in _CONNECTORS):
+            words.pop(0)
+            lead += 1
+        while words and (words[-1] in _NOT_ENTITIES or words[-1] in _CONNECTORS):
+            words.pop()
+        if not words:
+            continue
+        # A lone capitalized word at sentence start may just be grammar
+        # ("Researchers found..."). Keep it only if it's an uncommon word
+        # (Ozempic, Nvidia) or looks like a name (GPT-4o, NASA).
+        name = " ".join(words)
+        at_start = m.start() <= first_word and lead == 0
+        if at_start and len(words) == 1 and not re.search(r"\d|[A-Z].*[A-Z]|-", name):
+            if zipf_frequency(name.lower(), "en") >= COMMON_ZIPF:
+                continue
+        found.append(name)
+    return list(dict.fromkeys(found))
+
+
+def _present(entity: str, source_text: str) -> bool:
+    haystack = source_text.lower()
+    if entity.lower() in haystack:
+        return True
+    # Accept the head word for multi-word names ("Marie Curie" -> "Curie").
+    last = entity.split()[-1].lower()
+    return len(last) > 3 and re.search(rf"\b{re.escape(last)}\b", haystack) is not None
+
+
+def missing_entities(claim: str, source_text: str) -> list[str]:
+    return [e for e in named_entities(claim) if not _present(e, source_text)]

upheld/evals.py

@@ -0,0 +1,91 @@
+"""Measure a verifier against human labels.
+
+Input is JSONL, one labeled citation per line:
+  {"claim": "...", "evidence": "..." | ["...", ...], "label": "supported" | "unsupported",
+   "answer": "..."?, "question": "..."?}
+
+`answer` is the full response the claim came from (the claim must appear in
+it verbatim). Give it when you have it: verifiers trained on full responses
+judge claims better in context.
+
+"unsupported" is the positive class, since catching bad citations is the job.
+"""
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from upheld.verify import Verifier
+
+
+@dataclass
+class Example:
+    claim: str
+    evidence: list[str]
+    unsupported: bool
+    question: str | None = None
+    answer: str | None = None
+
+
+@dataclass
+class Metrics:
+    threshold: float
+    n: int
+    precision: float  # precision, recall and f1 are for the unsupported class
+    recall: float
+    f1: float
+    macro_f1: float  # mean F1 over both classes, as AttributionBench reports
+    accuracy: float
+    flagged_rate: float
+
+
+def load_examples(path: Path) -> list[Example]:
+    examples = []
+    for line_no, line in enumerate(path.read_text().splitlines(), 1):
+        if not line.strip():
+            continue
+        row = json.loads(line)
+        if row["label"] not in ("supported", "unsupported"):
+            raise ValueError(f"line {line_no}: label must be 'supported' or 'unsupported'")
+        evidence = row["evidence"] if isinstance(row["evidence"], list) else [row["evidence"]]
+        answer = row.get("answer")
+        if answer is not None and row["claim"] not in answer:
+            raise ValueError(f"line {line_no}: claim does not appear verbatim in answer")
+        examples.append(Example(row["claim"], evidence, row["label"] == "unsupported", row.get("question"), answer))
+    return examples
+
+
+def score_examples(examples: list[Example], verifier: Verifier) -> list[float]:
+    scores = []
+    for e in examples:
+        answer = e.answer or e.claim
+        start = answer.index(e.claim)
+        [result] = verifier.check(answer, [(start, start + len(e.claim))], e.evidence, e.question)
+        scores.append(result.score)
+    return scores
+
+
+def metrics_at(threshold: float, scores: list[float], examples: list[Example]) -> Metrics:
+    return metrics_for([s < threshold for s in scores], examples, threshold)
+
+
+def metrics_for(flagged: list[bool], examples: list[Example], threshold: float) -> Metrics:
+    """Metrics for a list of unsupported/supported decisions, one per example."""
+    actual = [e.unsupported for e in examples]
+    n = len(examples)
+    f1, precision, recall = _f1([f and a for f, a in zip(flagged, actual)], flagged, actual)
+    f1_supported, _, _ = _f1(
+        [not f and not a for f, a in zip(flagged, actual)], [not f for f in flagged], [not a for a in actual]
+    )
+    correct = sum(f == a for f, a in zip(flagged, actual))
+    return Metrics(
+        threshold, n, precision, recall, f1, (f1 + f1_supported) / 2, correct / n if n else 0.0, sum(flagged) / n if n else 0.0
+    )
+
+
+def _f1(true_positive: list[bool], predicted: list[bool], actual: list[bool]) -> tuple[float, float, float]:
+    tp = sum(true_positive)
+    precision = tp / sum(predicted) if sum(predicted) else 0.0
+    recall = tp / sum(actual) if sum(actual) else 0.0
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
+    return f1, precision, recall

upheld/evidence.py

@@ -0,0 +1,123 @@
+"""Find the passages in a source that are most relevant to a claim.
+
+Keyword search (BM25) misses paraphrases: an answer says "intended to stand
+for only 20 years" where the source says "slated for dismantling in 1909".
+Hybrid mode adds a small embedding model and merges the two rankings with
+reciprocal rank fusion, so a passage either method ranks highly makes the cut.
+"""
+
+import re
+from typing import Protocol
+
+import numpy as np
+from rank_bm25 import BM25Okapi
+
+STOPWORDS = frozenset(
+    "a an and are as at be been but by for from has have in is it its of on or that the this to was were will with "
+    "which who what when where how than then there their they them these those not no can could would should".split()
+)
+_WORD = re.compile(r"[a-z0-9]+(?:[.,'][a-z0-9]+)*")
+
+
+def tokenize(text: str) -> list[str]:
+    return [w for w in _WORD.findall(text.lower()) if w not in STOPWORDS]
+
+
+def chunk(text: str, size: int = 120, overlap: int = 30) -> list[str]:
+    """Split text into overlapping word windows, starting each window at a sentence boundary where possible."""
+    sentences = re.split(r"(?<=[.!?])\s+", text.strip())
+    chunks: list[str] = []
+    current: list[str] = []
+    for sentence in sentences:
+        words = sentence.split()
+        if current and len(current) + len(words) > size:
+            chunks.append(" ".join(current))
+            current = current[-overlap:] if overlap else []
+        current.extend(words)
+    if current:
+        chunks.append(" ".join(current))
+    return chunks
+
+
+class Embedder(Protocol):
+    name: str
+
+    def embed(self, texts: list[str], *, query: bool = False) -> np.ndarray:
+        """Return one L2-normalized vector per text."""
+        ...
+
+
+class SentenceEmbedder:
+    DEFAULT_MODEL = "BAAI/bge-small-en-v1.5"
+    # bge v1.5 recommends this prefix on short retrieval queries.
+    BGE_QUERY_PREFIX = "Represent this sentence for searching relevant passages: "
+
+    def __init__(self, model_name: str = DEFAULT_MODEL):
+        try:
+            from sentence_transformers import SentenceTransformer
+        except ImportError as e:
+            raise ImportError('hybrid retrieval needs the optional extra: pip install "upheld[dense]"') from e
+        self.model = SentenceTransformer(model_name)
+        self.name = model_name.rsplit("/", 1)[-1]
+        self.query_prefix = self.BGE_QUERY_PREFIX if "bge" in model_name.lower() else ""
+
+    def embed(self, texts: list[str], *, query: bool = False) -> np.ndarray:
+        if query:
+            texts = [self.query_prefix + t for t in texts]
+        return self.model.encode(texts, normalize_embeddings=True, batch_size=32, show_progress_bar=False)
+
+
+def make_embedder(retrieval: str) -> Embedder | None:
+    """`bm25` -> None, `hybrid` -> embedding model, `auto` -> hybrid if the dense extra is installed."""
+    if retrieval == "bm25":
+        return None
+    if retrieval == "hybrid":
+        return SentenceEmbedder()
+    if retrieval == "auto":
+        try:
+            return SentenceEmbedder()
+        except ImportError:
+            return None
+    raise ValueError(f"unknown retrieval: {retrieval} (choose auto, bm25 or hybrid)")
+
+
+def retrieval_name(embedder: Embedder | None) -> str:
+    return "bm25" if embedder is None else f"hybrid:bm25+{embedder.name}"
+
+
+def rrf(rankings: list[list[int]], k: int = 60) -> list[tuple[int, float]]:
+    """Reciprocal rank fusion: score each item by the sum of 1 / (k + rank) across rankings."""
+    scores: dict[int, float] = {}
+    for ranking in rankings:
+        for rank, item in enumerate(ranking, 1):
+            scores[item] = scores.get(item, 0.0) + 1 / (k + rank)
+    return sorted(scores.items(), key=lambda pair: pair[1], reverse=True)
+
+
+class SourceIndex:
+    """One source's passages, indexed once and searched once per claim."""
+
+    def __init__(self, text: str, embedder: Embedder | None = None):
+        self.passages = chunk(text)
+        self.embedder = embedder
+        self._bm25 = BM25Okapi([tokenize(p) for p in self.passages]) if self.passages else None
+        self._vectors: np.ndarray | None = None
+
+    def search(self, query: str, k: int = 3) -> list[tuple[str, float]]:
+        """Top-k passages with scores scaled so the best is 1.0."""
+        if len(self.passages) <= k:
+            return [(p, 1.0) for p in self.passages]
+        scores = self._bm25.get_scores(tokenize(query))
+        order = [int(i) for i in np.argsort(-scores)]
+        if self.embedder is None:
+            ranked = [(i, float(scores[i])) for i in order[:k]]
+        else:
+            if self._vectors is None:
+                self._vectors = self.embedder.embed(self.passages)
+            query_vector = self.embedder.embed([query], query=True)[0]
+            dense = [int(i) for i in np.argsort(-(self._vectors @ query_vector))]
+            # Passages with no keyword overlap don't get a BM25 rank.
+            keyword = [i for i in order if scores[i] > 0]
+            ranked = rrf([keyword, dense])[:k]
+        best = ranked[0][1] or 1.0
+        return [(self.passages[i], round(s / best, 3)) for i, s in ranked]