vecshift 0.1.0__py3-none-any.whl

vecshift/bench/generate.py

@@ -0,0 +1,106 @@
+"""Generate realistic search queries for documents with an LLM.
+
+Any OpenAI-compatible chat endpoint works. The generated queries can be saved and reused,
+so a benchmark only pays for them once.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import random
+from collections.abc import Callable, Sequence
+
+import httpx
+
+from vecshift.bench.corpus import Document, Query
+from vecshift.embeddings.providers import CONCURRENCY, MAX_ATTEMPTS, RETRY_STATUSES, api_key
+from vecshift.embeddings.spec import ModelSpec
+
+PROMPT = """Write one search query that a person might type to find the passage below.
+The passage must answer the query. Don't copy long phrases from it: use the words a
+searcher would use. Reply with the query only, on one line.
+
+Passage:
+{text}"""
+
+
+class GenerationError(Exception):
+    """Query generation failed. The message never contains credentials."""
+
+
+class QueryGenerator:
+    def __init__(self, spec: ModelSpec, client: httpx.AsyncClient | None = None) -> None:
+        if spec.provider == "hash" or not spec.url:
+            raise GenerationError("Query generation needs a chat model, such as openai/gpt-4o-mini")
+        key = api_key(spec, GenerationError)
+        self.spec = spec
+        self.tokens = 0
+        headers = {"Authorization": f"Bearer {key}"} if key else {}
+        self._client = client or httpx.AsyncClient(
+            timeout=120, headers=headers, verify=True, follow_redirects=False
+        )
+        if client is not None and key:
+            self._client.headers.update(headers)
+        self._owns_client = client is None
+        self._limit = asyncio.Semaphore(CONCURRENCY)
+
+    async def aclose(self) -> None:
+        if self._owns_client:
+            await self._client.aclose()
+
+    async def _one(self, doc: Document, language: str | None = None) -> Query | None:
+        prompt = PROMPT.format(text=doc.text[:4000])
+        if language:
+            prompt = prompt.replace(
+                "Reply with the query only", f"Write the query in {language}. Reply with it only"
+            )
+        payload = {
+            "model": self.spec.model,
+            "temperature": 0,
+            "max_tokens": 64,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+        delay = 1.0
+        async with self._limit:
+            for attempt in range(1, MAX_ATTEMPTS + 1):
+                try:
+                    r = await self._client.post(f"{self.spec.url}/chat/completions", json=payload)
+                except httpx.TransportError as exc:
+                    if attempt == MAX_ATTEMPTS:
+                        raise GenerationError(
+                            f"{self.spec.name}: couldn't reach {self.spec.url}"
+                        ) from exc
+                    await asyncio.sleep(delay)
+                    delay *= 2
+                    continue
+                if r.status_code in RETRY_STATUSES and attempt < MAX_ATTEMPTS:
+                    await asyncio.sleep(delay + random.uniform(0, 0.5))
+                    delay *= 2
+                    continue
+                if r.status_code >= 400:
+                    raise GenerationError(f"{self.spec.name}: HTTP {r.status_code}")
+                body = r.json()
+                self.tokens += int((body.get("usage") or {}).get("total_tokens") or 0)
+                text = str(body["choices"][0]["message"]["content"] or "")
+                line = text.strip().splitlines()[0] if text.strip() else ""
+                line = line.strip().strip('"“”').strip()
+                return Query(line, frozenset({doc.id})) if line else None
+        return None  # pragma: no cover
+
+    async def generate(
+        self,
+        docs: Sequence[Document],
+        n: int,
+        seed: int,
+        language: Callable[[Document], str | None] | None = None,
+    ) -> list[Query]:
+        """One query for each of ``n`` sampled documents.
+
+        ``language`` picks the language to write each query in, for cross-language tests
+        such as an Arabic question about an English passage.
+        """
+        chosen = random.Random(seed).sample(list(docs), min(n, len(docs)))
+        results = await asyncio.gather(
+            *(self._one(d, language(d) if language else None) for d in chosen)
+        )
+        return [q for q in results if q is not None]

vecshift/bench/html.py

@@ -0,0 +1,325 @@
+"""Render a benchmark as a self-contained HTML leaderboard."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable
+from datetime import datetime
+
+from vecshift.bench.runner import BenchResult, ModelResult
+from vecshift.html_kit import asset, compact, e, page, stamp
+
+VALUE_TOLERANCE = 0.02
+"""A model within this much recall@10 of the best counts as "as good" when picking value."""
+
+QUERY_SOURCES = {
+    "proxy": "proxy queries taken from the documents",
+    "generated": "written by an LLM",
+    "labeled": "your labeled queries",
+}
+
+
+def _money(value: float | None) -> str:
+    if value is None:
+        return "—"
+    if value == 0:
+        return "free"
+    if value < 0.01:
+        return "<$0.01"
+    return f"${value:,.2f}" if value < 1000 else f"${value:,.0f}"
+
+
+def _ms(value: float | None) -> str:
+    if value is None:
+        return "—"
+    return f"{value:.2f} ms" if value < 1 else f"{value:,.1f} ms"
+
+
+def _gb(value: float | None) -> str:
+    return "—" if value is None else f"{value:,.1f} GB" if value >= 1 else f"{value * 1000:,.0f} MB"
+
+
+def _value_pick(models: list[ModelResult]) -> ModelResult | None:
+    best = max(m.scores.recall_at_10 for m in models if m.scores)
+    close = [m for m in models if m.scores and m.scores.recall_at_10 >= best - VALUE_TOLERANCE]
+    priced = [m for m in close if m.cost_per_million_docs is not None]
+    pool = priced or close
+    return min(
+        pool,
+        key=lambda m: (
+            m.cost_per_million_docs if priced else 0,
+            m.gb_per_million_vectors or math.inf,
+        ),
+    )
+
+
+def _picks(models: list[ModelResult]) -> str:
+    scored = [m for m in models if m.scores]
+    if not scored:
+        return ""
+    best = max(scored, key=lambda m: m.scores.recall_at_10 if m.scores else 0)
+    value = _value_pick(scored)
+    timed = [m for m in scored if m.query_ms_p50 is not None]
+    fastest = min(timed, key=lambda m: m.query_ms_p50 or 0) if timed else None
+
+    def card(label: str, m: ModelResult | None, why: str) -> str:
+        if m is None:
+            return ""
+        return (
+            f'<div class="card pick"><div class="tile-label">{e(label)}</div>'
+            f'<div class="who">{e(m.name)}</div><div class="why">{e(why)}</div></div>'
+        )
+
+    if best.scores is None:  # pragma: no cover - scored models always have scores
+        return ""
+    value_why = ""
+    if value:
+        cost = value.cost_per_million_docs
+        price = "free to embed" if cost == 0 else f"{_money(cost)} per 1M documents"
+        value_why = f"{price}, {_gb(value.gb_per_million_vectors)} per 1M vectors"
+    return (
+        '<section aria-label="Picks"><div class="picks">'
+        + card("Best retrieval", best, f"recall@10 {best.scores.recall_at_10:.3f}")
+        + card("Best value", value, value_why)
+        + card(
+            "Fastest queries",
+            fastest,
+            f"{_ms(fastest.query_ms_p50)} median per query" if fastest else "",
+        )
+        + "</div></section>"
+    )
+
+
+def _tiles(result: BenchResult) -> str:
+    tiles = [
+        ("Documents", compact(result.documents), f"sampled from {result.source}"),
+        ("Queries", compact(result.queries), QUERY_SOURCES.get(result.query_source, "")),
+        ("Models", str(len(result.models)), "compared on the same data"),
+    ]
+    return (
+        '<section aria-label="Key numbers"><div class="tiles">'
+        + "".join(
+            f'<div class="card tile"><div class="tile-label">{e(label)}</div>'
+            f'<div class="tile-value">{e(value)}</div><div class="tile-caption">{e(caption)}</div>'
+            "</div>"
+            for label, value, caption in tiles
+        )
+        + "</div></section>"
+    )
+
+
+def _leaderboard(ranked: list[ModelResult]) -> str:
+    scored = [m for m in ranked if m.scores]
+
+    def best(get: Callable[[ModelResult], float | None], low: bool = False) -> float | None:
+        values = [v for m in scored if (v := get(m)) is not None]
+        if len(set(values)) < 2:  # nothing stands out when every model ties
+            return None
+        return min(values) if low else max(values)
+
+    def cell(value: float | None, target: float | None, text: str) -> str:
+        cls = "num best" if value is not None and value == target and len(scored) > 1 else "num"
+        return f'<td class="{cls}">{e(text)}</td>'
+
+    tops = {
+        "r10": best(lambda m: m.scores.recall_at_10 if m.scores else None),
+        "r1": best(lambda m: m.scores.recall_at_1 if m.scores else None),
+        "mrr": best(lambda m: m.scores.mrr_at_10 if m.scores else None),
+        "lat": best(lambda m: m.query_ms_p50, low=True),
+        "dps": best(lambda m: m.docs_per_second),
+        "cost": best(lambda m: m.cost_per_million_docs, low=True),
+        "gb": best(lambda m: m.gb_per_million_vectors, low=True),
+    }
+    rows = []
+    for rank, m in enumerate(ranked, 1):
+        name = f'<td class="model"><b>{e(m.name)}</b><code>{e(m.model_tag or m.spec)}</code></td>'
+        if not m.scores:
+            rows.append(
+                f'<tr class="failed"><td class="rank">&ndash;</td>{name}'
+                f'<td colspan="7">Failed: {e(m.error or "unknown error")}</td></tr>'
+            )
+            continue
+        s = m.scores
+        r10 = s.recall_at_10
+        r10_cls = "best" if r10 == tops["r10"] and len(scored) > 1 else ""
+        cost = _money(m.cost_per_million_docs)
+        if m.tokens_estimated and m.cost_per_million_docs:
+            cost = "~" + cost
+        rows.append(
+            f'<tr><td class="rank">{rank}</td>{name}'
+            f'<td class="num"><div class="recall"><span class="{r10_cls}">{r10:.3f}</span>'
+            f'<span class="track" aria-hidden="true"><span style="width:{r10 * 100:.1f}%">'
+            "</span></span></div></td>"
+            + cell(s.recall_at_1, tops["r1"], f"{s.recall_at_1:.3f}")
+            + cell(s.mrr_at_10, tops["mrr"], f"{s.mrr_at_10:.3f}")
+            + cell(m.query_ms_p50, tops["lat"], _ms(m.query_ms_p50))
+            + cell(
+                m.docs_per_second,
+                tops["dps"],
+                f"{m.docs_per_second:,.0f}" if m.docs_per_second else "cached",
+            )
+            + cell(m.cost_per_million_docs, tops["cost"], cost)
+            + cell(m.gb_per_million_vectors, tops["gb"], _gb(m.gb_per_million_vectors))
+            + "</tr>"
+        )
+    head = (
+        '<tr><th class="rank">#</th><th>Model</th><th class="num">Recall@10</th>'
+        '<th class="num">Recall@1</th><th class="num">MRR@10</th>'
+        '<th class="num">Query p50</th><th class="num">Docs/s</th>'
+        '<th class="num">Per 1M docs</th><th class="num">Per 1M vectors</th></tr>'
+    )
+    return (
+        '<section aria-labelledby="lb-title"><h2 id="lb-title">Leaderboard</h2>'
+        '<div class="card lb"><div class="table-scroll"><table>'
+        f"<thead>{head}</thead><tbody>{''.join(rows)}</tbody></table></div></div></section>"
+    )
+
+
+def _log_ticks(lo: float, hi: float) -> list[float]:
+    ticks = []
+    for exp in range(math.floor(math.log10(lo)) - 1, math.ceil(math.log10(hi)) + 1):
+        for step in (1, 2, 5):
+            t = step * 10.0**exp
+            if lo <= t <= hi:
+                ticks.append(t)
+    if len(ticks) > 6:
+        ticks = [t for t in ticks if str(t).lstrip("0.").startswith("1")] or ticks[::2]
+    return ticks
+
+
+def _scatter(
+    ranked: list[ModelResult],
+    title: str,
+    subtitle: str,
+    get_x: Callable[[ModelResult], float | None],
+    fmt_x: Callable[[float], str],
+    x_name: str,
+) -> str:
+    points = [
+        (rank, m, x, m.scores.recall_at_10)
+        for rank, m in enumerate(ranked, 1)
+        if m.scores and (x := get_x(m)) is not None and x > 0
+    ]
+    head = f"<figcaption><b>{e(title)}</b><span>{e(subtitle)}</span></figcaption>"
+    if len(points) < 2:
+        return (
+            f'<figure class="card">{head}<p class="statement">Needs two or more models</p></figure>'
+        )
+
+    xs = [p[2] for p in points]
+    lo, hi = min(xs) / 1.6, max(xs) * 1.6
+    span = math.log10(hi) - math.log10(lo)
+
+    def xpos(v: float) -> float:
+        return 100 * (math.log10(v) - math.log10(lo)) / span
+
+    top = min(1.0, math.ceil(max(p[3] for p in points) * 10 + 0.5) / 10)
+    ygrid = "".join(
+        f'<div class="gridline" style="bottom:{f * 100:.0f}%"></div>' for f in (0.5, 1.0)
+    )
+    yaxis = "".join(
+        f'<span style="bottom:{f * 100:.0f}%">{top * f:.2f}</span>' for f in (0.0, 0.5, 1.0)
+    )
+    ticks = _log_ticks(lo, hi)
+    vgrid = "".join(f'<div class="vline" style="left:{xpos(t):.2f}%"></div>' for t in ticks)
+    xlabels = "".join(f'<span style="left:{xpos(t):.2f}%">{e(fmt_x(t))}</span>' for t in ticks)
+    dots = "".join(
+        f'<div class="pt" tabindex="0" style="left:{xpos(x):.2f}%;bottom:{100 * y / top:.2f}%" '
+        f'data-tip-value="{y:.3f} recall@10" data-tip-label="{e(m.name)} · {e(fmt_x(x))}" '
+        f'aria-label="{rank}. {e(m.name)}: recall@10 {y:.3f}, {e(fmt_x(x))}">{rank}</div>'
+        for rank, m, x, y in points
+    )
+    rows = "".join(
+        f'<tr><td>{rank}. {e(m.name)}</td><td class="num">{e(fmt_x(x))}</td>'
+        f'<td class="num">{y:.3f}</td></tr>'
+        for rank, m, x, y in points
+    )
+    table = (
+        '<details class="table-view"><summary>View as table</summary><div class="table-scroll">'
+        f'<table><thead><tr><th>Model</th><th class="num">{e(x_name)}</th>'
+        f'<th class="num">Recall@10</th></tr></thead><tbody>{rows}</tbody></table></div></details>'
+    )
+    plot = (
+        f'<div class="scatter" role="img" aria-label="{e(title)}">'
+        f'<div class="yaxis" aria-hidden="true" style="height:220px">{yaxis}</div>'
+        f'<div class="plot">{ygrid}{vgrid}{dots}</div>'
+        f'<div class="xaxis log" aria-hidden="true">{xlabels}</div>'
+        f'<div class="axis-title" aria-hidden="true">{e(x_name)}, log scale</div></div>'
+    )
+    return f'<figure class="card">{head}{plot}{table}</figure>'
+
+
+def _charts(ranked: list[ModelResult]) -> str:
+    storage = _scatter(
+        ranked,
+        "Quality vs storage",
+        "Recall@10 against float32 storage for a million vectors. Up and left is better.",
+        lambda m: m.gb_per_million_vectors,
+        _gb,
+        "Storage per 1M vectors",
+    )
+    latency = _scatter(
+        ranked,
+        "Quality vs query speed",
+        "Recall@10 against median time to embed one query. Up and left is better.",
+        lambda m: m.query_ms_p50,
+        _ms,
+        "Query latency",
+    )
+    return (
+        '<section aria-labelledby="charts-title"><h2 id="charts-title">Trade-offs</h2>'
+        f'<div class="charts">{storage}{latency}</div>'
+        '<p class="chart-note">Points are numbered by leaderboard rank.</p></section>'
+    )
+
+
+def _method(result: BenchResult) -> str:
+    parts = [
+        f"<p>Each model embedded the same {result.documents:,} documents and "
+        f"{result.queries:,} queries ({e(QUERY_SOURCES.get(result.query_source, ''))}). "
+        "Search is exact cosine similarity over all documents, so the numbers measure the "
+        "models, not an index. Recall@10 is the share of relevant documents found in the top "
+        "10 results.</p>",
+        *(f"<p>{e(n)}</p>" for n in result.notes),
+        *(
+            f'<p class="err">✖ {e(m.name)} failed: {e(m.error)}</p>'
+            for m in result.models
+            if m.error
+        ),
+        "<p>Costs use the provider's reported token counts and the price per million tokens "
+        "in each model spec; a ~ marks an estimate.</p>",
+    ]
+    return (
+        '<section aria-labelledby="method-title"><h2 id="method-title">How this was measured</h2>'
+        f'<div class="card method">{"".join(parts)}</div></section>'
+    )
+
+
+def render_html(
+    result: BenchResult, *, version: str | None = None, generated_at: datetime | None = None
+) -> str:
+    when = stamp(generated_at)
+    ranked = result.ranked()
+    by = f"vecshift {version}" if version else "vecshift"
+    body = (
+        _tiles(result)
+        + _picks(result.models)
+        + _leaderboard(ranked)
+        + _charts(ranked)
+        + _method(result)
+    )
+    footer = (
+        f"<p>Generated by {e(by)} on {e(when)}.</p>"
+        "<p>This page holds scores and model names only, no document text. It loads nothing "
+        "from the network.</p>"
+    )
+    return page(
+        title=f"Embedding benchmark · {result.source}",
+        product="vecshift bench",
+        heading=f"Embedding models on {result.source}",
+        meta=[f"{len(result.models)} models", when],
+        body=body,
+        footer=footer,
+        generator=by,
+        extra_css=asset("bench.css"),
+    )

vecshift/bench/metrics.py

@@ -0,0 +1,50 @@
+"""Retrieval metrics computed with exact (brute-force) cosine search."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import numpy as np
+import numpy.typing as npt
+
+K = 10
+
+Matrix = npt.NDArray[np.float32]
+
+
+@dataclass(frozen=True, slots=True)
+class Scores:
+    recall_at_1: float
+    recall_at_10: float
+    mrr_at_10: float
+    ndcg_at_10: float
+
+
+def normalize(vectors: Sequence[Sequence[float]]) -> Matrix:
+    m = np.asarray(vectors, dtype=np.float32)
+    norms = np.linalg.norm(m, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    return m / norms
+
+
+def evaluate(docs: Matrix, queries: Matrix, relevant: Sequence[set[int]]) -> Scores:
+    """Score each query's top 10 documents against its relevant document indexes."""
+    k = min(K, docs.shape[0])
+    sims = queries @ docs.T
+    top = np.argpartition(-sims, k - 1, axis=1)[:, :k]
+    order = np.take_along_axis(sims, top, axis=1).argsort(axis=1)[:, ::-1]
+    ranked = np.take_along_axis(top, order, axis=1)
+
+    r1 = r10 = mrr = ndcg = 0.0
+    for row, rel in zip(ranked.tolist(), relevant, strict=True):
+        hits = [rank for rank, doc in enumerate(row, 1) if doc in rel]
+        r1 += (1 if hits and hits[0] == 1 else 0) / len(rel)
+        r10 += len(hits) / len(rel)
+        mrr += 1 / hits[0] if hits else 0.0
+        dcg = sum(1 / math.log2(rank + 1) for rank in hits)
+        ideal = sum(1 / math.log2(i + 1) for i in range(1, min(len(rel), k) + 1))
+        ndcg += dcg / ideal
+    n = len(relevant)
+    return Scores(r1 / n, r10 / n, mrr / n, ndcg / n)

vecshift/bench/runner.py

@@ -0,0 +1,183 @@
+"""Run a benchmark: embed, search, score, and measure each model."""
+
+from __future__ import annotations
+
+import statistics
+import time
+from collections.abc import Callable, Sequence
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+from vecshift.bench.corpus import Benchmark
+from vecshift.bench.metrics import Scores, evaluate, normalize
+from vecshift.embeddings import (
+    CachedEmbedder,
+    EmbeddingCache,
+    EmbeddingError,
+    ModelSpec,
+    create_embedder,
+)
+
+LATENCY_SAMPLES = 20
+CHARS_PER_TOKEN = 4
+
+
+@dataclass(slots=True)
+class ModelResult:
+    name: str
+    spec: str
+    dimensions: int | None = None
+    model_tag: str | None = None
+    scores: Scores | None = None
+    error: str | None = None
+    docs_per_second: float | None = None
+    """Documents embedded per second, or ``None`` when every document was cached."""
+    query_ms_p50: float | None = None
+    query_ms_p95: float | None = None
+    tokens_per_doc: float | None = None
+    tokens_estimated: bool = False
+    price_per_million_tokens: float | None = None
+    cached_docs: int = 0
+    local: bool = False
+
+    @property
+    def cost_per_million_docs(self) -> float | None:
+        if self.price_per_million_tokens is None or self.tokens_per_doc is None:
+            return None
+        return self.tokens_per_doc * self.price_per_million_tokens
+
+    @property
+    def gb_per_million_vectors(self) -> float | None:
+        """float32 storage for a million vectors, in GB."""
+        return self.dimensions * 4 / 1000 if self.dimensions else None
+
+    def to_dict(self) -> dict[str, Any]:
+        data = {k: v for k, v in asdict(self).items() if k != "scores"}
+        data["scores"] = asdict(self.scores) if self.scores else None
+        data["cost_per_million_docs"] = self.cost_per_million_docs
+        data["gb_per_million_vectors"] = self.gb_per_million_vectors
+        return data
+
+
+@dataclass(slots=True)
+class BenchResult:
+    source: str
+    documents: int
+    queries: int
+    query_source: str
+    notes: list[str] = field(default_factory=list)
+    models: list[ModelResult] = field(default_factory=list)
+
+    def ranked(self) -> list[ModelResult]:
+        """Best recall@10 first; failed models last."""
+        return sorted(
+            self.models,
+            key=lambda m: (m.scores is None, -(m.scores.recall_at_10 if m.scores else 0)),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "documents": self.documents,
+            "queries": self.queries,
+            "query_source": self.query_source,
+            "notes": self.notes,
+            "models": [m.to_dict() for m in self.ranked()],
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class PlanItem:
+    spec: ModelSpec
+    est_tokens: int
+    est_cost: float | None
+
+
+def plan(bench: Benchmark, specs: Sequence[ModelSpec]) -> list[PlanItem]:
+    """Estimated tokens and cost per model, before anything is sent."""
+    chars = sum(len(d.text) for d in bench.documents) + sum(len(q.text) for q in bench.queries)
+    chars += sum(len(q.text) for q in bench.queries[:LATENCY_SAMPLES])
+    tokens = chars // CHARS_PER_TOKEN
+    return [
+        PlanItem(s, tokens, None if s.price is None else tokens * s.price / 1_000_000)
+        for s in specs
+    ]
+
+
+def _percentile(values: Sequence[float], pct: float) -> float:
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, max(0, round(pct / 100 * (len(ordered) - 1))))
+    return ordered[index]
+
+
+async def _run_one(bench: Benchmark, spec: ModelSpec, cache: EmbeddingCache | None) -> ModelResult:
+    result = ModelResult(
+        name=spec.name,
+        spec=spec.safe_raw,
+        price_per_million_tokens=spec.price,
+        local=spec.is_local,
+    )
+    embedder = create_embedder(spec)
+    use_cache = cache is not None and spec.provider != "hash"
+    wrapped = CachedEmbedder(embedder, cache) if use_cache and cache else None
+    try:
+        doc_texts = [d.text for d in bench.documents]
+        start = time.perf_counter()
+        doc_vecs = await (wrapped or embedder).embed(doc_texts, "document")
+        elapsed = time.perf_counter() - start
+        fresh = wrapped.misses if wrapped else len(doc_texts)
+        result.cached_docs = len(doc_texts) - fresh
+        if fresh:
+            result.docs_per_second = fresh / elapsed if elapsed > 0 else None
+            result.tokens_per_doc = embedder.tokens / fresh
+            result.tokens_estimated = embedder.tokens_estimated
+        else:
+            result.tokens_per_doc = (
+                sum(len(t) for t in doc_texts) / len(doc_texts) / CHARS_PER_TOKEN
+            )
+            result.tokens_estimated = True
+
+        query_texts = [q.text for q in bench.queries]
+        query_vecs = await (wrapped or embedder).embed(query_texts, "query")
+
+        timings = []
+        for text in query_texts[:LATENCY_SAMPLES]:
+            t0 = time.perf_counter()
+            await embedder.embed([text], "query")
+            timings.append((time.perf_counter() - t0) * 1000)
+        if timings:
+            result.query_ms_p50 = statistics.median(timings)
+            result.query_ms_p95 = _percentile(timings, 95)
+
+        index = {d.id: i for i, d in enumerate(bench.documents)}
+        relevant = [{index[r] for r in q.relevant if r in index} for q in bench.queries]
+        result.scores = evaluate(normalize(doc_vecs), normalize(query_vecs), relevant)
+        result.dimensions = len(doc_vecs[0])
+        result.model_tag = embedder.fingerprint.model_tag
+    except EmbeddingError as exc:
+        result.error = str(exc)
+    finally:
+        await embedder.aclose()
+    return result
+
+
+async def run(
+    bench: Benchmark,
+    specs: Sequence[ModelSpec],
+    cache: EmbeddingCache | None,
+    source: str,
+    on_model: Callable[[ModelSpec], None] | None = None,
+) -> BenchResult:
+    """Benchmark each model in turn, so latency measurements don't compete."""
+    result = BenchResult(
+        source=source,
+        documents=len(bench.documents),
+        queries=len(bench.queries),
+        query_source=bench.query_source,
+        notes=list(bench.notes),
+    )
+    for spec in specs:
+        if on_model:
+            on_model(spec)
+        result.models.append(await _run_one(bench, spec, cache))
+    return result