chunkshop 0.3.0__py3-none-any.whl

chunkshop/__init__.py

@@ -0,0 +1,5 @@
+"""Reusable ingestion tool: source -> chunker -> embedder -> extractor -> pgvector table."""
+from chunkshop.config import CellConfig, load_config
+from chunkshop.pipeline import Pipeline
+
+__all__ = ["CellConfig", "load_config", "Pipeline"]

chunkshop/bakeoff/__init__.py

@@ -0,0 +1,25 @@
+"""chunkshop bakeoff: `chunkshop bakeoff` CLI subcommand + library surface.
+
+Promotes the `scripts/bench_matrix.py` hacking tool into a user-facing,
+config-driven chunker x embedder matrix evaluation that emits a leaderboard +
+a runnable `recommended.yaml` cell.
+"""
+from chunkshop.bakeoff.config import (
+    BakeoffConfig,
+    BakeoffResults,
+    BakeoffTargetConfig,
+    ComboResult,
+    GoldQuery,
+    MatrixConfig,
+    ScoringConfig,
+)
+
+__all__ = [
+    "BakeoffConfig",
+    "BakeoffResults",
+    "BakeoffTargetConfig",
+    "ComboResult",
+    "GoldQuery",
+    "MatrixConfig",
+    "ScoringConfig",
+]

chunkshop/bakeoff/config.py

@@ -0,0 +1,115 @@
+"""Pydantic config models for `chunkshop bakeoff` runs (SC-002, SC-003).
+
+One `BakeoffConfig` = one matrix evaluation: a corpus, a set of gold queries,
+and a cross-product of chunkers x embedders to rank with recall@k + MRR.
+Mirrors the existing `chunkshop.config.CellConfig` conventions (pydantic v2,
+`extra="forbid"`, discriminated unions) so typos in YAML fail at config-load,
+not at runtime.
+"""
+from __future__ import annotations
+
+from typing import Any, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from chunkshop.config import (
+    ChunkerConfig,
+    FastembedEmbedder,
+    FramerConfig,
+    IdentityFramerConfig,
+    RuntimeConfig,
+    SourceConfig,
+)
+
+
+class _Base(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class GoldQuery(_Base):
+    """One user-authored evaluation query. Doc-level gold only for MVP."""
+
+    query: str
+    gold_doc_id: str
+
+
+class MatrixConfig(_Base):
+    """Cross-product axes. N embedders x M chunkers = N*M combos."""
+
+    embedders: list[FastembedEmbedder] = Field(..., min_length=1)
+    chunkers: list[ChunkerConfig] = Field(..., min_length=1)
+
+
+class BakeoffTargetConfig(_Base):
+    """Where bakeoff tables land. One table per combo under `schema`."""
+
+    dsn_env: str
+    # `schema` is a pydantic reserved name; expose as `schema_name` in Python,
+    # accept `schema` in YAML via alias. Matches `TargetConfig`'s treatment.
+    schema_name: str = Field(alias="schema")
+
+
+class ScoringConfig(_Base):
+    """Retrieval-metric config. `k` controls recall cutoffs; `top_k` is pgvector LIMIT."""
+
+    k: list[int] = [1, 3, 5]
+    include_mrr: bool = True
+    top_k: int = 5
+
+    @field_validator("k")
+    @classmethod
+    def _ks_positive(cls, v: list[int]) -> list[int]:
+        if not v or any(k <= 0 for k in v):
+            raise ValueError("scoring.k must be a non-empty list of positive ints")
+        return sorted(set(v))
+
+
+class BakeoffConfig(_Base):
+    """Top-level bakeoff run config. One YAML = one factorial run."""
+
+    name: str
+    source: SourceConfig
+    framer: FramerConfig = Field(default_factory=IdentityFramerConfig)
+    gold_queries: Union[str, list[GoldQuery]]  # path to YAML/JSON file OR inline list
+    matrix: MatrixConfig
+    target: BakeoffTargetConfig
+    scoring: ScoringConfig = Field(default_factory=ScoringConfig)
+    output_dir: Optional[str] = None
+    runtime: Optional[RuntimeConfig] = None
+
+
+class ComboResult(_Base):
+    """Scored outcome for one (chunker, embedder) combo."""
+
+    chunker_key: str
+    embedder_key: str
+    chunker_label: str
+    embedder_label: str
+    table: str
+    ingest_chunks: int
+    ingest_wall_seconds: float
+    # Subset of ingest_wall_seconds spent inside the embedder. Lets the
+    # leaderboard distinguish "this combo is slow because of the embedder"
+    # from "this combo is slow because of the chunker / sink". 0.0 if the
+    # embedder didn't track timing.
+    ingest_embed_seconds: float = 0.0
+    aggregate: dict[str, float]
+    per_query: list[dict[str, Any]]
+
+
+class BakeoffResults(_Base):
+    """Full output of `run_bakeoff`. Round-trips through JSON via pydantic."""
+
+    run_name: str
+    started_at: str
+    corpus_label: str
+    n_queries: int
+    n_combos: int
+    combos: list[ComboResult]
+    gold_queries: list[dict[str, str]]
+    # Wall time per unique embedder spent embedding all gold queries during
+    # the scoring phase. Indicative of query-time latency at production
+    # scale: the value scaled by your expected QPS predicts CPU cost.
+    # Keys are embedder_key (same as ComboResult.embedder_key); values are
+    # seconds for embedding all `n_queries` queries in one batch.
+    query_embed_seconds_by_embedder: dict[str, float] = {}

chunkshop/bakeoff/gold.py

@@ -0,0 +1,40 @@
+"""Gold-query loader for bakeoff configs (SC-003).
+
+`BakeoffConfig.gold_queries` is `str | list[GoldQuery]` — a path to a YAML/JSON
+file on disk, or an inline list already validated by pydantic. This module
+resolves either shape to a concrete `list[GoldQuery]`.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Union
+
+import yaml
+
+from chunkshop.bakeoff.config import GoldQuery
+
+
+def load_gold_queries(spec: Union[str, list[GoldQuery]]) -> list[GoldQuery]:
+    """Resolve a `gold_queries` spec to a validated list of `GoldQuery`.
+
+    - `list[GoldQuery]` — returned as-is (already pydantic-validated).
+    - `str` — treated as a filesystem path. `.json` parses as JSON array;
+      anything else parses as a YAML list. Each element is run through
+      `GoldQuery.model_validate`.
+    """
+    if isinstance(spec, list):
+        return spec
+    path = Path(spec)
+    if not path.exists():
+        raise FileNotFoundError(f"gold_queries file not found: {path}")
+    text = path.read_text(encoding="utf-8")
+    if path.suffix.lower() == ".json":
+        raw = json.loads(text)
+    else:
+        raw = yaml.safe_load(text)
+    if not isinstance(raw, list):
+        raise ValueError(
+            f"gold_queries file must be a YAML/JSON list; got {type(raw).__name__}"
+        )
+    return [GoldQuery.model_validate(x) for x in raw]

chunkshop/bakeoff/keys.py

@@ -0,0 +1,53 @@
+"""Deterministic chunker + embedder key derivation for combo table names (SC-004).
+
+The bakeoff runner writes one Postgres table per combo: `{chunker_key}__{embedder_key}`
+under the target schema. Table names must be lowercase-underscore idents so
+pgvector operators + the sink's regex-allowlisted identifier validator both
+accept them. Derivations here are pure functions — same config in, same key out.
+"""
+from __future__ import annotations
+
+import re
+
+from chunkshop.config import (
+    ChunkerConfig,
+    FastembedEmbedder,
+    FixedOverlapChunker,
+    HierarchyChunker,
+    NeighborExpandChunker,
+    SentenceAwareChunker,
+)
+
+_ID_SAFE = re.compile(r"[^a-z0-9]+")
+
+
+def embedder_key(cfg: FastembedEmbedder) -> str:
+    """Model short name stripped of org prefix + punctuation.
+
+    `Xenova/bge-base-en-v1.5-int8` -> `bge_base_en_v1_5_int8`.
+    """
+    short = cfg.model_name.split("/")[-1].lower()
+    return _ID_SAFE.sub("_", short).strip("_")
+
+
+def chunker_key(cfg: ChunkerConfig) -> str:
+    """One deterministic key per chunker shape. Include params that change behavior.
+
+    `fixed_overlap` includes window/step so (w=300,s=150) and (w=500,s=100)
+    don't collide. `neighbor_expand` recurses into its `base` so the outer
+    window + the underlying strategy both land in the ident.
+    """
+    if isinstance(cfg, HierarchyChunker):
+        return "hierarchy"
+    if isinstance(cfg, SentenceAwareChunker):
+        return "sentence_aware"
+    if isinstance(cfg, FixedOverlapChunker):
+        return f"fixed_overlap_w{cfg.window_words}_s{cfg.step_words}"
+    if isinstance(cfg, NeighborExpandChunker):
+        return f"neighbor_expand_w{cfg.window}_over_{chunker_key(cfg.base)}"
+    raise ValueError(f"unknown chunker type for key derivation: {type(cfg).__name__}")
+
+
+def combo_table(chunker: ChunkerConfig, embedder: FastembedEmbedder) -> str:
+    """Build the combo's table name: `{chunker_key}__{embedder_key}`."""
+    return f"{chunker_key(chunker)}__{embedder_key(embedder)}"

chunkshop/bakeoff/output.py

@@ -0,0 +1,159 @@
+"""Output writers for bakeoff runs (SC-006, SC-007, SC-008).
+
+Three files land in `out_dir`:
+- `results.json` — raw BakeoffResults, round-trips via pydantic.
+- `report.md` — human-readable leaderboard + statistical-power honesty note.
+- `recommended.yaml` — the top-MRR combo rendered as a runnable chunkshop
+  `CellConfig`. User points `source` at their real corpus and runs `chunkshop
+  ingest` with it.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+from chunkshop.bakeoff.config import BakeoffConfig, BakeoffResults
+from chunkshop.bakeoff.keys import chunker_key, embedder_key
+
+
+def write_results_json(results: BakeoffResults, out_dir: Path) -> Path:
+    """Dump raw BakeoffResults to `{out_dir}/results.json`."""
+    out = out_dir / "results.json"
+    out.write_text(results.model_dump_json(indent=2))
+    return out
+
+
+def write_report_md(
+    cfg: BakeoffConfig, results: BakeoffResults, out_dir: Path
+) -> Path:
+    """Render a markdown leaderboard sorted by MRR desc + honesty note."""
+    ranked = sorted(results.combos, key=lambda c: -c.aggregate.get("mrr", 0))
+
+    header_cols = " | ".join(f"r@{k}" for k in cfg.scoring.k)
+    # Columns: # | Chunker | Embedder | r@k... | MRR | chunks | ingest_s | embed_s
+    sep_cells = "|".join(["---"] * (len(cfg.scoring.k) + 7))
+
+    lines = [
+        f"# Bakeoff report: {results.run_name}",
+        "",
+        f"- Run: {results.started_at}",
+        f"- Corpus: {results.corpus_label}",
+        f"- Queries: {results.n_queries}",
+        f"- Combos: {results.n_combos}",
+        "",
+        "## Leaderboard (sorted by MRR)",
+        "",
+        f"| # | Chunker | Embedder | {header_cols} | MRR | chunks | ingest_s | embed_s |",
+        f"|{sep_cells}|",
+    ]
+    for i, c in enumerate(ranked, start=1):
+        rk = [f"{c.aggregate.get(f'recall_at_{k}', 0):.3f}" for k in cfg.scoring.k]
+        mrr = f"{c.aggregate.get('mrr', 0):.3f}"
+        lines.append(
+            f"| {i} | `{c.chunker_label}` | `{c.embedder_label}` | "
+            + " | ".join(rk)
+            + f" | {mrr} | {c.ingest_chunks} | {c.ingest_wall_seconds:.2f} | "
+            f"{c.ingest_embed_seconds:.2f} |"
+        )
+
+    # Per-query detail: top-1 doc_id per combo, one row per (combo, query).
+    lines += [
+        "",
+        "## Per-query detail (top-1 hit per combo)",
+        "",
+        "| Chunker | Embedder | Query | Gold | Top-1 | MRR |",
+        "|---|---|---|---|---|---|",
+    ]
+    for c in ranked:
+        for pq in c.per_query:
+            top1 = pq["top_k"][0]["doc_id"] if pq.get("top_k") else "-"
+            lines.append(
+                f"| `{c.chunker_label}` | `{c.embedder_label}` | "
+                f"{pq['query']} | `{pq['gold_doc_id']}` | "
+                f"`{top1}` | {pq.get('mrr', 0):.3f} |"
+            )
+
+    # Query-time embedding cost. Each unique embedder embeds all gold
+    # queries once during scoring; the wall time is a proxy for the
+    # production query-time latency you'd see at scale.
+    if results.query_embed_seconds_by_embedder:
+        n = max(results.n_queries, 1)
+        lines += [
+            "",
+            "## Query-time embedding cost",
+            "",
+            f"Wall time to embed all {results.n_queries} gold queries, per "
+            "unique embedder. At production scale this scales by your "
+            "expected QPS — useful for choosing between a slower-but-better "
+            "embedder and a faster-but-worse one.",
+            "",
+            "| Embedder | total_s | per_query_ms |",
+            "|---|---|---|",
+        ]
+        for k, total in sorted(
+            results.query_embed_seconds_by_embedder.items(), key=lambda kv: kv[1]
+        ):
+            per_q_ms = (total / n) * 1000.0
+            lines.append(f"| `{k}` | {total:.3f} | {per_q_ms:.1f} |")
+
+    # Honesty note scaled to n_queries. Load-bearing — never drop this.
+    n = max(results.n_queries, 1)
+    lines += [
+        "",
+        "## Statistical power",
+        "",
+        f"{results.n_queries} queries means one query flipping moves aggregate recall by "
+        f"{1 / n:.3f}. Combos within ~{2 / n:.2f} of the leader are not reliably "
+        "distinguishable. Re-run with more queries or a larger corpus before treating "
+        "the leaderboard as a tournament result.",
+        "",
+    ]
+    out = out_dir / "report.md"
+    out.write_text("\n".join(lines))
+    return out
+
+
+def write_recommended_yaml(
+    cfg: BakeoffConfig, results: BakeoffResults, out_dir: Path
+) -> Path:
+    """Render the top-MRR combo as a runnable CellConfig YAML.
+
+    Round-trip through `CellConfig.model_validate` is covered by
+    test_bakeoff_output::test_recommended_yaml_parses_as_cell_config.
+    """
+    ranked = sorted(results.combos, key=lambda c: -c.aggregate.get("mrr", 0))
+    top = ranked[0]
+
+    winner_chunker = next(
+        c for c in cfg.matrix.chunkers if chunker_key(c) == top.chunker_key
+    )
+    winner_embedder = next(
+        e for e in cfg.matrix.embedders if embedder_key(e) == top.embedder_key
+    )
+
+    # Use pydantic's `by_alias=True` so `schema_name` dumps as `schema`
+    # (matches the YAML/chunkshop convention and round-trips through
+    # TargetConfig.model_validate).
+    recommended = {
+        "# NOTE": (
+            f"Top combo from bakeoff '{results.run_name}' "
+            f"(MRR={top.aggregate.get('mrr', 0):.3f}, "
+            f"r@1={top.aggregate.get('recall_at_1', 0):.3f}). "
+            "Point `source` at your real corpus before running `chunkshop ingest`."
+        ),
+        "cell_name": f"{results.run_name}_recommended",
+        "source": cfg.source.model_dump(exclude_none=True, by_alias=True),
+        "framer": cfg.framer.model_dump(exclude_none=True, by_alias=True),
+        "chunker": winner_chunker.model_dump(exclude_none=True, by_alias=True),
+        "embedder": winner_embedder.model_dump(exclude_none=True, by_alias=True),
+        "target": {
+            "dsn_env": cfg.target.dsn_env,
+            "schema": cfg.target.schema_name,
+            "table": f"{results.run_name}_production",
+            "mode": "overwrite",
+        },
+    }
+    out = out_dir / "recommended.yaml"
+    out.write_text(yaml.safe_dump(recommended, sort_keys=False))
+    return out

chunkshop/bakeoff/runner.py

@@ -0,0 +1,228 @@
+"""`run_bakeoff(cfg: BakeoffConfig) -> BakeoffResults` (SC-004, SC-005, SC-006).
+
+Serial cross-product over `matrix.embedders x matrix.chunkers`. For each combo:
+  1. Build a `CellConfig` and call `runner.run_cell` — reuses the full
+     source -> framer -> chunker -> embedder -> sink pipeline.
+  2. Count chunks that landed in the combo's table.
+Then, once per embedder (not per combo), embed all gold queries in a batch so
+N combos sharing an embedder share the same query vectors. For each combo run
+pgvector top-K against its table, score with `score.score_query`, aggregate,
+and return a `BakeoffResults`.
+
+Design decisions:
+- Serial, not parallel. MVP. Parallelism via subprocess orchestrator is an ASK
+  FIRST item in the brief.
+- DSN env var is read, not passed. Caller (CLI) sets `os.environ[dsn_env] = dsn`.
+- Schema creation happens naturally via `run_cell -> sink.create_table -> advisory
+  lock`. No pre-creation here.
+- Matrix size > 50 is caller's responsibility. CLI checks + prompts; runner
+  runs whatever the config says.
+"""
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+import numpy as np
+import psycopg
+from psycopg import sql
+
+from chunkshop.bakeoff.config import (
+    BakeoffConfig,
+    BakeoffResults,
+    ComboResult,
+)
+from chunkshop.bakeoff.gold import load_gold_queries
+from chunkshop.bakeoff.keys import chunker_key, combo_table, embedder_key
+from chunkshop.bakeoff.score import aggregate_scores, score_query
+from chunkshop.config import (
+    CellConfig,
+    NoneExtractor,
+    RuntimeConfig,
+    TargetConfig,
+)
+from chunkshop.embedders import load_embedder
+from chunkshop.runner import run_cell
+
+
+def _chunker_label(cfg) -> str:
+    """Human-readable chunker label for report tables."""
+    t = getattr(cfg, "type", type(cfg).__name__)
+    if t == "neighbor_expand":
+        base = _chunker_label(cfg.base)
+        return f"neighbor_expand(window={cfg.window}, base={base})"
+    if t == "fixed_overlap":
+        return f"fixed_overlap(window_words={cfg.window_words}, step_words={cfg.step_words})"
+    return t
+
+
+def _count_chunks(dsn: str, schema: str, table: str) -> int:
+    with psycopg.connect(dsn) as conn, conn.cursor() as cur:
+        cur.execute(
+            sql.SQL("SELECT COUNT(*) FROM {}.{}").format(
+                sql.Identifier(schema), sql.Identifier(table)
+            )
+        )
+        return cur.fetchone()[0]
+
+
+def _query_top_k(
+    dsn: str,
+    schema: str,
+    table: str,
+    query_vec: np.ndarray,
+    k: int,
+) -> list[tuple[str, int]]:
+    """pgvector top-K lookup. Returns [(doc_id, seq_num), ...] ordered by cosine distance."""
+    vec_str = "[" + ",".join(f"{x:.8f}" for x in query_vec.tolist()) + "]"
+    with psycopg.connect(dsn) as conn, conn.cursor() as cur:
+        cur.execute(
+            sql.SQL(
+                "SELECT doc_id, seq_num FROM {}.{} "
+                "ORDER BY embedding <=> %s::vector LIMIT %s"
+            ).format(sql.Identifier(schema), sql.Identifier(table)),
+            (vec_str, k),
+        )
+        return [(r[0], r[1]) for r in cur.fetchall()]
+
+
+def _build_cell_cfg(
+    bakeoff: BakeoffConfig,
+    chunker_cfg,
+    embedder_cfg,
+    table: str,
+) -> CellConfig:
+    """Translate (bakeoff config, one chunker, one embedder) into a runnable CellConfig."""
+    rt = bakeoff.runtime or RuntimeConfig()
+    return CellConfig(
+        cell_name=f"{bakeoff.name}__{chunker_key(chunker_cfg)}__{embedder_key(embedder_cfg)}",
+        source=bakeoff.source,
+        framer=bakeoff.framer,
+        chunker=chunker_cfg,
+        embedder=embedder_cfg,
+        extractor=NoneExtractor(),
+        target=TargetConfig(
+            dsn_env=bakeoff.target.dsn_env,
+            schema=bakeoff.target.schema_name,
+            table=table,
+            mode="overwrite",
+            hnsw=False,
+        ),
+        runtime=rt,
+    )
+
+
+def _corpus_label(bakeoff: BakeoffConfig) -> str:
+    """Best-effort human label for the corpus. `files.glob` -> the glob string; else source type."""
+    src = bakeoff.source
+    if getattr(src, "type", None) == "files":
+        return getattr(src, "glob", "files")
+    return getattr(src, "type", "unknown")
+
+
+def run_bakeoff(cfg: BakeoffConfig) -> BakeoffResults:
+    """Execute every (chunker, embedder) combo, score against gold, return results.
+
+    Caller must set `os.environ[cfg.target.dsn_env]` before calling — the sink
+    reads the DSN from there. Raises `RuntimeError` if the env var is unset.
+    """
+    if cfg.target.dsn_env not in os.environ:
+        raise RuntimeError(
+            f"DSN env var {cfg.target.dsn_env!r} is not set. The CLI sets it from "
+            "--dsn / $CHUNKSHOP_DSN before calling run_bakeoff."
+        )
+    dsn = os.environ[cfg.target.dsn_env]
+    schema = cfg.target.schema_name
+
+    gold = load_gold_queries(cfg.gold_queries)
+    combos_in = [(c, e) for c in cfg.matrix.chunkers for e in cfg.matrix.embedders]
+
+    started_at = time.strftime("%Y-%m-%d %H:%M:%S")
+
+    # ----- Phase 1: ingest every combo serially -----
+    ingest_meta: list[dict[str, Any]] = []
+    for c, e in combos_in:
+        table = combo_table(c, e)
+        cell_cfg = _build_cell_cfg(cfg, c, e, table)
+        t0 = time.time()
+        res = run_cell(cell_cfg)
+        wall = time.time() - t0
+        if res.error:
+            raise RuntimeError(
+                f"ingest failed for combo {table}: {res.error}"
+            )
+        n_chunks = _count_chunks(dsn, schema, table)
+        ingest_meta.append(
+            {"chunker": c, "embedder": e, "table": table,
+             "chunks": n_chunks,
+             "wall_seconds": round(wall, 2),
+             # Subset of wall_seconds: just the embedder's portion.
+             "embed_seconds": round(getattr(res, "embed_seconds", 0.0), 2)}
+        )
+
+    # ----- Phase 2: embed gold queries once per embedder (not once per combo) -----
+    # Two combos sharing an embedder share the same query vectors.
+    # Capture per-embedder query-embed wall time — that's a proxy for
+    # production query-time latency at scale.
+    query_vecs_by_emb_key: dict[str, np.ndarray] = {}
+    query_embed_seconds_by_emb_key: dict[str, float] = {}
+    for e in cfg.matrix.embedders:
+        k = embedder_key(e)
+        if k in query_vecs_by_emb_key:
+            continue
+        embedder = load_embedder(e)
+        t_qe = time.perf_counter()
+        vecs = embedder.embed([g.query for g in gold])
+        query_embed_seconds_by_emb_key[k] = round(time.perf_counter() - t_qe, 3)
+        query_vecs_by_emb_key[k] = vecs
+
+    # ----- Phase 3: score every combo -----
+    combo_results: list[ComboResult] = []
+    for meta in ingest_meta:
+        c = meta["chunker"]
+        e = meta["embedder"]
+        ck = chunker_key(c)
+        ek = embedder_key(e)
+        table = meta["table"]
+        vecs = query_vecs_by_emb_key[ek]
+
+        per_query: list[dict[str, Any]] = []
+        per_query_scores: list[dict[str, float]] = []
+        for i, g in enumerate(gold):
+            top = _query_top_k(dsn, schema, table, vecs[i], k=cfg.scoring.top_k)
+            doc_ids = [t[0] for t in top]
+            s = score_query(doc_ids, g.gold_doc_id, cfg.scoring.k)
+            per_query_scores.append(s)
+            per_query.append({
+                "query": g.query,
+                "gold_doc_id": g.gold_doc_id,
+                "top_k": [{"doc_id": t[0], "seq_num": t[1]} for t in top],
+                **s,
+            })
+
+        agg = aggregate_scores(per_query_scores)
+
+        combo_results.append(ComboResult(
+            chunker_key=ck,
+            embedder_key=ek,
+            chunker_label=_chunker_label(c),
+            embedder_label=e.model_name,
+            table=table,
+            ingest_chunks=meta["chunks"],
+            ingest_wall_seconds=meta["wall_seconds"],
+            ingest_embed_seconds=meta["embed_seconds"],
+            aggregate=agg,
+            per_query=per_query,
+        ))
+
+    return BakeoffResults(
+        run_name=cfg.name,
+        started_at=started_at,
+        corpus_label=_corpus_label(cfg),
+        n_queries=len(gold),
+        n_combos=len(combo_results),
+        combos=combo_results,
+        gold_queries=[{"query": g.query, "gold_doc_id": g.gold_doc_id} for g in gold],
+        query_embed_seconds_by_embedder=query_embed_seconds_by_emb_key,
+    )

chunkshop/bakeoff/score.py

@@ -0,0 +1,41 @@
+"""Pure scoring functions for bakeoff evaluation (SC-005).
+
+Given a ranked list of retrieved doc_ids for a query and a single gold doc_id,
+compute recall@k for each k in `k_values` and MRR. Aggregation across queries
+is a simple arithmetic mean. No external deps — kept here for easy unit test.
+"""
+from __future__ import annotations
+
+from typing import Iterable
+
+
+def score_query(
+    ranked_doc_ids: list[str],
+    gold_doc_id: str,
+    k_values: Iterable[int],
+) -> dict[str, float]:
+    """Score one query against one gold doc_id.
+
+    Returns `{recall_at_K: 0|1, ..., mrr: float}`. MRR uses 1/rank of the first
+    gold hit in the ranked list (unbounded — callers should slice to top-K
+    before scoring if they want bounded MRR).
+    """
+    result: dict[str, float] = {}
+    for k in k_values:
+        result[f"recall_at_{k}"] = 1 if gold_doc_id in ranked_doc_ids[:k] else 0
+    mrr = 0.0
+    for rank, did in enumerate(ranked_doc_ids, start=1):
+        if did == gold_doc_id:
+            mrr = 1.0 / rank
+            break
+    result["mrr"] = mrr
+    return result
+
+
+def aggregate_scores(per_query: list[dict[str, float]]) -> dict[str, float]:
+    """Arithmetic mean of each metric across all queries. Empty input -> {}."""
+    if not per_query:
+        return {}
+    n = len(per_query)
+    keys = per_query[0].keys()
+    return {k: sum(q[k] for q in per_query) / n for k in keys}