agent-pack-haystack 0.0.2__tar.gz → 1.0.0__tar.gz

agent_pack_haystack-1.0.0/CHANGELOG.md

@@ -0,0 +1,36 @@
+# Changelog
+
+## [integrations/agent_pack-v0.1.0] - 2026-09-02
+
+### 🚜 Refactor
+
+- [**breaking**] Agent Pack - align/standardize Agents configuration (#3894)
+
+
+## [integrations/agent_pack-v0.0.2] - 2026-08-26
+
+### 🚀 Features
+
+- Use `AgentTool` in Deep Research Agent (#3850)
+
+### 🐛 Bug Fixes
+
+- Fix new issues raised by ruff 0.16.0 (#3670)
+
+### 🧹 Chores
+
+- Re-standardize READMEs (#3768)
+
+
+## [integrations/agent_pack-v0.0.1] - 2026-07-20
+
+### 🚀 Features
+
+- Add advanced rag agent (#3600)
+
+### 🧹 Chores
+
+- Move agent pack to core integrations (#3591)
+- Prepare Agent pack for 3.0 release (#3611)
+
+<!-- generated by git-cliff -->

{agent_pack_haystack-0.0.2 → agent_pack_haystack-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: agent-pack-haystack
-Version: 0.0.2
+Version: 1.0.0
 Summary: Pre-configured Haystack Agents
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/agent_pack#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -12,15 +12,16 @@ Keywords: agents,haystack
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: haystack-ai>=3.1.0
+Requires-Dist: pyyaml>=6.0.1
 Description-Content-Type: text/markdown
 
 # agent-pack-haystack

agent_pack_haystack-1.0.0/examples/advanced_rag_eval.py

@@ -0,0 +1,306 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Mini evaluation harness for the Advanced RAG agent, run against the MultiHopRAG corpus.
+#
+# The corpus and its labelled questions come from `multihop_rag`, which chunks the news articles and works out
+# which chunks hold each question's quoted evidence. That gives exact ground truth: an eval case names the
+# documents an answer needs, so retrieval is scored as recall over those IDs rather than a metadata predicate.
+#
+# Reported per eval case:
+#
+# - Retrieval: recall over the documents the question's evidence lives in, from the run's accumulated `documents`.
+# - Process: whether metadata was inspected before any retrieval, per-tool call counts, how many retrievals used
+#   a filter, tool errors, steps, and wall-clock time. An eval case fails when it exceeds its tool budget.
+# - Citations: the answer must cite every document the question's evidence lives in, and every
+#   `[doc <short-id>]` it carries must resolve to a document the run returned. Whether the answer is *right* is
+#   not checked; that needs a judge this harness does not have yet.
+# - Token usage, per eval case and totalled, for comparing models and reasoning efforts.
+#
+# Run from `integrations/agent_pack` with `OPENAI_API_KEY` set. The corpus requires `datasets`:
+#
+#     hatch run test:python examples/advanced_rag_eval.py
+#     hatch run test:python examples/advanced_rag_eval.py --max-cases 5
+#     hatch run test:python examples/advanced_rag_eval.py --store opensearch
+#
+# `--store opensearch` reuses a populated index across runs, so repeat runs skip re-indexing. Set `OPENSEARCH_URL`
+# if it is not http://localhost:9200, and `OPENSEARCH_USERNAME` / `OPENSEARCH_PASSWORD` for a secured instance.
+
+import argparse
+import re
+import time
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Any
+
+from haystack import Document
+from haystack.components.agents import Agent
+from haystack.dataclasses import ChatMessage
+from multihop_rag import CORPUS_KEY, LabelledQuestion, build_eval_cases, prepare_corpus
+from util import build_bm25_retriever, preview
+
+from haystack_integrations.agent_pack.advanced_rag import create_advanced_rag_agent
+
+RETRIEVAL_TOOLS = ("search_documents", "fetch_documents_by_filter")
+METADATA_TOOLS = ("list_metadata_fields", "get_metadata_field_values", "get_metadata_field_range")
+_CITATION_RE = re.compile(r"\[doc ([0-9a-f]{4,16})[^]]*\]")
+
+
+@dataclass
+class EvalCase:
+    """
+    One eval case: a question, the documents that answer it, and the budgets the run may spend.
+
+    :param question: The question to put to the agent.
+    :param evidence: Ground truth, as `{chunk id: the quote found in that chunk}`. The keys are the documents the
+        answer needs, and the values are what the answer should be based on. For example,
+        {"a1b2c3...": "Tyreek Hill now needs to ...", "d4e5f6...": "The Dolphins went on to ..."}
+    :param tool_budgets: How many times the run may call a tool, or a group of tools sharing one allowance, as
+        `{tool name or names: limit}`. Exceeding any of them fails the eval case. Which tools these are depends
+        on the agent under evaluation, so the caller supplies them.
+    """
+
+    question: str
+    evidence: dict[str, str]
+    tool_budgets: dict[str | tuple[str, ...], int]
+
+    @property
+    def expected_document_ids(self) -> frozenset[str]:
+        """The chunks an answer needs, which are the ones its evidence was found in."""
+        return frozenset(self.evidence)
+
+
+@dataclass
+class ToolRunStats:
+    """
+    The tool calls one agent run made, and what they add up to.
+
+    :param calls: Every call the run made, in the order it made them, as `(tool name, the arguments it passed)`:
+
+            [("list_metadata_fields", {}), ("search_documents", {"query": "CRISPR", "filters": None})]
+
+    :param errors: The calls that came back an error, as `(tool name, what it said)`:
+
+            [("get_metadata_field_values", "field 'nope' does not exist in the store")]
+    """
+
+    calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list)
+    errors: list[tuple[str, str]] = field(default_factory=list)
+
+    @staticmethod
+    def _named(tools: str | tuple[str, ...]) -> set[str]:
+        """
+        Normalize one tool name or a group of them to a set.
+
+        :param tools: A tool name, or several of them.
+        :returns: The names as a set.
+        """
+        return {tools} if isinstance(tools, str) else set(tools)
+
+    def calls_to(self, tools: str | tuple[str, ...]) -> int:
+        """
+        Count the calls made to one tool, or to any of a group of them.
+
+        :param tools: A tool name, or several of them.
+        :returns: How many calls the run made to them.
+        """
+        wanted = self._named(tools=tools)
+        return sum(1 for name, _ in self.calls if name in wanted)
+
+    def called_before(self, tools: str | tuple[str, ...], other: str | tuple[str, ...]) -> bool:
+        """
+        Whether the run reached for one tool before it reached for another.
+
+        :param tools: The tool, or tools, that should come first.
+        :param other: The tool, or tools, they should come before.
+        :returns: True when one of `tools` was called and none of `other` was called before it. False when
+            `other` came first, and when neither was called at all.
+        """
+        wanted, after = self._named(tools=tools), self._named(tools=other)
+        for name, _ in self.calls:
+            if name in wanted:
+                return True
+            if name in after:
+                return False
+        return False
+
+
+def extract_tool_run_stats(messages: list[ChatMessage]) -> ToolRunStats:
+    """
+    Extract tool calls and error results from an agent run.
+
+    :param messages: The messages returned by `agent.run(...)`.
+    :returns: The extracted statistics.
+    """
+    return ToolRunStats(
+        calls=[(call.tool_name, call.arguments or {}) for message in messages for call in message.tool_calls],
+        errors=[
+            (result.origin.tool_name, result.result)
+            for message in messages
+            for result in message.tool_call_results
+            if result.error
+        ],
+    )
+
+
+def _sum_usage(total: dict[str, int], usage: dict[str, Any]) -> dict[str, int]:
+    """
+    Accumulate the numeric entries of a token-usage dict into a running total.
+
+    :param total: The running totals, updated in place.
+    :param usage: One run's `token_usage` dict (key names vary by generator).
+    :returns: The updated totals.
+    """
+    for key, value in (usage or {}).items():
+        if isinstance(value, int):
+            total[key] = total.get(key, 0) + value
+    return total
+
+
+def run_eval_case(agent: Agent, case: EvalCase, position: int, total: int) -> dict[str, Any]:
+    """
+    Run the agent on one eval case and print its report.
+
+    :param agent: The agent under evaluation.
+    :param case: The eval case to evaluate.
+    :param position: Which eval case this is, for the report heading.
+    :param total: How many eval cases there are, for the report heading.
+    :returns: A dict with `passed` (bool), `usage` (the run's token_usage dict), and `time` (s).
+    """
+    started = time.perf_counter()
+    result = agent.run(messages=[ChatMessage.from_user(case.question)])
+    elapsed = time.perf_counter() - started
+
+    # Everything the report needs comes out of the one run: its messages, its answer and its token usage.
+    tool_run_stats = extract_tool_run_stats(messages=result["messages"])
+    answer = result["last_message"].text or ""
+    usage = result.get("token_usage") or {}
+
+    # Recall on the retrieved documents: how many of the expected documents the Agent found.
+    retrieved_docs = result.get("documents") or []
+    retrieved_ids = {document.id for document in retrieved_docs}
+    found = case.expected_document_ids & retrieved_ids
+    recall = len(found) / len(case.expected_document_ids)
+
+    # Resolve each [doc <short-id>] the answer uses against what the Agent found. A reference matching nothing
+    # is a fake citation, and one the answer doesn't use is an uncited document. Both are failures.
+    cited_refs = _CITATION_RE.findall(answer)
+    resolved = [ref for ref in cited_refs if any(document.id.startswith(ref) for document in retrieved_docs)]
+    cited_ids = {document.id for document in retrieved_docs if any(document.id.startswith(r) for r in cited_refs)}
+    citations_ok = len(resolved) == len(cited_refs)
+    uncited = case.expected_document_ids - cited_ids
+
+    # An eval case passes only on all four: it found every expected document, cited every one of them, made no
+    # citation that does not resolve, and stayed inside its tool budget.
+    spent = {tools: tool_run_stats.calls_to(tools=tools) for tools in case.tool_budgets}
+    within_budget = all(used <= case.tool_budgets[tools] for tools, used in spent.items())
+    passed = recall == 1.0 and not uncited and citations_ok and within_budget
+
+    inspected_first = tool_run_stats.called_before(tools="list_metadata_fields", other=RETRIEVAL_TOOLS)
+    counts = Counter(name for name, _ in tool_run_stats.calls)
+    filtered_retrievals = sum(
+        1 for name, args in tool_run_stats.calls if name in RETRIEVAL_TOOLS and args.get("filters")
+    )
+    needed = len(case.expected_document_ids)
+    print(f"\n=== eval case {position}/{total}: {'PASS' if passed else 'FAIL'} ===")
+    print(f"  question: {case.question}")
+    print(f"  tools called: {dict(counts)}")
+    print(
+        f"  inspected metadata first: {inspected_first}   retrievals with a filter: {filtered_retrievals}   "
+        f"tool errors: {len(tool_run_stats.errors)}   steps: {result['step_count']}   time: {elapsed:.1f}s"
+    )
+    for tools, used in spent.items():
+        limit = case.tool_budgets[tools]
+        label = tools if isinstance(tools, str) else " + ".join(tools)
+        print(f"  tool budget: {label} {used}/{limit} -> {'ok' if used <= limit else 'EXCEEDED'}")
+    print(f"  retrieval: found {len(found)}/{needed} of the documents the answer needs, {len(retrieved_docs)} returned")
+    print(
+        f"  citations: cited {needed - len(uncited)}/{needed} of them, "
+        f"and {len(resolved)}/{len(cited_refs)} of the answer's references point at a returned document"
+    )
+
+    # Chunks of one article all carry its title, so they are grouped under it and told apart by chunk number.
+    # Articles keep the order they were first returned in, which is the order the run ranked them.
+    by_article: dict[str, list[Document]] = {}
+    for document in retrieved_docs:
+        by_article.setdefault(document.meta.get("title", ""), []).append(document)
+
+    print("\n  documents returned, grouped by article. -> marks one the answer needs:")
+    for title, documents in by_article.items():
+        print(f"    Title: {title}")
+        for document in sorted(documents, key=lambda chunk: chunk.meta.get("split_id", 0)):
+            needed_here = "-> " if document.id in case.expected_document_ids else "   "
+            snippet = preview(text=document.content or "", limit=80)
+            print(f"      {needed_here}chunk {document.meta.get('split_id'):>2}  [doc {document.id[:8]}]  {snippet}")
+    # Naming the quote, since the id alone says nothing about what the run failed to find or failed to use.
+    print()
+    for document_id in sorted(case.expected_document_ids - retrieved_ids):
+        print(
+            f"  needed but never retrieved: [doc {document_id[:8]}] "
+            f"{preview(text=case.evidence[document_id], limit=96)}"
+        )
+    for document_id in sorted(uncited & retrieved_ids):
+        print(
+            f"  needed and retrieved but not cited: [doc {document_id[:8]}] "
+            f"{preview(text=case.evidence[document_id], limit=96)}"
+        )
+
+    if usage:
+        print(f"  tokens: { {k: v for k, v in usage.items() if isinstance(v, int)} }")
+    for tool_name, message in tool_run_stats.errors:
+        print(f"  tool error: {tool_name} -> {preview(text=message, limit=120)}")
+    print("  answer:")
+    for line in answer.splitlines():
+        print(f"    {line}")
+    return {"passed": passed, "usage": usage, "time": elapsed}
+
+
+def main() -> None:
+    """Run the eval set and print per-eval-case reports plus a summary."""
+    parser = argparse.ArgumentParser(description="Mini evaluation harness for the Advanced RAG agent.")
+    parser.add_argument("--store", choices=("in_memory", "opensearch"), default="in_memory")
+    parser.add_argument("--max-cases", type=int, default=10, help="How many labelled questions to evaluate.")
+    parser.add_argument("--case-seed", type=int, default=0, help="Selects which eval cases are drawn from the dataset.")
+    arguments = parser.parse_args()
+
+    store, articles = prepare_corpus(backend=arguments.store)
+    chunks = sum(len(article.chunks) for article in articles.values())
+    print(f"{CORPUS_KEY} on {arguments.store}: {chunks} chunks from {len(articles)} articles")
+
+    labelled: list[LabelledQuestion] = build_eval_cases(
+        articles=articles, limit=arguments.max_cases, seed=arguments.case_seed
+    )
+    # Budgets for the tools this agent has. Lenient on purpose: too many retrievals is better than too few.
+    # Generous on purpose: a MultiHopRAG question needs evidence from several articles, so several searches are
+    # the expected shape of a good run rather than a sign of floundering.
+    budgets: dict[str | tuple[str, ...], int] = {METADATA_TOOLS: 8, RETRIEVAL_TOOLS: 12}
+    eval_cases = [
+        EvalCase(question=question.question, evidence=question.evidence, tool_budgets=budgets) for question in labelled
+    ]
+    print(f"eval cases: {len(eval_cases)} labelled from evidence")
+
+    # Build the advanced rag agent
+    agent = create_advanced_rag_agent(document_store=store, retriever=build_bm25_retriever(store=store))
+
+    # Run the eval cases
+    results = [
+        run_eval_case(agent=agent, case=case, position=position, total=len(eval_cases))
+        for position, case in enumerate(eval_cases, start=1)
+    ]
+
+    # Calculate total usage
+    total_usage: dict[str, int] = {}
+    for result in results:
+        _sum_usage(total=total_usage, usage=result["usage"])
+
+    # Print the results
+    passed = sum(result["passed"] for result in results)
+    print(f"\n=== {passed}/{len(results)} eval cases passed ===")
+    print(f"total time: {sum(result['time'] for result in results):.1f}s")
+    if total_usage:
+        print(f"total tokens: {total_usage}")
+
+
+if __name__ == "__main__":
+    main()

agent_pack_haystack-1.0.0/examples/multihop_rag.py

@@ -0,0 +1,309 @@
+# SPDX-FileCopyrightText: 2026-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Prepare the MultiHopRAG dataset as an evaluation set for harness optimization.
+#
+# The dataset pairs 609 news articles carrying real metadata — category, source, author, publication timestamp — with
+# 2,556 labelled queries whose supporting evidence is quoted verbatim and attributed to its article.
+#
+# Articles are long — a median of 7,836 characters, up to 71,034. So articles are split by word with overlap.
+#
+# Splitting would normally blur the ground truth, since evidence is attributed to an article and not a chunk.
+# So we ensure that a fact is contained wholly within a chunk. If a fact is split across two chunks, the query is
+# excluded from the evaluation set.
+#
+# Run this module directly to build the corpus and report what it produced:
+#
+#     hatch run test:python examples/multihop_rag.py
+
+import argparse
+import hashlib
+import os
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from haystack import Document
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.document_stores.types import DocumentStore, DuplicatePolicy
+from haystack.lazy_imports import LazyImport
+
+with LazyImport(message='Run "pip install datasets" to build the MultiHopRAG evaluation set.') as datasets_import:
+    from datasets import load_dataset
+
+with LazyImport(message='Run "pip install opensearch-haystack" to use an OpenSearch store.') as opensearch_import:
+    from haystack_integrations.document_stores.opensearch import OpenSearchDocumentStore
+
+
+@dataclass(frozen=True)
+class LabelledQuestion:
+    """
+    One question the dataset labels, with the evidence an answer needs.
+
+    :param question: The question as the dataset asks it.
+    :param evidence: The quoted evidence the answer is built from, by the chunk it was found in.
+    :param answer: The ground-truth answer, verbatim. How to check a reply against it is the harness's call: most
+        are a name or a number, but some are short words like "Yes", where a substring check may match for
+        reasons unrelated to the answer being right.
+    """
+
+    question: str
+    evidence: dict[str, str]
+    answer: str = ""
+
+    @property
+    def expected_document_ids(self) -> frozenset[str]:
+        """The chunks an answer needs, which are the ones its evidence was found in."""
+        return frozenset(self.evidence)
+
+
+DATASET_ID = "yixuantt/MultiHopRAG"
+CORPUS_KEY = "multihop-rag"
+
+# Word-based splitting with overlap. Measured over the whole dataset at this setting: no evidence fact is split
+# across a chunk boundary, so every one of the 6,084 is contained wholly within a chunk. However, the overlap
+# means 926 of them are present in two adjacent chunks, so a query can no longer retrieve just a single expected
+# document, so the query is dropped rather than scored loosely. So we end up with 1,432 of the 2,255 queries as
+# valid eval points.
+SPLIT_BY = "word"
+SPLIT_LENGTH = 350
+SPLIT_OVERLAP = 90
+
+# Carried from the article onto each of its chunks
+ARTICLE_METADATA = ("title", "category", "source", "author", "published_at", "url")
+
+# Queries whose evidence is present in the corpus. The dataset's fourth type, `null_query`, is deliberately
+# excluded for now to focus on answerable question types.
+ANSWERABLE_QUESTION_TYPES = ("comparison_query", "inference_query", "temporal_query")
+
+
+@dataclass(frozen=True)
+class Article:
+    """One article's body and its chunks, in the order they appear in it."""
+
+    body: str
+    chunks: tuple[Document, ...]
+
+
+def build_document_store(backend: str, index: str) -> DocumentStore:
+    """
+    Build an empty in-memory or OpenSearch document store under a named index.
+
+    :param backend: Either "in_memory" or "opensearch".
+    :param index: Names the index to open or create.
+    :returns: The empty store.
+    """
+    if backend == "opensearch":
+        opensearch_import.check()
+        url = os.environ.get("OPENSEARCH_URL", "http://localhost:9200")
+        return OpenSearchDocumentStore(
+            hosts=url,
+            index=index,
+            use_ssl=url.startswith("https"),
+            verify_certs=not url.startswith("https://localhost"),
+        )
+    return InMemoryDocumentStore(index=index)
+
+
+def prepare_corpus(
+    backend: Literal["in_memory", "opensearch"] = "in_memory", index: str = CORPUS_KEY
+) -> tuple[DocumentStore, dict[str, Article]]:
+    """
+    Build the chunked corpus and index it, reusing an already-populated store.
+
+    :param backend: Either "in_memory" or "opensearch".
+    :param index: Names the index holding the corpus.
+    :returns: The populated store and every article by title, each with the chunks covering it.
+    """
+    datasets_import.check()
+    store = build_document_store(backend=backend, index=index)
+
+    # Load the dataset, split the articles into chunks, and write them to the store.
+    documents = [
+        Document(content=row["body"], meta={field: row[field] for field in ARTICLE_METADATA})
+        for row in load_dataset(DATASET_ID, "corpus", split="train")
+        if (row["body"] or "").strip()
+    ]
+    splitter = DocumentSplitter(split_by=SPLIT_BY, split_length=SPLIT_LENGTH, split_overlap=SPLIT_OVERLAP)
+    chunks = splitter.run(documents=documents)["documents"]
+
+    # Write only when the store is empty or the corpus changed, so a repeated run does not reindex.
+    if store.count_documents() != len(chunks):
+        store.write_documents(documents=chunks, policy=DuplicatePolicy.OVERWRITE)
+
+    # Group the chunks under the article they were split from, using the id the splitter records on each.
+    by_source: dict[str, list[Document]] = {}
+    for chunk in chunks:
+        by_source.setdefault(chunk.meta["source_id"], []).append(chunk)
+    articles = {
+        document.meta["title"]: Article(
+            body=document.content or "",
+            chunks=tuple(sorted(by_source.get(document.id, []), key=lambda chunk: chunk.meta["split_idx_start"])),
+        )
+        for document in documents
+    }
+    return store, articles
+
+
+def _covering_chunks(article: Article, fact: str) -> list[Document] | None:
+    """
+    Find the chunks a quoted evidence fact is contained in.
+
+    A quote appearing more than once is ambiguous evidence, so if this occurs an empty list is returned to indicate
+    that this datapoint should be excluded.
+
+    :param article: The article the fact is attributed to.
+    :param fact: The quoted evidence.
+    :returns: The chunks holding every occurrence of the fact, or None when the quote is not in the article at
+        all. An empty list when the quote is there but no single chunk holds all of it: it either spans a chunk boundary
+        or is repeated in multiple chunks.
+    """
+    # Find all occurrences of the fact in the article.
+    occurrences = []
+    position = article.body.find(fact)
+    while position != -1:
+        occurrences.append(position)
+        position = article.body.find(fact, position + 1)
+
+    # If no occurrences we return None
+    if not occurrences:
+        return None
+
+    # Return all chunks that cover all occurrences of the fact. If a fact is split across two chunks, an empty list is
+    # returned.
+    return [
+        chunk
+        for chunk in article.chunks
+        if all(
+            chunk.meta["split_idx_start"] <= start
+            and start + len(fact) <= chunk.meta["split_idx_start"] + len(chunk.content or "")
+            for start in occurrences
+        )
+    ]
+
+
+def exact_eval_case_candidates(articles: dict[str, Article]) -> dict[str, list[LabelledQuestion]]:
+    """
+    Build every eval case whose expected documents can be stated exactly, grouped by question type.
+
+    A query is kept when each of its evidence facts is contained in exactly one chunk. Overlap puts some facts in
+    two adjacent chunks, and those queries are dropped.
+
+    :param articles: The corpus the eval cases will be scored against, by article title.
+    :returns: Eval cases per question type, ordered by question for reproducibility.
+    :raises ValueError: If any evidence fact is split across chunks, since no single chunk then holds it.
+    """
+    datasets_import.check()
+    candidates: dict[str, list[LabelledQuestion]] = {name: [] for name in ANSWERABLE_QUESTION_TYPES}
+    split_across_chunks = 0
+    for row in load_dataset(DATASET_ID, "MultiHopRAG", split="train"):
+        if row["question_type"] not in candidates:
+            continue
+
+        # Locate every fact this query cites. One chunk per fact keeps the query; anything else drops it.
+        evidence: dict[str, str] = {}
+        exact = True
+        for entry in row["evidence_list"]:
+            fact = entry["fact"]
+            holders = _covering_chunks(article=articles[entry["title"]], fact=fact)
+            # holders is [] when the fact is in the article but split across chunks, or present in several
+            split_across_chunks += holders == []
+            if holders is None or len(holders) != 1:
+                exact = False
+                continue
+            evidence[holders[0].id] = fact
+
+        if exact and evidence:
+            candidates[row["question_type"]].append(
+                LabelledQuestion(question=row["query"], evidence=evidence, answer=row["answer"])
+            )
+
+    # The splitting settings decide this, so it is reachable by editing them.
+    if split_across_chunks:
+        msg = (
+            f"{split_across_chunks} evidence facts are present but split across chunks, so no single chunk holds "
+            f"them. Lower SPLIT_LENGTH or raise SPLIT_OVERLAP until every fact fits inside one chunk."
+        )
+        raise ValueError(msg)
+    return {name: sorted(cases, key=lambda case: case.question) for name, cases in candidates.items()}
+
+
+def build_eval_cases(articles: dict[str, Article], limit: int, seed: int = 0) -> list[LabelledQuestion]:
+    """
+    Build a reproducible set of eval cases, spread evenly across the question types.
+
+    :param articles: The corpus the eval cases will be scored against, by article title.
+    :param limit: How many eval cases to select. Each one costs an Agent run per candidate measured.
+    :param seed: Seed for the selection, so the same evaluation set is rebuilt every time.
+    :returns: The selected eval cases, interleaved by question type so a truncated set stays balanced.
+    :raises ValueError: If `limit` is below one.
+    """
+    if limit < 1:
+        msg = "limit must be at least 1."
+        raise ValueError(msg)
+    candidates = exact_eval_case_candidates(articles=articles)
+
+    def rank(case: LabelledQuestion) -> str:
+        """Order eval cases by a hash of the seed and the question, which the same seed reproduces exactly."""
+        return hashlib.sha256(f"{seed}:{case.question}".encode()).hexdigest()
+
+    # Sorted by a hash rather than seeded with `random`, whose ordering for a given seed can change between Python
+    # versions.
+    ordered = {name: sorted(cases, key=rank) for name, cases in candidates.items()}
+
+    # Take one eval case per question type in turn, so stopping at the limit still leaves the types balanced.
+    selected: list[LabelledQuestion] = []
+    for position in range(max(len(pool) for pool in ordered.values())):
+        for name in ANSWERABLE_QUESTION_TYPES:
+            if position < len(ordered[name]):
+                selected.append(ordered[name][position])
+            if len(selected) == limit:
+                return selected
+    return selected
+
+
+def _report(store: DocumentStore, articles: dict[str, Article], cases: list[LabelledQuestion]) -> None:
+    """Describe what was built, including the checks that make the eval cases scoreable."""
+    chunks = [chunk for article in articles.values() for chunk in article.chunks]
+    lengths = sorted(len(chunk.content or "") for chunk in chunks)
+    metadata: dict[str, Any] = {field: {chunk.meta.get(field) for chunk in chunks} for field in ARTICLE_METADATA}
+    print(f"  articles:  {len(articles)}")
+    print(f"  chunks:    {len(chunks)} indexed={store.count_documents()} (mean {sum(lengths) // len(lengths)} chars)")
+    print(
+        f"  metadata:  category={len(metadata['category'])} source={len(metadata['source'])} "
+        f"author={len(metadata['author'])} distinct values"
+    )
+    print(f"  published: {min(metadata['published_at'])} .. {max(metadata['published_at'])}")
+    print(f"\n  eval cases selected: {len(cases)}")
+    # Listed rather than given as a range, since the sizes present need not be contiguous.
+    sizes = sorted({len(case.expected_document_ids) for case in cases})
+    listed = str(sizes[0]) if len(sizes) == 1 else f"{', '.join(str(size) for size in sizes[:-1])} or {sizes[-1]}"
+    print(f"    expected documents per eval case: {listed}")
+    print(f"    with a ground-truth answer:  {sum(1 for case in cases if case.answer)}")
+    print("    example questions:")
+    for case in cases[:3]:
+        print(f"      - {case.question[:96]}")
+
+
+def main() -> None:
+    """Build the corpus and evaluation set, and report what came out."""
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--store", choices=("in_memory", "opensearch"), default="in_memory")
+    parser.add_argument("--max-cases", type=int, default=20)
+    parser.add_argument("--case-seed", type=int, default=0)
+    arguments = parser.parse_args()
+
+    print(f"=== preparing {DATASET_ID} on {arguments.store} ===")
+    store, articles = prepare_corpus(backend=arguments.store)
+    cases = build_eval_cases(articles=articles, limit=arguments.max_cases, seed=arguments.case_seed)
+    _report(store=store, articles=articles, cases=cases)
+
+    available = exact_eval_case_candidates(articles=articles)
+    print(f"\n  eval cases available in total: {sum(len(pool) for pool in available.values())}")
+    for name, pool in available.items():
+        print(f"    {name:<20} {len(pool)}")
+
+
+if __name__ == "__main__":
+    main()