raglab 0.2.2__tar.gz → 0.2.3__tar.gz

{raglab-0.2.2 → raglab-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: raglab
-Version: 0.2.2
+Version: 0.2.3
 Summary: A medley of tools to make RAG-based applications.
 Project-URL: Homepage, https://github.com/thorwhalen/raglab
 Project-URL: Repository, https://github.com/thorwhalen/raglab
@@ -9,7 +9,7 @@ Author: thorwhalen
 License: mit
 License-File: LICENSE
 Requires-Python: >=3.10
-Requires-Dist: ir>=0.1.12
+Requires-Dist: ir>=0.1.14
 Provides-Extra: dev
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'

{raglab-0.2.2 → raglab-0.2.3}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "raglab"
-version = "0.2.2"
+version = "0.2.3"
 description = "A medley of tools to make RAG-based applications."
 readme = "README.md"
 requires-python = ">=3.10"
@@ -18,7 +18,7 @@ authors = [
 # strategies (Planner/Formulator/Evaluator) use `oa` lazily — the `llm` extra
 # below — so `import raglab` stays offline by default.
 dependencies = [
-    "ir>=0.1.12",
+    "ir>=0.1.14",
 ]
 
 [project.license]

{raglab-0.2.2 → raglab-0.2.3}/raglab/__init__.py

@@ -47,6 +47,7 @@ from .agent import (
     score_reranker,
     single_subtask_planner,
 )
+from .llm import EVALUATION_PROMPT, make_llm_evaluator, make_llm_formulator
 
 __all__ = [
     "Query",
@@ -69,4 +70,7 @@ __all__ = [
     "passthrough_evaluator",
     "score_reranker",
     "identity_citer",
+    "make_llm_formulator",
+    "make_llm_evaluator",
+    "EVALUATION_PROMPT",
 ]

{raglab-0.2.2 → raglab-0.2.3}/raglab/agent.py

@@ -30,6 +30,7 @@ from typing import Any, Protocol, runtime_checkable
 # ir owns the retrieval substrate: the Result type and the Retriever leaf
 # contract live there (one-way dependency, ir is the SSOT).
 from ir import Retriever, SearchHit
+from ir.base import best_per_artifact
 
 #: A retrieved item — ir's :class:`~ir.base.SearchHit` (ir_09's ``Result``):
 #: a *pointer + snippet* (``text``) with a ``score`` and ``metadata``.
@@ -188,13 +189,19 @@ def passthrough_evaluator(task: SubTask, results: Sequence[Result]) -> Judgement
 
 
 def score_reranker(results: Sequence[Result]) -> Sequence[Result]:
-    """Final ordering by descending ``score`` (the cross-source merge, v1).
+    """Cross-source merge (v1): one surface per artifact, ordered by descending score.
+
+    Delegates to :func:`ir.base.best_per_artifact` (ir is the SSOT for hit
+    operations): an artifact retrieved by several queries / sources / rounds —
+    common once the back-edge re-queries — survives once, at its highest score, so
+    the merged list carries no duplicate ``artifact_id``. Also the evaluator's
+    pre-selection rank, so :func:`ir.select` never sees duplicates either.
 
     Note: a plain score sort assumes comparable score scales across sources
     (true when they share an embedder + mode). A rank-based (RRF) cross-source
     merge for heterogeneous backends is a documented follow-up.
     """
-    return sorted(results, key=lambda r: float(getattr(r, "score", 0.0)), reverse=True)
+    return best_per_artifact(results)
 
 
 def identity_citer(results: Sequence[Result]) -> Sequence[Result]:

raglab-0.2.3/raglab/llm.py

@@ -0,0 +1,257 @@
+"""LLM-backed roles — the Formulator and Evaluator (ir_09 §8 steps 1 & 3).
+
+`raglab`'s no-LLM thin slice (:mod:`raglab.agent`) runs offline: an identity
+formulator and a pass-through evaluator. This module supplies the two LLM roles
+that turn that slice into a real agent — query understanding and the back-edge:
+
+- :func:`make_llm_formulator` — query rewrite / expand / HyDE. A thin adapter
+  over ir's :func:`ir.make_llm_formulator` (the ``formulate=`` seam, ir_09 §3):
+  ir owns the lazy-:mod:`oa` rewriter and the identity fallback; raglab only
+  adapts the shape ``str -> str | [str]`` to the role contract
+  ``(SubTask, source) -> [LowLevelQuery]``.
+- :func:`make_llm_evaluator` — sufficiency + refinement. ``ir.select`` owns the
+  *relevance* decision (the calibrated committed subset, ir_09 §3); the LLM owns
+  *sufficiency* — informed by ir's model-free :attr:`ir.Selection.sufficient`
+  hint, it judges whether the committed subset actually satisfies the goal and,
+  when it does not, emits a ``refinement`` SubTask. That refinement is the
+  **back-edge** that makes the loop an agent rather than a DAG.
+
+Two load-bearing boundaries (ir ↔ raglab), guarded here:
+
+1. A Formulator returns **queries, never SubTasks** — decomposition is the
+   Planner's job. :func:`make_llm_formulator` only ever yields
+   :class:`~raglab.agent.LowLevelQuery`\\ s.
+2. The **back-edge lives in raglab**, never in ir: ir derives a ``sufficient``
+   *signal* from its own selection; raglab's Evaluator is what reads it, decides,
+   and re-queries.
+
+Both builders mirror :func:`ir.select.make_llm_selector`: an injectable callable
+(a test double, or your own router), built lazily on :mod:`oa` only when omitted
+(so ``import raglab`` stays offline), with a **safe fallback** — a formulator must
+never make retrieval worse than the raw query, and an evaluator must never
+fabricate an endless loop (on any failure it returns no refinement, which is the
+loop's break condition).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any, Callable
+
+import ir
+
+from .agent import (
+    Evaluator,
+    Formulator,
+    Judgement,
+    LowLevelQuery,
+    Result,
+    SubTask,
+    score_reranker,
+)
+
+__all__ = ["make_llm_formulator", "make_llm_evaluator", "EVALUATION_PROMPT"]
+
+#: An ir-style query formulator: a query string -> one query or several.
+QueryFormulator = Callable[[str], "str | Sequence[str]"]
+
+#: An evaluator judge: ``(goal, results) -> (sufficient, refinement)``.
+#: ``sufficient`` ends the loop; a non-empty ``refinement`` query becomes the
+#: next sub-goal (the back-edge). A raw text reply is also accepted and parsed.
+Judge = Callable[..., "tuple[bool, str | None] | str"]
+
+#: Truncate each rendered result's text to this many chars in the judge prompt.
+DFLT_MAX_RESULT_CHARS = 500
+
+
+# --------------------------------------------------------------------------- #
+# LLM Formulator — adapt ir's query-level formulator to the role contract
+# --------------------------------------------------------------------------- #
+
+
+def make_llm_formulator(
+    *,
+    formulate: QueryFormulator | None = None,
+    params: Mapping[str, Any] | None = None,
+    **make_kwargs: Any,
+) -> Formulator:
+    """An LLM-backed raglab :class:`~raglab.agent.Formulator`.
+
+    Adapts an ir-style query formulator (``str -> str | [str]``) to the role
+    contract ``(SubTask, source) -> [LowLevelQuery]``: the sub-goal text is
+    rewritten / expanded into one or more search queries, each wrapped as a
+    :class:`~raglab.agent.LowLevelQuery` against ``source``.
+
+    Args:
+        formulate: an injectable ir-style formulator (a test double, or one you
+            built). When omitted it is built once via
+            :func:`ir.make_llm_formulator` (``**make_kwargs`` are forwarded —
+            e.g. ``n``, ``prompt``, ``rewriter``); that call is offline (``oa`` is
+            imported lazily only when the formulator is *invoked*), so
+            ``import raglab`` stays offline.
+        params: per-call retriever overrides (e.g. ``{"mode": "hybrid", "k": 5}``)
+            attached to every emitted :class:`~raglab.agent.LowLevelQuery`.
+
+    The boundary: this returns **queries, never SubTasks**. ir's formulator
+    already falls back to identity on any failure, so the emitted list is never
+    empty — a formulator must never make retrieval worse than the raw sub-goal.
+    """
+    extra = dict(params or {})
+    fn: QueryFormulator = (
+        formulate if formulate is not None else ir.make_llm_formulator(**make_kwargs)
+    )
+
+    def formulator(task: SubTask, source: str) -> list[LowLevelQuery]:
+        try:
+            out = fn(task.goal)
+        except Exception:
+            out = None  # a formulator must never make retrieval worse: fall back
+        queries = [out] if isinstance(out, str) else list(out or [])
+        queries = [q for q in queries if isinstance(q, str) and q.strip()] or [
+            task.goal
+        ]
+        return [
+            LowLevelQuery(source=source, query=q, params=dict(extra)) for q in queries
+        ]
+
+    return formulator
+
+
+# --------------------------------------------------------------------------- #
+# LLM Evaluator — ir.select owns relevance; the LLM owns sufficiency + the
+# back-edge (ir_09 §3/§4)
+# --------------------------------------------------------------------------- #
+
+#: Default prompt for :func:`make_llm_evaluator` — judge sufficiency, else emit a
+#: single improved query (the refinement that drives the back-edge).
+EVALUATION_PROMPT = """\
+A search agent is pursuing this goal:
+
+{goal}
+
+A calibrated selector reviewed the retrieved candidates and committed to the
+results below (it abstained if this list is empty):
+
+{results}
+
+Decide whether these results are SUFFICIENT to satisfy the goal.
+- If they are, reply with exactly: SUFFICIENT
+- If they are not, reply with: INSUFFICIENT
+  then, on the next line, a single improved search query that would retrieve what
+  is still missing. Keep it a terse search phrase — no prose, no numbering.
+"""
+
+
+def make_llm_evaluator(
+    *,
+    judge: Judge | None = None,
+    select_strategy: str = "conservative",
+    select_kwargs: Mapping[str, Any] | None = None,
+    prompt: str = EVALUATION_PROMPT,
+    max_result_chars: int = DFLT_MAX_RESULT_CHARS,
+    **prompt_function_kwargs: Any,
+) -> Evaluator:
+    """An LLM-backed raglab :class:`~raglab.agent.Evaluator` (turns on the back-edge).
+
+    Relevance is ir's: each round the accumulated results are passed through
+    :func:`ir.select` and the committed subset becomes the ``Judgement.relevant``
+    (so the LLM stays in its lane — LLM relevance is known-fragile, ir_01 §3).
+    Sufficiency is the LLM's: it reads the committed subset (informed by ir's
+    model-free :attr:`~ir.Selection.sufficient` hint via abstention) and decides
+    whether the goal is satisfied. When it is not, the judge's improved query
+    becomes a ``refinement`` :class:`~raglab.agent.SubTask` over the same sources
+    — the **back-edge**.
+
+    Args:
+        judge: an injectable ``(goal, results) -> (sufficient, refinement)``
+            callable (a test double, or your own router); a raw text reply is also
+            accepted and parsed. When omitted it is built lazily on :mod:`oa`.
+        select_strategy: ir selection strategy for the relevance decision
+            (default ``"conservative"`` — distractor-robust).
+        select_kwargs: extra args forwarded to :func:`ir.select`
+            (e.g. ``max_k``, ``rel``, ``min_score``).
+        max_result_chars: per-result text truncation in the judge prompt.
+
+    Safe fallback: any judge error returns ``refinement=None`` — the control
+    loop's break condition — so a failing judge can never fabricate an endless
+    loop. Sufficiency without a refinement query is likewise treated as a stop.
+    """
+    sel_kw = dict(select_kwargs or {})
+
+    def _ask_judge(goal: str, rendered: str) -> tuple[bool, str | None]:
+        fn = (
+            judge
+            if judge is not None
+            else _default_llm_judge(prompt, **prompt_function_kwargs)
+        )
+        return _normalize_verdict(fn(goal=goal, results=rendered))
+
+    def evaluator(task: SubTask, results: Sequence[Result]) -> Judgement:
+        # ``ir.select`` documents a best-first precondition; the loop accumulates
+        # hits across rounds/sources in arbitrary order, so rank them first (the
+        # same score ordering the final reranker uses).
+        ranked = score_reranker(results)
+        selection = ir.select(list(ranked), strategy=select_strategy, **sel_kw)
+        relevant = list(selection.selected)
+        rendered = (
+            _render_results(relevant, max_result_chars)
+            if relevant
+            else "(none — the selector abstained)"
+        )
+        try:
+            sufficient, refinement = _ask_judge(task.goal, rendered)
+        except Exception:
+            # Trust ir's model-free signal for the report, but never re-query on a
+            # judge failure: refinement=None is the loop's break condition.
+            return Judgement(
+                relevant=relevant, sufficient=selection.sufficient, refinement=None
+            )
+        if sufficient or not refinement:
+            return Judgement(relevant=relevant, sufficient=True, refinement=None)
+        return Judgement(
+            relevant=relevant,
+            sufficient=False,
+            refinement=SubTask(goal=refinement, sources=task.sources),
+        )
+
+    return evaluator
+
+
+def _render_results(results: Sequence[Result], max_chars: int) -> str:
+    """Render committed hits as ``- id: text`` lines for the judge prompt."""
+    return "\n".join(f"- {h.artifact_id}: {str(h.text)[:max_chars]}" for h in results)
+
+
+def _normalize_verdict(out: "tuple[bool, str | None] | str") -> tuple[bool, str | None]:
+    """Coerce a judge reply (a ``(sufficient, refinement)`` tuple, or raw text)."""
+    if isinstance(out, tuple):
+        sufficient, refinement = out
+        refinement = str(refinement).strip() if refinement else None
+        return bool(sufficient), (refinement or None)
+    return _parse_verdict(str(out or ""))
+
+
+def _parse_verdict(text: str) -> tuple[bool, str | None]:
+    """Parse a SUFFICIENT / INSUFFICIENT + refinement-query text reply."""
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    if not lines or lines[0].upper().startswith("SUFFICIENT"):
+        return True, None
+    refinement = lines[1] if len(lines) > 1 else None
+    return False, (refinement or None)
+
+
+def _default_llm_judge(prompt: str, **prompt_function_kwargs: Any) -> Judge:
+    """Build the default sufficiency judge on :mod:`oa` (lazy import)."""
+    import oa
+
+    fn = oa.prompt_function(
+        prompt,
+        egress=_parse_verdict,
+        name="evaluate_sufficiency",
+        **prompt_function_kwargs,
+    )
+
+    def judge(*, goal: str, results: str) -> tuple[bool, str | None]:
+        return fn(goal=goal, results=results)
+
+    return judge

{raglab-0.2.2 → raglab-0.2.3}/tests/test_agent.py

@@ -66,6 +66,28 @@ def test_cross_source_merge_reranks_by_score():
     assert [r.artifact_id for r in results] == ["b", "a"]  # by score desc
 
 
+def test_final_results_have_no_duplicate_artifacts():
+    # An artifact re-retrieved across back-edge rounds collapses to its best score.
+    retr = _fake_retriever(_hits(("a", 0.9), ("b", 0.4)))
+    rounds = {"n": 0}
+
+    def refining(task, results):
+        rounds["n"] += 1
+        if rounds["n"] < 2:
+            return Judgement(
+                list(results),
+                sufficient=False,
+                refinement=SubTask(task.goal, task.sources),
+            )
+        return Judgement(list(results), sufficient=True)
+
+    ids = [
+        r.artifact_id for r in make_search_agent({"s": retr}, evaluator=refining)("q")
+    ]
+    assert ids == ["a", "b"]  # best score per artifact, descending
+    assert len(ids) == len(set(ids))  # the re-query did not duplicate artifacts
+
+
 def test_passthrough_evaluator_does_not_loop():
     retr = _fake_retriever(_hits(("a", 0.9)))
     make_search_agent({"s": retr})("q")
@@ -133,6 +155,26 @@ def test_budget_bounds_a_never_sufficient_loop():
     assert len(retr.calls) == 3  # exactly max_rounds — the safety net holds
 
 
+def test_budget_caps_results_per_task_seen_by_evaluator():
+    retr = _fake_retriever(_hits(*[(f"a{i}", 1.0 - i / 100) for i in range(20)]))
+    seen = {}
+
+    def evaluator(task, results):
+        seen["n"] = len(results)
+        return Judgement(list(results), sufficient=True)
+
+    make_search_agent(
+        {"s": retr}, evaluator=evaluator, budget=Budget(max_results_per_task=5)
+    )("q")
+    assert seen["n"] == 5  # the evaluator sees at most max_results_per_task
+
+
+def test_budget_caps_sources_per_task():
+    retrs = {f"s{i}": _fake_retriever(_hits((f"a{i}", 0.5))) for i in range(6)}
+    make_search_agent(retrs, budget=Budget(max_sources_per_task=2))("q")
+    assert sum(1 for r in retrs.values() if r.calls) == 2  # only first 2 sources hit
+
+
 # ----- end-to-end over a REAL ir corpus (hermetic: light embedder) ---------- #
 
 

raglab-0.2.3/tests/test_llm_roles.py

@@ -0,0 +1,250 @@
+"""Tests for raglab's LLM-backed roles (the Formulator and Evaluator).
+
+Hermetic and deterministic: every "LLM" is an injected test double (no model, no
+network). The Formulator adapts an ir-style ``str -> [str]`` rewriter; the
+Evaluator delegates relevance to ``ir.select`` and sufficiency to the injected
+judge. One end-to-end demo wires a real ``ir`` corpus (the light, numpy-only
+embedder, in-memory store) and shows the **back-edge** recovering a gold document
+that single-shot retrieval misses.
+"""
+
+import ir
+from ir import SearchHit
+from ir.store import CorpusStore
+
+from raglab import (
+    Budget,
+    LowLevelQuery,
+    SubTask,
+    make_llm_evaluator,
+    make_llm_formulator,
+    make_search_agent,
+)
+
+
+def _hits(*specs):
+    """``(artifact_id, score)`` pairs -> ir.SearchHits (no corpus needed)."""
+    return [SearchHit(aid, "k", score, f"text {aid}", {}) for aid, score in specs]
+
+
+def _fake_retriever(hits):
+    """A Retriever that records its calls and returns canned hits."""
+    calls = []
+
+    def retrieve(query, **kw):
+        calls.append((query, kw))
+        return list(hits)
+
+    retrieve.calls = calls
+    return retrieve
+
+
+# ----- LLM Formulator: adapt str -> [str] to (SubTask, source) -> [LLQ] ------ #
+
+
+def test_llm_formulator_fans_out_one_llq_per_query():
+    formulator = make_llm_formulator(formulate=lambda q: [q, q + " alt"])
+    llqs = formulator(SubTask(goal="deploy", sources=("s",)), "s")
+    assert [q.query for q in llqs] == ["deploy", "deploy alt"]
+    assert all(isinstance(q, LowLevelQuery) and q.source == "s" for q in llqs)
+
+
+def test_llm_formulator_accepts_a_bare_string():
+    formulator = make_llm_formulator(formulate=lambda q: q + "!")
+    llqs = formulator(SubTask(goal="x", sources=("s",)), "s")
+    assert [q.query for q in llqs] == ["x!"]
+
+
+def test_llm_formulator_attaches_params_to_every_query():
+    formulator = make_llm_formulator(
+        formulate=lambda q: [q, q + " b"], params={"mode": "hybrid", "k": 5}
+    )
+    llqs = formulator(SubTask(goal="g", sources=("s",)), "s")
+    assert all(q.params == {"mode": "hybrid", "k": 5} for q in llqs)
+
+
+def test_llm_formulator_empty_output_falls_back_to_the_goal():
+    # A formulator must never make retrieval worse than the raw sub-goal.
+    formulator = make_llm_formulator(formulate=lambda q: [])
+    llqs = formulator(SubTask(goal="the goal", sources=("s",)), "s")
+    assert [q.query for q in llqs] == ["the goal"]
+
+
+def test_llm_formulator_swallows_a_raising_callable():
+    # A failing custom formulator must fall back to the goal, never propagate.
+    def boom(_q):
+        raise RuntimeError("rewriter down")
+
+    formulator = make_llm_formulator(formulate=boom)
+    llqs = formulator(SubTask(goal="the goal", sources=("s",)), "s")
+    assert [q.query for q in llqs] == ["the goal"]
+
+
+def test_llm_formulator_drives_the_agent_loop():
+    retr = _fake_retriever(_hits(("a", 0.9)))
+    formulator = make_llm_formulator(formulate=lambda q: [q, q + " expanded"])
+    make_search_agent({"s": retr}, formulator=formulator)("q")
+    assert [c[0] for c in retr.calls] == ["q", "q expanded"]
+
+
+# ----- LLM Evaluator: ir.select owns relevance, the LLM owns sufficiency ----- #
+
+
+def test_evaluator_relevance_comes_from_ir_select():
+    # Conservative selection keeps only the near-top hit; "b" is a distractor.
+    evaluator = make_llm_evaluator(judge=lambda **kw: (True, None))
+    judged = evaluator(SubTask("g", ("s",)), _hits(("a", 0.9), ("b", 0.1)))
+    assert [h.artifact_id for h in judged.relevant] == ["a"]
+    assert judged.sufficient and judged.refinement is None
+
+
+def test_evaluator_emits_a_refinement_when_insufficient():
+    evaluator = make_llm_evaluator(judge=lambda **kw: (False, "better query"))
+    judged = evaluator(SubTask("g", ("s1", "s2")), _hits(("a", 0.9)))
+    assert judged.sufficient is False
+    assert judged.refinement == SubTask(goal="better query", sources=("s1", "s2"))
+
+
+def test_evaluator_insufficient_without_a_query_stops_the_loop():
+    # Insufficient but no refinement query -> nothing better to try -> stop.
+    evaluator = make_llm_evaluator(judge=lambda **kw: (False, None))
+    judged = evaluator(SubTask("g", ("s",)), _hits(("a", 0.9)))
+    assert judged.sufficient is True and judged.refinement is None
+
+
+def test_evaluator_parses_a_raw_text_reply():
+    evaluator = make_llm_evaluator(
+        judge=lambda **kw: "INSUFFICIENT\nvector database filtering"
+    )
+    judged = evaluator(SubTask("g", ("s",)), _hits(("a", 0.9)))
+    assert judged.refinement.goal == "vector database filtering"
+
+
+def test_evaluator_judge_error_falls_back_to_signal_no_loop():
+    def boom(**kw):
+        raise RuntimeError("model down")
+
+    evaluator = make_llm_evaluator(judge=boom)
+    judged = evaluator(SubTask("g", ("s",)), _hits(("a", 0.9)))
+    # refinement=None is the loop's break condition: a judge error never spins.
+    assert judged.refinement is None
+    assert judged.sufficient is True  # ir.select committed to "a" -> sufficient
+
+
+def test_evaluator_renders_abstention_to_the_judge():
+    seen = {}
+
+    def judge(*, goal, results):
+        seen["results"] = results
+        return (True, None)
+
+    evaluator = make_llm_evaluator(judge=judge)
+    evaluator(SubTask("g", ("s",)), [])  # no results -> ir.select abstains
+    assert "abstained" in seen["results"]
+
+
+def test_evaluator_forwards_select_kwargs_to_ir_select():
+    # Two near-tied hits: conservative keeps only "a" by default, but a loose
+    # rel threshold (forwarded via select_kwargs) admits "b" too.
+    hits = _hits(("a", 0.9), ("b", 0.6))
+    strict = make_llm_evaluator(judge=lambda **kw: (True, None))
+    loose = make_llm_evaluator(
+        judge=lambda **kw: (True, None), select_kwargs={"rel": 0.5}
+    )
+    assert [h.artifact_id for h in strict(SubTask("g", ("s",)), hits).relevant] == ["a"]
+    assert [h.artifact_id for h in loose(SubTask("g", ("s",)), hits).relevant] == [
+        "a",
+        "b",
+    ]
+
+
+def test_evaluator_ranks_heterogeneous_results_before_selecting():
+    # Accumulated cross-source results arrive unordered; the evaluator must rank
+    # best-first before ir.select (which trusts input order).
+    captured = {}
+
+    def judge(*, goal, results):
+        captured["results"] = results
+        return (True, None)
+
+    evaluator = make_llm_evaluator(judge=judge, select_kwargs={"rel": 0.0})
+    unordered = _hits(("lo", 0.1), ("hi", 0.9), ("mid", 0.5))
+    judged = evaluator(SubTask("g", ("s",)), unordered)
+    assert [h.artifact_id for h in judged.relevant] == ["hi", "mid", "lo"]
+
+
+# ----- the back-edge end-to-end, wired through the agent loop ---------------- #
+
+
+def test_evaluator_back_edge_loops_until_sufficient():
+    retr = _fake_retriever(_hits(("a", 0.9)))
+    rounds = {"n": 0}
+
+    def judge(*, goal, results):
+        rounds["n"] += 1
+        if rounds["n"] < 2:
+            return (False, goal + " more")
+        return (True, None)
+
+    make_search_agent({"s": retr}, evaluator=make_llm_evaluator(judge=judge))("q")
+    assert rounds["n"] == 2  # looped once via the back-edge
+    assert [c[0] for c in retr.calls] == ["q", "q more"]  # refinement re-queried
+
+
+def test_evaluator_back_edge_is_bounded_by_budget():
+    retr = _fake_retriever(_hits(("a", 0.9)))
+    evaluator = make_llm_evaluator(judge=lambda **kw: (False, "again"))
+    make_search_agent({"s": retr}, evaluator=evaluator, budget=Budget(max_rounds=3))(
+        "q"
+    )
+    assert len(retr.calls) == 3  # the safety net holds even if never sufficient
+
+
+# ----- end-to-end over a REAL ir corpus (hermetic: light embedder) ---------- #
+
+
+def _light_corpus():
+    docs = {
+        "embed": "embed and cache model vectors",
+        "systemd": "configure systemd units and restart services",
+        "filtering": "narrow similarity search using metadata filters",
+    }
+    return ir.build(
+        ir.CorpusSource.from_mapping(docs, name="t", strategy=ir.WholeText()),
+        store=CorpusStore.memory(),
+        embedder="light",
+    )
+
+
+def test_back_edge_recovers_a_doc_single_shot_misses():
+    """A query that overlaps a distractor misses the gold; the refinement recovers it.
+
+    Deterministic with the light embedder: the round-1 query shares vocabulary
+    with ``embed`` (a positive-score distractor) but none with the gold
+    ``filtering``, so single-shot ranks ``embed`` first; the injected judge
+    declares it insufficient and reformulates to the gold doc's own vocabulary, so
+    round 2 retrieves ``filtering`` to the top via the back-edge.
+    """
+    corpus = _light_corpus()
+    sources = {"t": ir.as_retriever(corpus, k=3)}
+    vague = "cache model results"  # overlaps the `embed` distractor, not the gold
+    gold_query = "narrow similarity search using metadata filters"
+
+    # Baseline: single-shot (no LLM evaluator) surfaces the distractor, not the gold.
+    baseline = make_search_agent(sources)(vague)
+    assert baseline[0].artifact_id == "embed"
+
+    # With the back-edge: reformulate to the gold's vocabulary, then it wins.
+    rounds = {"n": 0}
+
+    def judge(*, goal, results):
+        rounds["n"] += 1
+        if rounds["n"] < 2:
+            return (False, gold_query)
+        return (True, None)
+
+    agent = make_search_agent(sources, evaluator=make_llm_evaluator(judge=judge))
+    results = agent(vague)
+    assert rounds["n"] == 2  # the back-edge fired
+    assert results[0].artifact_id == "filtering"  # gold recovered
+    assert isinstance(results[0], SearchHit)