raglab 0.2.6__tar.gz → 0.2.8__tar.gz

{raglab-0.2.6 → raglab-0.2.8}/.claude/CLAUDE.md

@@ -29,7 +29,10 @@ control loop, with concrete tools injected at the leaves.
   cyclic-graph runner. This back-edge is what makes it an agent vs a DAG.
 - **Budget governor** (`max_rounds` / `max_sources_per_task` / `max_results_per_task`);
   termination as a **separately injected policy** (ir_09 §9), not folded into the
-  evaluator.
+  evaluator. Pool bounding is an injected `pool_cap` policy (default
+  `rank_fair_cap`: per-source best-first, round-robin). It must return hits
+  untouched — never pre-rank the pool with the RRF reranker (fused scores must not
+  re-enter the pool).
 - **Source registry** = a live `Mapping[name, Retriever]` across *heterogeneous*
   backends (ir corpora + web/SQL/graph); cross-source merge + global rerank at
   the fan-in point.

{raglab-0.2.6 → raglab-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: raglab
-Version: 0.2.6
+Version: 0.2.8
 Summary: A medley of tools to make RAG-based applications.
 Project-URL: Homepage, https://thorwhalen.github.io/raglab/
 Project-URL: Repository, https://github.com/thorwhalen/raglab
@@ -68,7 +68,7 @@ import raglab
 # Register named ir corpora as the agent's sources, then search across them.
 sources = raglab.ir_sources("skills", "reports", mode="hybrid")
 agent = raglab.make_search_agent(sources)
-results = agent("how do I deploy the app")     # -> list[ir.SearchHit], ranked
+results = agent("how do I deploy the app")  # -> list[ir.SearchHit], ranked
 ```
 
 A self-contained, runnable example (build a tiny corpus, search it):
@@ -80,18 +80,18 @@ import raglab
 
 docs = {
     "deploy": "deploy the app to the server with systemd units",
-    "embed":  "embed text with a model and cache the vectors",
+    "embed": "embed text with a model and cache the vectors",
     "search": "vector similarity search with metadata filters",
 }
 corpus = ir.build(
     ir.CorpusSource.from_mapping(docs, name="t", strategy=ir.WholeText()),
     store=CorpusStore.memory(),
-    embedder="light",          # numpy-only, no model download
+    embedder="light",  # numpy-only, no model download
 )
 
 agent = raglab.make_search_agent({"t": ir.as_retriever(corpus, k=3)})
 results = agent("deploy the app to the server")
-results[0].artifact_id        # -> "deploy"
+results[0].artifact_id  # -> "deploy"
 ```
 
 `sources` is any `Mapping[name, Retriever]`. Build it with
@@ -147,8 +147,8 @@ import raglab
 
 agent = raglab.make_search_agent(
     sources,
-    formulator=raglab.make_llm_formulator(),   # rewrite/expand the query (HyDE)
-    evaluator=raglab.make_llm_evaluator(),      # sufficiency + the back-edge
+    formulator=raglab.make_llm_formulator(),  # rewrite/expand the query (HyDE)
+    evaluator=raglab.make_llm_evaluator(),  # sufficiency + the back-edge
 )
 results = agent("how do I deploy the app")
 ```
@@ -210,14 +210,19 @@ Every role is just a callable matching a `Protocol`, so you can inject your own:
 ```python
 from raglab import SubTask, LowLevelQuery, Judgement, make_search_agent
 
+
 def my_planner(query, sources):
     # split into two sub-goals over all sources
-    return [SubTask(goal=g, sources=tuple(sources))
-            for g in (query.text, query.text + " examples")]
+    return [
+        SubTask(goal=g, sources=tuple(sources))
+        for g in (query.text, query.text + " examples")
+    ]
+
 
 def my_formulator(task, source):
     return [LowLevelQuery(source=source, query=task.goal, params={"k": 10})]
 
+
 agent = make_search_agent(sources, planner=my_planner, formulator=my_formulator)
 ```
 

{raglab-0.2.6 → raglab-0.2.8}/README.md

@@ -34,7 +34,7 @@ import raglab
 # Register named ir corpora as the agent's sources, then search across them.
 sources = raglab.ir_sources("skills", "reports", mode="hybrid")
 agent = raglab.make_search_agent(sources)
-results = agent("how do I deploy the app")     # -> list[ir.SearchHit], ranked
+results = agent("how do I deploy the app")  # -> list[ir.SearchHit], ranked
 ```
 
 A self-contained, runnable example (build a tiny corpus, search it):
@@ -46,18 +46,18 @@ import raglab
 
 docs = {
     "deploy": "deploy the app to the server with systemd units",
-    "embed":  "embed text with a model and cache the vectors",
+    "embed": "embed text with a model and cache the vectors",
     "search": "vector similarity search with metadata filters",
 }
 corpus = ir.build(
     ir.CorpusSource.from_mapping(docs, name="t", strategy=ir.WholeText()),
     store=CorpusStore.memory(),
-    embedder="light",          # numpy-only, no model download
+    embedder="light",  # numpy-only, no model download
 )
 
 agent = raglab.make_search_agent({"t": ir.as_retriever(corpus, k=3)})
 results = agent("deploy the app to the server")
-results[0].artifact_id        # -> "deploy"
+results[0].artifact_id  # -> "deploy"
 ```
 
 `sources` is any `Mapping[name, Retriever]`. Build it with
@@ -113,8 +113,8 @@ import raglab
 
 agent = raglab.make_search_agent(
     sources,
-    formulator=raglab.make_llm_formulator(),   # rewrite/expand the query (HyDE)
-    evaluator=raglab.make_llm_evaluator(),      # sufficiency + the back-edge
+    formulator=raglab.make_llm_formulator(),  # rewrite/expand the query (HyDE)
+    evaluator=raglab.make_llm_evaluator(),  # sufficiency + the back-edge
 )
 results = agent("how do I deploy the app")
 ```
@@ -176,14 +176,19 @@ Every role is just a callable matching a `Protocol`, so you can inject your own:
 ```python
 from raglab import SubTask, LowLevelQuery, Judgement, make_search_agent
 
+
 def my_planner(query, sources):
     # split into two sub-goals over all sources
-    return [SubTask(goal=g, sources=tuple(sources))
-            for g in (query.text, query.text + " examples")]
+    return [
+        SubTask(goal=g, sources=tuple(sources))
+        for g in (query.text, query.text + " examples")
+    ]
+
 
 def my_formulator(task, source):
     return [LowLevelQuery(source=source, query=task.goal, params={"k": 10})]
 
+
 agent = make_search_agent(sources, planner=my_planner, formulator=my_formulator)
 ```
 

{raglab-0.2.6 → raglab-0.2.8}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "raglab"
-version = "0.2.6"
+version = "0.2.8"
 description = "A medley of tools to make RAG-based applications."
 readme = "README.md"
 requires-python = ">=3.10"

{raglab-0.2.6 → raglab-0.2.8}/raglab/__init__.py

@@ -30,32 +30,44 @@ from .agent import (
     Citer,
     Evaluator,
     Formulator,
+    GradedLabel,
     Judgement,
     LowLevelQuery,
     Planner,
+    PoolCap,
     Query,
     Reranker,
     Result,
     Retriever,
     SingleContextAgent,
     SubTask,
+    arrival_order_cap,
     identity_citer,
     identity_formulator,
     ir_sources,
     make_rrf_reranker,
     make_search_agent,
     passthrough_evaluator,
+    rank_fair_cap,
     rrf_reranker,
     score_reranker,
     single_subtask_planner,
 )
 from .llm import EVALUATION_PROMPT, make_llm_evaluator, make_llm_formulator
+from .judging import (
+    DEFAULT_LEVELS,
+    DEFAULT_TOPICAL_RUBRIC,
+    GRADING_PROMPT,
+    apply_gates,
+    make_graded_rubric_evaluator,
+)
 
 __all__ = [
     "Query",
     "SubTask",
     "LowLevelQuery",
     "Judgement",
+    "GradedLabel",
     "Result",
     "Retriever",
     "Planner",
@@ -64,6 +76,9 @@ __all__ = [
     "Reranker",
     "Citer",
     "Budget",
+    "PoolCap",
+    "rank_fair_cap",
+    "arrival_order_cap",
     "SingleContextAgent",
     "make_search_agent",
     "ir_sources",
@@ -77,4 +92,9 @@ __all__ = [
     "make_llm_formulator",
     "make_llm_evaluator",
     "EVALUATION_PROMPT",
+    "DEFAULT_LEVELS",
+    "DEFAULT_TOPICAL_RUBRIC",
+    "GRADING_PROMPT",
+    "apply_gates",
+    "make_graded_rubric_evaluator",
 ]

{raglab-0.2.6 → raglab-0.2.8}/raglab/agent.py

@@ -31,6 +31,7 @@ Dependency direction is one-way: `raglab` imports `ir`; `ir` never imports
 
 from __future__ import annotations
 
+import math
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
 from typing import Any, Protocol, runtime_checkable
@@ -51,6 +52,7 @@ __all__ = [
     "SubTask",
     "LowLevelQuery",
     "Judgement",
+    "GradedLabel",
     "Result",
     "Retriever",
     "Planner",
@@ -59,6 +61,9 @@ __all__ = [
     "Reranker",
     "Citer",
     "Budget",
+    "PoolCap",
+    "rank_fair_cap",
+    "arrival_order_cap",
     "SingleContextAgent",
     "make_search_agent",
     "ir_sources",
@@ -107,6 +112,25 @@ class LowLevelQuery:
     params: Mapping[str, Any] = field(default_factory=dict)
 
 
+@dataclass(frozen=True)
+class GradedLabel:
+    """A graded verdict on one candidate (raglab#8): a level, three axes, the evidence.
+
+    ``level`` is one of the evaluator's ordered levels (best first). The axes are
+    Self-RAG's: ``is_about`` (topical, ISREL), ``is_supported`` (the content
+    actually establishes it, not a lexical mention: ISSUP) and ``is_useful``
+    (ISUSE). ``evidence_quote`` is a verbatim span of the candidate's content
+    backing the level; ``facet`` names the sub-goal it was judged against.
+    """
+
+    level: str
+    is_about: bool
+    is_supported: bool
+    is_useful: bool
+    evidence_quote: str = ""
+    facet: str | None = None
+
+
 @dataclass(frozen=True)
 class Judgement:
     """An evaluator's verdict over a round's results.
@@ -114,12 +138,15 @@ class Judgement:
     ``relevant`` is the kept subset; ``sufficient`` says whether to stop; a
     non-``None`` ``refinement`` is the **back-edge** — the next sub-task to
     re-query with. The pass-through default returns ``sufficient=True`` and no
-    refinement (no loop).
+    refinement (no loop). ``graded`` is optional and additive: a graded
+    evaluator maps each judged ``artifact_id`` to its :class:`GradedLabel`;
+    binary evaluators leave it empty.
     """
 
     relevant: Sequence[Result]
     sufficient: bool
     refinement: SubTask | None = None
+    graded: Mapping[str, GradedLabel] = field(default_factory=dict)
 
 
 # --------------------------------------------------------------------------- #
@@ -178,6 +205,70 @@ class Budget:
     max_results_per_task: int = 50
 
 
+@runtime_checkable
+class PoolCap(Protocol):
+    """Bound the pool the evaluator sees to at most ``limit`` results.
+
+    A governor decision (which hits survive the budget), injected like every other
+    role so a strategy can be swapped without touching the loop. It must return
+    hits **unchanged** — never re-scored — because the pool keeps raw per-source
+    magnitudes until the fan-in reranker.
+    """
+
+    def __call__(self, results: Sequence[Result], limit: int) -> list[Result]: ...
+
+
+def _rank_key(hit: Result) -> float:
+    """A hit's raw score for ordering *within its source*; a missing or NaN score is last."""
+    score = hit.score
+    if isinstance(score, (int, float)) and not math.isnan(score):
+        return float(score)
+    return -math.inf
+
+
+def rank_fair_cap(results: Sequence[Result], limit: int) -> list[Result]:
+    """Keep at most ``limit`` hits, taking the best of each source in turn (the default).
+
+    Groups the pool by ``hit.source`` (in first-arrival order; untagged hits are one
+    group), collapses each group to one hit per artifact (its best surface, as
+    :func:`ir.base.best_per_artifact` does), orders the group best-first by its own raw
+    score -- sound, since one source is one score scale -- and deals hits round-robin
+    across the groups. So a late source's strong hits are never cut before the evaluator
+    sees them, a duplicate never takes a slot, and no score ever crosses a source
+    boundary or is rewritten (unlike pre-ranking with the RRF reranker, whose fused
+    ordinal scores must never re-enter the pool). The output is in that dealt order
+    whatever the pool's size. A missing or NaN score sorts last in its source.
+    """
+    if limit <= 0:
+        return []
+    groups: dict[str | None, dict[str, Result]] = {}
+    for hit in results:
+        best = groups.setdefault(hit.source, {})
+        cur = best.get(hit.artifact_id)
+        if cur is None or _rank_key(hit) > _rank_key(cur):
+            best[hit.artifact_id] = hit
+    queues = [
+        sorted(best.values(), key=_rank_key, reverse=True) for best in groups.values()
+    ]
+    capped: list[Result] = []
+    for depth in range(max((len(q) for q in queues), default=0)):
+        for queue in queues:
+            if depth < len(queue):
+                capped.append(queue[depth])
+                if len(capped) == limit:
+                    return capped
+    return capped
+
+
+def arrival_order_cap(results: Sequence[Result], limit: int) -> list[Result]:
+    """Keep the first ``limit`` hits in arrival order (source order x query order).
+
+    The pre-governor behaviour: a late source's strong hits can be cut before the
+    evaluator sees them. Kept as an explicit opt-in (``pool_cap=arrival_order_cap``).
+    """
+    return list(results[: max(0, limit)])
+
+
 # --------------------------------------------------------------------------- #
 # Default role implementations (the no-LLM thin slice) — ir_09 §8 step 2
 # --------------------------------------------------------------------------- #
@@ -306,6 +397,7 @@ class SingleContextAgent:
     reranker: Reranker = rrf_reranker
     citer: Citer = identity_citer
     budget: Budget = field(default_factory=Budget)
+    pool_cap: PoolCap = rank_fair_cap
 
     def __call__(self, query: str | Query) -> list[Result]:
         """Run the agent for *query*; returns the final ranked, cited results."""
@@ -317,9 +409,15 @@ class SingleContextAgent:
         return list(self.citer(ranked))
 
     def _run_task(self, task: SubTask) -> list[Result]:
-        found: list[Result] = []
+        # `kept` is what the evaluator judged relevant last round: governor-approved,
+        # carried into the next round untouched. Only a round's *new* hits compete for
+        # the rest of the budget (`pool_cap`), so a re-query can never evict a hit the
+        # evaluator already kept, nor spend a slot re-offering it.
+        kept: list[Result] = []
         current = task
+        limit = self.budget.max_results_per_task
         for _ in range(max(1, self.budget.max_rounds)):
+            found: list[Result] = []
             for source in current.sources[: self.budget.max_sources_per_task]:
                 retriever = self.sources.get(source)
                 if retriever is None:
@@ -333,12 +431,15 @@ class SingleContextAgent:
                     found.extend(
                         tag_source(retriever(llq.query, **dict(llq.params)), source)
                     )
-            judged = self.evaluator(current, found[: self.budget.max_results_per_task])
-            found = list(judged.relevant)
+            seen = {(h.source, h.artifact_id) for h in kept}
+            fresh = [h for h in found if (h.source, h.artifact_id) not in seen]
+            pool = kept + self.pool_cap(fresh, max(0, limit - len(kept)))
+            judged = self.evaluator(current, pool)
+            kept = list(judged.relevant)
             if judged.sufficient or judged.refinement is None:
                 break
             current = judged.refinement  # the back-edge: re-query
-        return found
+        return kept
 
 
 def make_search_agent(
@@ -350,6 +451,7 @@ def make_search_agent(
     reranker: Reranker | None = None,
     citer: Citer | None = None,
     budget: Budget | None = None,
+    pool_cap: PoolCap | None = None,
 ) -> SingleContextAgent:
     """Build a :class:`SingleContextAgent` over *sources* with smart defaults.
 
@@ -372,6 +474,7 @@ def make_search_agent(
         reranker=reranker or rrf_reranker,
         citer=citer or identity_citer,
         budget=budget or Budget(),
+        pool_cap=pool_cap or rank_fair_cap,
     )
 
 

raglab-0.2.8/raglab/judging.py

@@ -0,0 +1,325 @@
+"""Graded rubric judging — every candidate gets a level, three axes and evidence (raglab#8).
+
+A binary relevant/not cannot say "mentions the word but is not about it". The
+evaluator built by :func:`make_graded_rubric_evaluator` asks an injected judge,
+for **each** candidate in the round's pool, for the three Self-RAG axes
+(``is_about``, ``is_supported``, ``is_useful``), a level on an ordered scale and
+a verbatim ``evidence_quote`` from the candidate's disclosed content. The judge
+proposes; deterministic gates here dispose:
+
+- a level above ``unsupported_cap`` needs ``is_supported`` — topical vocabulary
+  without supporting content is capped (the direct fix for on-word distractors);
+- with ``require_evidence_quote``, it also needs a quote found verbatim in the
+  content the judge was shown (whitespace-insensitive);
+- ``calibration_offset`` (levels, learned from a gold slice) shifts every level
+  down (positive: the judge over-grades) or up;
+- the level options are shown in a fresh random order on every call, against
+  position bias.
+
+The result is a :class:`~raglab.agent.Judgement` whose additive ``graded`` map
+carries every label, whose ``relevant`` keeps the candidates at or above
+``relevant_level``, and whose back-edge comes from an injected ``refine`` policy
+(none by default: the loop stops). As in :mod:`raglab.llm`, the default judge is
+built lazily on :mod:`oa`, so ``import raglab`` stays offline, and a judge that
+fails on a candidate leaves it ungraded rather than guessing.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+import random
+import re
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any
+
+import ir
+from ir.base import best_per_artifact
+
+from .agent import Evaluator, GradedLabel, Judgement, Result, SubTask
+
+__all__ = [
+    "DEFAULT_LEVELS",
+    "DEFAULT_TOPICAL_RUBRIC",
+    "GRADING_PROMPT",
+    "make_graded_rubric_evaluator",
+    "apply_gates",
+]
+
+#: Ordered levels, best first.
+DEFAULT_LEVELS: tuple[str, ...] = ("core", "strong", "uses-tools", "tangential", "none")
+
+#: What the default levels mean, for a "is this package about TOPIC" question.
+DEFAULT_TOPICAL_RUBRIC = """\
+- core: the candidate's main purpose IS the topic.
+- strong: a substantial, central part of the candidate is about the topic.
+- uses-tools: the candidate builds on tools or libraries of the topic (its dependencies or imports establish it) without being about it.
+- tangential: the topic is mentioned, or its vocabulary appears, but the content does not establish it.
+- none: unrelated.
+"""
+
+#: The judge's prompt; ``{levels}`` is shown in a fresh random order each call.
+GRADING_PROMPT = """\
+Goal: {goal}
+
+Rubric (levels, in no particular order):
+{levels}
+
+Definitions:
+{rubric}
+
+Candidate {artifact_id}:
+---
+{content}
+---
+
+Answer with one JSON object and nothing else, with keys:
+"is_about" (bool: is the candidate topically about the goal?),
+"is_supported" (bool: does its content - code, dependencies, imports, full text - actually establish it, beyond a mention of the words?),
+"is_useful" (bool: would it help someone pursuing the goal?),
+"evidence_quote" (a short span copied VERBATIM from the candidate's content that supports your level),
+"level" (one of the rubric's levels).
+"""
+
+#: A graded judge: keyword arguments ``goal, artifact_id, content, levels, rubric``
+#: -> a mapping with the label's keys, or the raw JSON text of one.
+GradedJudge = Callable[..., "Mapping[str, Any] | str"]
+
+#: A refinement policy: ``(task, graded) -> the next SubTask, or None to stop``.
+Refine = Callable[[SubTask, Mapping[str, GradedLabel]], "SubTask | None"]
+
+#: Truncate each candidate's disclosed content to this many chars in the prompt.
+DFLT_MAX_CONTENT_CHARS = 4000
+
+#: A shorter evidence quote ("a", "the") would be found in almost any content.
+DFLT_MIN_QUOTE_CHARS = 8
+
+_log = logging.getLogger(__name__)
+
+
+def _normalize_space(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip().casefold()
+
+
+def apply_gates(
+    proposed: GradedLabel,
+    content: str,
+    *,
+    levels: Sequence[str] = DEFAULT_LEVELS,
+    unsupported_cap: str = "tangential",
+    require_evidence_quote: bool = True,
+    calibration_offset: float | None = None,
+    min_quote_chars: int = DFLT_MIN_QUOTE_CHARS,
+) -> GradedLabel:
+    """The level a judge's proposal is allowed, after calibration and the gates.
+
+    The quote must be at least ``min_quote_chars`` long and found in ``content``,
+    ignoring case and runs of whitespace (judges re-case and re-wrap what they
+    copy). ``calibration_offset`` is in levels, rounded half up, so a shift is
+    the same at every level.
+
+    >>> label = GradedLabel('core', True, False, True, 'a DAG of tasks')
+    >>> apply_gates(label, 'Runs a DAG of tasks asynchronously.').level
+    'tangential'
+    >>> ok = GradedLabel('core', True, True, True, 'graph  library')
+    >>> apply_gates(ok, 'A graph library.').level, apply_gates(ok, 'Nothing here.').level
+    ('core', 'tangential')
+    >>> apply_gates(ok, 'A graph library.', calibration_offset=1.4).level
+    'strong'
+    """
+    cap = levels.index(unsupported_cap)
+    rank = levels.index(proposed.level)
+    if calibration_offset:
+        rank = min(max(math.floor(rank + calibration_offset + 0.5), 0), len(levels) - 1)
+    quote = _normalize_space(proposed.evidence_quote)
+    quoted = len(quote) >= min_quote_chars and quote in _normalize_space(content)
+    if rank < cap and (
+        not proposed.is_supported or (require_evidence_quote and not quoted)
+    ):
+        rank = cap
+    return GradedLabel(
+        level=levels[rank],
+        is_about=proposed.is_about,
+        is_supported=proposed.is_supported,
+        is_useful=proposed.is_useful,
+        evidence_quote=proposed.evidence_quote if quoted else "",
+        facet=proposed.facet,
+    )
+
+
+def _parse_label(out: Mapping[str, Any] | str, levels: Sequence[str]) -> GradedLabel:
+    """A judge's reply as a :class:`GradedLabel`; raises ``ValueError`` when unusable."""
+    if isinstance(out, str):
+        match = re.search(r"\{.*\}", out, flags=re.DOTALL)
+        if not match:
+            raise ValueError(f"no JSON object in the judge's reply: {out[:200]!r}")
+        out = json.loads(match.group(0))
+    by_name = {lv.casefold(): lv for lv in levels}
+    level = by_name.get(str(out.get("level", "")).strip().casefold())
+    if level is None:
+        raise ValueError(f"level {out.get('level')!r} is not one of {list(levels)}")
+
+    def flag(key: str) -> bool:
+        value = out.get(key)
+        if isinstance(value, str):
+            return value.strip().lower() in ("true", "yes")
+        return bool(value)
+
+    return GradedLabel(
+        level=level,
+        is_about=flag("is_about"),
+        is_supported=flag("is_supported"),
+        is_useful=flag("is_useful"),
+        evidence_quote=str(out.get("evidence_quote") or ""),
+    )
+
+
+def _contents(
+    hits: Sequence[Result], level: str, disclose_kwargs: Mapping[str, Any]
+) -> list[str]:
+    """Each hit's disclosed content at ``level``, its snippet when nothing loads."""
+    selection = ir.Selection(
+        selected=list(hits), candidates=list(hits), abstained=False, reason="graded"
+    )
+    disclosed = ir.disclose(selection, level=level, **dict(disclose_kwargs))
+    return [d.body or h.text for h, d in zip(hits, disclosed, strict=True)]
+
+
+def make_graded_rubric_evaluator(
+    *,
+    levels: Sequence[str] = DEFAULT_LEVELS,
+    rubric: str = DEFAULT_TOPICAL_RUBRIC,
+    judge: GradedJudge | None = None,
+    disclose_level: str = "body",
+    disclose_kwargs: Mapping[str, Any] | None = None,
+    require_evidence_quote: bool = True,
+    unsupported_cap: str = "tangential",
+    calibration_offset: float | None = None,
+    min_quote_chars: int = DFLT_MIN_QUOTE_CHARS,
+    relevant_level: str = "uses-tools",
+    refine: Refine | None = None,
+    rng: random.Random | None = None,
+    prompt: str = GRADING_PROMPT,
+    max_content_chars: int = DFLT_MAX_CONTENT_CHARS,
+    **prompt_function_kwargs: Any,
+) -> Evaluator:
+    """An :class:`~raglab.agent.Evaluator` that grades every candidate on a rubric.
+
+    Args:
+        levels: the ordered levels, best first.
+        rubric: what each level means, shown to the judge.
+        judge: an injectable graded judge (a test double, or your own router);
+            built lazily on :mod:`oa` when omitted.
+        disclose_level, disclose_kwargs: what the judge reads — each candidate's
+            content through :func:`ir.disclose` at that level (``store=`` /
+            ``loader=`` go in ``disclose_kwargs``); a candidate with nothing to
+            load is judged on its snippet.
+        require_evidence_quote, unsupported_cap, calibration_offset,
+            min_quote_chars: the gates, see :func:`apply_gates`.
+        relevant_level: the lowest level kept in ``Judgement.relevant``.
+        refine: the back-edge policy ``(task, graded) -> SubTask | None``; by
+            default none, so the loop stops after one round.
+        rng: the source of the per-call level order (seed it for reproducible
+            prompts).
+
+    Every candidate is judged once per goal: one judge call per (source,
+    artifact) the first time it appears in a pool for that goal, its label
+    remembered for later rounds, so a kept hit is not re-judged (nor lost to a
+    later failing call). ``Judgement.graded`` is keyed by ``artifact_id``; an id
+    found in several sources carries its best label. A reply that does not parse
+    leaves the candidate ungraded (logged); when every judge call of a round
+    raises, the first error is re-raised rather than returning an empty,
+    "sufficient" round. This evaluator is for labelling a pool, not for cheap
+    filtering.
+    """
+    levels = tuple(levels)
+    for name, value in (
+        ("unsupported_cap", unsupported_cap),
+        ("relevant_level", relevant_level),
+    ):
+        if value not in levels:
+            raise ValueError(f"{name}={value!r} is not one of {list(levels)}")
+    keep = levels.index(relevant_level)
+    rng = rng if rng is not None else random.Random()
+    disclose_kwargs = dict(disclose_kwargs or {})
+    state: dict[str, GradedJudge] = {}
+    memo: dict[tuple[str, str | None, str], GradedLabel] = {}
+
+    def ask(**kwargs: Any) -> Mapping[str, Any] | str:
+        if judge is not None:
+            return judge(**kwargs)
+        if "judge" not in state:
+            state["judge"] = _default_graded_judge(prompt, **prompt_function_kwargs)
+        return state["judge"](**kwargs)
+
+    def evaluator(task: SubTask, results: Sequence[Result]) -> Judgement:
+        hits = best_per_artifact(results)
+        todo = [h for h in hits if (task.goal, h.source, h.artifact_id) not in memo]
+        contents = _contents(todo, disclose_level, disclose_kwargs) if todo else []
+        errors: list[Exception] = []
+        for hit, content in zip(todo, contents, strict=True):
+            content = str(content)[:max_content_chars]
+            shown = list(levels)
+            rng.shuffle(shown)
+            try:
+                reply = ask(
+                    goal=task.goal,
+                    artifact_id=hit.artifact_id,
+                    content=content,
+                    levels="\n".join(f"- {lv}" for lv in shown),
+                    rubric=rubric,
+                )
+            except Exception as error:  # the judge itself failed: auth, network, limits
+                _log.warning("judge failed on %s: %r", hit.artifact_id, error)
+                errors.append(error)
+                continue
+            try:
+                proposed = _parse_label(reply, levels)
+            except (ValueError, TypeError, AttributeError) as error:
+                _log.warning("unusable judge reply on %s: %s", hit.artifact_id, error)
+                continue  # left ungraded: never guess a level
+            memo[(task.goal, hit.source, hit.artifact_id)] = apply_gates(
+                proposed,
+                content,
+                levels=levels,
+                unsupported_cap=unsupported_cap,
+                require_evidence_quote=require_evidence_quote,
+                calibration_offset=calibration_offset,
+                min_quote_chars=min_quote_chars,
+            )
+        if todo and len(errors) == len(todo):
+            raise errors[0]
+        labels = {
+            (h.source, h.artifact_id): memo[key]
+            for h in hits
+            if (key := (task.goal, h.source, h.artifact_id)) in memo
+        }
+        graded: dict[str, GradedLabel] = {}
+        for (_, aid), label in labels.items():
+            if aid not in graded or levels.index(label.level) < levels.index(
+                graded[aid].level
+            ):
+                graded[aid] = label
+        relevant = [
+            h
+            for h in hits
+            if (label := labels.get((h.source, h.artifact_id))) is not None
+            and levels.index(label.level) <= keep
+        ]
+        refinement = refine(task, graded) if refine is not None else None
+        return Judgement(
+            relevant=relevant,
+            sufficient=refinement is None,
+            refinement=refinement,
+            graded=graded,
+        )
+
+    return evaluator
+
+
+def _default_graded_judge(prompt: str, **prompt_function_kwargs: Any) -> GradedJudge:
+    """Build the default graded judge on :mod:`oa` (lazy import)."""
+    import oa
+
+    return oa.prompt_function(prompt, name="grade_candidate", **prompt_function_kwargs)

{raglab-0.2.6 → raglab-0.2.8}/tests/test_fanin.py

@@ -286,3 +286,144 @@ def test_score_reranker_remains_the_homogeneous_opt_in():
     pytest.importorskip("ir")
     pool = _hits(("a", 0.3), source="s1") + _hits(("b", 0.9), source="s2")
     assert [h.artifact_id for h in raglab.score_reranker(pool)] == ["b", "a"]
+
+
+# ----- the budget cap: rank-fair, scale-safe (raglab#5) ---------------------- #
+
+
+def _capturing_evaluator(seen):
+    def evaluator(task, results):
+        seen.append(list(results))
+        return Judgement(list(results), sufficient=True)
+
+    return evaluator
+
+
+def test_small_cap_lets_both_disparate_sources_top_hits_through():
+    # source "big" arrives first with many BM25-scale hits; "cos" arrives last with
+    # cosine-scale hits. Arrival-order truncation would never show "cos" at all.
+    big = _hits(*[(f"b{i}", 40.0 - i) for i in range(10)], source="big")
+    cos = _hits(("c_best", 0.95), ("c2", 0.4), source="cos")
+    seen = []
+    make_search_agent(
+        {"big": _fake_retriever(big), "cos": _fake_retriever(cos)},
+        evaluator=_capturing_evaluator(seen),
+        budget=raglab.Budget(max_results_per_task=4),
+    )("q")
+    pool = seen[0]
+    assert len(pool) == 4
+    ids = [h.artifact_id for h in pool]
+    assert ids[:2] == ["b0", "c_best"]  # each source's best, dealt round-robin
+    assert set(ids) == {"b0", "b1", "c_best", "c2"}
+
+
+def test_capped_pool_keeps_raw_per_source_magnitudes():
+    big = _hits(*[(f"b{i}", 40.0 - i) for i in range(10)], source="big")
+    cos = _hits(("c_best", 0.95), ("c2", 0.4), source="cos")
+    originals = {(h.source, h.artifact_id): h for h in big + cos}
+    seen = []
+    make_search_agent(
+        {"big": _fake_retriever(big), "cos": _fake_retriever(cos)},
+        evaluator=_capturing_evaluator(seen),
+        budget=raglab.Budget(max_results_per_task=3),
+    )("q")
+    for hit in seen[0]:
+        assert hit == originals[(hit.source, hit.artifact_id)]  # untouched
+        assert "source_rank" not in hit.metadata
+        assert "source_score" not in hit.metadata  # no fused-score stamp either
+
+
+def test_rank_fair_cap_orders_within_source_by_its_own_score():
+    # one source, scores out of arrival order (e.g. two formulator queries)
+    pool = _hits(("lo", 0.1), ("hi", 0.9), ("mid", 0.5), source="s")
+    assert [h.artifact_id for h in raglab.rank_fair_cap(pool, 2)] == ["hi", "mid"]
+    # untagged hits are one group of their own
+    mixed = _hits(("u1", 3.0), ("u2", 2.0)) + _hits(("t1", 0.5), source="t")
+    assert [h.artifact_id for h in raglab.rank_fair_cap(mixed, 2)] == ["u1", "t1"]
+
+
+def test_rank_fair_cap_within_limit_and_degenerate_limits():
+    pool = _hits(("a", 0.1), ("b", 0.9), source="s")
+    # within budget nothing is cut, and the order is the same dealt order as above it
+    assert [h.artifact_id for h in raglab.rank_fair_cap(pool, 5)] == ["b", "a"]
+    assert raglab.rank_fair_cap(pool, 0) == []
+    assert raglab.rank_fair_cap(pool, -1) == []
+    uneven = _hits(*[(f"x{i}", 1.0 - i / 10) for i in range(5)], source="x") + _hits(
+        ("y0", 9.0), source="y"
+    )
+    # a short source runs dry; the long one fills the rest
+    assert [h.artifact_id for h in raglab.rank_fair_cap(uneven, 4)] == [
+        "x0",
+        "y0",
+        "x1",
+        "x2",
+    ]
+
+
+def test_pool_cap_is_injectable_and_arrival_order_is_the_opt_out():
+    big = _hits(*[(f"b{i}", 40.0 - i) for i in range(10)], source="big")
+    cos = _hits(("c_best", 0.95), source="cos")
+    seen = []
+    make_search_agent(
+        {"big": _fake_retriever(big), "cos": _fake_retriever(cos)},
+        evaluator=_capturing_evaluator(seen),
+        budget=raglab.Budget(max_results_per_task=3),
+        pool_cap=raglab.arrival_order_cap,
+    )("q")
+    assert [h.artifact_id for h in seen[0]] == ["b0", "b1", "b2"]
+    assert isinstance(raglab.rank_fair_cap, raglab.PoolCap)
+
+
+def test_cap_collapses_duplicates_and_tolerates_missing_scores():
+    dup = _hits(("a", 0.9), ("a", 0.5), ("b", 0.8), ("c", 0.7), source="s")
+    assert [h.artifact_id for h in raglab.rank_fair_cap(dup, 3)] == ["a", "b", "c"]
+    assert raglab.rank_fair_cap(dup, 3)[0].score == 0.9  # the best surface survives
+    odd = _hits(("nan", float("nan")), ("none", None), ("ok", 0.1), source="s")
+    assert raglab.rank_fair_cap(odd, 1)[0].artifact_id == "ok"
+
+
+def _refine_once(seen, keep):
+    """An evaluator that keeps only `keep` in round one and asks for a re-query."""
+
+    def evaluator(task, results):
+        seen.append(list(results))
+        if len(seen) == 1:
+            kept = [h for h in results if h.artifact_id in keep]
+            return Judgement(
+                kept, sufficient=False, refinement=SubTask("again", task.sources)
+            )
+        return Judgement(list(results), sufficient=True)
+
+    return evaluator
+
+
+def _rounds(*per_round):
+    """A retriever returning a different canned hit list on each call."""
+    calls = iter(per_round)
+    return lambda query, **kw: list(next(calls))
+
+
+def test_back_edge_never_evicts_a_hit_the_evaluator_kept():
+    round1 = _hits(
+        ("weak_but_relevant", 0.1), ("junk1", 0.9), ("junk2", 0.8), source="s"
+    )
+    round2 = _hits(("new1", 0.95), ("new2", 0.85), ("new3", 0.7), source="s")
+    seen = []
+    out = make_search_agent(
+        {"s": _rounds(round1, round2)},
+        evaluator=_refine_once(seen, {"weak_but_relevant"}),
+        budget=raglab.Budget(max_results_per_task=3),
+    )("q")
+    assert [h.artifact_id for h in seen[1]] == ["weak_but_relevant", "new1", "new2"]
+    assert "weak_but_relevant" in {h.artifact_id for h in out}
+
+
+def test_back_edge_does_not_spend_slots_on_re_retrieved_hits():
+    same = _hits(("a", 0.9), ("b", 0.8), ("c", 0.7), source="s")
+    seen = []
+    make_search_agent(
+        {"s": _rounds(same, same)},
+        evaluator=_refine_once(seen, {"a"}),
+        budget=raglab.Budget(max_results_per_task=3),
+    )("q")
+    assert [h.artifact_id for h in seen[1]] == ["a", "b", "c"]  # no second "a"

raglab-0.2.8/tests/test_judging.py

@@ -0,0 +1,220 @@
+"""Tests for the graded rubric evaluator (raglab#8).
+
+Hermetic: the judge is a scripted test double keyed by artifact id; no model,
+no network. Content comes from each hit's snippet (nothing to load from disk).
+"""
+
+import json
+import random
+
+from ir import SearchHit
+
+from raglab import (
+    DEFAULT_LEVELS,
+    Budget,
+    GradedLabel,
+    Judgement,
+    SubTask,
+    make_graded_rubric_evaluator,
+    make_search_agent,
+)
+
+TASK = SubTask(goal="graph library", sources=("pkgs",))
+
+
+def _hit(aid, text, score=1.0):
+    return SearchHit(aid, "k", score, text, {})
+
+
+def _judge(replies):
+    """A judge returning ``replies[artifact_id]`` and recording what it was shown."""
+    calls = []
+
+    def judge(**kw):
+        calls.append(kw)
+        reply = replies[kw["artifact_id"]]
+        if isinstance(reply, Exception):
+            raise reply
+        return reply
+
+    judge.calls = calls
+    return judge
+
+
+def _reply(level, *, about=True, supported=True, useful=True, quote=""):
+    return {
+        "level": level,
+        "is_about": about,
+        "is_supported": supported,
+        "is_useful": useful,
+        "evidence_quote": quote,
+    }
+
+
+POOL = [
+    _hit("graphy", "A graph library: nodes, edges, traversals."),
+    _hit("au", "An async task framework; jobs form a DAG."),
+    _hit("plotter", "Draws charts with networkx graph layouts."),
+]
+
+
+def test_every_candidate_is_graded_and_the_supported_gate_caps_on_word_distractors():
+    judge = _judge(
+        {
+            "graphy": _reply("core", quote="A graph library"),
+            # on-word but not supported: the judge over-grades, the gate caps it
+            "au": _reply("strong", supported=False, quote="jobs form a DAG"),
+            "plotter": _reply("uses-tools", quote="networkx graph layouts"),
+        }
+    )
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+    assert isinstance(judged, Judgement)
+    assert {aid: g.level for aid, g in judged.graded.items()} == {
+        "graphy": "core",
+        "au": "tangential",
+        "plotter": "uses-tools",
+    }
+    assert [h.artifact_id for h in judged.relevant] == ["graphy", "plotter"]
+    assert judged.sufficient and judged.refinement is None
+    assert len(judge.calls) == 3
+
+
+def test_a_quote_not_found_verbatim_caps_the_level_unless_quotes_are_optional():
+    judge = _judge({"graphy": _reply("core", quote="the best graph toolkit")})
+    strict = make_graded_rubric_evaluator(judge=judge)(TASK, POOL[:1])
+    assert strict.graded["graphy"].level == "tangential"
+    assert strict.graded["graphy"].evidence_quote == ""
+    lax = make_graded_rubric_evaluator(judge=judge, require_evidence_quote=False)(
+        TASK, POOL[:1]
+    )
+    assert lax.graded["graphy"].level == "core"
+
+
+def test_the_calibration_offset_demotes_an_over_grading_judge():
+    judge = _judge({"graphy": _reply("core", quote="graph library")})
+    ev = make_graded_rubric_evaluator(judge=judge, calibration_offset=1.0)
+    assert ev(TASK, POOL[:1]).graded["graphy"].level == "strong"
+
+
+def test_a_failing_or_unparseable_judge_leaves_the_candidate_ungraded():
+    judge = _judge(
+        {
+            "graphy": json.dumps(_reply("core", quote="graph library")),  # raw text ok
+            "au": RuntimeError("rate limited"),
+            "plotter": '{"level": "excellent"}',  # not a level
+        }
+    )
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+    assert set(judged.graded) == {"graphy"}
+    assert [h.artifact_id for h in judged.relevant] == ["graphy"]
+
+
+def test_level_options_are_shown_in_a_fresh_order_each_call():
+    judge = _judge({h.artifact_id: _reply("none") for h in POOL})
+    ev = make_graded_rubric_evaluator(judge=judge, rng=random.Random(0))
+    for _ in range(4):
+        ev(TASK, POOL)
+    orders = {c["levels"] for c in judge.calls}
+    assert len(orders) > 1
+    assert all(sorted(o.split("\n")) == sorted(f"- {lv}" for lv in DEFAULT_LEVELS) for o in orders)
+
+
+def test_the_refine_policy_drives_the_back_edge_through_the_agent():
+    corpus = {
+        "graph library": [_hit("au", "An async task framework; jobs form a DAG.")],
+        "graph library nodes edges": [_hit("graphy", "A graph library: nodes, edges.")],
+    }
+
+    def retriever(query, **kw):
+        return list(corpus.get(query, []))
+
+    judge = _judge(
+        {
+            "au": _reply("core", supported=False, quote="jobs form a DAG"),
+            "graphy": _reply("core", quote="A graph library"),
+        }
+    )
+
+    def refine(task, graded):
+        if any(g.level in ("core", "strong") for g in graded.values()):
+            return None
+        return SubTask(goal=task.goal + " nodes edges", sources=task.sources)
+
+    agent = make_search_agent(
+        {"pkgs": retriever},
+        evaluator=make_graded_rubric_evaluator(judge=judge, refine=refine),
+        budget=Budget(max_rounds=3),
+    )
+    assert [h.artifact_id for h in agent("graph library")] == ["graphy"]
+
+
+def test_binary_judgements_are_unchanged():
+    assert Judgement(relevant=[], sufficient=True).graded == {}
+    label = GradedLabel("core", True, True, True)
+    assert label.evidence_quote == "" and label.facet is None
+
+
+def test_unknown_level_names_are_refused():
+    import pytest
+
+    with pytest.raises(ValueError, match="relevant_level"):
+        make_graded_rubric_evaluator(judge=_judge({}), relevant_level="great")
+
+
+def test_the_same_id_in_two_sources_is_judged_per_source():
+    a = SearchHit("x", "k", 1.0, "A graph library for nodes.", {}, source="A")
+    b = SearchHit("x", "k", 1.0, "An audio decoder.", {}, source="B")
+
+    def judge(**kw):
+        if "graph" in kw["content"]:
+            return _reply("core", quote="A graph library")
+        return _reply("none")
+
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, [a, b])
+    assert [h.source for h in judged.relevant] == ["A"]
+    assert judged.graded["x"].level == "core"  # the best label across sources
+
+
+def test_custom_levels_match_whatever_their_case():
+    judge = _judge({"graphy": _reply("core", quote="A graph library")})
+    ev = make_graded_rubric_evaluator(
+        judge=judge, levels=("Core", "Tangential", "None"),
+        unsupported_cap="Tangential", relevant_level="Core",
+    )
+    assert ev(TASK, POOL[:1]).graded["graphy"].level == "Core"
+
+
+def test_a_judge_that_fails_on_every_candidate_raises():
+    import pytest
+
+    judge = _judge({h.artifact_id: RuntimeError("no key") for h in POOL})
+    with pytest.raises(RuntimeError, match="no key"):
+        make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+
+
+def test_a_candidate_is_judged_once_per_goal_and_its_label_kept():
+    judge = _judge({"graphy": _reply("core", quote="A graph library")})
+    ev = make_graded_rubric_evaluator(judge=judge)
+    ev(TASK, POOL[:1])
+    judge_down = ev(TASK, POOL[:1])  # no new call: the label is remembered
+    assert len(judge.calls) == 1
+    assert [h.artifact_id for h in judge_down.relevant] == ["graphy"]
+
+
+def test_a_half_level_offset_shifts_every_level_alike():
+    from raglab import apply_gates
+
+    content = "A graph library."
+    shifted = [
+        apply_gates(GradedLabel(lv, True, True, True, "graph library"), content,
+                    calibration_offset=0.5).level
+        for lv in DEFAULT_LEVELS[:-1]
+    ]
+    assert shifted == list(DEFAULT_LEVELS[1:])
+
+
+def test_a_one_word_quote_is_not_evidence():
+    from raglab import apply_gates
+
+    label = GradedLabel("core", True, True, True, "a")
+    assert apply_gates(label, "A graph library.").level == "tangential"