raglab 0.2.7__tar.gz → 0.2.9__tar.gz

{raglab-0.2.7 → raglab-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: raglab
-Version: 0.2.7
+Version: 0.2.9
 Summary: A medley of tools to make RAG-based applications.
 Project-URL: Homepage, https://thorwhalen.github.io/raglab/
 Project-URL: Repository, https://github.com/thorwhalen/raglab

{raglab-0.2.7 → raglab-0.2.9}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "raglab"
-version = "0.2.7"
+version = "0.2.9"
 description = "A medley of tools to make RAG-based applications."
 readme = "README.md"
 requires-python = ">=3.10"

{raglab-0.2.7 → raglab-0.2.9}/raglab/__init__.py

@@ -30,6 +30,7 @@ from .agent import (
     Citer,
     Evaluator,
     Formulator,
+    GradedLabel,
     Judgement,
     LowLevelQuery,
     Planner,
@@ -53,12 +54,20 @@ from .agent import (
     single_subtask_planner,
 )
 from .llm import EVALUATION_PROMPT, make_llm_evaluator, make_llm_formulator
+from .judging import (
+    DEFAULT_LEVELS,
+    DEFAULT_TOPICAL_RUBRIC,
+    GRADING_PROMPT,
+    apply_gates,
+    make_graded_rubric_evaluator,
+)
 
 __all__ = [
     "Query",
     "SubTask",
     "LowLevelQuery",
     "Judgement",
+    "GradedLabel",
     "Result",
     "Retriever",
     "Planner",
@@ -83,4 +92,9 @@ __all__ = [
     "make_llm_formulator",
     "make_llm_evaluator",
     "EVALUATION_PROMPT",
+    "DEFAULT_LEVELS",
+    "DEFAULT_TOPICAL_RUBRIC",
+    "GRADING_PROMPT",
+    "apply_gates",
+    "make_graded_rubric_evaluator",
 ]

{raglab-0.2.7 → raglab-0.2.9}/raglab/agent.py

@@ -52,6 +52,7 @@ __all__ = [
     "SubTask",
     "LowLevelQuery",
     "Judgement",
+    "GradedLabel",
     "Result",
     "Retriever",
     "Planner",
@@ -111,6 +112,25 @@ class LowLevelQuery:
     params: Mapping[str, Any] = field(default_factory=dict)
 
 
+@dataclass(frozen=True)
+class GradedLabel:
+    """A graded verdict on one candidate (raglab#8): a level, three axes, the evidence.
+
+    ``level`` is one of the evaluator's ordered levels (best first). The axes are
+    Self-RAG's: ``is_about`` (topical, ISREL), ``is_supported`` (the content
+    actually establishes it, not a lexical mention: ISSUP) and ``is_useful``
+    (ISUSE). ``evidence_quote`` is a verbatim span of the candidate's content
+    backing the level; ``facet`` names the sub-goal it was judged against.
+    """
+
+    level: str
+    is_about: bool
+    is_supported: bool
+    is_useful: bool
+    evidence_quote: str = ""
+    facet: str | None = None
+
+
 @dataclass(frozen=True)
 class Judgement:
     """An evaluator's verdict over a round's results.
@@ -118,12 +138,15 @@ class Judgement:
     ``relevant`` is the kept subset; ``sufficient`` says whether to stop; a
     non-``None`` ``refinement`` is the **back-edge** — the next sub-task to
     re-query with. The pass-through default returns ``sufficient=True`` and no
-    refinement (no loop).
+    refinement (no loop). ``graded`` is optional and additive: a graded
+    evaluator maps each judged ``artifact_id`` to its :class:`GradedLabel`;
+    binary evaluators leave it empty.
     """
 
     relevant: Sequence[Result]
     sufficient: bool
     refinement: SubTask | None = None
+    graded: Mapping[str, GradedLabel] = field(default_factory=dict)
 
 
 # --------------------------------------------------------------------------- #

raglab-0.2.9/raglab/judging.py

@@ -0,0 +1,325 @@
+"""Graded rubric judging — every candidate gets a level, three axes and evidence (raglab#8).
+
+A binary relevant/not cannot say "mentions the word but is not about it". The
+evaluator built by :func:`make_graded_rubric_evaluator` asks an injected judge,
+for **each** candidate in the round's pool, for the three Self-RAG axes
+(``is_about``, ``is_supported``, ``is_useful``), a level on an ordered scale and
+a verbatim ``evidence_quote`` from the candidate's disclosed content. The judge
+proposes; deterministic gates here dispose:
+
+- a level above ``unsupported_cap`` needs ``is_supported`` — topical vocabulary
+  without supporting content is capped (the direct fix for on-word distractors);
+- with ``require_evidence_quote``, it also needs a quote found verbatim in the
+  content the judge was shown (whitespace-insensitive);
+- ``calibration_offset`` (levels, learned from a gold slice) shifts every level
+  down (positive: the judge over-grades) or up;
+- the level options are shown in a fresh random order on every call, against
+  position bias.
+
+The result is a :class:`~raglab.agent.Judgement` whose additive ``graded`` map
+carries every label, whose ``relevant`` keeps the candidates at or above
+``relevant_level``, and whose back-edge comes from an injected ``refine`` policy
+(none by default: the loop stops). As in :mod:`raglab.llm`, the default judge is
+built lazily on :mod:`oa`, so ``import raglab`` stays offline, and a judge that
+fails on a candidate leaves it ungraded rather than guessing.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+import random
+import re
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any
+
+import ir
+from ir.base import best_per_artifact
+
+from .agent import Evaluator, GradedLabel, Judgement, Result, SubTask
+
+__all__ = [
+    "DEFAULT_LEVELS",
+    "DEFAULT_TOPICAL_RUBRIC",
+    "GRADING_PROMPT",
+    "make_graded_rubric_evaluator",
+    "apply_gates",
+]
+
+#: Ordered levels, best first.
+DEFAULT_LEVELS: tuple[str, ...] = ("core", "strong", "uses-tools", "tangential", "none")
+
+#: What the default levels mean, for a "is this package about TOPIC" question.
+DEFAULT_TOPICAL_RUBRIC = """\
+- core: the candidate's main purpose IS the topic.
+- strong: a substantial, central part of the candidate is about the topic.
+- uses-tools: the candidate builds on tools or libraries of the topic (its dependencies or imports establish it) without being about it.
+- tangential: the topic is mentioned, or its vocabulary appears, but the content does not establish it.
+- none: unrelated.
+"""
+
+#: The judge's prompt; ``{levels}`` is shown in a fresh random order each call.
+GRADING_PROMPT = """\
+Goal: {goal}
+
+Rubric (levels, in no particular order):
+{levels}
+
+Definitions:
+{rubric}
+
+Candidate {artifact_id}:
+---
+{content}
+---
+
+Answer with one JSON object and nothing else, with keys:
+"is_about" (bool: is the candidate topically about the goal?),
+"is_supported" (bool: does its content - code, dependencies, imports, full text - actually establish it, beyond a mention of the words?),
+"is_useful" (bool: would it help someone pursuing the goal?),
+"evidence_quote" (a short span copied VERBATIM from the candidate's content that supports your level),
+"level" (one of the rubric's levels).
+"""
+
+#: A graded judge: keyword arguments ``goal, artifact_id, content, levels, rubric``
+#: -> a mapping with the label's keys, or the raw JSON text of one.
+GradedJudge = Callable[..., "Mapping[str, Any] | str"]
+
+#: A refinement policy: ``(task, graded) -> the next SubTask, or None to stop``.
+Refine = Callable[[SubTask, Mapping[str, GradedLabel]], "SubTask | None"]
+
+#: Truncate each candidate's disclosed content to this many chars in the prompt.
+DFLT_MAX_CONTENT_CHARS = 4000
+
+#: A shorter evidence quote ("a", "the") would be found in almost any content.
+DFLT_MIN_QUOTE_CHARS = 8
+
+_log = logging.getLogger(__name__)
+
+
+def _normalize_space(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip().casefold()
+
+
+def apply_gates(
+    proposed: GradedLabel,
+    content: str,
+    *,
+    levels: Sequence[str] = DEFAULT_LEVELS,
+    unsupported_cap: str = "tangential",
+    require_evidence_quote: bool = True,
+    calibration_offset: float | None = None,
+    min_quote_chars: int = DFLT_MIN_QUOTE_CHARS,
+) -> GradedLabel:
+    """The level a judge's proposal is allowed, after calibration and the gates.
+
+    The quote must be at least ``min_quote_chars`` long and found in ``content``,
+    ignoring case and runs of whitespace (judges re-case and re-wrap what they
+    copy). ``calibration_offset`` is in levels, rounded half up, so a shift is
+    the same at every level.
+
+    >>> label = GradedLabel('core', True, False, True, 'a DAG of tasks')
+    >>> apply_gates(label, 'Runs a DAG of tasks asynchronously.').level
+    'tangential'
+    >>> ok = GradedLabel('core', True, True, True, 'graph  library')
+    >>> apply_gates(ok, 'A graph library.').level, apply_gates(ok, 'Nothing here.').level
+    ('core', 'tangential')
+    >>> apply_gates(ok, 'A graph library.', calibration_offset=1.4).level
+    'strong'
+    """
+    cap = levels.index(unsupported_cap)
+    rank = levels.index(proposed.level)
+    if calibration_offset:
+        rank = min(max(math.floor(rank + calibration_offset + 0.5), 0), len(levels) - 1)
+    quote = _normalize_space(proposed.evidence_quote)
+    quoted = len(quote) >= min_quote_chars and quote in _normalize_space(content)
+    if rank < cap and (
+        not proposed.is_supported or (require_evidence_quote and not quoted)
+    ):
+        rank = cap
+    return GradedLabel(
+        level=levels[rank],
+        is_about=proposed.is_about,
+        is_supported=proposed.is_supported,
+        is_useful=proposed.is_useful,
+        evidence_quote=proposed.evidence_quote if quoted else "",
+        facet=proposed.facet,
+    )
+
+
+def _parse_label(out: Mapping[str, Any] | str, levels: Sequence[str]) -> GradedLabel:
+    """A judge's reply as a :class:`GradedLabel`; raises ``ValueError`` when unusable."""
+    if isinstance(out, str):
+        match = re.search(r"\{.*\}", out, flags=re.DOTALL)
+        if not match:
+            raise ValueError(f"no JSON object in the judge's reply: {out[:200]!r}")
+        out = json.loads(match.group(0))
+    by_name = {lv.casefold(): lv for lv in levels}
+    level = by_name.get(str(out.get("level", "")).strip().casefold())
+    if level is None:
+        raise ValueError(f"level {out.get('level')!r} is not one of {list(levels)}")
+
+    def flag(key: str) -> bool:
+        value = out.get(key)
+        if isinstance(value, str):
+            return value.strip().lower() in ("true", "yes")
+        return bool(value)
+
+    return GradedLabel(
+        level=level,
+        is_about=flag("is_about"),
+        is_supported=flag("is_supported"),
+        is_useful=flag("is_useful"),
+        evidence_quote=str(out.get("evidence_quote") or ""),
+    )
+
+
+def _contents(
+    hits: Sequence[Result], level: str, disclose_kwargs: Mapping[str, Any]
+) -> list[str]:
+    """Each hit's disclosed content at ``level``, its snippet when nothing loads."""
+    selection = ir.Selection(
+        selected=list(hits), candidates=list(hits), abstained=False, reason="graded"
+    )
+    disclosed = ir.disclose(selection, level=level, **dict(disclose_kwargs))
+    return [d.body or h.text for h, d in zip(hits, disclosed, strict=True)]
+
+
+def make_graded_rubric_evaluator(
+    *,
+    levels: Sequence[str] = DEFAULT_LEVELS,
+    rubric: str = DEFAULT_TOPICAL_RUBRIC,
+    judge: GradedJudge | None = None,
+    disclose_level: str = "body",
+    disclose_kwargs: Mapping[str, Any] | None = None,
+    require_evidence_quote: bool = True,
+    unsupported_cap: str = "tangential",
+    calibration_offset: float | None = None,
+    min_quote_chars: int = DFLT_MIN_QUOTE_CHARS,
+    relevant_level: str = "uses-tools",
+    refine: Refine | None = None,
+    rng: random.Random | None = None,
+    prompt: str = GRADING_PROMPT,
+    max_content_chars: int = DFLT_MAX_CONTENT_CHARS,
+    **prompt_function_kwargs: Any,
+) -> Evaluator:
+    """An :class:`~raglab.agent.Evaluator` that grades every candidate on a rubric.
+
+    Args:
+        levels: the ordered levels, best first.
+        rubric: what each level means, shown to the judge.
+        judge: an injectable graded judge (a test double, or your own router);
+            built lazily on :mod:`oa` when omitted.
+        disclose_level, disclose_kwargs: what the judge reads — each candidate's
+            content through :func:`ir.disclose` at that level (``store=`` /
+            ``loader=`` go in ``disclose_kwargs``); a candidate with nothing to
+            load is judged on its snippet.
+        require_evidence_quote, unsupported_cap, calibration_offset,
+            min_quote_chars: the gates, see :func:`apply_gates`.
+        relevant_level: the lowest level kept in ``Judgement.relevant``.
+        refine: the back-edge policy ``(task, graded) -> SubTask | None``; by
+            default none, so the loop stops after one round.
+        rng: the source of the per-call level order (seed it for reproducible
+            prompts).
+
+    Every candidate is judged once per goal: one judge call per (source,
+    artifact) the first time it appears in a pool for that goal, its label
+    remembered for later rounds, so a kept hit is not re-judged (nor lost to a
+    later failing call). ``Judgement.graded`` is keyed by ``artifact_id``; an id
+    found in several sources carries its best label. A reply that does not parse
+    leaves the candidate ungraded (logged); when every judge call of a round
+    raises, the first error is re-raised rather than returning an empty,
+    "sufficient" round. This evaluator is for labelling a pool, not for cheap
+    filtering.
+    """
+    levels = tuple(levels)
+    for name, value in (
+        ("unsupported_cap", unsupported_cap),
+        ("relevant_level", relevant_level),
+    ):
+        if value not in levels:
+            raise ValueError(f"{name}={value!r} is not one of {list(levels)}")
+    keep = levels.index(relevant_level)
+    rng = rng if rng is not None else random.Random()
+    disclose_kwargs = dict(disclose_kwargs or {})
+    state: dict[str, GradedJudge] = {}
+    memo: dict[tuple[str, str | None, str], GradedLabel] = {}
+
+    def ask(**kwargs: Any) -> Mapping[str, Any] | str:
+        if judge is not None:
+            return judge(**kwargs)
+        if "judge" not in state:
+            state["judge"] = _default_graded_judge(prompt, **prompt_function_kwargs)
+        return state["judge"](**kwargs)
+
+    def evaluator(task: SubTask, results: Sequence[Result]) -> Judgement:
+        hits = best_per_artifact(results)
+        todo = [h for h in hits if (task.goal, h.source, h.artifact_id) not in memo]
+        contents = _contents(todo, disclose_level, disclose_kwargs) if todo else []
+        errors: list[Exception] = []
+        for hit, content in zip(todo, contents, strict=True):
+            content = str(content)[:max_content_chars]
+            shown = list(levels)
+            rng.shuffle(shown)
+            try:
+                reply = ask(
+                    goal=task.goal,
+                    artifact_id=hit.artifact_id,
+                    content=content,
+                    levels="\n".join(f"- {lv}" for lv in shown),
+                    rubric=rubric,
+                )
+            except Exception as error:  # the judge itself failed: auth, network, limits
+                _log.warning("judge failed on %s: %r", hit.artifact_id, error)
+                errors.append(error)
+                continue
+            try:
+                proposed = _parse_label(reply, levels)
+            except (ValueError, TypeError, AttributeError) as error:
+                _log.warning("unusable judge reply on %s: %s", hit.artifact_id, error)
+                continue  # left ungraded: never guess a level
+            memo[(task.goal, hit.source, hit.artifact_id)] = apply_gates(
+                proposed,
+                content,
+                levels=levels,
+                unsupported_cap=unsupported_cap,
+                require_evidence_quote=require_evidence_quote,
+                calibration_offset=calibration_offset,
+                min_quote_chars=min_quote_chars,
+            )
+        if todo and len(errors) == len(todo):
+            raise errors[0]
+        labels = {
+            (h.source, h.artifact_id): memo[key]
+            for h in hits
+            if (key := (task.goal, h.source, h.artifact_id)) in memo
+        }
+        graded: dict[str, GradedLabel] = {}
+        for (_, aid), label in labels.items():
+            if aid not in graded or levels.index(label.level) < levels.index(
+                graded[aid].level
+            ):
+                graded[aid] = label
+        relevant = [
+            h
+            for h in hits
+            if (label := labels.get((h.source, h.artifact_id))) is not None
+            and levels.index(label.level) <= keep
+        ]
+        refinement = refine(task, graded) if refine is not None else None
+        return Judgement(
+            relevant=relevant,
+            sufficient=refinement is None,
+            refinement=refinement,
+            graded=graded,
+        )
+
+    return evaluator
+
+
+def _default_graded_judge(prompt: str, **prompt_function_kwargs: Any) -> GradedJudge:
+    """Build the default graded judge on :mod:`oa` (lazy import)."""
+    import oa
+
+    return oa.prompt_function(prompt, name="grade_candidate", **prompt_function_kwargs)

raglab-0.2.9/raglab/recipes/__init__.py

@@ -0,0 +1,5 @@
+"""Recipes: whole workflows composed from raglab's roles, ready to call."""
+
+from .corpus_labeling import CorpusLabeling, FacetCoverage, label_corpus
+
+__all__ = ["CorpusLabeling", "FacetCoverage", "label_corpus"]

raglab-0.2.9/raglab/recipes/corpus_labeling.py

@@ -0,0 +1,219 @@
+"""Label a corpus against a topic, facet by facet, with a graded judge (raglab#8, part B).
+
+:func:`label_corpus` composes existing pieces and adds no orchestrator:
+
+- a planner that emits one :class:`~raglab.agent.SubTask` per **facet** of the
+  topic, for broad recall;
+- a formulator (identity by default; :func:`raglab.make_llm_formulator` for
+  query expansion);
+- the graded rubric evaluator (:func:`raglab.make_graded_rubric_evaluator`),
+  which judges **every** retrieved candidate on its disclosed content;
+- a **coverage** check per facet (enough candidates at ``coverage_level``),
+  which, when a facet looks thin and a ``reformulate`` policy is given, emits a
+  refinement sub-task: the back-edge of :class:`~raglab.agent.SingleContextAgent`.
+
+The deliverable is a frozen :class:`CorpusLabeling`: each judged artifact's best
+:class:`~raglab.agent.GradedLabel` (its ``facet`` set) and each facet's coverage.
+Nothing here touches ``ir``; the corpus is read through a retriever
+(:func:`ir.as_retriever` by default).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping
+from dataclasses import dataclass, field, replace
+from typing import Any
+
+import ir
+
+from ..agent import (
+    Budget,
+    Formulator,
+    GradedLabel,
+    Judgement,
+    Query,
+    Result,
+    Retriever,
+    SingleContextAgent,
+    SubTask,
+    identity_formulator,
+)
+from ..judging import DEFAULT_LEVELS, make_graded_rubric_evaluator
+
+__all__ = ["CorpusLabeling", "FacetCoverage", "label_corpus"]
+
+#: A gap-driven reformulation: ``(facet goal, labels so far) -> a new query, or None``.
+Reformulate = Callable[[str, Mapping[str, GradedLabel]], "str | None"]
+
+#: The registry key the corpus's retriever is registered under.
+SOURCE = "corpus"
+
+#: Candidates retrieved per query by the default retriever: labelling wants recall.
+DFLT_K = 50
+
+
+@dataclass(frozen=True)
+class FacetCoverage:
+    """How well one facet was covered: candidates judged, those at the coverage level, rounds run."""
+
+    facet: str
+    judged: int
+    covered: int
+    rounds: int
+    sufficient: bool
+
+
+@dataclass(frozen=True)
+class CorpusLabeling:
+    """The graded labels of a corpus for one topic, and each facet's coverage."""
+
+    labels: Mapping[str, GradedLabel] = field(default_factory=dict)
+    coverage: Mapping[str, FacetCoverage] = field(default_factory=dict)
+    levels: tuple[str, ...] = DEFAULT_LEVELS
+
+    def at_least(self, level: str) -> list[str]:
+        """The artifact ids labelled ``level`` or better, on the levels they were labelled on."""
+        cut = self.levels.index(level)
+        return [
+            aid for aid, g in self.labels.items() if self.levels.index(g.level) <= cut
+        ]
+
+
+def _facet_goals(facets: Mapping[str, str] | Iterable[str]) -> dict[str, str]:
+    goals = dict(facets) if isinstance(facets, Mapping) else {f: f for f in facets}
+    if not goals:
+        raise ValueError("label_corpus needs at least one facet")
+    if len(set(goals.values())) < len(goals):
+        raise ValueError(
+            f"two facets share a goal: {goals}; give each facet its own goal"
+        )
+    return goals
+
+
+def label_corpus(
+    corpus: Any,
+    *,
+    facets: Mapping[str, str] | Iterable[str],
+    retriever: Retriever | None = None,
+    formulator: Formulator = identity_formulator,
+    judge: Callable[..., Any] | None = None,
+    reformulate: Reformulate | None = None,
+    levels: tuple[str, ...] = DEFAULT_LEVELS,
+    coverage_level: str = "strong",
+    min_covered: int = 1,
+    budget: Budget | None = None,
+    k: int = DFLT_K,
+    **evaluator_kwargs: Any,
+) -> CorpusLabeling:
+    """Label ``corpus`` for each of ``facets`` with a graded judge; see the module.
+
+    Args:
+        corpus: an ``ir`` corpus or corpus name, read through
+            ``ir.as_retriever(corpus, k=k)`` unless ``retriever`` is given.
+        facets: the topic's facets, as ``{name: goal}`` or names (a name is its
+            own goal). One sub-task each.
+        retriever: inject a :data:`~raglab.agent.Retriever` instead (a test
+            double, a federated source); ``corpus`` is then not read.
+        formulator: the Formulator role (identity, or
+            :func:`raglab.make_llm_formulator`).
+        judge: the graded judge (built lazily on :mod:`oa` when omitted).
+        reformulate: gap-driven reformulation; when a facet has fewer than
+            ``min_covered`` labels at ``coverage_level`` it gets a new query, which
+            becomes the facet's next sub-task. ``None`` (default): one round each.
+        budget: the loop bounds (``max_rounds`` caps the reformulations).
+        **evaluator_kwargs: forwarded to
+            :func:`~raglab.judging.make_graded_rubric_evaluator` (``rubric``,
+            ``calibration_offset``, ``disclose_kwargs``, ``rng``, ...).
+    """
+    goals = _facet_goals(facets)
+    levels = tuple(levels)
+    named = {
+        "coverage_level": coverage_level,
+        **{
+            key: evaluator_kwargs.get(key, default)
+            for key, default in (
+                ("unsupported_cap", "tangential"),
+                ("relevant_level", "uses-tools"),
+            )
+        },
+    }
+    for key, value in named.items():
+        if value not in levels:
+            raise ValueError(
+                f"{key}={value!r} is not one of {list(levels)}: with custom levels, "
+                "pass coverage_level, unsupported_cap and relevant_level from them"
+            )
+    cut = levels.index(coverage_level)
+    source = retriever if retriever is not None else ir.as_retriever(corpus, k=k)
+    facet_of: dict[str, str] = {goal: name for name, goal in goals.items()}
+    seen: dict[str, dict[str, GradedLabel]] = {name: {} for name in goals}
+    rounds: dict[str, int] = dict.fromkeys(goals, 0)
+
+    def covered(name: str) -> int:
+        return sum(levels.index(g.level) <= cut for g in seen[name].values())
+
+    max_rounds = (budget if budget is not None else Budget()).max_rounds
+
+    def refine(task: SubTask, name: str) -> SubTask | None:
+        if (
+            reformulate is None
+            or covered(name) >= min_covered
+            or rounds[name] >= max_rounds
+        ):
+            return None  # covered, no policy, or no round left to run a new query in
+        query = reformulate(goals[name], dict(seen[name]))
+        if not query or query in facet_of:
+            return None  # nothing new to ask: stop rather than loop
+        facet_of[query] = name
+        return SubTask(goal=query, sources=task.sources)
+
+    graded_evaluator = make_graded_rubric_evaluator(
+        judge=judge, levels=levels, **evaluator_kwargs
+    )
+
+    def evaluator(task: SubTask, results: list[Result]) -> Judgement:
+        name = facet_of[task.goal]
+        rounds[name] += 1
+        judged = graded_evaluator(_as_facet_goal(task, goals[name]), results)
+        for aid, label in judged.graded.items():
+            label = replace(label, facet=name)
+            best = seen[name].get(aid)
+            if best is None or levels.index(label.level) < levels.index(best.level):
+                seen[name][aid] = label
+        refinement = refine(task, name)
+        return replace(judged, sufficient=refinement is None, refinement=refinement)
+
+    def _as_facet_goal(task: SubTask, goal: str) -> SubTask:
+        # The judge grades against the facet's goal, whatever query retrieved the pool.
+        return task if task.goal == goal else replace(task, goal=goal)
+
+    def planner(query: Query, sources: Mapping[str, Retriever]) -> list[SubTask]:
+        return [SubTask(goal=goal, sources=(SOURCE,)) for goal in goals.values()]
+
+    agent = SingleContextAgent(
+        sources={SOURCE: source},
+        planner=planner,
+        formulator=formulator,
+        evaluator=evaluator,
+        budget=budget if budget is not None else Budget(),
+    )
+    agent(Query(text=" / ".join(goals.values())))
+
+    labels: dict[str, GradedLabel] = {}
+    for per_facet in seen.values():
+        for aid, label in per_facet.items():
+            if aid not in labels or levels.index(label.level) < levels.index(
+                labels[aid].level
+            ):
+                labels[aid] = label
+    coverage = {
+        name: FacetCoverage(
+            facet=name,
+            judged=len(seen[name]),
+            covered=covered(name),
+            rounds=rounds[name],
+            sufficient=covered(name) >= min_covered,
+        )
+        for name in goals
+    }
+    return CorpusLabeling(labels=labels, coverage=coverage, levels=levels)

raglab-0.2.9/tests/test_corpus_labeling.py

@@ -0,0 +1,117 @@
+"""Tests for the corpus-labeling recipe (raglab#8, part B). Hermetic: a fake retriever and a scripted judge."""
+
+import pytest
+from ir import SearchHit
+
+from raglab import Budget
+from raglab.recipes import CorpusLabeling, label_corpus
+
+DOCS = {
+    "graphy": "A graph library: nodes, edges, traversals.",
+    "au": "An async task framework; jobs form a DAG.",
+    "layout": "Graph layout algorithms for drawing networks.",
+    "decoder": "An audio decoder.",
+}
+INDEX = {
+    "graph data structures": ["graphy", "au"],
+    "graph drawing": ["decoder"],
+    "network layout algorithms": ["layout"],
+}
+
+
+def _retriever(query, **kw):
+    return [SearchHit(aid, "k", 1.0 - i / 10, DOCS[aid], {}) for i, aid in enumerate(INDEX.get(query, []))]
+
+
+def _judge(*, goal, artifact_id, content, **kw):
+    level = {"graphy": "core", "au": "strong", "layout": "core", "decoder": "none"}[artifact_id]
+    supported = artifact_id != "au"
+    quote = content[:20]
+    return {"level": level, "is_about": True, "is_supported": supported, "is_useful": supported, "evidence_quote": quote}
+
+
+def test_every_facet_is_labelled_and_the_gate_holds():
+    result = label_corpus(
+        None,
+        facets={"structures": "graph data structures", "drawing": "graph drawing"},
+        retriever=_retriever,
+        judge=_judge,
+    )
+    assert isinstance(result, CorpusLabeling)
+    assert {aid: g.level for aid, g in result.labels.items()} == {
+        "graphy": "core",
+        "au": "tangential",
+        "decoder": "none",
+    }
+    assert result.labels["graphy"].facet == "structures"
+    assert result.coverage["structures"].sufficient
+    assert not result.coverage["drawing"].sufficient
+    assert result.at_least("strong") == ["graphy"]
+
+
+def test_a_thin_facet_is_reformulated_and_judged_against_its_own_goal():
+    goals_seen = []
+
+    def judge(**kw):
+        goals_seen.append(kw["goal"])
+        return _judge(**kw)
+
+    result = label_corpus(
+        None,
+        facets={"drawing": "graph drawing"},
+        retriever=_retriever,
+        judge=judge,
+        reformulate=lambda goal, labels: "network layout algorithms",
+        budget=Budget(max_rounds=3),
+    )
+    assert result.labels["layout"].level == "core"
+    assert result.coverage["drawing"].rounds == 2 and result.coverage["drawing"].sufficient
+    assert set(goals_seen) == {"graph drawing"}
+
+
+def test_a_reformulation_that_repeats_itself_stops():
+    result = label_corpus(
+        None,
+        facets=["graph drawing"],
+        retriever=_retriever,
+        judge=_judge,
+        reformulate=lambda goal, labels: goal,
+        budget=Budget(max_rounds=5),
+    )
+    assert result.coverage["graph drawing"].rounds == 1
+
+
+def test_facets_are_required():
+    with pytest.raises(ValueError, match="at least one facet"):
+        label_corpus(None, facets=[], retriever=_retriever, judge=_judge)
+
+
+def test_facets_with_one_goal_are_refused():
+    with pytest.raises(ValueError, match="share a goal"):
+        label_corpus(None, facets={"x": "same", "y": "same"}, retriever=_retriever, judge=_judge)
+
+
+def test_no_reformulation_is_asked_for_once_the_rounds_are_spent():
+    asked = []
+
+    def reformulate(goal, labels):
+        asked.append(goal)
+        return f"query {len(asked)}"
+
+    label_corpus(None, facets=["graph drawing"], retriever=_retriever, judge=_judge,
+                 reformulate=reformulate, budget=Budget(max_rounds=2))
+    assert len(asked) == 1
+
+
+def test_custom_levels_need_their_own_level_names():
+    def judge(**kw):
+        return {"level": "hi", "is_about": True, "is_supported": True, "is_useful": True,
+                "evidence_quote": kw["content"][:20]}
+
+    with pytest.raises(ValueError, match="unsupported_cap"):
+        label_corpus(None, facets=["graph data structures"], retriever=_retriever, judge=judge,
+                     levels=("hi", "lo"), coverage_level="hi")
+    result = label_corpus(None, facets=["graph data structures"], retriever=_retriever, judge=judge,
+                          levels=("hi", "lo"), coverage_level="hi",
+                          unsupported_cap="lo", relevant_level="hi")
+    assert sorted(result.at_least("hi")) == ["au", "graphy"]

raglab-0.2.9/tests/test_judging.py

@@ -0,0 +1,220 @@
+"""Tests for the graded rubric evaluator (raglab#8).
+
+Hermetic: the judge is a scripted test double keyed by artifact id; no model,
+no network. Content comes from each hit's snippet (nothing to load from disk).
+"""
+
+import json
+import random
+
+from ir import SearchHit
+
+from raglab import (
+    DEFAULT_LEVELS,
+    Budget,
+    GradedLabel,
+    Judgement,
+    SubTask,
+    make_graded_rubric_evaluator,
+    make_search_agent,
+)
+
+TASK = SubTask(goal="graph library", sources=("pkgs",))
+
+
+def _hit(aid, text, score=1.0):
+    return SearchHit(aid, "k", score, text, {})
+
+
+def _judge(replies):
+    """A judge returning ``replies[artifact_id]`` and recording what it was shown."""
+    calls = []
+
+    def judge(**kw):
+        calls.append(kw)
+        reply = replies[kw["artifact_id"]]
+        if isinstance(reply, Exception):
+            raise reply
+        return reply
+
+    judge.calls = calls
+    return judge
+
+
+def _reply(level, *, about=True, supported=True, useful=True, quote=""):
+    return {
+        "level": level,
+        "is_about": about,
+        "is_supported": supported,
+        "is_useful": useful,
+        "evidence_quote": quote,
+    }
+
+
+POOL = [
+    _hit("graphy", "A graph library: nodes, edges, traversals."),
+    _hit("au", "An async task framework; jobs form a DAG."),
+    _hit("plotter", "Draws charts with networkx graph layouts."),
+]
+
+
+def test_every_candidate_is_graded_and_the_supported_gate_caps_on_word_distractors():
+    judge = _judge(
+        {
+            "graphy": _reply("core", quote="A graph library"),
+            # on-word but not supported: the judge over-grades, the gate caps it
+            "au": _reply("strong", supported=False, quote="jobs form a DAG"),
+            "plotter": _reply("uses-tools", quote="networkx graph layouts"),
+        }
+    )
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+    assert isinstance(judged, Judgement)
+    assert {aid: g.level for aid, g in judged.graded.items()} == {
+        "graphy": "core",
+        "au": "tangential",
+        "plotter": "uses-tools",
+    }
+    assert [h.artifact_id for h in judged.relevant] == ["graphy", "plotter"]
+    assert judged.sufficient and judged.refinement is None
+    assert len(judge.calls) == 3
+
+
+def test_a_quote_not_found_verbatim_caps_the_level_unless_quotes_are_optional():
+    judge = _judge({"graphy": _reply("core", quote="the best graph toolkit")})
+    strict = make_graded_rubric_evaluator(judge=judge)(TASK, POOL[:1])
+    assert strict.graded["graphy"].level == "tangential"
+    assert strict.graded["graphy"].evidence_quote == ""
+    lax = make_graded_rubric_evaluator(judge=judge, require_evidence_quote=False)(
+        TASK, POOL[:1]
+    )
+    assert lax.graded["graphy"].level == "core"
+
+
+def test_the_calibration_offset_demotes_an_over_grading_judge():
+    judge = _judge({"graphy": _reply("core", quote="graph library")})
+    ev = make_graded_rubric_evaluator(judge=judge, calibration_offset=1.0)
+    assert ev(TASK, POOL[:1]).graded["graphy"].level == "strong"
+
+
+def test_a_failing_or_unparseable_judge_leaves_the_candidate_ungraded():
+    judge = _judge(
+        {
+            "graphy": json.dumps(_reply("core", quote="graph library")),  # raw text ok
+            "au": RuntimeError("rate limited"),
+            "plotter": '{"level": "excellent"}',  # not a level
+        }
+    )
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+    assert set(judged.graded) == {"graphy"}
+    assert [h.artifact_id for h in judged.relevant] == ["graphy"]
+
+
+def test_level_options_are_shown_in_a_fresh_order_each_call():
+    judge = _judge({h.artifact_id: _reply("none") for h in POOL})
+    ev = make_graded_rubric_evaluator(judge=judge, rng=random.Random(0))
+    for _ in range(4):
+        ev(TASK, POOL)
+    orders = {c["levels"] for c in judge.calls}
+    assert len(orders) > 1
+    assert all(sorted(o.split("\n")) == sorted(f"- {lv}" for lv in DEFAULT_LEVELS) for o in orders)
+
+
+def test_the_refine_policy_drives_the_back_edge_through_the_agent():
+    corpus = {
+        "graph library": [_hit("au", "An async task framework; jobs form a DAG.")],
+        "graph library nodes edges": [_hit("graphy", "A graph library: nodes, edges.")],
+    }
+
+    def retriever(query, **kw):
+        return list(corpus.get(query, []))
+
+    judge = _judge(
+        {
+            "au": _reply("core", supported=False, quote="jobs form a DAG"),
+            "graphy": _reply("core", quote="A graph library"),
+        }
+    )
+
+    def refine(task, graded):
+        if any(g.level in ("core", "strong") for g in graded.values()):
+            return None
+        return SubTask(goal=task.goal + " nodes edges", sources=task.sources)
+
+    agent = make_search_agent(
+        {"pkgs": retriever},
+        evaluator=make_graded_rubric_evaluator(judge=judge, refine=refine),
+        budget=Budget(max_rounds=3),
+    )
+    assert [h.artifact_id for h in agent("graph library")] == ["graphy"]
+
+
+def test_binary_judgements_are_unchanged():
+    assert Judgement(relevant=[], sufficient=True).graded == {}
+    label = GradedLabel("core", True, True, True)
+    assert label.evidence_quote == "" and label.facet is None
+
+
+def test_unknown_level_names_are_refused():
+    import pytest
+
+    with pytest.raises(ValueError, match="relevant_level"):
+        make_graded_rubric_evaluator(judge=_judge({}), relevant_level="great")
+
+
+def test_the_same_id_in_two_sources_is_judged_per_source():
+    a = SearchHit("x", "k", 1.0, "A graph library for nodes.", {}, source="A")
+    b = SearchHit("x", "k", 1.0, "An audio decoder.", {}, source="B")
+
+    def judge(**kw):
+        if "graph" in kw["content"]:
+            return _reply("core", quote="A graph library")
+        return _reply("none")
+
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, [a, b])
+    assert [h.source for h in judged.relevant] == ["A"]
+    assert judged.graded["x"].level == "core"  # the best label across sources
+
+
+def test_custom_levels_match_whatever_their_case():
+    judge = _judge({"graphy": _reply("core", quote="A graph library")})
+    ev = make_graded_rubric_evaluator(
+        judge=judge, levels=("Core", "Tangential", "None"),
+        unsupported_cap="Tangential", relevant_level="Core",
+    )
+    assert ev(TASK, POOL[:1]).graded["graphy"].level == "Core"
+
+
+def test_a_judge_that_fails_on_every_candidate_raises():
+    import pytest
+
+    judge = _judge({h.artifact_id: RuntimeError("no key") for h in POOL})
+    with pytest.raises(RuntimeError, match="no key"):
+        make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+
+
+def test_a_candidate_is_judged_once_per_goal_and_its_label_kept():
+    judge = _judge({"graphy": _reply("core", quote="A graph library")})
+    ev = make_graded_rubric_evaluator(judge=judge)
+    ev(TASK, POOL[:1])
+    judge_down = ev(TASK, POOL[:1])  # no new call: the label is remembered
+    assert len(judge.calls) == 1
+    assert [h.artifact_id for h in judge_down.relevant] == ["graphy"]
+
+
+def test_a_half_level_offset_shifts_every_level_alike():
+    from raglab import apply_gates
+
+    content = "A graph library."
+    shifted = [
+        apply_gates(GradedLabel(lv, True, True, True, "graph library"), content,
+                    calibration_offset=0.5).level
+        for lv in DEFAULT_LEVELS[:-1]
+    ]
+    assert shifted == list(DEFAULT_LEVELS[1:])
+
+
+def test_a_one_word_quote_is_not_evidence():
+    from raglab import apply_gates
+
+    label = GradedLabel("core", True, True, True, "a")
+    assert apply_gates(label, "A graph library.").level == "tangential"