raglab 0.2.7__tar.gz → 0.2.8__tar.gz

{raglab-0.2.7 → raglab-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: raglab
-Version: 0.2.7
+Version: 0.2.8
 Summary: A medley of tools to make RAG-based applications.
 Project-URL: Homepage, https://thorwhalen.github.io/raglab/
 Project-URL: Repository, https://github.com/thorwhalen/raglab

{raglab-0.2.7 → raglab-0.2.8}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "raglab"
-version = "0.2.7"
+version = "0.2.8"
 description = "A medley of tools to make RAG-based applications."
 readme = "README.md"
 requires-python = ">=3.10"

{raglab-0.2.7 → raglab-0.2.8}/raglab/__init__.py

@@ -30,6 +30,7 @@ from .agent import (
     Citer,
     Evaluator,
     Formulator,
+    GradedLabel,
     Judgement,
     LowLevelQuery,
     Planner,
@@ -53,12 +54,20 @@ from .agent import (
     single_subtask_planner,
 )
 from .llm import EVALUATION_PROMPT, make_llm_evaluator, make_llm_formulator
+from .judging import (
+    DEFAULT_LEVELS,
+    DEFAULT_TOPICAL_RUBRIC,
+    GRADING_PROMPT,
+    apply_gates,
+    make_graded_rubric_evaluator,
+)
 
 __all__ = [
     "Query",
     "SubTask",
     "LowLevelQuery",
     "Judgement",
+    "GradedLabel",
     "Result",
     "Retriever",
     "Planner",
@@ -83,4 +92,9 @@ __all__ = [
     "make_llm_formulator",
     "make_llm_evaluator",
     "EVALUATION_PROMPT",
+    "DEFAULT_LEVELS",
+    "DEFAULT_TOPICAL_RUBRIC",
+    "GRADING_PROMPT",
+    "apply_gates",
+    "make_graded_rubric_evaluator",
 ]

{raglab-0.2.7 → raglab-0.2.8}/raglab/agent.py

@@ -52,6 +52,7 @@ __all__ = [
     "SubTask",
     "LowLevelQuery",
     "Judgement",
+    "GradedLabel",
     "Result",
     "Retriever",
     "Planner",
@@ -111,6 +112,25 @@ class LowLevelQuery:
     params: Mapping[str, Any] = field(default_factory=dict)
 
 
+@dataclass(frozen=True)
+class GradedLabel:
+    """A graded verdict on one candidate (raglab#8): a level, three axes, the evidence.
+
+    ``level`` is one of the evaluator's ordered levels (best first). The axes are
+    Self-RAG's: ``is_about`` (topical, ISREL), ``is_supported`` (the content
+    actually establishes it, not a lexical mention: ISSUP) and ``is_useful``
+    (ISUSE). ``evidence_quote`` is a verbatim span of the candidate's content
+    backing the level; ``facet`` names the sub-goal it was judged against.
+    """
+
+    level: str
+    is_about: bool
+    is_supported: bool
+    is_useful: bool
+    evidence_quote: str = ""
+    facet: str | None = None
+
+
 @dataclass(frozen=True)
 class Judgement:
     """An evaluator's verdict over a round's results.
@@ -118,12 +138,15 @@ class Judgement:
     ``relevant`` is the kept subset; ``sufficient`` says whether to stop; a
     non-``None`` ``refinement`` is the **back-edge** — the next sub-task to
     re-query with. The pass-through default returns ``sufficient=True`` and no
-    refinement (no loop).
+    refinement (no loop). ``graded`` is optional and additive: a graded
+    evaluator maps each judged ``artifact_id`` to its :class:`GradedLabel`;
+    binary evaluators leave it empty.
     """
 
     relevant: Sequence[Result]
     sufficient: bool
     refinement: SubTask | None = None
+    graded: Mapping[str, GradedLabel] = field(default_factory=dict)
 
 
 # --------------------------------------------------------------------------- #

raglab-0.2.8/raglab/judging.py

@@ -0,0 +1,325 @@
+"""Graded rubric judging — every candidate gets a level, three axes and evidence (raglab#8).
+
+A binary relevant/not cannot say "mentions the word but is not about it". The
+evaluator built by :func:`make_graded_rubric_evaluator` asks an injected judge,
+for **each** candidate in the round's pool, for the three Self-RAG axes
+(``is_about``, ``is_supported``, ``is_useful``), a level on an ordered scale and
+a verbatim ``evidence_quote`` from the candidate's disclosed content. The judge
+proposes; deterministic gates here dispose:
+
+- a level above ``unsupported_cap`` needs ``is_supported`` — topical vocabulary
+  without supporting content is capped (the direct fix for on-word distractors);
+- with ``require_evidence_quote``, it also needs a quote found verbatim in the
+  content the judge was shown (whitespace-insensitive);
+- ``calibration_offset`` (levels, learned from a gold slice) shifts every level
+  down (positive: the judge over-grades) or up;
+- the level options are shown in a fresh random order on every call, against
+  position bias.
+
+The result is a :class:`~raglab.agent.Judgement` whose additive ``graded`` map
+carries every label, whose ``relevant`` keeps the candidates at or above
+``relevant_level``, and whose back-edge comes from an injected ``refine`` policy
+(none by default: the loop stops). As in :mod:`raglab.llm`, the default judge is
+built lazily on :mod:`oa`, so ``import raglab`` stays offline, and a judge that
+fails on a candidate leaves it ungraded rather than guessing.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import math
+import random
+import re
+from collections.abc import Callable, Mapping, Sequence
+from typing import Any
+
+import ir
+from ir.base import best_per_artifact
+
+from .agent import Evaluator, GradedLabel, Judgement, Result, SubTask
+
+__all__ = [
+    "DEFAULT_LEVELS",
+    "DEFAULT_TOPICAL_RUBRIC",
+    "GRADING_PROMPT",
+    "make_graded_rubric_evaluator",
+    "apply_gates",
+]
+
+#: Ordered levels, best first.
+DEFAULT_LEVELS: tuple[str, ...] = ("core", "strong", "uses-tools", "tangential", "none")
+
+#: What the default levels mean, for a "is this package about TOPIC" question.
+DEFAULT_TOPICAL_RUBRIC = """\
+- core: the candidate's main purpose IS the topic.
+- strong: a substantial, central part of the candidate is about the topic.
+- uses-tools: the candidate builds on tools or libraries of the topic (its dependencies or imports establish it) without being about it.
+- tangential: the topic is mentioned, or its vocabulary appears, but the content does not establish it.
+- none: unrelated.
+"""
+
+#: The judge's prompt; ``{levels}`` is shown in a fresh random order each call.
+GRADING_PROMPT = """\
+Goal: {goal}
+
+Rubric (levels, in no particular order):
+{levels}
+
+Definitions:
+{rubric}
+
+Candidate {artifact_id}:
+---
+{content}
+---
+
+Answer with one JSON object and nothing else, with keys:
+"is_about" (bool: is the candidate topically about the goal?),
+"is_supported" (bool: does its content - code, dependencies, imports, full text - actually establish it, beyond a mention of the words?),
+"is_useful" (bool: would it help someone pursuing the goal?),
+"evidence_quote" (a short span copied VERBATIM from the candidate's content that supports your level),
+"level" (one of the rubric's levels).
+"""
+
+#: A graded judge: keyword arguments ``goal, artifact_id, content, levels, rubric``
+#: -> a mapping with the label's keys, or the raw JSON text of one.
+GradedJudge = Callable[..., "Mapping[str, Any] | str"]
+
+#: A refinement policy: ``(task, graded) -> the next SubTask, or None to stop``.
+Refine = Callable[[SubTask, Mapping[str, GradedLabel]], "SubTask | None"]
+
+#: Truncate each candidate's disclosed content to this many chars in the prompt.
+DFLT_MAX_CONTENT_CHARS = 4000
+
+#: A shorter evidence quote ("a", "the") would be found in almost any content.
+DFLT_MIN_QUOTE_CHARS = 8
+
+_log = logging.getLogger(__name__)
+
+
+def _normalize_space(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip().casefold()
+
+
+def apply_gates(
+    proposed: GradedLabel,
+    content: str,
+    *,
+    levels: Sequence[str] = DEFAULT_LEVELS,
+    unsupported_cap: str = "tangential",
+    require_evidence_quote: bool = True,
+    calibration_offset: float | None = None,
+    min_quote_chars: int = DFLT_MIN_QUOTE_CHARS,
+) -> GradedLabel:
+    """The level a judge's proposal is allowed, after calibration and the gates.
+
+    The quote must be at least ``min_quote_chars`` long and found in ``content``,
+    ignoring case and runs of whitespace (judges re-case and re-wrap what they
+    copy). ``calibration_offset`` is in levels, rounded half up, so a shift is
+    the same at every level.
+
+    >>> label = GradedLabel('core', True, False, True, 'a DAG of tasks')
+    >>> apply_gates(label, 'Runs a DAG of tasks asynchronously.').level
+    'tangential'
+    >>> ok = GradedLabel('core', True, True, True, 'graph  library')
+    >>> apply_gates(ok, 'A graph library.').level, apply_gates(ok, 'Nothing here.').level
+    ('core', 'tangential')
+    >>> apply_gates(ok, 'A graph library.', calibration_offset=1.4).level
+    'strong'
+    """
+    cap = levels.index(unsupported_cap)
+    rank = levels.index(proposed.level)
+    if calibration_offset:
+        rank = min(max(math.floor(rank + calibration_offset + 0.5), 0), len(levels) - 1)
+    quote = _normalize_space(proposed.evidence_quote)
+    quoted = len(quote) >= min_quote_chars and quote in _normalize_space(content)
+    if rank < cap and (
+        not proposed.is_supported or (require_evidence_quote and not quoted)
+    ):
+        rank = cap
+    return GradedLabel(
+        level=levels[rank],
+        is_about=proposed.is_about,
+        is_supported=proposed.is_supported,
+        is_useful=proposed.is_useful,
+        evidence_quote=proposed.evidence_quote if quoted else "",
+        facet=proposed.facet,
+    )
+
+
+def _parse_label(out: Mapping[str, Any] | str, levels: Sequence[str]) -> GradedLabel:
+    """A judge's reply as a :class:`GradedLabel`; raises ``ValueError`` when unusable."""
+    if isinstance(out, str):
+        match = re.search(r"\{.*\}", out, flags=re.DOTALL)
+        if not match:
+            raise ValueError(f"no JSON object in the judge's reply: {out[:200]!r}")
+        out = json.loads(match.group(0))
+    by_name = {lv.casefold(): lv for lv in levels}
+    level = by_name.get(str(out.get("level", "")).strip().casefold())
+    if level is None:
+        raise ValueError(f"level {out.get('level')!r} is not one of {list(levels)}")
+
+    def flag(key: str) -> bool:
+        value = out.get(key)
+        if isinstance(value, str):
+            return value.strip().lower() in ("true", "yes")
+        return bool(value)
+
+    return GradedLabel(
+        level=level,
+        is_about=flag("is_about"),
+        is_supported=flag("is_supported"),
+        is_useful=flag("is_useful"),
+        evidence_quote=str(out.get("evidence_quote") or ""),
+    )
+
+
+def _contents(
+    hits: Sequence[Result], level: str, disclose_kwargs: Mapping[str, Any]
+) -> list[str]:
+    """Each hit's disclosed content at ``level``, its snippet when nothing loads."""
+    selection = ir.Selection(
+        selected=list(hits), candidates=list(hits), abstained=False, reason="graded"
+    )
+    disclosed = ir.disclose(selection, level=level, **dict(disclose_kwargs))
+    return [d.body or h.text for h, d in zip(hits, disclosed, strict=True)]
+
+
+def make_graded_rubric_evaluator(
+    *,
+    levels: Sequence[str] = DEFAULT_LEVELS,
+    rubric: str = DEFAULT_TOPICAL_RUBRIC,
+    judge: GradedJudge | None = None,
+    disclose_level: str = "body",
+    disclose_kwargs: Mapping[str, Any] | None = None,
+    require_evidence_quote: bool = True,
+    unsupported_cap: str = "tangential",
+    calibration_offset: float | None = None,
+    min_quote_chars: int = DFLT_MIN_QUOTE_CHARS,
+    relevant_level: str = "uses-tools",
+    refine: Refine | None = None,
+    rng: random.Random | None = None,
+    prompt: str = GRADING_PROMPT,
+    max_content_chars: int = DFLT_MAX_CONTENT_CHARS,
+    **prompt_function_kwargs: Any,
+) -> Evaluator:
+    """An :class:`~raglab.agent.Evaluator` that grades every candidate on a rubric.
+
+    Args:
+        levels: the ordered levels, best first.
+        rubric: what each level means, shown to the judge.
+        judge: an injectable graded judge (a test double, or your own router);
+            built lazily on :mod:`oa` when omitted.
+        disclose_level, disclose_kwargs: what the judge reads — each candidate's
+            content through :func:`ir.disclose` at that level (``store=`` /
+            ``loader=`` go in ``disclose_kwargs``); a candidate with nothing to
+            load is judged on its snippet.
+        require_evidence_quote, unsupported_cap, calibration_offset,
+            min_quote_chars: the gates, see :func:`apply_gates`.
+        relevant_level: the lowest level kept in ``Judgement.relevant``.
+        refine: the back-edge policy ``(task, graded) -> SubTask | None``; by
+            default none, so the loop stops after one round.
+        rng: the source of the per-call level order (seed it for reproducible
+            prompts).
+
+    Every candidate is judged once per goal: one judge call per (source,
+    artifact) the first time it appears in a pool for that goal, its label
+    remembered for later rounds, so a kept hit is not re-judged (nor lost to a
+    later failing call). ``Judgement.graded`` is keyed by ``artifact_id``; an id
+    found in several sources carries its best label. A reply that does not parse
+    leaves the candidate ungraded (logged); when every judge call of a round
+    raises, the first error is re-raised rather than returning an empty,
+    "sufficient" round. This evaluator is for labelling a pool, not for cheap
+    filtering.
+    """
+    levels = tuple(levels)
+    for name, value in (
+        ("unsupported_cap", unsupported_cap),
+        ("relevant_level", relevant_level),
+    ):
+        if value not in levels:
+            raise ValueError(f"{name}={value!r} is not one of {list(levels)}")
+    keep = levels.index(relevant_level)
+    rng = rng if rng is not None else random.Random()
+    disclose_kwargs = dict(disclose_kwargs or {})
+    state: dict[str, GradedJudge] = {}
+    memo: dict[tuple[str, str | None, str], GradedLabel] = {}
+
+    def ask(**kwargs: Any) -> Mapping[str, Any] | str:
+        if judge is not None:
+            return judge(**kwargs)
+        if "judge" not in state:
+            state["judge"] = _default_graded_judge(prompt, **prompt_function_kwargs)
+        return state["judge"](**kwargs)
+
+    def evaluator(task: SubTask, results: Sequence[Result]) -> Judgement:
+        hits = best_per_artifact(results)
+        todo = [h for h in hits if (task.goal, h.source, h.artifact_id) not in memo]
+        contents = _contents(todo, disclose_level, disclose_kwargs) if todo else []
+        errors: list[Exception] = []
+        for hit, content in zip(todo, contents, strict=True):
+            content = str(content)[:max_content_chars]
+            shown = list(levels)
+            rng.shuffle(shown)
+            try:
+                reply = ask(
+                    goal=task.goal,
+                    artifact_id=hit.artifact_id,
+                    content=content,
+                    levels="\n".join(f"- {lv}" for lv in shown),
+                    rubric=rubric,
+                )
+            except Exception as error:  # the judge itself failed: auth, network, limits
+                _log.warning("judge failed on %s: %r", hit.artifact_id, error)
+                errors.append(error)
+                continue
+            try:
+                proposed = _parse_label(reply, levels)
+            except (ValueError, TypeError, AttributeError) as error:
+                _log.warning("unusable judge reply on %s: %s", hit.artifact_id, error)
+                continue  # left ungraded: never guess a level
+            memo[(task.goal, hit.source, hit.artifact_id)] = apply_gates(
+                proposed,
+                content,
+                levels=levels,
+                unsupported_cap=unsupported_cap,
+                require_evidence_quote=require_evidence_quote,
+                calibration_offset=calibration_offset,
+                min_quote_chars=min_quote_chars,
+            )
+        if todo and len(errors) == len(todo):
+            raise errors[0]
+        labels = {
+            (h.source, h.artifact_id): memo[key]
+            for h in hits
+            if (key := (task.goal, h.source, h.artifact_id)) in memo
+        }
+        graded: dict[str, GradedLabel] = {}
+        for (_, aid), label in labels.items():
+            if aid not in graded or levels.index(label.level) < levels.index(
+                graded[aid].level
+            ):
+                graded[aid] = label
+        relevant = [
+            h
+            for h in hits
+            if (label := labels.get((h.source, h.artifact_id))) is not None
+            and levels.index(label.level) <= keep
+        ]
+        refinement = refine(task, graded) if refine is not None else None
+        return Judgement(
+            relevant=relevant,
+            sufficient=refinement is None,
+            refinement=refinement,
+            graded=graded,
+        )
+
+    return evaluator
+
+
+def _default_graded_judge(prompt: str, **prompt_function_kwargs: Any) -> GradedJudge:
+    """Build the default graded judge on :mod:`oa` (lazy import)."""
+    import oa
+
+    return oa.prompt_function(prompt, name="grade_candidate", **prompt_function_kwargs)

raglab-0.2.8/tests/test_judging.py

@@ -0,0 +1,220 @@
+"""Tests for the graded rubric evaluator (raglab#8).
+
+Hermetic: the judge is a scripted test double keyed by artifact id; no model,
+no network. Content comes from each hit's snippet (nothing to load from disk).
+"""
+
+import json
+import random
+
+from ir import SearchHit
+
+from raglab import (
+    DEFAULT_LEVELS,
+    Budget,
+    GradedLabel,
+    Judgement,
+    SubTask,
+    make_graded_rubric_evaluator,
+    make_search_agent,
+)
+
+TASK = SubTask(goal="graph library", sources=("pkgs",))
+
+
+def _hit(aid, text, score=1.0):
+    return SearchHit(aid, "k", score, text, {})
+
+
+def _judge(replies):
+    """A judge returning ``replies[artifact_id]`` and recording what it was shown."""
+    calls = []
+
+    def judge(**kw):
+        calls.append(kw)
+        reply = replies[kw["artifact_id"]]
+        if isinstance(reply, Exception):
+            raise reply
+        return reply
+
+    judge.calls = calls
+    return judge
+
+
+def _reply(level, *, about=True, supported=True, useful=True, quote=""):
+    return {
+        "level": level,
+        "is_about": about,
+        "is_supported": supported,
+        "is_useful": useful,
+        "evidence_quote": quote,
+    }
+
+
+POOL = [
+    _hit("graphy", "A graph library: nodes, edges, traversals."),
+    _hit("au", "An async task framework; jobs form a DAG."),
+    _hit("plotter", "Draws charts with networkx graph layouts."),
+]
+
+
+def test_every_candidate_is_graded_and_the_supported_gate_caps_on_word_distractors():
+    judge = _judge(
+        {
+            "graphy": _reply("core", quote="A graph library"),
+            # on-word but not supported: the judge over-grades, the gate caps it
+            "au": _reply("strong", supported=False, quote="jobs form a DAG"),
+            "plotter": _reply("uses-tools", quote="networkx graph layouts"),
+        }
+    )
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+    assert isinstance(judged, Judgement)
+    assert {aid: g.level for aid, g in judged.graded.items()} == {
+        "graphy": "core",
+        "au": "tangential",
+        "plotter": "uses-tools",
+    }
+    assert [h.artifact_id for h in judged.relevant] == ["graphy", "plotter"]
+    assert judged.sufficient and judged.refinement is None
+    assert len(judge.calls) == 3
+
+
+def test_a_quote_not_found_verbatim_caps_the_level_unless_quotes_are_optional():
+    judge = _judge({"graphy": _reply("core", quote="the best graph toolkit")})
+    strict = make_graded_rubric_evaluator(judge=judge)(TASK, POOL[:1])
+    assert strict.graded["graphy"].level == "tangential"
+    assert strict.graded["graphy"].evidence_quote == ""
+    lax = make_graded_rubric_evaluator(judge=judge, require_evidence_quote=False)(
+        TASK, POOL[:1]
+    )
+    assert lax.graded["graphy"].level == "core"
+
+
+def test_the_calibration_offset_demotes_an_over_grading_judge():
+    judge = _judge({"graphy": _reply("core", quote="graph library")})
+    ev = make_graded_rubric_evaluator(judge=judge, calibration_offset=1.0)
+    assert ev(TASK, POOL[:1]).graded["graphy"].level == "strong"
+
+
+def test_a_failing_or_unparseable_judge_leaves_the_candidate_ungraded():
+    judge = _judge(
+        {
+            "graphy": json.dumps(_reply("core", quote="graph library")),  # raw text ok
+            "au": RuntimeError("rate limited"),
+            "plotter": '{"level": "excellent"}',  # not a level
+        }
+    )
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+    assert set(judged.graded) == {"graphy"}
+    assert [h.artifact_id for h in judged.relevant] == ["graphy"]
+
+
+def test_level_options_are_shown_in_a_fresh_order_each_call():
+    judge = _judge({h.artifact_id: _reply("none") for h in POOL})
+    ev = make_graded_rubric_evaluator(judge=judge, rng=random.Random(0))
+    for _ in range(4):
+        ev(TASK, POOL)
+    orders = {c["levels"] for c in judge.calls}
+    assert len(orders) > 1
+    assert all(sorted(o.split("\n")) == sorted(f"- {lv}" for lv in DEFAULT_LEVELS) for o in orders)
+
+
+def test_the_refine_policy_drives_the_back_edge_through_the_agent():
+    corpus = {
+        "graph library": [_hit("au", "An async task framework; jobs form a DAG.")],
+        "graph library nodes edges": [_hit("graphy", "A graph library: nodes, edges.")],
+    }
+
+    def retriever(query, **kw):
+        return list(corpus.get(query, []))
+
+    judge = _judge(
+        {
+            "au": _reply("core", supported=False, quote="jobs form a DAG"),
+            "graphy": _reply("core", quote="A graph library"),
+        }
+    )
+
+    def refine(task, graded):
+        if any(g.level in ("core", "strong") for g in graded.values()):
+            return None
+        return SubTask(goal=task.goal + " nodes edges", sources=task.sources)
+
+    agent = make_search_agent(
+        {"pkgs": retriever},
+        evaluator=make_graded_rubric_evaluator(judge=judge, refine=refine),
+        budget=Budget(max_rounds=3),
+    )
+    assert [h.artifact_id for h in agent("graph library")] == ["graphy"]
+
+
+def test_binary_judgements_are_unchanged():
+    assert Judgement(relevant=[], sufficient=True).graded == {}
+    label = GradedLabel("core", True, True, True)
+    assert label.evidence_quote == "" and label.facet is None
+
+
+def test_unknown_level_names_are_refused():
+    import pytest
+
+    with pytest.raises(ValueError, match="relevant_level"):
+        make_graded_rubric_evaluator(judge=_judge({}), relevant_level="great")
+
+
+def test_the_same_id_in_two_sources_is_judged_per_source():
+    a = SearchHit("x", "k", 1.0, "A graph library for nodes.", {}, source="A")
+    b = SearchHit("x", "k", 1.0, "An audio decoder.", {}, source="B")
+
+    def judge(**kw):
+        if "graph" in kw["content"]:
+            return _reply("core", quote="A graph library")
+        return _reply("none")
+
+    judged = make_graded_rubric_evaluator(judge=judge)(TASK, [a, b])
+    assert [h.source for h in judged.relevant] == ["A"]
+    assert judged.graded["x"].level == "core"  # the best label across sources
+
+
+def test_custom_levels_match_whatever_their_case():
+    judge = _judge({"graphy": _reply("core", quote="A graph library")})
+    ev = make_graded_rubric_evaluator(
+        judge=judge, levels=("Core", "Tangential", "None"),
+        unsupported_cap="Tangential", relevant_level="Core",
+    )
+    assert ev(TASK, POOL[:1]).graded["graphy"].level == "Core"
+
+
+def test_a_judge_that_fails_on_every_candidate_raises():
+    import pytest
+
+    judge = _judge({h.artifact_id: RuntimeError("no key") for h in POOL})
+    with pytest.raises(RuntimeError, match="no key"):
+        make_graded_rubric_evaluator(judge=judge)(TASK, POOL)
+
+
+def test_a_candidate_is_judged_once_per_goal_and_its_label_kept():
+    judge = _judge({"graphy": _reply("core", quote="A graph library")})
+    ev = make_graded_rubric_evaluator(judge=judge)
+    ev(TASK, POOL[:1])
+    judge_down = ev(TASK, POOL[:1])  # no new call: the label is remembered
+    assert len(judge.calls) == 1
+    assert [h.artifact_id for h in judge_down.relevant] == ["graphy"]
+
+
+def test_a_half_level_offset_shifts_every_level_alike():
+    from raglab import apply_gates
+
+    content = "A graph library."
+    shifted = [
+        apply_gates(GradedLabel(lv, True, True, True, "graph library"), content,
+                    calibration_offset=0.5).level
+        for lv in DEFAULT_LEVELS[:-1]
+    ]
+    assert shifted == list(DEFAULT_LEVELS[1:])
+
+
+def test_a_one_word_quote_is_not_evidence():
+    from raglab import apply_gates
+
+    label = GradedLabel("core", True, True, True, "a")
+    assert apply_gates(label, "A graph library.").level == "tangential"