evalstone 0.1.0__py3-none-any.whl

evalstone/checks/deterministic.py

@@ -0,0 +1,279 @@
+"""Built-in deterministic checks.
+
+Pure code, no model, free, repeatable. Each factory returns a ``FunctionCheck``
+you can hand to a suite or register under a key of your choosing. Where a
+check compares against a reference, the reference comes from
+``sample.expected`` — the whole value, or a dotted ``path`` into it — unless a
+literal ``expected``/``needles`` is given.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from collections.abc import Callable, Iterable
+from typing import TYPE_CHECKING, Any
+
+import jsonschema
+from jsonschema.validators import validator_for
+
+from evalstone.checks._base import CheckFn, FunctionCheck
+from evalstone.score import CheckKind, Score
+
+if TYPE_CHECKING:
+    from evalstone.dataset import Sample
+    from evalstone.output import Output
+
+_MISSING = object()
+
+
+def _dig(value: Any, path: str | None) -> Any:
+    """Follow a dotted path into dicts/lists; ``_MISSING`` when absent."""
+    if path is None:
+        return value
+    cur = value
+    for part in path.split("."):
+        if isinstance(cur, dict) and part in cur:
+            cur = cur[part]
+        elif isinstance(cur, list) and part.isdecimal() and int(part) < len(cur):
+            cur = cur[int(part)]
+        else:
+            return _MISSING
+    return cur
+
+
+def _reference(sample: Sample, path: str | None, literal: Any) -> Any:
+    if literal is not _MISSING:
+        return literal
+    ref = _dig(sample.expected, path)
+    if ref is _MISSING or ref is None:
+        where = f"expected.{path}" if path else "expected"
+        raise ValueError(f"sample {sample.id} has no {where}")
+    return ref
+
+
+def _as_json(output: Output) -> Any:
+    """The output as a JSON value: parse ``str`` values, pass everything else through."""
+    if isinstance(output.value, str):
+        return json.loads(output.value)
+    return output.value
+
+
+def _as_number(value: Any) -> float:
+    if isinstance(value, bool):
+        raise TypeError("boolean is not a number")
+    if isinstance(value, int | float):
+        return float(value)
+    if isinstance(value, str):
+        return float(value.strip())
+    raise TypeError(f"not a number: {value!r}")
+
+
+# ── factories ───────────────────────────────────────────────────────────────
+
+
+def exact(
+    *,
+    key: str = "exact",
+    expected: Any = _MISSING,
+    path: str | None = None,
+    case_sensitive: bool = True,
+    strip: bool = True,
+) -> FunctionCheck:
+    """The output equals the reference. Strings are compared as text (optionally
+    case-insensitive and stripped); anything else by JSON equality."""
+
+    def fn(sample: Sample, output: Output) -> Score:
+        ref = _reference(sample, path, expected)
+        if isinstance(ref, str):
+            got = output.text
+            if strip:
+                got, ref = got.strip(), ref.strip()
+            if not case_sensitive:
+                got, ref = got.lower(), ref.lower()
+            ok = got == ref
+        else:
+            try:
+                ok = _as_json(output) == ref
+            except json.JSONDecodeError:
+                return Score.fail(f"output is not valid JSON; expected {ref!r}")
+        return Score.from_bool(ok, "" if ok else f"expected {ref!r}, got {output.text!r}")
+
+    return FunctionCheck(key, fn, kind=CheckKind.DETERMINISTIC)
+
+
+def contains(
+    *,
+    key: str = "contains",
+    needles: Iterable[str] | str = _MISSING,  # type: ignore[assignment]
+    path: str | None = "must_include",
+    require_all: bool = True,
+    case_sensitive: bool = False,
+) -> FunctionCheck:
+    """Every (or any) needle appears in the output text. Needles default to
+    ``sample.expected.must_include`` (a string or a list). An empty needle set
+    passes vacuously."""
+    literal: Any = _MISSING
+    if needles is not _MISSING:  # materialise once: an iterator must not be consumed per sample
+        literal = [needles] if isinstance(needles, str) else [str(n) for n in needles]
+
+    def fn(sample: Sample, output: Output) -> Score:
+        ref = _reference(sample, path, literal)
+        wanted = [ref] if isinstance(ref, str) else [str(n) for n in ref]
+        if not wanted:
+            return Score.ok("no needles")
+        hay = output.text if case_sensitive else output.text.lower()
+        hits = [n for n in wanted if (n if case_sensitive else n.lower()) in hay]
+        missing = [n for n in wanted if n not in hits]
+        ok = not missing if require_all else bool(hits)
+        value = len(hits) / len(wanted)
+        reason = "" if ok else f"missing {missing!r}"
+        return Score(value=value, passed=ok, reason=reason)
+
+    return FunctionCheck(key, fn, kind=CheckKind.DETERMINISTIC)
+
+
+def regex(
+    pattern: str,
+    *,
+    key: str = "regex",
+    flags: int = 0,
+    must_match: bool = True,
+) -> FunctionCheck:
+    """The output text matches (or, with ``must_match=False``, does not match) ``pattern``."""
+    compiled = re.compile(pattern, flags)
+
+    def fn(sample: Sample, output: Output) -> Score:
+        found = compiled.search(output.text) is not None
+        ok = found if must_match else not found
+        verb = "did not match" if must_match else "matched"
+        return Score.from_bool(ok, "" if ok else f"{verb} /{pattern}/")
+
+    return FunctionCheck(key, fn, kind=CheckKind.DETERMINISTIC)
+
+
+def json_schema(schema: dict[str, Any] | type, *, key: str = "json_schema") -> FunctionCheck:
+    """The output is valid JSON and conforms to ``schema`` — a JSON Schema dict (the draft
+    named by ``$schema``, 2020-12 when absent) or a pydantic model class. A malformed
+    schema is rejected here, not once per sample at run time."""
+    validate: Callable[[Any], None]
+    if isinstance(schema, dict):
+        cls = validator_for(schema, default=jsonschema.Draft202012Validator)
+        try:
+            cls.check_schema(schema)
+        except jsonschema.SchemaError as exc:
+            raise ValueError(f"json_schema {key!r}: invalid schema: {exc.message}") from None
+        validator = cls(schema)
+
+        def validate(value: Any) -> None:
+            errors = sorted(validator.iter_errors(value), key=lambda e: list(e.path))
+            if errors:
+                first = errors[0]
+                where = "/".join(str(p) for p in first.path) or "<root>"
+                raise ValueError(f"{where}: {first.message}")
+
+    else:
+        from pydantic import TypeAdapter, ValidationError
+
+        adapter = TypeAdapter(schema)
+
+        def validate(value: Any) -> None:
+            try:
+                adapter.validate_python(value)
+            except ValidationError as exc:
+                first = exc.errors()[0]
+                where = "/".join(str(p) for p in first["loc"]) or "<root>"
+                raise ValueError(f"{where}: {first['msg']}") from None
+
+    def fn(sample: Sample, output: Output) -> Score:
+        try:
+            value = _as_json(output)
+        except json.JSONDecodeError as exc:
+            return Score.fail(f"not valid JSON: {exc.msg}")
+        try:
+            validate(value)
+        except ValueError as exc:
+            return Score.fail(f"schema: {exc}")
+        return Score.ok()
+
+    return FunctionCheck(key, fn, kind=CheckKind.DETERMINISTIC)
+
+
+def numeric_tolerance(
+    *,
+    key: str = "numeric_tolerance",
+    expected: Any = _MISSING,
+    path: str | None = None,
+    output_path: str | None = None,
+    abs_tol: float = 0.0,
+    rel_tol: float = 0.0,
+) -> FunctionCheck:
+    """The output number is within ``abs_tol`` / ``rel_tol`` of the reference number.
+    ``output_path`` digs into a JSON output; otherwise the whole output is the number."""
+
+    def fn(sample: Sample, output: Output) -> Score:
+        ref = _as_number(_reference(sample, path, expected))
+        if output_path is None:
+            raw = output.value
+        else:
+            try:
+                raw = _dig(_as_json(output), output_path)
+            except json.JSONDecodeError as exc:
+                return Score.fail(f"not valid JSON: {exc.msg}")
+        if raw is _MISSING:
+            return Score.fail(f"output has no {output_path}")
+        try:
+            got = _as_number(raw)
+        except (TypeError, ValueError):
+            return Score.fail(f"output is not a number: {raw!r}")
+        ok = math.isclose(got, ref, rel_tol=rel_tol, abs_tol=abs_tol)
+        reason = "" if ok else f"expected {ref} ± ({abs_tol} abs, {rel_tol} rel), got {got}"
+        return Score.from_bool(ok, reason)
+
+    return FunctionCheck(key, fn, kind=CheckKind.DETERMINISTIC)
+
+
+def length(
+    *,
+    key: str = "length",
+    max_chars: int | None = None,
+    max_words: int | None = None,
+    min_chars: int = 0,
+) -> FunctionCheck:
+    """The output text is at least ``min_chars`` long (0 by default — pass ``min_chars=1``
+    to reject empty output) and under the given limits."""
+
+    def fn(sample: Sample, output: Output) -> Score:
+        text = output.text.strip()
+        chars, words = len(text), len(text.split())
+        if chars < min_chars:
+            return Score.fail(f"{chars} chars < min {min_chars}")
+        if max_chars is not None and chars > max_chars:
+            return Score.fail(f"{chars} chars > max {max_chars}")
+        if max_words is not None and words > max_words:
+            return Score.fail(f"{words} words > max {max_words}")
+        return Score.ok()
+
+    return FunctionCheck(key, fn, kind=CheckKind.DETERMINISTIC)
+
+
+def python(
+    fn: CheckFn,
+    key: str | None = None,
+    *,
+    kind: CheckKind | str = CheckKind.DETERMINISTIC,
+    version: int = 1,
+) -> FunctionCheck:
+    """Any ``fn(sample, output[, ctx]) -> Score | bool | float`` as a check, unregistered.
+
+    The key defaults to the function's name; lambdas, partials and callable
+    objects have no usable name, so they need an explicit ``key``.
+    """
+    name = getattr(fn, "__name__", None)
+    if key is None and (not name or name == "<lambda>"):
+        raise ValueError("python(): give an explicit key for a lambda, partial or callable object")
+    return FunctionCheck(key or name, fn, kind=CheckKind(kind), version=version)
+
+
+__all__ = ["contains", "exact", "json_schema", "length", "numeric_tolerance", "python", "regex"]

evalstone/checks/human.py

@@ -0,0 +1,124 @@
+"""Human labels — the check of last resort, and the ground truth judges are calibrated against.
+
+A label is a person's verdict on one (sample, output) pair, stored as a JSONL
+line and keyed by the sample id **and a hash of the output text**, so a label
+never silently applies to a changed answer. ``evalstone label <report>`` writes
+them; the ``human`` check reads them: a labelled output gets the label's
+score, an unlabelled one is an error (never a pass).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from evalstone._json import sha256
+from evalstone.checks._base import FunctionCheck
+from evalstone.dataset import Sample
+from evalstone.output import Output
+from evalstone.score import CheckKind, Score
+
+
+def output_hash(output: Output | str) -> str:
+    text = output if isinstance(output, str) else output.text
+    return sha256(text)[:16]
+
+
+class Label(BaseModel):
+    """One human verdict."""
+
+    sample_id: str
+    output_hash: str
+    value: float = Field(ge=0.0, le=1.0)
+    passed: bool
+    notes: str = ""
+    check: str | None = None  # which rubric/check the grade is for; None = overall
+    labeler: str = ""
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    raw: Any = None  # the original grade (e.g. 4 on a 1-5 scale)
+
+    @property
+    def key(self) -> tuple[str, str, str | None]:
+        return (self.sample_id, self.output_hash, self.check)
+
+
+class LabelStore:
+    """Labels in a JSONL file. Later lines win for the same (sample, output, check)."""
+
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self._labels: dict[tuple[str, str, str | None], Label] = {}
+        if self.path.exists():
+            with self.path.open(encoding="utf-8") as fh:
+                for lineno, line in enumerate(fh, start=1):
+                    if not line.strip():
+                        continue
+                    try:
+                        label = Label.model_validate_json(line)
+                    except ValueError as exc:
+                        raise ValueError(f"{self.path}:{lineno}: invalid label: {exc}") from exc
+                    self._labels[label.key] = label
+
+    def __len__(self) -> int:
+        return len(self._labels)
+
+    def __iter__(self) -> Iterable[Label]:
+        return iter(self._labels.values())
+
+    def get(self, sample_id: str, output: Output | str, check: str | None = None) -> Label | None:
+        return self._labels.get((sample_id, output_hash(output), check))
+
+    def add(self, label: Label) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open("a", encoding="utf-8") as fh:
+            fh.write(label.model_dump_json() + "\n")
+        self._labels[label.key] = label
+
+
+def human(
+    *, key: str = "human", labels: str | Path = "labels.jsonl", check: str | None = None
+) -> FunctionCheck:
+    """Score from a person's label for this (sample, output); unlabelled → error.
+
+    ``check`` selects labels recorded for a specific rubric (``evalstone label
+    --check``); ``None`` uses the overall grade.
+    """
+    store_path = Path(labels)
+    cached: dict[str, Any] = {}
+
+    def store() -> LabelStore:
+        # One parse per file version: a stat per call (so labels added while a
+        # long-lived server runs are seen), a re-read only when the file changed.
+        try:
+            st = store_path.stat()
+            stamp: tuple[int, int] | None = (st.st_mtime_ns, st.st_size)
+        except FileNotFoundError:
+            stamp = None
+        if cached.get("stamp", ...) != stamp:
+            cached["store"] = LabelStore(store_path)
+            cached["stamp"] = stamp
+        return cached["store"]
+
+    def fn(sample: Sample, output: Output) -> Score:
+        label = store().get(sample.id, output, check)
+        if label is None:
+            return Score.errored(
+                f"no human label for {sample.id} @ {output_hash(output)} "
+                f"(run `evalstone label <report> --out {store_path}`)"
+            )
+        reason = label.notes or ("labelled pass" if label.passed else "labelled fail")
+        return Score(
+            value=label.value,
+            passed=label.passed,
+            reason=reason,
+            raw={"labeler": label.labeler, "grade": label.raw, "at": label.created_at.isoformat()},
+        )
+
+    return FunctionCheck(key, fn, kind=CheckKind.HUMAN)
+
+
+__all__ = ["Label", "LabelStore", "human", "output_hash"]

evalstone/checks/judge.py

@@ -0,0 +1,219 @@
+"""LLM-as-judge: a rubric, a small scale, structured JSON back, cached by content.
+
+Follows the practice in Evals 101 chapter 6: specific criteria, a defined
+scale, the reference included only when asked for, temperature left at the
+provider default (the cache makes re-runs free anyway), and **an error is
+never a pass** (timeouts, refusals, unparseable verdicts → ``Score.errored``).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from evalstone._json import dumps, sha256
+from evalstone.checks._base import FunctionCheck
+from evalstone.dataset import Sample
+from evalstone.output import Output
+from evalstone.providers import ChatMessage, ProviderError
+from evalstone.score import CheckKind, Context, Score
+
+PROMPT_VERSION = 1
+
+SYSTEM_TEMPLATE = """You are a strict evaluator grading one answer produced by an AI system.
+
+RUBRIC:
+{rubric}
+
+Score the answer from 1 to {scale} according to the rubric ({scale} = fully meets it, 1 = fails it).
+Ignore formatting and length unless the rubric mentions them.
+Reply with ONLY a JSON object: {{"score": <integer 1-{scale}>, "reason": "<one sentence>"}}"""
+
+USER_TEMPLATE = """INPUT:
+{input}
+{reference}
+ANSWER:
+{output}"""
+
+
+def _json_text(value: Any) -> str:
+    return value if isinstance(value, str) else dumps(value, indent=2)
+
+
+def extract_json(text: str) -> dict[str, Any]:
+    """The first JSON object in a model reply, tolerating prose around it."""
+    match = re.search(r"\{.*\}", text, flags=re.DOTALL)
+    if not match:
+        raise ValueError(f"no JSON object in judge reply: {text[:120]!r}")
+    try:
+        data = json.loads(match.group(0))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"invalid JSON in judge reply: {exc.msg}") from None
+    if not isinstance(data, dict):
+        raise ValueError(f"judge reply is not a JSON object: {text[:120]!r}")
+    return data
+
+
+def parse_verdict(text: str, scale: int) -> tuple[int, str]:
+    """Pull ``{"score": n, "reason": ...}`` out of a model reply."""
+    data = extract_json(text)
+    try:
+        score = round(float(data["score"]))
+    except (KeyError, TypeError, ValueError):
+        raise ValueError(f"judge reply has no numeric score: {data!r}"[:200]) from None
+    if not 1 <= score <= scale:
+        raise ValueError(f"judge score {score} outside 1..{scale}")
+    return score, str(data.get("reason", "")).strip()
+
+
+class JudgeCheck(FunctionCheck):
+    """A ``FunctionCheck`` whose function calls a model; exposes the judge version hash."""
+
+    def __init__(
+        self,
+        key: str,
+        fn: Any,
+        *,
+        version_hash: str,
+        model: str | None,
+        rubric: str,
+        scale: int,
+        pass_at: float,
+    ):
+        super().__init__(key, fn, kind=CheckKind.JUDGE)
+        # rubric + scale + prompt version; the resolved model is folded in at run time
+        self.version_hash = version_hash
+        self.model = model
+        self.rubric = rubric
+        self.scale = scale
+        self.pass_at = pass_at
+
+
+def judge(
+    key: str,
+    rubric: str,
+    *,
+    model: str | None = None,
+    scale: int = 5,
+    pass_level: int | None = None,
+    threshold: float | None = None,
+    include_reference: bool = False,
+    max_tokens: int = 256,
+) -> JudgeCheck:
+    """Grade an output against ``rubric`` with a model.
+
+    The model answers with a level ``n`` in ``1..scale``; ``value = (n-1)/(scale-1)``.
+    The verdict passes at or above ``pass_level`` (a rubric level; default: the
+    second-highest level — 4 on a 1-5 scale — and the top level on a 1-2 yes/no
+    scale) or, equivalently, at or above ``threshold`` (a 0..1 fraction of the
+    normalised value). Give one or the other.
+    ``include_reference`` adds ``sample.expected`` to the prompt — only do this when
+    similarity to the reference is the criterion.
+    """
+    if scale < 2:
+        raise ValueError("scale must be at least 2")
+    if pass_level is not None and threshold is not None:
+        raise ValueError("give pass_level (a rubric level) or threshold (0..1), not both")
+    if threshold is not None:
+        if not 0.0 <= threshold <= 1.0:
+            raise ValueError(
+                f"threshold must be a 0..1 fraction of the normalised score, got {threshold}; "
+                "to pass at a rubric level use pass_level="
+            )
+        pass_at = threshold
+    else:
+        level = pass_level if pass_level is not None else (scale - 1 if scale >= 3 else scale)
+        if not 1 <= level <= scale:
+            raise ValueError(f"pass_level must be within 1..{scale}, got {level}")
+        pass_at = (level - 1) / (scale - 1)
+    identity = {"v": PROMPT_VERSION, "rubric": rubric, "scale": scale, "ref": include_reference}
+    version_hash = sha256(identity)[:16]
+
+    def fn(sample: Sample, output: Output, ctx: Context) -> Score:
+        providers = ctx.providers
+        if providers is None:
+            return Score.errored("judge: no providers configured (ctx.providers is None)")
+        try:
+            resolved = "/".join(providers.split(model))
+        except ProviderError as exc:
+            return Score.errored(str(exc))
+        judge_version = sha256({**identity, "model": resolved})[:16]
+        reference = ""
+        if include_reference and sample.expected is not None:
+            reference = (
+                f"\nREFERENCE (what a good answer must satisfy):\n{_json_text(sample.expected)}\n"
+            )
+        user = USER_TEMPLATE.format(
+            input=_json_text(sample.input), reference=reference, output=output.text
+        )
+        system = SYSTEM_TEMPLATE.format(rubric=rubric.strip(), scale=scale)
+
+        def to_score(n: int, reason: str, *, raw: dict[str, Any], cost: float, ms: float) -> Score:
+            # value/passed are derived here, so a cached raw verdict serves every threshold.
+            value = (n - 1) / (scale - 1)
+            return Score(
+                value=value,
+                passed=value >= pass_at,
+                reason=f"{n}/{scale}: {reason}" if reason else f"{n}/{scale}",
+                cost_usd=cost,
+                latency_ms=ms,
+                raw={**raw, "score": n, "scale": scale, "judge_version": judge_version},
+            )
+
+        # The cache holds the model's raw verdict (level + reason), keyed by the
+        # resolved judge version and the exact prompt — never a derived pass/fail.
+        cache_key = sha256({"j": judge_version, "u": user})
+        cached = ctx.cache.get(cache_key) if ctx.cache is not None else None
+        if cached is not None and cached.raw and "score" in cached.raw:
+            raw = {k: v for k, v in cached.raw.items() if k not in {"score", "scale"}}
+            return to_score(
+                int(cached.raw["score"]),
+                _verdict_reason(cached),
+                raw={**raw, "cached": True},
+                cost=0.0,
+                ms=0.0,
+            )
+        try:
+            resp = providers.complete(
+                model,
+                {
+                    "system": system,
+                    "messages": [ChatMessage(role="user", content=user)],
+                    "max_tokens": max_tokens,
+                },
+            )
+        except ProviderError as exc:
+            return Score.errored(str(exc))
+        try:
+            n, reason = parse_verdict(resp.text, scale)
+        except ValueError as exc:
+            return Score.errored(
+                f"judge: {exc}", cost_usd=resp.cost_usd or 0.0, latency_ms=resp.latency_ms
+            )
+        score = to_score(
+            n,
+            reason,
+            raw={
+                "model": resp.model,
+                "tokens": [resp.input_tokens, resp.output_tokens],
+                "unpriced": resp.cost_usd is None,
+                "verdict_reason": reason,
+            },
+            cost=resp.cost_usd or 0.0,
+            ms=resp.latency_ms,
+        )
+        if ctx.cache is not None:
+            ctx.cache.set(cache_key, score)
+        return score
+
+    return JudgeCheck(
+        key, fn, version_hash=version_hash, model=model, rubric=rubric, scale=scale, pass_at=pass_at
+    )
+
+
+def _verdict_reason(score: Score) -> str:
+    return str((score.raw or {}).get("verdict_reason", ""))
+
+
+__all__ = ["PROMPT_VERSION", "JudgeCheck", "extract_json", "judge", "parse_verdict"]

evalstone/checks/rag.py

@@ -0,0 +1,70 @@
+"""RAG checks — the three questions a retrieval-augmented answer can fail.
+
+Each is a ``judge`` with a narrow rubric. The sample input is expected to carry
+the question and the retrieved context (``{"question": …, "context": …}`` by
+default; the field names are configurable). Measuring them separately tells
+you whether to fix the retriever or the prompt (Evals 101, chapter 4).
+"""
+
+from __future__ import annotations
+
+from evalstone.checks.judge import JudgeCheck, judge
+
+_FAITHFULNESS = """You are checking FAITHFULNESS of an answer to its source material.
+The INPUT contains the field `{context}` — the only material the answer may rely on.
+Grade whether every factual claim in the ANSWER is supported by `{context}`.
+5 = every claim supported · 4 = one minor unsupported detail · 3 = one substantive unsupported
+claim · 2 = several unsupported claims · 1 = the answer contradicts the context or is fabricated.
+Ignore style, tone and whether the answer is complete; judge only support by the context."""
+
+_CONTEXT_RELEVANCE = """You are checking CONTEXT RELEVANCE for a retrieval system.
+The INPUT contains `{question}` (what was asked) and `{context}` (what retrieval returned).
+Grade whether `{context}` contains the information needed to answer `{question}`.
+5 = fully answers it · 4 = answers it with minor gaps · 3 = partially relevant · 2 = tangential ·
+1 = unrelated. Ignore the ANSWER entirely; judge only the retrieved context."""
+
+_ANSWER_RELEVANCE = """You are checking ANSWER RELEVANCE.
+The INPUT contains `{question}`. Grade whether the ANSWER actually addresses that question.
+5 = directly and completely addresses it · 4 = addresses it with a minor omission ·
+3 = partially addresses it · 2 = mostly off-topic · 1 = does not address it.
+Ignore factual accuracy (another check covers that); judge only whether the question was
+answered."""
+
+
+def faithfulness(
+    key: str = "faithfulness",
+    *,
+    context: str = "context",
+    model: str | None = None,
+    threshold: float | None = None,
+) -> JudgeCheck:
+    """Every claim in the answer is supported by ``sample.input[context]``."""
+    return judge(key, _FAITHFULNESS.format(context=context), model=model, threshold=threshold)
+
+
+def context_relevance(
+    key: str = "context_relevance",
+    *,
+    question: str = "question",
+    context: str = "context",
+    model: str | None = None,
+    threshold: float | None = None,
+) -> JudgeCheck:
+    """The retrieved ``context`` can answer the ``question``."""
+    rubric = _CONTEXT_RELEVANCE.format(question=question, context=context)
+    return judge(key, rubric, model=model, threshold=threshold)
+
+
+def answer_relevance(
+    key: str = "answer_relevance",
+    *,
+    question: str = "question",
+    model: str | None = None,
+    threshold: float | None = None,
+) -> JudgeCheck:
+    """The answer addresses the ``question``."""
+    rubric = _ANSWER_RELEVANCE.format(question=question)
+    return judge(key, rubric, model=model, threshold=threshold)
+
+
+__all__ = ["answer_relevance", "context_relevance", "faithfulness"]