contextattr 0.1.0__py3-none-any.whl

contextattr/prompt/units.py

@@ -0,0 +1,271 @@
+"""The input format of docs/interface.md §2: Prompt, Message, Unit, Group and Response, with their checks."""
+
+from __future__ import annotations
+
+import dataclasses
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from functools import cached_property
+from types import MappingProxyType
+from typing import Union
+
+LABELS = ("positive", "negative")
+
+
+def _need_str(value, what: str, *, empty: bool = True) -> None:
+    if not isinstance(value, str):
+        raise TypeError(f"{what} must be a string, not {type(value).__name__}")
+    if not empty and not value:
+        raise ValueError(f"{what} is empty")
+
+
+def _need_opt_str(value, what: str) -> None:
+    if value is not None:
+        _need_str(value, what)
+
+
+def _need_sequence(value, what: str) -> None:
+    if isinstance(value, (str, bytes, Mapping)) or not isinstance(value, Iterable):
+        raise TypeError(f"{what} must be a list, not {type(value).__name__}")
+
+
+@dataclass(frozen=True)
+class Unit:
+    """A removable piece of a prompt: `text`, then `sep`, which goes with it when it is deleted and stays when it is
+    replaced. `replace` is this unit's replacement text, overriding the run's removal setting."""
+
+    id: str
+    text: str
+    sep: str = ""
+    tag: str | None = None
+    replace: str | None = None
+    group: str | None = None  # the id of the Group holding it, set by the Group
+
+    def __post_init__(self) -> None:
+        _need_str(self.id, "unit id", empty=False)
+        _need_str(self.text, f"unit {self.id!r}: text", empty=False)
+        _need_str(self.sep, f"unit {self.id!r}: sep")
+        _need_opt_str(self.tag, f"unit {self.id!r}: tag")
+        _need_opt_str(self.replace, f"unit {self.id!r}: replace")
+        _need_opt_str(self.group, f"unit {self.id!r}: group")
+
+
+@dataclass(frozen=True)
+class Group:
+    """Units that belong together, such as a document cut into sentences. `lead` comes before them and `sep` after;
+    both stay while any of the units shows (kept, or replaced) and go with the last one deleted. `tag` and `replace`
+    are given to each unit that has none of its own."""
+
+    id: str
+    units: tuple[Unit, ...]
+    lead: str = ""
+    sep: str = ""
+    tag: str | None = None
+    replace: str | None = None
+
+    def __post_init__(self) -> None:
+        _need_str(self.id, "group id", empty=False)
+        _need_str(self.lead, f"group {self.id!r}: lead")
+        _need_str(self.sep, f"group {self.id!r}: sep")
+        _need_opt_str(self.tag, f"group {self.id!r}: tag")
+        _need_opt_str(self.replace, f"group {self.id!r}: replace")
+        _need_sequence(self.units, f"group {self.id!r}: units")
+        units = []
+        for u in self.units:
+            if not isinstance(u, Unit):
+                raise TypeError(f"group {self.id!r}: units must be Unit objects, not {type(u).__name__}")
+            if u.group not in (None, self.id):
+                raise ValueError(f"group {self.id!r}: unit {u.id!r} already belongs to group {u.group!r}")
+            units.append(dataclasses.replace(
+                u, group=self.id,
+                tag=self.tag if u.tag is None else u.tag,
+                replace=self.replace if u.replace is None else u.replace))
+        if not units:
+            raise ValueError(f"group {self.id!r} has no units")
+        object.__setattr__(self, "units", tuple(units))
+
+
+Piece = Union[str, Unit, Group]
+
+
+def _pieces(content, what: str) -> tuple[Piece, ...]:
+    """A message's content or a text prompt: a string (all fixed), or a list of strings, units and groups."""
+    if isinstance(content, str):
+        return (content,)
+    _need_sequence(content, what)
+    out = []
+    for p in content:
+        if isinstance(p, Unit) and p.group is not None:
+            raise ValueError(f"{what}: unit {p.id!r} names group {p.group!r} but is not inside it")
+        if not isinstance(p, (str, Unit, Group)):
+            raise TypeError(f"{what}: a piece is a string, a Unit or a Group, not {type(p).__name__}")
+        out.append(p)
+    return tuple(out)
+
+
+@dataclass(frozen=True)
+class Message:
+    """One chat message: its role and its content, a sequence of fixed strings, units and groups."""
+
+    role: str
+    content: tuple[Piece, ...]
+
+    def __post_init__(self) -> None:
+        _need_str(self.role, "message role", empty=False)
+        object.__setattr__(self, "content", _pieces(self.content, f"{self.role} message"))
+
+
+@dataclass(frozen=True)
+class Response:
+    """A reply of the model: `label` is positive, negative or None; `span` [start, end) marks the characters that
+    matter."""
+
+    id: str
+    text: str
+    label: str | None = None
+    span: tuple[int, int] | None = None
+
+    def __post_init__(self) -> None:
+        _need_str(self.id, "response id", empty=False)
+        _need_str(self.text, f"response {self.id!r}: text", empty=False)
+        if self.label is not None and self.label not in LABELS:
+            raise ValueError(f"response {self.id!r}: label must be positive, negative or None, not {self.label!r}")
+        if self.span is not None:
+            _need_sequence(self.span, f"response {self.id!r}: span")
+            span = tuple(self.span)
+            if len(span) != 2 or not all(isinstance(x, int) and not isinstance(x, bool) for x in span):
+                raise TypeError(f"response {self.id!r}: span must be two integers [start, end)")
+            start, end = span
+            if not 0 <= start < end <= len(self.text):
+                raise ValueError(f"response {self.id!r}: span {list(span)} must satisfy 0 <= start < end <= "
+                                 f"{len(self.text)}, the length of its text")
+            object.__setattr__(self, "span", span)
+
+
+@dataclass(frozen=True)
+class Prompt:
+    """A prompt with its removable units: chat `messages` or a plain `text`, never both. Messages may be given as
+    dicts {"role", "content"} and responses as dicts; everything not marked as a unit is fixed."""
+
+    id: str
+    messages: tuple[Message, ...] | None = None
+    text: tuple[Piece, ...] | None = None
+    reference: str | None = None
+    responses: tuple[Response, ...] = ()
+
+    def __post_init__(self) -> None:
+        _need_str(self.id, "prompt id", empty=False)
+        where = f"prompt {self.id!r}"
+        if (self.messages is None) == (self.text is None):
+            raise ValueError(f"{where}: give either messages or text")
+        if self.messages is not None:
+            _need_sequence(self.messages, f"{where}: messages")
+            messages = []
+            for m in self.messages:
+                if isinstance(m, Mapping):
+                    if set(m) != {"role", "content"}:
+                        raise ValueError(f"{where}: a message has the fields role and content, not {sorted(m)}")
+                    m = Message(m["role"], m["content"])
+                elif not isinstance(m, Message):
+                    raise TypeError(f"{where}: a message is a Message or a dict, not {type(m).__name__}")
+                messages.append(m)
+            if not messages:
+                raise ValueError(f"{where}: messages is empty")
+            object.__setattr__(self, "messages", tuple(messages))
+        else:
+            object.__setattr__(self, "text", _pieces(self.text, f"{where}: text"))
+        _need_opt_str(self.reference, f"{where}: reference")
+        _need_sequence(self.responses, f"{where}: responses")
+        responses = tuple(r if isinstance(r, Response) else Response(**r) if isinstance(r, Mapping) else
+                          _not_a_response(r, where) for r in self.responses)
+        ids = [r.id for r in responses]
+        if len(set(ids)) != len(ids):
+            raise ValueError(f"{where}: response ids must be unique: {ids}")
+        object.__setattr__(self, "responses", responses)
+
+        seen: set[str] = set()
+        for p in self._top_pieces():
+            for name in ([p.id, *(u.id for u in p.units)] if isinstance(p, Group) else
+                         [p.id] if isinstance(p, Unit) else []):
+                if name in seen:
+                    raise ValueError(f"{where}: the id {name!r} is used twice (unit and group ids must be unique)")
+                seen.add(name)
+        if not self.units:
+            raise ValueError(f"{where} has no units: mark at least one piece as a unit")
+
+    def _top_pieces(self):
+        if self.messages is not None:
+            for m in self.messages:
+                yield from m.content
+        else:
+            yield from self.text
+
+    @cached_property
+    def units(self) -> tuple[Unit, ...]:
+        """Every unit in order: message by message, piece by piece, a group's units in place."""
+        out: list[Unit] = []
+        for p in self._top_pieces():
+            if isinstance(p, Unit):
+                out.append(p)
+            elif isinstance(p, Group):
+                out.extend(p.units)
+        return tuple(out)
+
+    @property
+    def n(self) -> int:
+        """The number of units: the length of a mask."""
+        return len(self.units)
+
+    @property
+    def is_chat(self) -> bool:
+        return self.messages is not None
+
+    @cached_property
+    def unit_ids(self) -> tuple[str, ...]:
+        return tuple(u.id for u in self.units)
+
+    @cached_property
+    def _index(self) -> Mapping[str, int]:
+        return MappingProxyType({u.id: i for i, u in enumerate(self.units)})
+
+    @cached_property
+    def groups(self) -> Mapping[str, tuple[int, ...]]:
+        """Each group's id → the positions of its units in `units`."""
+        out: dict[str, list[int]] = {}
+        for i, u in enumerate(self.units):
+            if u.group is not None:
+                out.setdefault(u.group, []).append(i)
+        return MappingProxyType({g: tuple(ix) for g, ix in out.items()})
+
+    def index(self, unit_id: str) -> int:
+        """The position of a unit in `units`, and so in a mask."""
+        try:
+            return self._index[unit_id]
+        except KeyError:
+            raise KeyError(f"prompt {self.id!r} has no unit {unit_id!r}") from None
+
+    def mask(self, *, keep: Iterable[str] | str | None = None,
+             drop: Iterable[str] | str | None = None) -> tuple[bool, ...]:
+        """A mask from ids of units or groups: keep only `keep`, or keep every unit except `drop`."""
+        if (keep is None) == (drop is None):
+            raise ValueError("give either keep or drop")
+        names = keep if keep is not None else drop
+        chosen: set[int] = set()
+        for name in [names] if isinstance(names, str) else names:
+            if name in self.groups:
+                chosen.update(self.groups[name])
+            elif name in self._index:
+                chosen.add(self._index[name])
+            else:
+                raise KeyError(f"prompt {self.id!r} has no unit or group {name!r}")
+        return tuple((i in chosen) == (keep is not None) for i in range(self.n))
+
+    def render(self, mask=None, replace: str | None = None):
+        """The prompt as the model sees it with only the units `mask` keeps (render.py)."""
+        from .render import render
+
+        return render(self, mask, replace)
+
+
+def _not_a_response(value, where: str):
+    raise TypeError(f"{where}: a response is a Response or a dict, not {type(value).__name__}")

contextattr/report.py

@@ -0,0 +1,120 @@
+"""Reports (docs/interface.md §8): one self-contained HTML page (no script, no file beside it) showing each prompt with
+its units shaded by score where they stand, then the scores, the top interactions, the removal check and what the
+explanation cost."""
+
+from __future__ import annotations
+
+import html
+import math
+
+from .prompt import Group, Unit
+
+CSS = """
+:root { --bg: #ffffff; --fg: #1f2328; --muted: #57606a; --line: #d0d7de; --card: #f6f8fa;
+        --up: 214, 69, 65; --down: 47, 109, 214; }
+@media (prefers-color-scheme: dark) {
+  :root { --bg: #0d1117; --fg: #e6edf3; --muted: #9198a1; --line: #30363d; --card: #161b22;
+          --up: 248, 81, 73; --down: 88, 166, 255; } }
+* { box-sizing: border-box; }
+body { margin: 0; padding: 24px 16px; background: var(--bg); color: var(--fg);
+       font: 15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; }
+main { max-width: 960px; margin: 0 auto; }
+h1 { font-size: 22px; margin: 0 0 4px; } h2 { font-size: 18px; margin: 32px 0 6px; }
+.meta, .legend { color: var(--muted); font-size: 13px; }
+.prompt { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 12px 14px;
+          white-space: pre-wrap; overflow-wrap: anywhere; margin: 10px 0; }
+.role { display: block; font-size: 12px; text-transform: uppercase; letter-spacing: .04em; color: var(--muted);
+        margin-top: 6px; }
+.u { border-radius: 3px; padding: 0 1px; }
+table { border-collapse: collapse; width: 100%; font-size: 14px; margin: 8px 0; }
+th, td { text-align: left; padding: 4px 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
+td.num { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
+.wrap { overflow-x: auto; }
+"""
+
+
+def page(explanations, title: str = "ContextAttr report") -> str:
+    """The HTML page of a list of explanations."""
+    parts = [f"<!doctype html><html lang='en'><head><meta charset='utf-8'><meta name='viewport' "
+             f"content='width=device-width, initial-scale=1'><title>{_e(title)}</title><style>{CSS}</style></head>"
+             f"<body><main><h1>{_e(title)}</h1><p class='legend'>A unit's shade shows its score: "
+             f"<span class='u' style='background: rgba(var(--up), .45)'>pushes the value up</span>, "
+             f"<span class='u' style='background: rgba(var(--down), .45)'>pushes it down</span>; hover for the "
+             f"number.</p>"]
+    for e in explanations:
+        parts.append(_section(e))
+    parts.append("</main></body></html>")
+    return "".join(parts)
+
+
+def write(explanations, path) -> None:
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(page(explanations))
+
+
+def _section(e) -> str:
+    scores = dict(e.scores)
+    top = max([abs(v) for v in scores.values()] + [1e-300])
+    noise = e.noise["std"] if e.noise else None
+    out = [f"<h2>{_e(e.prompt_id or 'attribute')}</h2><p class='meta'>{_e(e.method)} · target {_e(e.target or '')} · "
+           f"{e.masks} masks · R² {_fmt(e.r2)} on {e.test_masks} held-out masks · noise {_fmt(noise)} · "
+           f"{e.requests.get('requests', 0)} requests</p>"]
+    if e.prompt is not None:
+        out.append(f"<div class='prompt'>{_prompt_html(e.prompt, scores, top)}</div>")
+    rows = sorted(scores.items(), key=lambda t: -t[1])
+    texts = dict(zip(e.unit_ids, e.unit_texts or [""] * len(e.unit_ids)))
+    out.append("<div class='wrap'><table><tr><th>unit</th><th>score</th><th>text</th></tr>")
+    for uid, v in rows:
+        out.append(f"<tr><td>{_e(uid)}</td><td class='num'>{v:+.4f}</td><td>{_e(_short(texts.get(uid, '')))}</td></tr>")
+    out.append("</table></div>")
+    inter = e.top_interactions(10)
+    if inter:
+        out.append("<div class='wrap'><table><tr><th>units together</th><th>score</th></tr>")
+        out += [f"<tr><td>{_e(' & '.join(s))}</td><td class='num'>{w:+.4f}</td></tr>" for s, w in inter]
+        out.append("</table></div>")
+    if e.removal:
+        out.append("<div class='wrap'><table><tr><th>removing the top r</th><th>units removed</th>"
+                   "<th>the value moves by</th><th>Rem(r)</th></tr>")
+        for r in e.removal:
+            removed = ", ".join(e.unit_ids[i] for i in r["removed"])
+            out.append(f"<tr><td>{r['r']}</td><td>{_e(removed)}</td><td class='num'>{r['change']:+.4f}</td>"
+                       f"<td class='num'>{_fmt(r['rem'])}</td></tr>")
+        out.append("</table></div>")
+    return "".join(out)
+
+
+def _prompt_html(prompt, scores, top) -> str:
+    def unit(u: Unit) -> str:
+        v = scores.get(u.id, 0.0)
+        alpha = 0.6 * abs(v) / top
+        colour = "--up" if v >= 0 else "--down"
+        return (f"<span class='u' title='{_e(u.id)}: {v:+.4f}' style='background: rgba(var({colour}), "
+                f"{alpha:.3f})'>{_e(u.text)}</span>{_e(u.sep)}")
+
+    def pieces(content) -> str:
+        out = []
+        for p in content:
+            if isinstance(p, str):
+                out.append(_e(p))
+            elif isinstance(p, Unit):
+                out.append(unit(p))
+            elif isinstance(p, Group):
+                out.append(_e(p.lead) + "".join(unit(u) for u in p.units) + _e(p.sep))
+        return "".join(out)
+
+    if prompt.messages is None:
+        return pieces(prompt.text)
+    return "".join(f"<span class='role'>{_e(m.role)}</span>{pieces(m.content)}" for m in prompt.messages)
+
+
+def _e(text) -> str:
+    return html.escape(str(text), quote=True)
+
+
+def _short(text: str, width: int = 120) -> str:
+    text = text.replace("\n", " ")
+    return text if len(text) <= width else text[: width - 1] + "…"
+
+
+def _fmt(x) -> str:
+    return "n/a" if x is None or (isinstance(x, float) and math.isnan(x)) else f"{x:.3g}"

contextattr/responses/__init__.py

@@ -0,0 +1,6 @@
+"""Responses and labels (docs/interface.md §4): checks that label a response, and replies sampled from the model."""
+
+from .checks import Verdict, describe, normalise, reference, regex, verdict
+from .sample import sample
+
+__all__ = ["Verdict", "describe", "normalise", "reference", "regex", "sample", "verdict"]

contextattr/responses/checks.py

@@ -0,0 +1,156 @@
+"""Checks: what labels a response (docs/interface.md §4). A check is any function check(prompt, response) giving a
+Verdict, a label ("positive", "negative" or None), True for negative and False for positive, or a number (a judge's
+score, a classifier's probability); `verdict` reads any of these; `describe` names a check in a run's settings."""
+
+from __future__ import annotations
+
+import functools
+import hashlib
+import math
+import numbers
+import re
+import string
+import types
+from dataclasses import dataclass
+
+import numpy as np
+
+from ..prompt import LABELS, Prompt
+
+
+@dataclass(frozen=True)
+class Verdict:
+    """A check's answer: a label, or a number, and optionally the span [start, end) of the response that matters."""
+
+    label: str | None = None
+    score: float | None = None
+    span: tuple[int, int] | None = None
+
+    def __post_init__(self) -> None:
+        if self.label is not None and self.label not in LABELS:
+            raise ValueError(f"a verdict's label is positive, negative or None, not {self.label!r}")
+        if self.score is not None and not math.isfinite(self.score):
+            raise ValueError("a verdict's score must be finite")
+
+    @property
+    def value(self) -> float | None:
+        """What Sampled averages: 1 for negative, 0 for positive, else the score; None when there is neither."""
+        if self.score is not None:
+            return float(self.score)
+        return None if self.label is None else float(self.label == "negative")
+
+
+def verdict(result) -> Verdict:
+    """Any check's answer as a Verdict."""
+    if isinstance(result, Verdict):
+        return result
+    if result is None or isinstance(result, str):
+        return Verdict(label=result)
+    if isinstance(result, bool):
+        return Verdict(label="negative" if result else "positive")
+    if isinstance(result, numbers.Real):
+        return Verdict(score=float(result))
+    raise TypeError(f"a check gives a Verdict, a label, a bool or a number, not {type(result).__name__}")
+
+
+def regex(pattern: str, *, negative: bool = True, flags: int = re.IGNORECASE):
+    """A check by pattern: a response that matches it is negative (or positive, with negative=False) and the match is
+    its span; one that does not match is the other label."""
+    compiled = re.compile(pattern, flags)
+
+    def check(prompt: Prompt, response: str) -> Verdict:
+        m = compiled.search(response)
+        hit, miss = ("negative", "positive") if negative else ("positive", "negative")
+        if m is None:
+            return Verdict(label=miss)
+        return Verdict(label=hit, span=(m.start(), m.end()) if m.end() > m.start() else None)
+
+    shown = "" if flags == re.IGNORECASE else f", flags={re.RegexFlag(flags)!r}"
+    check.__name__ = f"regex({pattern!r}, negative={negative}{shown})"
+    check.named = True
+    return check
+
+
+def reference(*, contains: bool = True):
+    """A check against the prompt's `reference`: positive when the response, normalised (lower case, no punctuation,
+    single spaces), contains the normalised reference (or equals it, with contains=False); negative otherwise."""
+
+    def check(prompt: Prompt, response: str) -> Verdict:
+        if prompt.reference is None:
+            raise ValueError(f"prompt {prompt.id!r} has no reference to check against")
+        ref, got = normalise(prompt.reference), normalise(response)
+        ok = (ref in got) if contains else (ref == got)
+        return Verdict(label="positive" if ok else "negative")
+
+    check.__name__ = f"reference(contains={contains})"
+    check.named = True
+    return check
+
+
+def describe(check) -> str:
+    """A check's name in a run's settings and an explanation's target, so that a resumed run cannot mix two checks.
+    regex's and reference's names carry their arguments. Any other check (a function, a bound method, a callable
+    object) is named by its name and a digest of its code (bytecode, constants and names, not line numbers) and of the
+    values it holds: its closure's and defaults', or its object's attributes, with numbers, strings and patterns by
+    value and other objects (a client, a model) by type. A check that depends on anything else should carry it in its
+    __name__ and set `named = True`."""
+    if getattr(check, "named", False) is True:
+        return check.__name__
+    if isinstance(check, functools.partial):
+        return f"partial({describe(check.func)}, {_held(check.args)}, {_held(check.keywords)})"
+    fn = getattr(check, "__func__", check)                       # a bound method: its function, and its object
+    if isinstance(fn, types.FunctionType):
+        owner = getattr(check, "__self__", None)
+        state = [_code(fn.__code__), [_cell(c) for c in fn.__closure__ or ()], _held(fn.__defaults__),
+                 _held(fn.__kwdefaults__), _held(getattr(owner, "__dict__", None))]
+        name = fn.__qualname__
+    else:
+        call = getattr(type(check).__call__, "__code__", None)
+        state = [_code(call) if call is not None else "", _held(getattr(check, "__dict__", None))]
+        name = type(check).__qualname__
+    return f"{name} [{hashlib.sha256(repr(state).encode()).hexdigest()[:12]}]"
+
+
+def _code(code: types.CodeType) -> str:
+    def const(c):
+        if isinstance(c, types.CodeType):
+            return _code(c)
+        if isinstance(c, (tuple, frozenset)):                     # a set's order changes with the hash seed
+            parts = [const(x) for x in c]
+            return repr(sorted(parts) if isinstance(c, frozenset) else parts)
+        return repr(c)
+
+    return repr((code.co_code.hex(), [const(c) for c in code.co_consts], code.co_names, code.co_varnames))
+
+
+def _cell(cell) -> str:
+    try:
+        return _held(cell.cell_contents)
+    except ValueError:                                           # a cell not yet filled
+        return "<empty>"
+
+
+def _held(value) -> str:
+    if value is None or isinstance(value, (str, bytes, numbers.Number)):
+        return repr(value)
+    if isinstance(value, re.Pattern):
+        return f"re.compile({value.pattern!r}, {int(value.flags)})"
+    if isinstance(value, np.ndarray):
+        digest = hashlib.sha256(np.ascontiguousarray(value).tobytes()).hexdigest()
+        return f"array({value.dtype}, {value.shape}, {digest})"
+    if isinstance(value, (tuple, list)):
+        return "[" + ", ".join(_held(v) for v in value) + "]"
+    if isinstance(value, (set, frozenset)):
+        return "{" + ", ".join(sorted(_held(v) for v in value)) + "}"
+    if isinstance(value, dict):
+        return "{" + ", ".join(sorted(f"{_held(k)}: {_held(v)}" for k, v in value.items())) + "}"
+    if isinstance(value, types.FunctionType):                    # a function it calls: by its code, not recursing
+        return value.__name__ if getattr(value, "named", False) is True else _code(value.__code__)
+    return f"<{type(value).__qualname__}>"
+
+
+_PUNCTUATION = str.maketrans({c: " " for c in string.punctuation})
+
+
+def normalise(text: str) -> str:
+    return " ".join(text.lower().translate(_PUNCTUATION).split())

contextattr/responses/sample.py

@@ -0,0 +1,36 @@
+"""Responses sampled from the full prompt and labelled (docs/interface.md §4): `sample` adds k replies of the model to
+a prompt's responses, each labelled by a check, so that scored targets (LogProb, Pooled, Contrast) can explain them."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from ..cache import Cache
+from ..prompt import Prompt, Response
+from ..wrapper.contract import Sampling
+from .checks import verdict
+
+DEFAULT = Sampling(n=8, temperature=0.7, top_p=0.95, max_tokens=256, seed=0)
+
+
+def sample(prompt: Prompt, model, sampling: Sampling = DEFAULT, check=None, *, cache: Cache | str | None = None,
+           replace: str | None = None) -> Prompt:
+    """The prompt with `sampling.n` replies to its full text added to its responses, as sample-0, sample-1, …, each
+    labelled by `check` (none: unlabelled). Empty replies are left out. The replies are cached like every answer."""
+    from ..value import Value     # noqa: PLC0415 - value imports the targets, which import this package
+    from ..targets.sampled import Sampled
+
+    replies = Value(prompt, model, Sampled(lambda p, r: 0.0, samples=sampling.n, sampling=sampling), replace=replace,
+                    cache=cache).replies(np.ones((1, prompt.n), dtype=bool))[0]
+    taken = {r.id for r in prompt.responses}
+    added = []
+    for k, text in enumerate(replies):
+        if not text.strip():
+            continue
+        rid = f"sample-{k}"
+        if rid in taken:
+            raise ValueError(f"prompt {prompt.id!r} already has a response {rid!r}")
+        v = verdict(check(prompt, text)) if check is not None else None
+        added.append(Response(rid, text, label=None if v is None else v.label, span=None if v is None else v.span))
+    return Prompt(prompt.id, messages=prompt.messages, text=prompt.text, reference=prompt.reference,
+                  responses=(*prompt.responses, *added))