contextattr 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
contextattr/value.py ADDED
@@ -0,0 +1,201 @@
1
+ """The value function (docs/interface.md §1, §5): masks → one number per masked prompt.
2
+
3
+ Through a wrapper and a target, each distinct request goes to the wrapper once, in batches, and every answer is
4
+ cached first: scored targets ask `score` for each (masked prompt, text), the sampled target asks `generate` for the
5
+ replies to each masked prompt (the check is applied after the cache, so changing it asks the model nothing). Or
6
+ through the user's own function of masked prompts, cached when it has a name.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import copy
12
+ import math
13
+
14
+ import numpy as np
15
+
16
+ from .cache import Cache
17
+ from .prompt import Prompt
18
+ from .prompt.render import render
19
+ from .targets.scored import reduce_scored
20
+ from .wrapper.contract import (Sampling, Scored, WrapperError, call_generate, call_score, can_generate, can_score,
21
+ model_name)
22
+
23
+
24
+ def as_masks(masks, n: int) -> np.ndarray:
25
+ """Check masks: an (m × n) array of 0/1 or booleans, one row per mask, 1 keeping a unit."""
26
+ a = np.asarray(masks)
27
+ if a.ndim != 2 or a.shape[1] != n:
28
+ raise ValueError(f"masks must be an (m × {n}) array of 0/1, one row per mask, not shape {a.shape}")
29
+ if a.dtype == bool:
30
+ return a
31
+ if not np.issubdtype(a.dtype, np.number) or not np.isin(a, (0, 1)).all():
32
+ raise ValueError("masks hold only 0/1 or True/False")
33
+ return a.astype(bool)
34
+
35
+
36
+ class Value:
37
+ """f(m) for masks m. Called with an (m × n) array of 0/1 it gives m values; with per_response=True (scored
38
+ targets), an (m × items) array, one column per scored text (see `items`)."""
39
+
40
+ def __init__(self, prompt: Prompt, model=None, target=None, *, function=None, name: str | None = None,
41
+ replace: str | None = None, cache: Cache | str | None = None, batch_size: int = 32,
42
+ per_response: bool = False):
43
+ if not isinstance(prompt, Prompt):
44
+ raise TypeError(f"Value needs a Prompt, not {type(prompt).__name__}")
45
+ if (model is None) == (function is None):
46
+ raise ValueError("Value needs a model and a target, or a function")
47
+ if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size < 1:
48
+ raise ValueError(f"batch_size must be a positive integer, not {batch_size!r}")
49
+ if replace is not None and not isinstance(replace, str):
50
+ raise TypeError("replace must be a string or None")
51
+ self.prompt, self.model, self.target, self.function = prompt, model, target, function
52
+ self.replace, self.batch_size, self.per_response = replace, batch_size, per_response
53
+ self.cache = cache if isinstance(cache, Cache) else Cache(cache)
54
+ self.stats = {"requests": 0, "reused": 0, "calls": 0}
55
+ if model is not None:
56
+ needs = getattr(target, "needs", None)
57
+ if needs not in ("score", "generate"):
58
+ raise TypeError(f"target must be a target such as LogProb, Contrast or Sampled, not "
59
+ f"{type(target).__name__}")
60
+ self.name = model_name(model)
61
+ if needs == "score" and not can_score(model):
62
+ raise WrapperError(f"{type(target).__name__} needs the wrapper's score, which {self.name} lacks "
63
+ f"(a wrapper that can only generate serves Sampled)")
64
+ if needs == "generate" and not can_generate(model):
65
+ raise WrapperError(f"{type(target).__name__} needs the wrapper's generate, which {self.name} lacks")
66
+ if needs == "generate" and per_response:
67
+ raise ValueError("per_response applies to scored targets")
68
+ self.items = target.items(prompt) if needs == "score" else []
69
+ if needs == "generate":
70
+ self.stats["generations"] = 0
71
+ else:
72
+ if not callable(function):
73
+ raise TypeError("function must be callable: masked prompts → numbers")
74
+ if target is not None or per_response:
75
+ raise ValueError("a function gives the value itself: no target, no per_response")
76
+ if name is not None and (not isinstance(name, str) or not name):
77
+ raise ValueError("name must be a non-empty string; it keys the cache")
78
+ self.name, self.items = name, []
79
+
80
+ @property
81
+ def n(self) -> int:
82
+ return self.prompt.n
83
+
84
+ def __call__(self, masks) -> np.ndarray:
85
+ rows = as_masks(masks, self.n)
86
+ return self._values(self._render(rows), per_response=self.per_response)
87
+
88
+ def replies(self, masks, repeat: int = 0) -> list[list[str]]:
89
+ """The sampled target's replies to each masked prompt (cached like everything else)."""
90
+ if getattr(self.target, "needs", None) != "generate":
91
+ raise ValueError("replies come from a sampled target")
92
+ rendered = self._render(as_masks(masks, self.n))
93
+ sampling = self.target.sampling_for(repeat)
94
+ requests = [self._request("generate", p, None, repeat, sampling) for p in rendered]
95
+ answers = self._ask(requests, lambda batch: self._generate(batch, sampling))
96
+ return [answers[key] for key, _, _ in requests]
97
+
98
+ def noise(self, repeats: int = 3) -> dict:
99
+ """The spread of the full prompt's value: from the main requests and from `repeats` fresh ones, each
100
+ cached under its own key, so a rerun gives the same figures. A sampled target samples a repeat k with seed
101
+ + k, so its spread is the sampling's."""
102
+ if isinstance(repeats, bool) or not isinstance(repeats, int) or repeats < 1:
103
+ raise ValueError("repeats must be a positive integer")
104
+ full = self._render(np.ones((1, self.n), dtype=bool))
105
+ values = [float(self._values(full, repeat=k)[0]) for k in range(repeats + 1)]
106
+ return {"values": values, "std": float(np.std(values, ddof=1))}
107
+
108
+ # -- inside ---------------------------------------------------------------------------------------------------
109
+
110
+ def _render(self, rows: np.ndarray) -> list:
111
+ done: dict[bytes, object] = {}
112
+ out = []
113
+ for row in rows:
114
+ key = row.tobytes()
115
+ if key not in done:
116
+ done[key] = render(self.prompt, row.tolist(), self.replace)
117
+ out.append(done[key])
118
+ return out
119
+
120
+ def _values(self, rendered: list, repeat: int = 0, per_response: bool = False) -> np.ndarray:
121
+ needs = getattr(self.target, "needs", None)
122
+ if self.model is not None and needs == "score":
123
+ requests = [[self._request("score", p, item.text, repeat) for item in self.items] for p in rendered]
124
+ answers = self._ask(sum(requests, []), self._score)
125
+ columns = np.array([[reduce_scored(answers[key], item) for (key, _, _), item in zip(row, self.items)]
126
+ for row in requests], dtype=float).reshape(len(rendered), len(self.items))
127
+ out = columns if per_response else self.target.combine(columns, self.items)
128
+ elif self.model is not None:
129
+ sampling = self.target.sampling_for(repeat)
130
+ requests = [self._request("generate", p, None, repeat, sampling) for p in rendered]
131
+ answers = self._ask(requests, lambda batch: self._generate(batch, sampling))
132
+ out = np.array([self.target.value(self.prompt, answers[key]) for key, _, _ in requests], dtype=float)
133
+ elif self.name is None: # an unnamed function: nothing is cached
134
+ out = np.array(self._apply([(None, p, None) for p in rendered]), dtype=float)
135
+ else:
136
+ requests = [self._request("function", p, None, repeat) for p in rendered]
137
+ answers = self._ask(requests, self._apply)
138
+ out = np.array([answers[key] for key, _, _ in requests], dtype=float)
139
+ if not np.isfinite(out).all():
140
+ raise ValueError("some values are not finite: a response with probability 1 under logodds, or a "
141
+ "function giving inf or nan")
142
+ return out
143
+
144
+ def _request(self, kind: str, prompt, text, repeat: int, sampling: Sampling | None = None):
145
+ req = {"kind": kind, "model" if kind != "function" else "name": self.name, "prompt": prompt}
146
+ if text is not None:
147
+ req["response"] = text
148
+ if sampling is not None:
149
+ req["sampling"] = sampling.as_dict()
150
+ if repeat:
151
+ req["repeat"] = repeat
152
+ return (Cache.key(req), prompt, text)
153
+
154
+ def _ask(self, requests: list, run) -> dict:
155
+ """Each distinct request answered once: from the cache, else by `run` in batches, each stored at once."""
156
+ distinct = {key: (key, p, t) for key, p, t in requests}
157
+ found = self.cache.get_many(distinct)
158
+ self.stats["reused"] += len(found)
159
+ missing = [r for key, r in distinct.items() if key not in found]
160
+ kind = "function" if self.model is None else self.target.needs
161
+ for i in range(0, len(missing), self.batch_size):
162
+ batch = missing[i:i + self.batch_size]
163
+ fresh = dict(zip((key for key, _, _ in batch), run(batch)))
164
+ self.cache.put_many(kind, fresh)
165
+ found.update(fresh)
166
+ if self.model is not None and self.target.needs == "score":
167
+ return {key: Scored(tuple(a["logprobs"]), None if a["tokens"] is None else tuple(a["tokens"]))
168
+ for key, a in found.items()}
169
+ return found
170
+
171
+ def _score(self, batch: list) -> list:
172
+ answers = call_score(self.model, [copy.deepcopy(p) for _, p, _ in batch], [t for _, _, t in batch])
173
+ self.stats["requests"] += len(batch)
174
+ self.stats["calls"] += 1
175
+ return [{"logprobs": list(a.logprobs), "tokens": None if a.tokens is None else list(a.tokens)}
176
+ for a in answers]
177
+
178
+ def _generate(self, batch: list, sampling: Sampling) -> list:
179
+ replies = call_generate(self.model, [copy.deepcopy(p) for _, p, _ in batch], sampling)
180
+ self.stats["requests"] += len(batch)
181
+ self.stats["calls"] += 1
182
+ self.stats["generations"] = self.stats.get("generations", 0) + len(batch) * sampling.n
183
+ return replies
184
+
185
+ def _apply(self, batch: list) -> list:
186
+ values = []
187
+ for i in range(0, len(batch), self.batch_size):
188
+ chunk = batch[i:i + self.batch_size]
189
+ out = self.function([copy.deepcopy(p) for _, p, _ in chunk])
190
+ self.stats["requests"] += len(chunk)
191
+ self.stats["calls"] += 1
192
+ try:
193
+ out = [float(x) for x in out]
194
+ except (TypeError, ValueError):
195
+ raise TypeError("the function must give one number per masked prompt") from None
196
+ if len(out) != len(chunk):
197
+ raise ValueError(f"the function gave {len(out)} numbers for {len(chunk)} masked prompts")
198
+ if not all(math.isfinite(x) for x in out):
199
+ raise ValueError("the function gave a value that is not finite")
200
+ values.extend(out)
201
+ return values
@@ -0,0 +1,9 @@
1
+ """What a user's wrapper must do, the check that it does, and a toy wrapper (docs/wrapper.md)."""
2
+
3
+ from .check import ModelReport, check_model
4
+ from .contract import (Generator, Rendered, Sampling, Scored, Scorer, WrapperError, call_generate, call_score,
5
+ can_generate, can_score, model_name)
6
+ from .toy import ToyModel
7
+
8
+ __all__ = ["Generator", "ModelReport", "Rendered", "Sampling", "Scored", "Scorer", "ToyModel", "WrapperError",
9
+ "call_generate", "call_score", "can_generate", "can_score", "check_model", "model_name"]
@@ -0,0 +1,163 @@
1
+ """check_model (docs/wrapper.md): about a dozen calls that tell a user whether their wrapper meets the contract.
2
+
3
+ Every problem is reported, none raised: a check FAILs when the wrapper breaks the contract or its answers cannot
4
+ serve attribution, and gives a note when something is legal but worth knowing (answers that vary, seeds ignored).
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import math
10
+ import time
11
+ from dataclasses import dataclass, field
12
+
13
+ from ..prompt import Prompt, Unit
14
+ from .contract import (Sampling, Scored, WrapperError, call_generate, call_score, can_generate, can_score,
15
+ model_name)
16
+
17
+ OK, FAIL, NOTE, INFO, SKIP = "ok", "FAIL", "note", "info", "skip"
18
+ DEFAULT_RESPONSE = "It is hot and humid, because Carnival falls in the Brazilian summer."
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class Check:
23
+ name: str
24
+ status: str
25
+ detail: str
26
+
27
+
28
+ @dataclass
29
+ class ModelReport:
30
+ """What check_model found: each check, and the targets the wrapper supports."""
31
+
32
+ model: str
33
+ checks: list[Check] = field(default_factory=list)
34
+ scores: bool = False
35
+ generates: bool = False
36
+ spans: bool = False
37
+
38
+ @property
39
+ def ok(self) -> bool:
40
+ """Ready for attribution: a name, a working method, and no failed check."""
41
+ return (self.scores or self.generates) and not any(c.status == FAIL for c in self.checks)
42
+
43
+ @property
44
+ def targets(self) -> list[str]:
45
+ """The targets this wrapper can serve: the scored ones need `score`, Sampled needs only `generate`."""
46
+ return (["LogProb", "Pooled", "Contrast", "Choice"] if self.scores else []) + \
47
+ (["Sampled"] if self.generates else [])
48
+
49
+ def add(self, name: str, status: str, detail: str) -> None:
50
+ self.checks.append(Check(name, status, detail))
51
+
52
+ def __str__(self) -> str:
53
+ width = max([len(c.name) for c in self.checks] + [0])
54
+ lines = [f"check_model: {self.model}"]
55
+ lines += [f" {c.status:<4} {c.name:<{width}} {c.detail}" for c in self.checks]
56
+ spans = (" (spans: yes)" if self.spans else " (spans: no)") if self.scores else ""
57
+ lines.append(f" targets: {', '.join(self.targets) or 'none yet'}{spans}")
58
+ lines.append(" ready: every check passed" if self.ok else " NOT ready for attribution: see the FAIL lines")
59
+ return "\n".join(lines)
60
+
61
+
62
+ def default_prompt() -> Prompt:
63
+ """The prompt check_model uses unless given one: the interface's Rio example."""
64
+ return Prompt("check", messages=[
65
+ {"role": "system", "content": "Answer in one sentence."},
66
+ {"role": "user", "content": [
67
+ "Documents:\n",
68
+ Unit("d1", "Rio Carnival takes place before Lent, in February or March.", sep="\n"),
69
+ Unit("d2", "Summer in Brazil runs from December to March.", sep="\n"),
70
+ Unit("d3", "Winter in Brazil runs from June to September.", sep="\n"),
71
+ "Question: What is the weather like during Rio Carnival?"]}])
72
+
73
+
74
+ def check_model(model, prompt: Prompt | None = None, response: str | None = None) -> ModelReport:
75
+ """Test a wrapper against the contract on `prompt` (default: the Rio example; give a text prompt for a model that
76
+ takes plain text) and `response` (default: the model's own greedy reply, else a fixed sentence)."""
77
+ prompt = prompt if prompt is not None else default_prompt()
78
+ full, empty = prompt.render(), prompt.render([0] * prompt.n)
79
+ try:
80
+ name = model_name(model)
81
+ except WrapperError as e:
82
+ report = ModelReport(type(model).__name__)
83
+ report.add("name", FAIL, str(e))
84
+ return report
85
+ report = ModelReport(name)
86
+ report.add("name", OK, repr(name))
87
+ has = [m for m, ok in (("score", can_score(model)), ("generate", can_generate(model))) if ok]
88
+ report.add("methods", OK if has else FAIL, ", ".join(has) if has else "neither score nor generate")
89
+ timings: list[tuple[float, int]] = []
90
+
91
+ def timed(fn, *args):
92
+ start = time.perf_counter()
93
+ out = fn(*args)
94
+ timings.append((time.perf_counter() - start, len(args[1])))
95
+ return out
96
+
97
+ reply = None
98
+ if "generate" in has:
99
+ try:
100
+ greedy = timed(call_generate, model, [full, full], Sampling(n=2, temperature=0.0))
101
+ report.generates = True
102
+ report.add("generate", OK, f"2 greedy replies to each of 2 prompts, e.g. {greedy[0][0][:60]!r}")
103
+ again = timed(call_generate, model, [full], Sampling(n=1, temperature=0.0))
104
+ same = len({greedy[0][0], greedy[0][1], greedy[1][0], greedy[1][1], again[0][0]}) == 1
105
+ report.add("greedy repeats", OK if same else NOTE,
106
+ "identical" if same else "greedy replies vary: sampled targets carry that noise")
107
+ s1 = timed(call_generate, model, [full], Sampling(n=1, temperature=1.0, seed=7))
108
+ s2 = timed(call_generate, model, [full], Sampling(n=1, temperature=1.0, seed=7))
109
+ report.add("seed", OK if s1 == s2 else NOTE, "honoured" if s1 == s2 else
110
+ "two calls with seed 7 differ: a rerun samples anew where the cache has no answer")
111
+ reply = greedy[0][0] if greedy[0][0].strip() else None
112
+ except Exception as e: # noqa: BLE001 - every failure is reported, not raised
113
+ report.add("generate", FAIL, f"{type(e).__name__}: {e}")
114
+ if "score" in has:
115
+ r = response if response is not None else (reply or DEFAULT_RESPONSE)
116
+ try:
117
+ a = timed(call_score, model, [full, full], [r, r])
118
+ b = timed(call_score, model, [full], [r])
119
+ if not a[0].logprobs:
120
+ raise WrapperError("score gave no tokens for a non-empty response")
121
+ report.scores = True
122
+ report.add("score", OK, f"{len(a[0].logprobs)} tokens, finite log-probabilities at most 0")
123
+ diff = max(_diff(a[0], a[1]), _diff(a[0], b[0]))
124
+ report.add("score repeats", OK if diff == 0 else NOTE, "identical" if diff == 0 else
125
+ f"differ by up to {diff:.2g}: the noise check measures this")
126
+ if a[0].tokens is None:
127
+ report.add("token texts", NOTE, "none given: targets cannot use spans")
128
+ elif "".join(a[0].tokens) == r:
129
+ report.spans = True
130
+ report.add("token texts", OK, "join back to the response: targets can use spans")
131
+ else:
132
+ report.add("token texts", FAIL, "given, but they do not join back to the response exactly")
133
+ c = timed(call_score, model, [empty], [r])
134
+ m_full, m_empty = _mean(a[0]), _mean(c[0])
135
+ report.add("prompt matters", OK if m_full != m_empty else FAIL,
136
+ f"{m_full:.3f} per token with every unit, {m_empty:.3f} with none" +
137
+ ("" if m_full != m_empty else ": the score ignores the prompt; does it reach the model?"))
138
+ words = r.split()
139
+ backwards = " ".join(reversed(words))
140
+ if len(set(words)) < 2 or backwards == r:
141
+ report.add("reply above reversed", SKIP, "the response has too few distinct words")
142
+ else:
143
+ d = timed(call_score, model, [full, full], [r, backwards])
144
+ m1, m2 = _mean(d[0]), _mean(d[1])
145
+ report.add("reply above reversed", OK if m1 > m2 else NOTE,
146
+ f"{m1:.3f} per token, {m2:.3f} with its words reversed" + ("" if m1 > m2 else
147
+ ": a model blind to word order, or the response not placed as the model's reply"))
148
+ except Exception as e: # noqa: BLE001
149
+ report.add("score", FAIL, f"{type(e).__name__}: {e}")
150
+ if timings:
151
+ seconds, requests = sum(t for t, _ in timings), sum(k for _, k in timings)
152
+ report.add("time", INFO, f"{seconds / requests:.3g} s per request over {len(timings)} calls")
153
+ return report
154
+
155
+
156
+ def _mean(s: Scored) -> float:
157
+ return math.fsum(s.logprobs) / len(s.logprobs) if s.logprobs else math.nan
158
+
159
+
160
+ def _diff(x: Scored, y: Scored) -> float:
161
+ if len(x.logprobs) != len(y.logprobs):
162
+ return math.inf
163
+ return max((abs(p - q) for p, q in zip(x.logprobs, y.logprobs)), default=0.0)
@@ -0,0 +1,166 @@
1
+ """What a user's wrapper must do (docs/wrapper.md), and the checks applied to every answer it gives.
2
+
3
+ A wrapper is any object with a `name` and at least one of two methods:
4
+
5
+ score(prompts, responses) -> list[Scored] the log-probability of each response token, given its prompt
6
+ generate(prompts, sampling) -> list[list[str]] sampling.n replies to each prompt
7
+
8
+ Prompts reach it rendered (`Rendered`): a string, or chat messages [{"role": ..., "content": "..."}]. The wrapper
9
+ applies its model's own chat template, batches as it likes, and raises on errors; it never truncates a prompt.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ import numbers
16
+ from collections.abc import Sequence
17
+ from dataclasses import dataclass
18
+ from typing import Protocol, Union, runtime_checkable
19
+
20
+ Rendered = Union[str, list[dict[str, str]]]
21
+
22
+ # A log-probability above 0 is impossible; rounding in low precision can give a hair above it.
23
+ LOGPROB_TOLERANCE = 1e-3
24
+
25
+
26
+ class WrapperError(RuntimeError):
27
+ """A wrapper broke the contract: its answer cannot be used."""
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class Sampling:
32
+ """How `generate` samples: `n` replies per prompt; temperature 0 is greedy."""
33
+
34
+ n: int = 1
35
+ temperature: float = 0.0
36
+ top_p: float = 1.0
37
+ max_tokens: int = 256
38
+ seed: int | None = 0
39
+ stop: tuple[str, ...] = ()
40
+
41
+ def __post_init__(self) -> None:
42
+ if not _is_int(self.n) or self.n < 1:
43
+ raise ValueError(f"Sampling.n must be an integer of at least 1, not {self.n!r}")
44
+ if not _is_real(self.temperature) or not 0 <= self.temperature < math.inf:
45
+ raise ValueError(f"Sampling.temperature must be a finite number of at least 0, not {self.temperature!r}")
46
+ if not _is_real(self.top_p) or not 0 < self.top_p <= 1:
47
+ raise ValueError(f"Sampling.top_p must lie in (0, 1], not {self.top_p!r}")
48
+ if not _is_int(self.max_tokens) or self.max_tokens < 1:
49
+ raise ValueError(f"Sampling.max_tokens must be an integer of at least 1, not {self.max_tokens!r}")
50
+ if self.seed is not None and not _is_int(self.seed):
51
+ raise ValueError(f"Sampling.seed must be an integer or None, not {self.seed!r}")
52
+ stop = (self.stop,) if isinstance(self.stop, str) else tuple(self.stop)
53
+ if not all(isinstance(s, str) and s for s in stop):
54
+ raise ValueError(f"Sampling.stop must hold non-empty strings, not {self.stop!r}")
55
+ object.__setattr__(self, "stop", stop)
56
+
57
+ def as_dict(self) -> dict:
58
+ return {"n": self.n, "temperature": float(self.temperature), "top_p": float(self.top_p),
59
+ "max_tokens": self.max_tokens, "seed": self.seed, "stop": list(self.stop)}
60
+
61
+
62
+ @dataclass(frozen=True)
63
+ class Scored:
64
+ """What `score` gives for one response: the natural-log probability of each response token, in order, and
65
+ optionally each token's text; the texts must join back to the response exactly, and are needed only for spans."""
66
+
67
+ logprobs: tuple[float, ...]
68
+ tokens: tuple[str, ...] | None = None
69
+
70
+ def __post_init__(self) -> None:
71
+ if isinstance(self.logprobs, (str, bytes)):
72
+ raise TypeError("Scored.logprobs must be a sequence of numbers")
73
+ try:
74
+ logprobs = tuple(float(x) for x in self.logprobs)
75
+ except (TypeError, ValueError):
76
+ raise TypeError("Scored.logprobs must be a sequence of numbers") from None
77
+ for k, x in enumerate(logprobs):
78
+ if not math.isfinite(x) or x > LOGPROB_TOLERANCE:
79
+ raise ValueError(f"Scored.logprobs[{k}] is {x}: a log-probability must be finite and at most 0")
80
+ object.__setattr__(self, "logprobs", logprobs)
81
+ if self.tokens is not None:
82
+ if isinstance(self.tokens, (str, bytes)):
83
+ raise TypeError("Scored.tokens must be a sequence of strings, one per token")
84
+ tokens = tuple(self.tokens)
85
+ if not all(isinstance(t, str) for t in tokens):
86
+ raise TypeError("Scored.tokens must be a sequence of strings, one per token")
87
+ if len(tokens) != len(logprobs):
88
+ raise ValueError(f"Scored has {len(logprobs)} log-probabilities but {len(tokens)} token texts")
89
+ object.__setattr__(self, "tokens", tokens)
90
+
91
+
92
+ @runtime_checkable
93
+ class Scorer(Protocol):
94
+ """A wrapper that can score given responses."""
95
+
96
+ name: str
97
+
98
+ def score(self, prompts: Sequence[Rendered], responses: Sequence[str]) -> list[Scored]: ...
99
+
100
+
101
+ @runtime_checkable
102
+ class Generator(Protocol):
103
+ """A wrapper that can generate replies."""
104
+
105
+ name: str
106
+
107
+ def generate(self, prompts: Sequence[Rendered], sampling: Sampling) -> list[list[str]]: ...
108
+
109
+
110
+ def model_name(model) -> str:
111
+ """The wrapper's name, which every cache key holds; it must change whenever the answers could change."""
112
+ name = getattr(model, "name", None)
113
+ if not isinstance(name, str) or not name:
114
+ raise WrapperError(f"a wrapper needs a non-empty string `name`; {type(model).__name__} has {name!r}")
115
+ return name
116
+
117
+
118
+ def can_score(model) -> bool:
119
+ return callable(getattr(model, "score", None))
120
+
121
+
122
+ def can_generate(model) -> bool:
123
+ return callable(getattr(model, "generate", None))
124
+
125
+
126
+ def call_score(model, prompts: list[Rendered], responses: list[str]) -> list[Scored]:
127
+ """Ask the wrapper to score, and check its answer's shape."""
128
+ name = model_name(model)
129
+ out = model.score(prompts, responses)
130
+ if isinstance(out, (str, bytes, dict)) or not hasattr(out, "__len__"):
131
+ raise WrapperError(f"{name}.score returned {type(out).__name__}, not a list of Scored")
132
+ out = list(out)
133
+ if len(out) != len(prompts):
134
+ raise WrapperError(f"{name}.score returned {len(out)} answers for {len(prompts)} requests")
135
+ for k, s in enumerate(out):
136
+ if not isinstance(s, Scored):
137
+ raise WrapperError(f"{name}.score answer {k} is {type(s).__name__}, not Scored")
138
+ return out
139
+
140
+
141
+ def call_generate(model, prompts: list[Rendered], sampling: Sampling) -> list[list[str]]:
142
+ """Ask the wrapper to generate, and check that every prompt got exactly sampling.n strings."""
143
+ name = model_name(model)
144
+ out = model.generate(prompts, sampling)
145
+ if isinstance(out, (str, bytes, dict)) or not hasattr(out, "__len__"):
146
+ raise WrapperError(f"{name}.generate returned {type(out).__name__}, not a list of lists of strings")
147
+ out = list(out)
148
+ if len(out) != len(prompts):
149
+ raise WrapperError(f"{name}.generate returned replies for {len(out)} prompts, not {len(prompts)}")
150
+ replies = []
151
+ for k, r in enumerate(out):
152
+ if isinstance(r, (str, bytes)) or not hasattr(r, "__len__"):
153
+ raise WrapperError(f"{name}.generate answer {k} is {type(r).__name__}, not a list of strings")
154
+ r = list(r)
155
+ if len(r) != sampling.n or not all(isinstance(x, str) for x in r):
156
+ raise WrapperError(f"{name}.generate answer {k} must hold {sampling.n} strings (sampling.n)")
157
+ replies.append(r)
158
+ return replies
159
+
160
+
161
+ def _is_int(x) -> bool:
162
+ return isinstance(x, numbers.Integral) and not isinstance(x, bool)
163
+
164
+
165
+ def _is_real(x) -> bool:
166
+ return isinstance(x, numbers.Real) and not isinstance(x, bool)
@@ -0,0 +1,56 @@
1
+ """Your model's wrapper, to fill in (`contextattr template my_wrapper.py` wrote this file). ContextAttr reaches a
2
+ model only through an object like this one: a `name`, and `generate`, `score` or both.
3
+
4
+ 1. Fill in `generate`, `score` or both, and delete the one you leave out.
5
+ generate writes replies to a prompt. Every model and API can do this. Serves the Sampled target.
6
+ score gives the log-probability of each token of a reply you supply. Possible where the backend returns
7
+ log-probabilities of given text: a model in your own process, a vLLM or SGLang server. Serves
8
+ LogProb, Pooled, Contrast and Choice, which are cheaper and repeat exactly: write it if you can.
9
+ 2. Run this file: `python my_wrapper.py`. check_model calls your wrapper about a dozen times and says what is wrong.
10
+ 3. Use it: `ca.explain(prompt, MyModel(), ...)`, or `use = "my_wrapper.py:MyModel"` in a run file.
11
+
12
+ The guide, with two complete wrappers to copy from:
13
+ https://github.com/Surya-Rayala/contextattr/blob/main/docs/wrapper.md
14
+ """
15
+
16
+ from contextattr import Sampling, Scored, check_model # noqa: F401
17
+
18
+
19
+ class MyModel:
20
+ # Every answer is cached under this name, so change it whenever the answers could change: another model,
21
+ # revision, precision or prompt format.
22
+ name = "my-model"
23
+
24
+ def __init__(self):
25
+ ... # load the model, or open the client of your server or API
26
+
27
+ def generate(self, prompts, sampling: Sampling):
28
+ """Reply to each prompt.
29
+
30
+ prompts a list. Each prompt is chat messages, [{"role": "user", "content": "..."}, ...], or a plain string.
31
+ sampling .n replies per prompt, .temperature (0: always the most likely token), .top_p, .max_tokens,
32
+ .seed, .stop.
33
+ return one list per prompt, holding exactly sampling.n replies (the reply's text only).
34
+ For two prompts and n = 1: [["first reply"], ["second reply"]]
35
+ """
36
+ raise NotImplementedError("fill in generate, or delete it")
37
+
38
+ def score(self, prompts, responses):
39
+ """Say how likely the model is to give responses[i] as its reply to prompts[i].
40
+
41
+ prompts as for generate.
42
+ responses a list of strings, one per prompt.
43
+ return one Scored per pair: Scored(logprobs), the natural-log probability of each token of the response,
44
+ in order. For a response of three tokens: Scored([-0.2, -1.3, -0.1])
45
+
46
+ Format the prompt as your model expects (its chat template), put the response where the model's reply would
47
+ begin, and read the log-probability the model gives each token of the response. Leave out the end-of-turn
48
+ token. Optional: Scored(logprobs, tokens), with each token's text, lets a target cover part of a response.
49
+ """
50
+ raise NotImplementedError("fill in score, or delete it")
51
+
52
+ # In both: raise an error on a failure or on a prompt that is too long. Never cut a prompt short.
53
+
54
+
55
+ if __name__ == "__main__":
56
+ print(check_model(MyModel()))