contextattr 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contextattr/__init__.py +22 -0
- contextattr/__main__.py +7 -0
- contextattr/cache.py +64 -0
- contextattr/cli.py +181 -0
- contextattr/evaluate.py +137 -0
- contextattr/methods/__init__.py +27 -0
- contextattr/methods/base.py +86 -0
- contextattr/methods/indices.py +59 -0
- contextattr/methods/interactions.py +215 -0
- contextattr/methods/kernelshap.py +114 -0
- contextattr/methods/linear.py +104 -0
- contextattr/methods/loo.py +31 -0
- contextattr/prompt/__init__.py +9 -0
- contextattr/prompt/io.py +222 -0
- contextattr/prompt/render.py +70 -0
- contextattr/prompt/split.py +153 -0
- contextattr/prompt/units.py +271 -0
- contextattr/report.py +120 -0
- contextattr/responses/__init__.py +6 -0
- contextattr/responses/checks.py +156 -0
- contextattr/responses/sample.py +36 -0
- contextattr/result.py +120 -0
- contextattr/run.py +306 -0
- contextattr/targets/__init__.py +7 -0
- contextattr/targets/sampled.py +58 -0
- contextattr/targets/scored.py +177 -0
- contextattr/value.py +201 -0
- contextattr/wrapper/__init__.py +9 -0
- contextattr/wrapper/check.py +163 -0
- contextattr/wrapper/contract.py +166 -0
- contextattr/wrapper/template.py +56 -0
- contextattr/wrapper/toy.py +129 -0
- contextattr-0.1.0.dist-info/METADATA +129 -0
- contextattr-0.1.0.dist-info/RECORD +38 -0
- contextattr-0.1.0.dist-info/WHEEL +5 -0
- contextattr-0.1.0.dist-info/entry_points.txt +2 -0
- contextattr-0.1.0.dist-info/licenses/LICENSE +21 -0
- contextattr-0.1.0.dist-info/top_level.txt +1 -0
contextattr/value.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""The value function (docs/interface.md §1, §5): masks → one number per masked prompt.
|
|
2
|
+
|
|
3
|
+
Through a wrapper and a target, each distinct request goes to the wrapper once, in batches, and every answer is
|
|
4
|
+
cached first: scored targets ask `score` for each (masked prompt, text), the sampled target asks `generate` for the
|
|
5
|
+
replies to each masked prompt (the check is applied after the cache, so changing it asks the model nothing). Or
|
|
6
|
+
through the user's own function of masked prompts, cached when it has a name.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import copy
|
|
12
|
+
import math
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
|
|
16
|
+
from .cache import Cache
|
|
17
|
+
from .prompt import Prompt
|
|
18
|
+
from .prompt.render import render
|
|
19
|
+
from .targets.scored import reduce_scored
|
|
20
|
+
from .wrapper.contract import (Sampling, Scored, WrapperError, call_generate, call_score, can_generate, can_score,
|
|
21
|
+
model_name)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def as_masks(masks, n: int) -> np.ndarray:
|
|
25
|
+
"""Check masks: an (m × n) array of 0/1 or booleans, one row per mask, 1 keeping a unit."""
|
|
26
|
+
a = np.asarray(masks)
|
|
27
|
+
if a.ndim != 2 or a.shape[1] != n:
|
|
28
|
+
raise ValueError(f"masks must be an (m × {n}) array of 0/1, one row per mask, not shape {a.shape}")
|
|
29
|
+
if a.dtype == bool:
|
|
30
|
+
return a
|
|
31
|
+
if not np.issubdtype(a.dtype, np.number) or not np.isin(a, (0, 1)).all():
|
|
32
|
+
raise ValueError("masks hold only 0/1 or True/False")
|
|
33
|
+
return a.astype(bool)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Value:
|
|
37
|
+
"""f(m) for masks m. Called with an (m × n) array of 0/1 it gives m values; with per_response=True (scored
|
|
38
|
+
targets), an (m × items) array, one column per scored text (see `items`)."""
|
|
39
|
+
|
|
40
|
+
def __init__(self, prompt: Prompt, model=None, target=None, *, function=None, name: str | None = None,
|
|
41
|
+
replace: str | None = None, cache: Cache | str | None = None, batch_size: int = 32,
|
|
42
|
+
per_response: bool = False):
|
|
43
|
+
if not isinstance(prompt, Prompt):
|
|
44
|
+
raise TypeError(f"Value needs a Prompt, not {type(prompt).__name__}")
|
|
45
|
+
if (model is None) == (function is None):
|
|
46
|
+
raise ValueError("Value needs a model and a target, or a function")
|
|
47
|
+
if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size < 1:
|
|
48
|
+
raise ValueError(f"batch_size must be a positive integer, not {batch_size!r}")
|
|
49
|
+
if replace is not None and not isinstance(replace, str):
|
|
50
|
+
raise TypeError("replace must be a string or None")
|
|
51
|
+
self.prompt, self.model, self.target, self.function = prompt, model, target, function
|
|
52
|
+
self.replace, self.batch_size, self.per_response = replace, batch_size, per_response
|
|
53
|
+
self.cache = cache if isinstance(cache, Cache) else Cache(cache)
|
|
54
|
+
self.stats = {"requests": 0, "reused": 0, "calls": 0}
|
|
55
|
+
if model is not None:
|
|
56
|
+
needs = getattr(target, "needs", None)
|
|
57
|
+
if needs not in ("score", "generate"):
|
|
58
|
+
raise TypeError(f"target must be a target such as LogProb, Contrast or Sampled, not "
|
|
59
|
+
f"{type(target).__name__}")
|
|
60
|
+
self.name = model_name(model)
|
|
61
|
+
if needs == "score" and not can_score(model):
|
|
62
|
+
raise WrapperError(f"{type(target).__name__} needs the wrapper's score, which {self.name} lacks "
|
|
63
|
+
f"(a wrapper that can only generate serves Sampled)")
|
|
64
|
+
if needs == "generate" and not can_generate(model):
|
|
65
|
+
raise WrapperError(f"{type(target).__name__} needs the wrapper's generate, which {self.name} lacks")
|
|
66
|
+
if needs == "generate" and per_response:
|
|
67
|
+
raise ValueError("per_response applies to scored targets")
|
|
68
|
+
self.items = target.items(prompt) if needs == "score" else []
|
|
69
|
+
if needs == "generate":
|
|
70
|
+
self.stats["generations"] = 0
|
|
71
|
+
else:
|
|
72
|
+
if not callable(function):
|
|
73
|
+
raise TypeError("function must be callable: masked prompts → numbers")
|
|
74
|
+
if target is not None or per_response:
|
|
75
|
+
raise ValueError("a function gives the value itself: no target, no per_response")
|
|
76
|
+
if name is not None and (not isinstance(name, str) or not name):
|
|
77
|
+
raise ValueError("name must be a non-empty string; it keys the cache")
|
|
78
|
+
self.name, self.items = name, []
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def n(self) -> int:
|
|
82
|
+
return self.prompt.n
|
|
83
|
+
|
|
84
|
+
def __call__(self, masks) -> np.ndarray:
|
|
85
|
+
rows = as_masks(masks, self.n)
|
|
86
|
+
return self._values(self._render(rows), per_response=self.per_response)
|
|
87
|
+
|
|
88
|
+
def replies(self, masks, repeat: int = 0) -> list[list[str]]:
|
|
89
|
+
"""The sampled target's replies to each masked prompt (cached like everything else)."""
|
|
90
|
+
if getattr(self.target, "needs", None) != "generate":
|
|
91
|
+
raise ValueError("replies come from a sampled target")
|
|
92
|
+
rendered = self._render(as_masks(masks, self.n))
|
|
93
|
+
sampling = self.target.sampling_for(repeat)
|
|
94
|
+
requests = [self._request("generate", p, None, repeat, sampling) for p in rendered]
|
|
95
|
+
answers = self._ask(requests, lambda batch: self._generate(batch, sampling))
|
|
96
|
+
return [answers[key] for key, _, _ in requests]
|
|
97
|
+
|
|
98
|
+
def noise(self, repeats: int = 3) -> dict:
|
|
99
|
+
"""The spread of the full prompt's value: from the main requests and from `repeats` fresh ones, each
|
|
100
|
+
cached under its own key, so a rerun gives the same figures. A sampled target samples a repeat k with seed
|
|
101
|
+
+ k, so its spread is the sampling's."""
|
|
102
|
+
if isinstance(repeats, bool) or not isinstance(repeats, int) or repeats < 1:
|
|
103
|
+
raise ValueError("repeats must be a positive integer")
|
|
104
|
+
full = self._render(np.ones((1, self.n), dtype=bool))
|
|
105
|
+
values = [float(self._values(full, repeat=k)[0]) for k in range(repeats + 1)]
|
|
106
|
+
return {"values": values, "std": float(np.std(values, ddof=1))}
|
|
107
|
+
|
|
108
|
+
# -- inside ---------------------------------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
def _render(self, rows: np.ndarray) -> list:
|
|
111
|
+
done: dict[bytes, object] = {}
|
|
112
|
+
out = []
|
|
113
|
+
for row in rows:
|
|
114
|
+
key = row.tobytes()
|
|
115
|
+
if key not in done:
|
|
116
|
+
done[key] = render(self.prompt, row.tolist(), self.replace)
|
|
117
|
+
out.append(done[key])
|
|
118
|
+
return out
|
|
119
|
+
|
|
120
|
+
def _values(self, rendered: list, repeat: int = 0, per_response: bool = False) -> np.ndarray:
|
|
121
|
+
needs = getattr(self.target, "needs", None)
|
|
122
|
+
if self.model is not None and needs == "score":
|
|
123
|
+
requests = [[self._request("score", p, item.text, repeat) for item in self.items] for p in rendered]
|
|
124
|
+
answers = self._ask(sum(requests, []), self._score)
|
|
125
|
+
columns = np.array([[reduce_scored(answers[key], item) for (key, _, _), item in zip(row, self.items)]
|
|
126
|
+
for row in requests], dtype=float).reshape(len(rendered), len(self.items))
|
|
127
|
+
out = columns if per_response else self.target.combine(columns, self.items)
|
|
128
|
+
elif self.model is not None:
|
|
129
|
+
sampling = self.target.sampling_for(repeat)
|
|
130
|
+
requests = [self._request("generate", p, None, repeat, sampling) for p in rendered]
|
|
131
|
+
answers = self._ask(requests, lambda batch: self._generate(batch, sampling))
|
|
132
|
+
out = np.array([self.target.value(self.prompt, answers[key]) for key, _, _ in requests], dtype=float)
|
|
133
|
+
elif self.name is None: # an unnamed function: nothing is cached
|
|
134
|
+
out = np.array(self._apply([(None, p, None) for p in rendered]), dtype=float)
|
|
135
|
+
else:
|
|
136
|
+
requests = [self._request("function", p, None, repeat) for p in rendered]
|
|
137
|
+
answers = self._ask(requests, self._apply)
|
|
138
|
+
out = np.array([answers[key] for key, _, _ in requests], dtype=float)
|
|
139
|
+
if not np.isfinite(out).all():
|
|
140
|
+
raise ValueError("some values are not finite: a response with probability 1 under logodds, or a "
|
|
141
|
+
"function giving inf or nan")
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
def _request(self, kind: str, prompt, text, repeat: int, sampling: Sampling | None = None):
|
|
145
|
+
req = {"kind": kind, "model" if kind != "function" else "name": self.name, "prompt": prompt}
|
|
146
|
+
if text is not None:
|
|
147
|
+
req["response"] = text
|
|
148
|
+
if sampling is not None:
|
|
149
|
+
req["sampling"] = sampling.as_dict()
|
|
150
|
+
if repeat:
|
|
151
|
+
req["repeat"] = repeat
|
|
152
|
+
return (Cache.key(req), prompt, text)
|
|
153
|
+
|
|
154
|
+
def _ask(self, requests: list, run) -> dict:
|
|
155
|
+
"""Each distinct request answered once: from the cache, else by `run` in batches, each stored at once."""
|
|
156
|
+
distinct = {key: (key, p, t) for key, p, t in requests}
|
|
157
|
+
found = self.cache.get_many(distinct)
|
|
158
|
+
self.stats["reused"] += len(found)
|
|
159
|
+
missing = [r for key, r in distinct.items() if key not in found]
|
|
160
|
+
kind = "function" if self.model is None else self.target.needs
|
|
161
|
+
for i in range(0, len(missing), self.batch_size):
|
|
162
|
+
batch = missing[i:i + self.batch_size]
|
|
163
|
+
fresh = dict(zip((key for key, _, _ in batch), run(batch)))
|
|
164
|
+
self.cache.put_many(kind, fresh)
|
|
165
|
+
found.update(fresh)
|
|
166
|
+
if self.model is not None and self.target.needs == "score":
|
|
167
|
+
return {key: Scored(tuple(a["logprobs"]), None if a["tokens"] is None else tuple(a["tokens"]))
|
|
168
|
+
for key, a in found.items()}
|
|
169
|
+
return found
|
|
170
|
+
|
|
171
|
+
def _score(self, batch: list) -> list:
|
|
172
|
+
answers = call_score(self.model, [copy.deepcopy(p) for _, p, _ in batch], [t for _, _, t in batch])
|
|
173
|
+
self.stats["requests"] += len(batch)
|
|
174
|
+
self.stats["calls"] += 1
|
|
175
|
+
return [{"logprobs": list(a.logprobs), "tokens": None if a.tokens is None else list(a.tokens)}
|
|
176
|
+
for a in answers]
|
|
177
|
+
|
|
178
|
+
def _generate(self, batch: list, sampling: Sampling) -> list:
|
|
179
|
+
replies = call_generate(self.model, [copy.deepcopy(p) for _, p, _ in batch], sampling)
|
|
180
|
+
self.stats["requests"] += len(batch)
|
|
181
|
+
self.stats["calls"] += 1
|
|
182
|
+
self.stats["generations"] = self.stats.get("generations", 0) + len(batch) * sampling.n
|
|
183
|
+
return replies
|
|
184
|
+
|
|
185
|
+
def _apply(self, batch: list) -> list:
|
|
186
|
+
values = []
|
|
187
|
+
for i in range(0, len(batch), self.batch_size):
|
|
188
|
+
chunk = batch[i:i + self.batch_size]
|
|
189
|
+
out = self.function([copy.deepcopy(p) for _, p, _ in chunk])
|
|
190
|
+
self.stats["requests"] += len(chunk)
|
|
191
|
+
self.stats["calls"] += 1
|
|
192
|
+
try:
|
|
193
|
+
out = [float(x) for x in out]
|
|
194
|
+
except (TypeError, ValueError):
|
|
195
|
+
raise TypeError("the function must give one number per masked prompt") from None
|
|
196
|
+
if len(out) != len(chunk):
|
|
197
|
+
raise ValueError(f"the function gave {len(out)} numbers for {len(chunk)} masked prompts")
|
|
198
|
+
if not all(math.isfinite(x) for x in out):
|
|
199
|
+
raise ValueError("the function gave a value that is not finite")
|
|
200
|
+
values.extend(out)
|
|
201
|
+
return values
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""What a user's wrapper must do, the check that it does, and a toy wrapper (docs/wrapper.md)."""
|
|
2
|
+
|
|
3
|
+
from .check import ModelReport, check_model
|
|
4
|
+
from .contract import (Generator, Rendered, Sampling, Scored, Scorer, WrapperError, call_generate, call_score,
|
|
5
|
+
can_generate, can_score, model_name)
|
|
6
|
+
from .toy import ToyModel
|
|
7
|
+
|
|
8
|
+
__all__ = ["Generator", "ModelReport", "Rendered", "Sampling", "Scored", "Scorer", "ToyModel", "WrapperError",
|
|
9
|
+
"call_generate", "call_score", "can_generate", "can_score", "check_model", "model_name"]
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""check_model (docs/wrapper.md): about a dozen calls that tell a user whether their wrapper meets the contract.
|
|
2
|
+
|
|
3
|
+
Every problem is reported, none raised: a check FAILs when the wrapper breaks the contract or its answers cannot
|
|
4
|
+
serve attribution, and gives a note when something is legal but worth knowing (answers that vary, seeds ignored).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import time
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
from ..prompt import Prompt, Unit
|
|
14
|
+
from .contract import (Sampling, Scored, WrapperError, call_generate, call_score, can_generate, can_score,
|
|
15
|
+
model_name)
|
|
16
|
+
|
|
17
|
+
OK, FAIL, NOTE, INFO, SKIP = "ok", "FAIL", "note", "info", "skip"
|
|
18
|
+
DEFAULT_RESPONSE = "It is hot and humid, because Carnival falls in the Brazilian summer."
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class Check:
|
|
23
|
+
name: str
|
|
24
|
+
status: str
|
|
25
|
+
detail: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class ModelReport:
|
|
30
|
+
"""What check_model found: each check, and the targets the wrapper supports."""
|
|
31
|
+
|
|
32
|
+
model: str
|
|
33
|
+
checks: list[Check] = field(default_factory=list)
|
|
34
|
+
scores: bool = False
|
|
35
|
+
generates: bool = False
|
|
36
|
+
spans: bool = False
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def ok(self) -> bool:
|
|
40
|
+
"""Ready for attribution: a name, a working method, and no failed check."""
|
|
41
|
+
return (self.scores or self.generates) and not any(c.status == FAIL for c in self.checks)
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def targets(self) -> list[str]:
|
|
45
|
+
"""The targets this wrapper can serve: the scored ones need `score`, Sampled needs only `generate`."""
|
|
46
|
+
return (["LogProb", "Pooled", "Contrast", "Choice"] if self.scores else []) + \
|
|
47
|
+
(["Sampled"] if self.generates else [])
|
|
48
|
+
|
|
49
|
+
def add(self, name: str, status: str, detail: str) -> None:
|
|
50
|
+
self.checks.append(Check(name, status, detail))
|
|
51
|
+
|
|
52
|
+
def __str__(self) -> str:
|
|
53
|
+
width = max([len(c.name) for c in self.checks] + [0])
|
|
54
|
+
lines = [f"check_model: {self.model}"]
|
|
55
|
+
lines += [f" {c.status:<4} {c.name:<{width}} {c.detail}" for c in self.checks]
|
|
56
|
+
spans = (" (spans: yes)" if self.spans else " (spans: no)") if self.scores else ""
|
|
57
|
+
lines.append(f" targets: {', '.join(self.targets) or 'none yet'}{spans}")
|
|
58
|
+
lines.append(" ready: every check passed" if self.ok else " NOT ready for attribution: see the FAIL lines")
|
|
59
|
+
return "\n".join(lines)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def default_prompt() -> Prompt:
|
|
63
|
+
"""The prompt check_model uses unless given one: the interface's Rio example."""
|
|
64
|
+
return Prompt("check", messages=[
|
|
65
|
+
{"role": "system", "content": "Answer in one sentence."},
|
|
66
|
+
{"role": "user", "content": [
|
|
67
|
+
"Documents:\n",
|
|
68
|
+
Unit("d1", "Rio Carnival takes place before Lent, in February or March.", sep="\n"),
|
|
69
|
+
Unit("d2", "Summer in Brazil runs from December to March.", sep="\n"),
|
|
70
|
+
Unit("d3", "Winter in Brazil runs from June to September.", sep="\n"),
|
|
71
|
+
"Question: What is the weather like during Rio Carnival?"]}])
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def check_model(model, prompt: Prompt | None = None, response: str | None = None) -> ModelReport:
|
|
75
|
+
"""Test a wrapper against the contract on `prompt` (default: the Rio example; give a text prompt for a model that
|
|
76
|
+
takes plain text) and `response` (default: the model's own greedy reply, else a fixed sentence)."""
|
|
77
|
+
prompt = prompt if prompt is not None else default_prompt()
|
|
78
|
+
full, empty = prompt.render(), prompt.render([0] * prompt.n)
|
|
79
|
+
try:
|
|
80
|
+
name = model_name(model)
|
|
81
|
+
except WrapperError as e:
|
|
82
|
+
report = ModelReport(type(model).__name__)
|
|
83
|
+
report.add("name", FAIL, str(e))
|
|
84
|
+
return report
|
|
85
|
+
report = ModelReport(name)
|
|
86
|
+
report.add("name", OK, repr(name))
|
|
87
|
+
has = [m for m, ok in (("score", can_score(model)), ("generate", can_generate(model))) if ok]
|
|
88
|
+
report.add("methods", OK if has else FAIL, ", ".join(has) if has else "neither score nor generate")
|
|
89
|
+
timings: list[tuple[float, int]] = []
|
|
90
|
+
|
|
91
|
+
def timed(fn, *args):
|
|
92
|
+
start = time.perf_counter()
|
|
93
|
+
out = fn(*args)
|
|
94
|
+
timings.append((time.perf_counter() - start, len(args[1])))
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
reply = None
|
|
98
|
+
if "generate" in has:
|
|
99
|
+
try:
|
|
100
|
+
greedy = timed(call_generate, model, [full, full], Sampling(n=2, temperature=0.0))
|
|
101
|
+
report.generates = True
|
|
102
|
+
report.add("generate", OK, f"2 greedy replies to each of 2 prompts, e.g. {greedy[0][0][:60]!r}")
|
|
103
|
+
again = timed(call_generate, model, [full], Sampling(n=1, temperature=0.0))
|
|
104
|
+
same = len({greedy[0][0], greedy[0][1], greedy[1][0], greedy[1][1], again[0][0]}) == 1
|
|
105
|
+
report.add("greedy repeats", OK if same else NOTE,
|
|
106
|
+
"identical" if same else "greedy replies vary: sampled targets carry that noise")
|
|
107
|
+
s1 = timed(call_generate, model, [full], Sampling(n=1, temperature=1.0, seed=7))
|
|
108
|
+
s2 = timed(call_generate, model, [full], Sampling(n=1, temperature=1.0, seed=7))
|
|
109
|
+
report.add("seed", OK if s1 == s2 else NOTE, "honoured" if s1 == s2 else
|
|
110
|
+
"two calls with seed 7 differ: a rerun samples anew where the cache has no answer")
|
|
111
|
+
reply = greedy[0][0] if greedy[0][0].strip() else None
|
|
112
|
+
except Exception as e: # noqa: BLE001 - every failure is reported, not raised
|
|
113
|
+
report.add("generate", FAIL, f"{type(e).__name__}: {e}")
|
|
114
|
+
if "score" in has:
|
|
115
|
+
r = response if response is not None else (reply or DEFAULT_RESPONSE)
|
|
116
|
+
try:
|
|
117
|
+
a = timed(call_score, model, [full, full], [r, r])
|
|
118
|
+
b = timed(call_score, model, [full], [r])
|
|
119
|
+
if not a[0].logprobs:
|
|
120
|
+
raise WrapperError("score gave no tokens for a non-empty response")
|
|
121
|
+
report.scores = True
|
|
122
|
+
report.add("score", OK, f"{len(a[0].logprobs)} tokens, finite log-probabilities at most 0")
|
|
123
|
+
diff = max(_diff(a[0], a[1]), _diff(a[0], b[0]))
|
|
124
|
+
report.add("score repeats", OK if diff == 0 else NOTE, "identical" if diff == 0 else
|
|
125
|
+
f"differ by up to {diff:.2g}: the noise check measures this")
|
|
126
|
+
if a[0].tokens is None:
|
|
127
|
+
report.add("token texts", NOTE, "none given: targets cannot use spans")
|
|
128
|
+
elif "".join(a[0].tokens) == r:
|
|
129
|
+
report.spans = True
|
|
130
|
+
report.add("token texts", OK, "join back to the response: targets can use spans")
|
|
131
|
+
else:
|
|
132
|
+
report.add("token texts", FAIL, "given, but they do not join back to the response exactly")
|
|
133
|
+
c = timed(call_score, model, [empty], [r])
|
|
134
|
+
m_full, m_empty = _mean(a[0]), _mean(c[0])
|
|
135
|
+
report.add("prompt matters", OK if m_full != m_empty else FAIL,
|
|
136
|
+
f"{m_full:.3f} per token with every unit, {m_empty:.3f} with none" +
|
|
137
|
+
("" if m_full != m_empty else ": the score ignores the prompt; does it reach the model?"))
|
|
138
|
+
words = r.split()
|
|
139
|
+
backwards = " ".join(reversed(words))
|
|
140
|
+
if len(set(words)) < 2 or backwards == r:
|
|
141
|
+
report.add("reply above reversed", SKIP, "the response has too few distinct words")
|
|
142
|
+
else:
|
|
143
|
+
d = timed(call_score, model, [full, full], [r, backwards])
|
|
144
|
+
m1, m2 = _mean(d[0]), _mean(d[1])
|
|
145
|
+
report.add("reply above reversed", OK if m1 > m2 else NOTE,
|
|
146
|
+
f"{m1:.3f} per token, {m2:.3f} with its words reversed" + ("" if m1 > m2 else
|
|
147
|
+
": a model blind to word order, or the response not placed as the model's reply"))
|
|
148
|
+
except Exception as e: # noqa: BLE001
|
|
149
|
+
report.add("score", FAIL, f"{type(e).__name__}: {e}")
|
|
150
|
+
if timings:
|
|
151
|
+
seconds, requests = sum(t for t, _ in timings), sum(k for _, k in timings)
|
|
152
|
+
report.add("time", INFO, f"{seconds / requests:.3g} s per request over {len(timings)} calls")
|
|
153
|
+
return report
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _mean(s: Scored) -> float:
|
|
157
|
+
return math.fsum(s.logprobs) / len(s.logprobs) if s.logprobs else math.nan
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _diff(x: Scored, y: Scored) -> float:
|
|
161
|
+
if len(x.logprobs) != len(y.logprobs):
|
|
162
|
+
return math.inf
|
|
163
|
+
return max((abs(p - q) for p, q in zip(x.logprobs, y.logprobs)), default=0.0)
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""What a user's wrapper must do (docs/wrapper.md), and the checks applied to every answer it gives.
|
|
2
|
+
|
|
3
|
+
A wrapper is any object with a `name` and at least one of two methods:
|
|
4
|
+
|
|
5
|
+
score(prompts, responses) -> list[Scored] the log-probability of each response token, given its prompt
|
|
6
|
+
generate(prompts, sampling) -> list[list[str]] sampling.n replies to each prompt
|
|
7
|
+
|
|
8
|
+
Prompts reach it rendered (`Rendered`): a string, or chat messages [{"role": ..., "content": "..."}]. The wrapper
|
|
9
|
+
applies its model's own chat template, batches as it likes, and raises on errors; it never truncates a prompt.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
import numbers
|
|
16
|
+
from collections.abc import Sequence
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Protocol, Union, runtime_checkable
|
|
19
|
+
|
|
20
|
+
Rendered = Union[str, list[dict[str, str]]]
|
|
21
|
+
|
|
22
|
+
# A log-probability above 0 is impossible; rounding in low precision can give a hair above it.
|
|
23
|
+
LOGPROB_TOLERANCE = 1e-3
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class WrapperError(RuntimeError):
|
|
27
|
+
"""A wrapper broke the contract: its answer cannot be used."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Sampling:
|
|
32
|
+
"""How `generate` samples: `n` replies per prompt; temperature 0 is greedy."""
|
|
33
|
+
|
|
34
|
+
n: int = 1
|
|
35
|
+
temperature: float = 0.0
|
|
36
|
+
top_p: float = 1.0
|
|
37
|
+
max_tokens: int = 256
|
|
38
|
+
seed: int | None = 0
|
|
39
|
+
stop: tuple[str, ...] = ()
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
if not _is_int(self.n) or self.n < 1:
|
|
43
|
+
raise ValueError(f"Sampling.n must be an integer of at least 1, not {self.n!r}")
|
|
44
|
+
if not _is_real(self.temperature) or not 0 <= self.temperature < math.inf:
|
|
45
|
+
raise ValueError(f"Sampling.temperature must be a finite number of at least 0, not {self.temperature!r}")
|
|
46
|
+
if not _is_real(self.top_p) or not 0 < self.top_p <= 1:
|
|
47
|
+
raise ValueError(f"Sampling.top_p must lie in (0, 1], not {self.top_p!r}")
|
|
48
|
+
if not _is_int(self.max_tokens) or self.max_tokens < 1:
|
|
49
|
+
raise ValueError(f"Sampling.max_tokens must be an integer of at least 1, not {self.max_tokens!r}")
|
|
50
|
+
if self.seed is not None and not _is_int(self.seed):
|
|
51
|
+
raise ValueError(f"Sampling.seed must be an integer or None, not {self.seed!r}")
|
|
52
|
+
stop = (self.stop,) if isinstance(self.stop, str) else tuple(self.stop)
|
|
53
|
+
if not all(isinstance(s, str) and s for s in stop):
|
|
54
|
+
raise ValueError(f"Sampling.stop must hold non-empty strings, not {self.stop!r}")
|
|
55
|
+
object.__setattr__(self, "stop", stop)
|
|
56
|
+
|
|
57
|
+
def as_dict(self) -> dict:
|
|
58
|
+
return {"n": self.n, "temperature": float(self.temperature), "top_p": float(self.top_p),
|
|
59
|
+
"max_tokens": self.max_tokens, "seed": self.seed, "stop": list(self.stop)}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True)
|
|
63
|
+
class Scored:
|
|
64
|
+
"""What `score` gives for one response: the natural-log probability of each response token, in order, and
|
|
65
|
+
optionally each token's text; the texts must join back to the response exactly, and are needed only for spans."""
|
|
66
|
+
|
|
67
|
+
logprobs: tuple[float, ...]
|
|
68
|
+
tokens: tuple[str, ...] | None = None
|
|
69
|
+
|
|
70
|
+
def __post_init__(self) -> None:
|
|
71
|
+
if isinstance(self.logprobs, (str, bytes)):
|
|
72
|
+
raise TypeError("Scored.logprobs must be a sequence of numbers")
|
|
73
|
+
try:
|
|
74
|
+
logprobs = tuple(float(x) for x in self.logprobs)
|
|
75
|
+
except (TypeError, ValueError):
|
|
76
|
+
raise TypeError("Scored.logprobs must be a sequence of numbers") from None
|
|
77
|
+
for k, x in enumerate(logprobs):
|
|
78
|
+
if not math.isfinite(x) or x > LOGPROB_TOLERANCE:
|
|
79
|
+
raise ValueError(f"Scored.logprobs[{k}] is {x}: a log-probability must be finite and at most 0")
|
|
80
|
+
object.__setattr__(self, "logprobs", logprobs)
|
|
81
|
+
if self.tokens is not None:
|
|
82
|
+
if isinstance(self.tokens, (str, bytes)):
|
|
83
|
+
raise TypeError("Scored.tokens must be a sequence of strings, one per token")
|
|
84
|
+
tokens = tuple(self.tokens)
|
|
85
|
+
if not all(isinstance(t, str) for t in tokens):
|
|
86
|
+
raise TypeError("Scored.tokens must be a sequence of strings, one per token")
|
|
87
|
+
if len(tokens) != len(logprobs):
|
|
88
|
+
raise ValueError(f"Scored has {len(logprobs)} log-probabilities but {len(tokens)} token texts")
|
|
89
|
+
object.__setattr__(self, "tokens", tokens)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@runtime_checkable
|
|
93
|
+
class Scorer(Protocol):
|
|
94
|
+
"""A wrapper that can score given responses."""
|
|
95
|
+
|
|
96
|
+
name: str
|
|
97
|
+
|
|
98
|
+
def score(self, prompts: Sequence[Rendered], responses: Sequence[str]) -> list[Scored]: ...
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@runtime_checkable
|
|
102
|
+
class Generator(Protocol):
|
|
103
|
+
"""A wrapper that can generate replies."""
|
|
104
|
+
|
|
105
|
+
name: str
|
|
106
|
+
|
|
107
|
+
def generate(self, prompts: Sequence[Rendered], sampling: Sampling) -> list[list[str]]: ...
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def model_name(model) -> str:
|
|
111
|
+
"""The wrapper's name, which every cache key holds; it must change whenever the answers could change."""
|
|
112
|
+
name = getattr(model, "name", None)
|
|
113
|
+
if not isinstance(name, str) or not name:
|
|
114
|
+
raise WrapperError(f"a wrapper needs a non-empty string `name`; {type(model).__name__} has {name!r}")
|
|
115
|
+
return name
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def can_score(model) -> bool:
|
|
119
|
+
return callable(getattr(model, "score", None))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def can_generate(model) -> bool:
|
|
123
|
+
return callable(getattr(model, "generate", None))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def call_score(model, prompts: list[Rendered], responses: list[str]) -> list[Scored]:
|
|
127
|
+
"""Ask the wrapper to score, and check its answer's shape."""
|
|
128
|
+
name = model_name(model)
|
|
129
|
+
out = model.score(prompts, responses)
|
|
130
|
+
if isinstance(out, (str, bytes, dict)) or not hasattr(out, "__len__"):
|
|
131
|
+
raise WrapperError(f"{name}.score returned {type(out).__name__}, not a list of Scored")
|
|
132
|
+
out = list(out)
|
|
133
|
+
if len(out) != len(prompts):
|
|
134
|
+
raise WrapperError(f"{name}.score returned {len(out)} answers for {len(prompts)} requests")
|
|
135
|
+
for k, s in enumerate(out):
|
|
136
|
+
if not isinstance(s, Scored):
|
|
137
|
+
raise WrapperError(f"{name}.score answer {k} is {type(s).__name__}, not Scored")
|
|
138
|
+
return out
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def call_generate(model, prompts: list[Rendered], sampling: Sampling) -> list[list[str]]:
|
|
142
|
+
"""Ask the wrapper to generate, and check that every prompt got exactly sampling.n strings."""
|
|
143
|
+
name = model_name(model)
|
|
144
|
+
out = model.generate(prompts, sampling)
|
|
145
|
+
if isinstance(out, (str, bytes, dict)) or not hasattr(out, "__len__"):
|
|
146
|
+
raise WrapperError(f"{name}.generate returned {type(out).__name__}, not a list of lists of strings")
|
|
147
|
+
out = list(out)
|
|
148
|
+
if len(out) != len(prompts):
|
|
149
|
+
raise WrapperError(f"{name}.generate returned replies for {len(out)} prompts, not {len(prompts)}")
|
|
150
|
+
replies = []
|
|
151
|
+
for k, r in enumerate(out):
|
|
152
|
+
if isinstance(r, (str, bytes)) or not hasattr(r, "__len__"):
|
|
153
|
+
raise WrapperError(f"{name}.generate answer {k} is {type(r).__name__}, not a list of strings")
|
|
154
|
+
r = list(r)
|
|
155
|
+
if len(r) != sampling.n or not all(isinstance(x, str) for x in r):
|
|
156
|
+
raise WrapperError(f"{name}.generate answer {k} must hold {sampling.n} strings (sampling.n)")
|
|
157
|
+
replies.append(r)
|
|
158
|
+
return replies
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _is_int(x) -> bool:
|
|
162
|
+
return isinstance(x, numbers.Integral) and not isinstance(x, bool)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _is_real(x) -> bool:
|
|
166
|
+
return isinstance(x, numbers.Real) and not isinstance(x, bool)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Your model's wrapper, to fill in (`contextattr template my_wrapper.py` wrote this file). ContextAttr reaches a
|
|
2
|
+
model only through an object like this one: a `name`, and `generate`, `score` or both.
|
|
3
|
+
|
|
4
|
+
1. Fill in `generate`, `score` or both, and delete the one you leave out.
|
|
5
|
+
generate writes replies to a prompt. Every model and API can do this. Serves the Sampled target.
|
|
6
|
+
score gives the log-probability of each token of a reply you supply. Possible where the backend returns
|
|
7
|
+
log-probabilities of given text: a model in your own process, a vLLM or SGLang server. Serves
|
|
8
|
+
LogProb, Pooled, Contrast and Choice, which are cheaper and repeat exactly: write it if you can.
|
|
9
|
+
2. Run this file: `python my_wrapper.py`. check_model calls your wrapper about a dozen times and says what is wrong.
|
|
10
|
+
3. Use it: `ca.explain(prompt, MyModel(), ...)`, or `use = "my_wrapper.py:MyModel"` in a run file.
|
|
11
|
+
|
|
12
|
+
The guide, with two complete wrappers to copy from:
|
|
13
|
+
https://github.com/Surya-Rayala/contextattr/blob/main/docs/wrapper.md
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from contextattr import Sampling, Scored, check_model # noqa: F401
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class MyModel:
|
|
20
|
+
# Every answer is cached under this name, so change it whenever the answers could change: another model,
|
|
21
|
+
# revision, precision or prompt format.
|
|
22
|
+
name = "my-model"
|
|
23
|
+
|
|
24
|
+
def __init__(self):
|
|
25
|
+
... # load the model, or open the client of your server or API
|
|
26
|
+
|
|
27
|
+
def generate(self, prompts, sampling: Sampling):
|
|
28
|
+
"""Reply to each prompt.
|
|
29
|
+
|
|
30
|
+
prompts a list. Each prompt is chat messages, [{"role": "user", "content": "..."}, ...], or a plain string.
|
|
31
|
+
sampling .n replies per prompt, .temperature (0: always the most likely token), .top_p, .max_tokens,
|
|
32
|
+
.seed, .stop.
|
|
33
|
+
return one list per prompt, holding exactly sampling.n replies (the reply's text only).
|
|
34
|
+
For two prompts and n = 1: [["first reply"], ["second reply"]]
|
|
35
|
+
"""
|
|
36
|
+
raise NotImplementedError("fill in generate, or delete it")
|
|
37
|
+
|
|
38
|
+
def score(self, prompts, responses):
|
|
39
|
+
"""Say how likely the model is to give responses[i] as its reply to prompts[i].
|
|
40
|
+
|
|
41
|
+
prompts as for generate.
|
|
42
|
+
responses a list of strings, one per prompt.
|
|
43
|
+
return one Scored per pair: Scored(logprobs), the natural-log probability of each token of the response,
|
|
44
|
+
in order. For a response of three tokens: Scored([-0.2, -1.3, -0.1])
|
|
45
|
+
|
|
46
|
+
Format the prompt as your model expects (its chat template), put the response where the model's reply would
|
|
47
|
+
begin, and read the log-probability the model gives each token of the response. Leave out the end-of-turn
|
|
48
|
+
token. Optional: Scored(logprobs, tokens), with each token's text, lets a target cover part of a response.
|
|
49
|
+
"""
|
|
50
|
+
raise NotImplementedError("fill in score, or delete it")
|
|
51
|
+
|
|
52
|
+
# In both: raise an error on a failure or on a prompt that is too long. Never cut a prompt short.
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
if __name__ == "__main__":
|
|
56
|
+
print(check_model(MyModel()))
|