contextattr 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
contextattr/result.py ADDED
@@ -0,0 +1,120 @@
1
+ """Explanation: what explain and attribute give (docs/interface.md §8)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from dataclasses import dataclass
7
+
8
+ import numpy as np
9
+
10
+ from .methods import indices
11
+ from .methods.base import Surrogate
12
+ from .prompt import Prompt, prompt_from_dict, prompt_to_dict
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class Explanation:
17
+ """The score of each unit (positive: it pushes the value up), how faithful the surrogate is on held-out masks
18
+ (R², SPEX eq. 7), the noise of the value, and what it cost in masks and requests."""
19
+
20
+ method: str
21
+ unit_ids: tuple[str, ...]
22
+ surrogate: Surrogate
23
+ r2: float | None
24
+ masks: int
25
+ test_masks: int
26
+ requests: dict
27
+ noise: dict | None = None
28
+ prompt_id: str | None = None
29
+ unit_texts: tuple[str, ...] | None = None
30
+ tags: tuple[str | None, ...] | None = None
31
+ target: str | None = None
32
+ removal: list[dict] | None = None
33
+ prompt: Prompt | None = None
34
+
35
+ @property
36
+ def scores(self) -> dict[str, float]:
37
+ """Each unit's id → its score."""
38
+ return dict(zip(self.unit_ids, self.surrogate.scores.tolist()))
39
+
40
+ @property
41
+ def interactions(self) -> dict[tuple[str, ...], float]:
42
+ """Each set of two or more units (by id) → what they do together (interaction methods only)."""
43
+ return {tuple(self.unit_ids[i] for i in s): w for s, w in self.surrogate.interactions.items()}
44
+
45
+ def shapley(self) -> dict[str, float]:
46
+ """Each unit's Shapley value, read from the whole surrogate (App. C): its share of every term it is in."""
47
+ return dict(zip(self.unit_ids, indices.shapley(self.surrogate).tolist()))
48
+
49
+ def banzhaf(self) -> dict[str, float]:
50
+ """Each unit's Banzhaf value, read from the whole surrogate (App. C)."""
51
+ return dict(zip(self.unit_ids, indices.banzhaf(self.surrogate).tolist()))
52
+
53
+ def shapley_taylor(self, order: int = 2) -> dict[tuple[str, ...], float]:
54
+ """The Shapley-Taylor index of an order, by unit ids (App. C, eq. 58)."""
55
+ return {tuple(self.unit_ids[i] for i in s): w for s, w in indices.shapley_taylor(self.surrogate, order).items()}
56
+
57
+ def top_interactions(self, k: int = 5) -> list[tuple[tuple[str, ...], float]]:
58
+ """The k interactions of largest absolute value."""
59
+ return sorted(self.interactions.items(), key=lambda t: -abs(t[1]))[:k]
60
+
61
+ def top(self, k: int = 5) -> list[tuple[str, float]]:
62
+ """The k units with the highest scores: those that push the value up most."""
63
+ order = np.argsort(-self.surrogate.scores, kind="stable")[:k]
64
+ return [(self.unit_ids[i], float(self.surrogate.scores[i])) for i in order]
65
+
66
+ def table(self, width: int = 60) -> str:
67
+ """A plain table of every unit, highest score first."""
68
+ noise = self.noise["std"] if self.noise else None
69
+ head = (f"{self.prompt_id or 'attribute'} · {self.method} · {self.masks} masks · "
70
+ f"R² {_fmt(self.r2)} on {self.test_masks} test masks · noise {_fmt(noise)} · "
71
+ f"{self.requests.get('requests', 0)} requests")
72
+ rows = []
73
+ for i in np.argsort(-self.surrogate.scores, kind="stable"):
74
+ text = "" if self.unit_texts is None else self.unit_texts[i].replace("\n", " ")
75
+ text = text if len(text) <= width else text[: width - 1] + "…"
76
+ rows.append(f" {self.unit_ids[i]:<12} {self.surrogate.scores[i]:+10.4f} {text}")
77
+ for s, w in self.top_interactions(5):
78
+ rows.append(f" {' & '.join(s):<12} {w:+10.4f} (together)")
79
+ for r in self.removal or []:
80
+ removed = ", ".join(self.unit_ids[i] for i in r["removed"])
81
+ rows.append(f" removing {removed}: the value moves by {r['change']:+.4f} (Rem {_fmt(r['rem'])})")
82
+ return "\n".join([head, *rows])
83
+
84
+ def as_dict(self) -> dict:
85
+ return {"prompt": self.prompt_id, "method": self.method, "target": self.target, "units": list(self.unit_ids),
86
+ "tags": None if self.tags is None else list(self.tags), "scores": self.surrogate.scores.tolist(),
87
+ "intercept": self.surrogate.intercept,
88
+ "interactions": [{"units": list(s), "score": w} for s, w in self.interactions.items()],
89
+ "r2": self.r2, "masks": self.masks, "test_masks": self.test_masks, "requests": dict(self.requests),
90
+ "noise": self.noise, "removal": self.removal,
91
+ "unit_texts": None if self.unit_texts is None else list(self.unit_texts)}
92
+
93
+ @classmethod
94
+ def from_dict(cls, d: dict, prompt: dict | Prompt | None = None) -> Explanation:
95
+ """The Explanation as_dict gave, with its prompt when given (as a Prompt or its JSON object)."""
96
+ units = list(d["units"])
97
+ surrogate = Surrogate(intercept=d["intercept"], scores=d["scores"], interactions={
98
+ tuple(units.index(u) for u in t["units"]): t["score"] for t in d["interactions"]})
99
+ if isinstance(prompt, dict):
100
+ prompt = prompt_from_dict(prompt)
101
+ return cls(method=d["method"], unit_ids=tuple(units), surrogate=surrogate, r2=d["r2"], masks=d["masks"],
102
+ test_masks=d["test_masks"], requests=dict(d["requests"]), noise=d["noise"], prompt_id=d["prompt"],
103
+ unit_texts=None if d.get("unit_texts") is None else tuple(d["unit_texts"]),
104
+ tags=None if d["tags"] is None else tuple(d["tags"]), target=d["target"], removal=d["removal"],
105
+ prompt=prompt)
106
+
107
+ def to_html(self, path=None) -> str:
108
+ """This explanation as an HTML page (report.py), written to `path` when given."""
109
+ from .report import page, write # noqa: PLC0415
110
+
111
+ if path is not None:
112
+ write([self], path)
113
+ return page([self])
114
+
115
+ def __str__(self) -> str:
116
+ return self.table()
117
+
118
+
119
+ def _fmt(x) -> str:
120
+ return "n/a" if x is None or (isinstance(x, float) and math.isnan(x)) else f"{x:.3g}"
contextattr/run.py ADDED
@@ -0,0 +1,306 @@
1
+ """The ways in (docs/interface.md §1, §8): explain a prompt through a model and a target, or through the user's own
2
+ function of masked prompts; attribute any function of masks; and Run, a prompt file explained with one setup."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import hashlib
7
+ import json
8
+ import os
9
+ import time
10
+ from datetime import datetime, timezone
11
+ from pathlib import Path
12
+
13
+ import numpy as np
14
+
15
+ from .cache import Cache
16
+ from .evaluate import faithfulness, held_out_masks, removal as removal_curve
17
+ from .methods import ShapiqMethod, get_method
18
+ from .prompt import Prompt, prompt_to_dict, read_prompts
19
+ from .responses.checks import describe
20
+ from .result import Explanation
21
+ from .targets import Contrast, LogProb, Pooled
22
+ from .value import Value
23
+ from .wrapper.contract import Sampling, model_name
24
+
25
+
26
+ def explain(prompt: Prompt, model=None, target=None, *, value=None, name: str | None = None,
27
+ method="contextcite", budget: int = 64, test_masks: int = 128, seed: int = 0,
28
+ replace: str | None = None, cache: Cache | str | None = None, batch_size: int = 32,
29
+ noise_repeats: int = 3, removal: tuple[int, ...] = ()) -> Explanation:
30
+ """Explain one prompt. The method's masks go through the model and the target (default: LogProb of the prompt's
31
+ only response) or through `value`, a function of masked prompts; the surrogate is fitted and then checked on
32
+ `test_masks` held-out masks, the same for every method run with this seed, so they are asked once and cached.
33
+ `noise_repeats` fresh values of the full prompt give the noise (0 skips it); `removal`, e.g. (1, 2, 3), the top-r
34
+ removal of SPEX eq. 8, each r one more request."""
35
+ if value is not None:
36
+ if model is not None or target is not None:
37
+ raise ValueError("give a model and a target, or a value function, not both")
38
+ v = Value(prompt, function=value, name=name, replace=replace, cache=cache, batch_size=batch_size)
39
+ target_name = name or "a function"
40
+ else:
41
+ if model is None:
42
+ raise ValueError("explain needs a model (with a target) or a value function")
43
+ target = target if target is not None else LogProb()
44
+ v = Value(prompt, model, target, replace=replace, cache=cache, batch_size=batch_size)
45
+ target_name = repr(target)
46
+ return _run(v, prompt.n, method, budget, test_masks, seed, noise_repeats, removal, prompt=prompt,
47
+ target=target_name)
48
+
49
+
50
+ def attribute(f, n: int, *, method="contextcite", budget: int = 64, test_masks: int = 128, seed: int = 0,
51
+ unit_ids=None, removal: tuple[int, ...] = ()) -> Explanation:
52
+ """Explain any function of masks: f takes an (m × n) array of 0/1, one row per mask, and gives m numbers (inputs
53
+ that are not text, such as image patches or table columns)."""
54
+ if isinstance(n, bool) or not isinstance(n, (int, np.integer)) or n < 1:
55
+ raise ValueError("n must be a positive number of units")
56
+
57
+ def v(masks):
58
+ out = np.asarray(f(np.array(masks, dtype=bool)), dtype=float).reshape(-1) # a copy: f may write into it
59
+ if len(out) != len(masks) or not np.isfinite(out).all():
60
+ raise ValueError("f must give one finite number per mask")
61
+ return out
62
+
63
+ v.stats = {}
64
+ ids = tuple(str(i) for i in range(n)) if unit_ids is None else tuple(unit_ids)
65
+ if len(ids) != n:
66
+ raise ValueError(f"unit_ids has {len(ids)} names for {n} units")
67
+ return _run(v, n, method, budget, test_masks, seed, 0, removal, unit_ids=ids)
68
+
69
+
70
+ def _run(v, n, method, budget, test_masks, seed, noise_repeats, removal, *, prompt=None, target=None, unit_ids=None):
71
+ m = get_method(method)
72
+ surrogate, used = m.run(v, n, budget, np.random.default_rng(seed))
73
+ r2 = None
74
+ if test_masks:
75
+ held_out = held_out_masks(n, test_masks, seed)
76
+ r2 = faithfulness(surrogate.predict(held_out), v(held_out))
77
+ noise = v.noise(noise_repeats) if noise_repeats else None
78
+ curve = removal_curve(surrogate, v, tuple(removal)) if removal else None
79
+ units = prompt.units if prompt is not None else None
80
+ return Explanation(method=repr(m), unit_ids=unit_ids or tuple(u.id for u in units), surrogate=surrogate, r2=r2,
81
+ masks=used, test_masks=test_masks or 0, requests=dict(getattr(v, "stats", {})), removal=curve,
82
+ noise=noise, prompt_id=None if prompt is None else prompt.id,
83
+ unit_texts=None if units is None else tuple(u.text for u in units),
84
+ tags=None if units is None else tuple(u.tag for u in units), target=target,
85
+ prompt=prompt)
86
+
87
+
88
+ class Run:
89
+ """A prompt file explained with one setup (docs/interface.md §8). `plan()` counts masks, requests and tokens before
90
+ any call, and `plan(measure=k)` times k real requests; `execute(out)` explains every prompt into the run directory
91
+ `out` (run.json, cache.sqlite, explanations.jsonl, summary.json, report.html) and resumes a run that stopped.
92
+
93
+ With `sample`, each prompt first gets sample.n replies of the model, labelled by `check`. Without `target`, each
94
+ prompt's responses choose it: Contrast with both labels, Pooled with one, LogProb with a single response."""
95
+
96
+ def __init__(self, model, prompts, *, target=None, sample: Sampling | None = None, check=None,
97
+ method="contextcite", budget: int = 64, test_masks: int = 128, seed: int = 0,
98
+ replace: str | None = None, batch_size: int = 32, noise_repeats: int = 3,
99
+ removal: tuple[int, ...] = ()):
100
+ self.prompts = read_prompts(prompts) if isinstance(prompts, (str, os.PathLike)) else list(prompts)
101
+ ids = [p.id for p in self.prompts]
102
+ if not ids or len(set(ids)) != len(ids) or not all(isinstance(p, Prompt) for p in self.prompts):
103
+ raise ValueError("Run needs one or more prompts with unique ids")
104
+ if sample is not None and not isinstance(sample, Sampling):
105
+ raise TypeError("sample must be a Sampling")
106
+ self.model, self.target, self.sample, self.check = model, target, sample, check
107
+ self.method, self.budget, self.test_masks, self.seed = method, budget, test_masks, seed
108
+ self.replace, self.batch_size, self.noise_repeats, self.removal = replace, batch_size, noise_repeats, tuple(removal)
109
+ get_method(method) # an unknown method fails now, not at the first prompt
110
+
111
+ def target_for(self, prompt: Prompt):
112
+ if self.target is not None:
113
+ return self.target
114
+ labels = {r.label for r in prompt.responses}
115
+ if {"negative", "positive"} <= labels:
116
+ return Contrast()
117
+ for label in ("negative", "positive"):
118
+ if label in labels:
119
+ return Pooled(label)
120
+ if len(prompt.responses) == 1:
121
+ return LogProb()
122
+ raise ValueError(f"prompt {prompt.id!r}: no target given, and its responses choose none (give target=, or "
123
+ f"responses with labels, or sample= with check=)")
124
+
125
+ def settings(self) -> dict:
126
+ """What decides the results; a run directory keeps one set of them."""
127
+ return {"model": model_name(self.model), "method": repr(get_method(self.method)), "budget": self.budget,
128
+ "test_masks": self.test_masks, "seed": self.seed, "replace": self.replace,
129
+ "noise_repeats": self.noise_repeats, "removal": list(self.removal),
130
+ "target": None if self.target is None else repr(self.target),
131
+ "sample": None if self.sample is None else self.sample.as_dict(),
132
+ "check": None if self.check is None else describe(self.check)}
133
+
134
+ def plan(self, measure: int = 0) -> dict:
135
+ """Masks, requests and tokens per prompt and in all, counted before any call: an upper bound, since repeated
136
+ masks are asked once (and a prompt of n units has only 2^n masks). Tokens are estimated as characters / 4 of
137
+ the full prompt and response; masked prompts are shorter. measure=k sends k real requests of the first prompt
138
+ (with a cache of its own) and projects the run's time from their seconds per request."""
139
+ rows = []
140
+ for p in self.prompts:
141
+ m = get_method(self.method)
142
+ masks = self.budget if isinstance(m, ShapiqMethod) else len(m.masks(p.n, self.budget,
143
+ np.random.default_rng(self.seed)))
144
+ distinct = min(masks + (self.test_masks or 0), 2 ** min(p.n, 60)) + (self.noise_repeats or 0)
145
+ sampled = getattr(self.target, "needs", None) == "generate"
146
+ texts = 1 if sampled else (len(p.responses) + self.sample.n if self.sample else
147
+ len(self.target_for(p).items(p)))
148
+ chars = len(json.dumps(p.render(), ensure_ascii=False))
149
+ reply = 4 * 256 if sampled else max([len(r.text) for r in p.responses] + [400])
150
+ requests = distinct * texts + (1 if self.sample else 0)
151
+ generations = (distinct * self.target.samples if sampled else 0) + (self.sample.n if self.sample else 0)
152
+ rows.append({"prompt": p.id, "units": p.n, "masks": masks, "test_masks": self.test_masks or 0,
153
+ "requests": requests, "generations": generations,
154
+ "tokens": int(requests * (chars + reply) / 4)})
155
+ total = {k: sum(r[k] for r in rows) for k in ("masks", "requests", "generations", "tokens")}
156
+ out = {"prompts": rows, "total": total, "measured": None}
157
+ if measure:
158
+ p = self.prompts[0]
159
+ q = p if not self.sample else Prompt(p.id, messages=p.messages, text=p.text, reference=p.reference,
160
+ responses=p.responses or ())
161
+ target = self.target if self.target is not None else (self.target_for(q) if q.responses else None)
162
+ if target is None:
163
+ raise ValueError("measure needs a target or responses on the first prompt")
164
+ v = Value(q, self.model, target, replace=self.replace, batch_size=self.batch_size)
165
+ masks = np.random.default_rng(self.seed).random((measure, q.n)) < 0.5
166
+ start = time.perf_counter()
167
+ v(masks)
168
+ seconds = time.perf_counter() - start
169
+ per = seconds / max(v.stats["requests"], 1)
170
+ out["measured"] = {"requests": v.stats["requests"], "seconds": round(seconds, 3),
171
+ "seconds_per_request": per, "projected_seconds": round(per * total["requests"], 1)}
172
+ return out
173
+
174
+ def execute(self, out) -> Results:
175
+ """Explain every prompt into the run directory `out` and give their explanations, in the order of the
176
+ prompts. A directory that holds a run resumes it: a prompt explained there already is read back, unless it
177
+ has changed since (refused: give a new directory or the prompt a new id), and a last line cut short by a kill
178
+ is removed and its prompt explained again. Each line of explanations.jsonl holds the prompt as explained (with
179
+ sampled responses), a digest of the prompt as given, the seconds it took and the explanation."""
180
+ out = Path(out)
181
+ out.mkdir(parents=True, exist_ok=True)
182
+ settings = self.settings()
183
+ meta = out / "run.json"
184
+ if meta.exists():
185
+ stored = json.loads(meta.read_text(encoding="utf-8"))["settings"]
186
+ if stored != settings:
187
+ raise ValueError(f"{out} holds a run with other settings; give a new directory\n"
188
+ f" stored: {stored}\n now: {settings}")
189
+ else:
190
+ from . import __version__ # noqa: PLC0415
191
+ meta.write_text(json.dumps({"settings": settings, "contextattr": __version__,
192
+ "started_utc": _utc()}, indent=2) + "\n", encoding="utf-8")
193
+ lines = out / "explanations.jsonl"
194
+ done = _rows(lines, repair=True)
195
+ given = {p.id: _digest(p) for p in self.prompts}
196
+ changed = [pid for pid, d in given.items() if pid in done and done[pid].get("input") != d]
197
+ if changed:
198
+ raise ValueError(f"{out}: prompt {', '.join(map(repr, changed))} changed since it was explained there; "
199
+ f"give a new directory, or the prompt a new id")
200
+ cache = Cache(out / "cache.sqlite")
201
+ from .responses import sample as sample_responses # noqa: PLC0415
202
+ for p in self.prompts:
203
+ if p.id in done:
204
+ continue
205
+ start = time.perf_counter()
206
+ q = sample_responses(p, self.model, self.sample, self.check, cache=cache, replace=self.replace) \
207
+ if self.sample else p
208
+ e = explain(q, self.model, self.target_for(q), method=self.method, budget=self.budget,
209
+ test_masks=self.test_masks, seed=self.seed, replace=self.replace, cache=cache,
210
+ batch_size=self.batch_size, noise_repeats=self.noise_repeats, removal=self.removal)
211
+ row = {"prompt": prompt_to_dict(q), "input": given[p.id], "seconds": round(time.perf_counter() - start, 3),
212
+ "explanation": e.as_dict()}
213
+ with open(lines, "a", encoding="utf-8") as f:
214
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
215
+ done[p.id] = row
216
+ cache.close()
217
+ results = Results([Explanation.from_dict(done[p.id]["explanation"], done[p.id]["prompt"])
218
+ for p in self.prompts])
219
+ (out / "summary.json").write_text(json.dumps(results.summary(), indent=2) + "\n", encoding="utf-8")
220
+ results.to_html(out / "report.html")
221
+ return results
222
+
223
+
224
+ class Results:
225
+ """The explanations of a run, with a summary over prompts."""
226
+
227
+ def __init__(self, explanations: list[Explanation]):
228
+ self.explanations = list(explanations)
229
+
230
+ @classmethod
231
+ def load(cls, out) -> Results:
232
+ """Every explanation a run directory holds, the prompts of earlier runs into it included (a last line cut short
233
+ is left out)."""
234
+ path = Path(out) / "explanations.jsonl"
235
+ if not path.exists():
236
+ raise FileNotFoundError(f"{path} does not exist")
237
+ return cls([Explanation.from_dict(r["explanation"], r["prompt"]) for r in _rows(path).values()])
238
+
239
+ def summary(self, by: str = "tag", top: int = 3) -> dict:
240
+ """For each tag (by="tag") or unit id (by="unit"): in how many prompts it occurs, its units' mean score, and the
241
+ share of those prompts in which one of its units is among the `top` highest scores."""
242
+ if by not in ("tag", "unit"):
243
+ raise ValueError("by is tag or unit")
244
+ acc: dict = {}
245
+ for e in self.explanations:
246
+ keys = e.tags if by == "tag" else e.unit_ids
247
+ if keys is None:
248
+ continue
249
+ best = {u for u, _ in e.top(top)}
250
+ seen = set()
251
+ for uid, key, score in zip(e.unit_ids, keys, e.surrogate.scores):
252
+ if key is None:
253
+ continue
254
+ a = acc.setdefault(key, {"prompts": 0, "units": 0, "score_sum": 0.0, "in_top": 0})
255
+ a["units"] += 1
256
+ a["score_sum"] += float(score)
257
+ if key not in seen:
258
+ seen.add(key)
259
+ a["prompts"] += 1
260
+ a["in_top"] += any(u in best for u, k in zip(e.unit_ids, keys) if k == key)
261
+ return {key: {"prompts": a["prompts"], "units": a["units"], "mean_score": a["score_sum"] / a["units"],
262
+ f"share_in_top_{top}": a["in_top"] / a["prompts"]} for key, a in sorted(acc.items())}
263
+
264
+ def to_html(self, path=None, title: str = "ContextAttr report") -> str:
265
+ from .report import page # noqa: PLC0415
266
+
267
+ text = page(self.explanations, title)
268
+ if path is not None:
269
+ Path(path).write_text(text, encoding="utf-8")
270
+ return text
271
+
272
+ def __len__(self) -> int:
273
+ return len(self.explanations)
274
+
275
+
276
+ def _digest(prompt: Prompt) -> str:
277
+ """The prompt as given, units, texts and responses: what a resumed run compares."""
278
+ text = json.dumps(prompt_to_dict(prompt), sort_keys=True, ensure_ascii=False)
279
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
280
+
281
+
282
+ def _rows(path: Path, repair: bool = False) -> dict:
283
+ """The rows of explanations.jsonl by prompt id, the first of each. A last line without its newline was cut short
284
+ by a kill while it was written: it is left out, and with `repair` removed from the file. Any other line that is not
285
+ a row means the file is damaged."""
286
+ if not path.exists():
287
+ return {}
288
+ data = path.read_bytes()
289
+ whole, newline, cut = data.rpartition(b"\n")
290
+ if cut and repair:
291
+ with open(path, "r+b") as f:
292
+ f.truncate(len(whole) + len(newline))
293
+ rows: dict = {}
294
+ for k, line in enumerate((whole + newline).decode("utf-8").splitlines(), 1):
295
+ if not line.strip():
296
+ continue
297
+ try:
298
+ row = json.loads(line)
299
+ rows.setdefault(row["prompt"]["id"], row)
300
+ except (ValueError, KeyError, TypeError) as error:
301
+ raise ValueError(f"{path}: line {k} is not a row of a run ({error}); the file is damaged") from error
302
+ return rows
303
+
304
+
305
+ def _utc() -> str:
306
+ return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -0,0 +1,7 @@
1
+ """The number read from the model for each masked prompt (docs/interface.md §5)."""
2
+
3
+ from .sampled import Sampled
4
+ from .scored import REDUCES, Choice, Contrast, Item, LogProb, Pooled, ScoredTarget, logodds, reduce_scored
5
+
6
+ __all__ = ["REDUCES", "Choice", "Contrast", "Item", "LogProb", "Pooled", "Sampled", "ScoredTarget", "logodds",
7
+ "reduce_scored"]
@@ -0,0 +1,58 @@
1
+ """The sampled target (docs/interface.md §5): it needs only the wrapper's `generate`, so it works with any model,
2
+ an API's included. For each masked prompt, k fresh replies; the value is the share a check flags as negative, or the
3
+ mean of a numeric check (a judge's score, a classifier's probability).
4
+
5
+ Every masked prompt is sampled with the same seed (common random numbers), so differences between masks are not
6
+ swamped by the luck of the draw where the backend honours seeds; the noise check samples with other seeds."""
7
+
8
+ from __future__ import annotations
9
+
10
+ import dataclasses
11
+ import math
12
+ from dataclasses import dataclass
13
+
14
+ from ..prompt import Prompt
15
+ from ..responses.checks import describe, verdict
16
+ from ..wrapper.contract import Sampling
17
+
18
+ DEFAULT = Sampling(n=4, temperature=0.7, top_p=0.95, max_tokens=256, seed=0)
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class Sampled:
23
+ """`samples` replies to each masked prompt, sampled as `sampling` says (default: temperature 0.7, top_p 0.95,
24
+ 256 tokens, seed 0), and the mean of the check's values over them (negative 1, positive 0, or its number)."""
25
+
26
+ check: object
27
+ samples: int = 4
28
+ sampling: Sampling | None = None
29
+
30
+ needs = "generate"
31
+
32
+ def __post_init__(self) -> None:
33
+ if not callable(self.check):
34
+ raise TypeError("Sampled needs a check: a function (prompt, response) → label, bool or number")
35
+ if isinstance(self.samples, bool) or not isinstance(self.samples, int) or self.samples < 1:
36
+ raise ValueError("Sampled: samples must be a positive integer")
37
+
38
+ def sampling_for(self, repeat: int = 0) -> Sampling:
39
+ """The sampling of every masked prompt, with `samples` replies; a noise repeat k samples with seed + k."""
40
+ base = self.sampling or DEFAULT
41
+ seed = None if base.seed is None else base.seed + repeat
42
+ return dataclasses.replace(base, n=self.samples, seed=seed)
43
+
44
+ def value(self, prompt: Prompt, replies: list[str]) -> float:
45
+ values = [verdict(self.check(prompt, r)).value for r in replies]
46
+ values = [v for v in values if v is not None]
47
+ if not values:
48
+ raise ValueError(f"the check labelled none of the {len(replies)} replies to a masked prompt of "
49
+ f"{prompt.id!r}")
50
+ return math.fsum(values) / len(values)
51
+
52
+ def __repr__(self) -> str:
53
+ """The check (responses.describe) and the sampling every masked prompt gets, so that a run's settings and an
54
+ explanation's target say both."""
55
+ s = self.sampling_for(0)
56
+ stop = f", stop={list(s.stop)!r}" if s.stop else ""
57
+ return (f"Sampled({describe(self.check)}, samples={self.samples}, temperature={float(s.temperature)!r}, "
58
+ f"top_p={float(s.top_p)!r}, max_tokens={s.max_tokens}, seed={s.seed}{stop})")
@@ -0,0 +1,177 @@
1
+ """Targets read from `score` (docs/interface.md §5): the log-probability of given responses under each masked prompt,
2
+ with nothing generated. A target names the texts to score (its items) and how their numbers combine into the value."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import math
7
+ from dataclasses import dataclass
8
+
9
+ import numpy as np
10
+
11
+ from ..prompt import Prompt, Response
12
+ from ..wrapper.contract import Scored
13
+
14
+ REDUCES = ("mean", "sum", "logodds")
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class Item:
19
+ """One text scored under every masked prompt: its id, its label, the span that counts, and how its token
20
+ log-probabilities become one number."""
21
+
22
+ id: str
23
+ text: str
24
+ label: str | None = None
25
+ span: tuple[int, int] | None = None
26
+ reduce: str = "mean"
27
+
28
+
29
+ class ScoredTarget:
30
+ """A target that needs the wrapper's `score`."""
31
+
32
+ needs = "score"
33
+
34
+ def items(self, prompt: Prompt) -> list[Item]:
35
+ raise NotImplementedError
36
+
37
+ def combine(self, columns: np.ndarray, items: list[Item]) -> np.ndarray:
38
+ """(masks × items) numbers → one value per mask."""
39
+ raise NotImplementedError
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class LogProb(ScoredTarget):
44
+ """The log-probability of one response, or of its span when it has one: per token on average ("mean", the
45
+ negative log-perplexity of SPEX §3), in total ("sum"), or as the logit of its probability ("logodds",
46
+ ContextCite §3). `response` is a response id of the prompt, a Response, or None for the prompt's only one."""
47
+
48
+ response: str | Response | None = None
49
+ reduce: str = "mean"
50
+
51
+ def __post_init__(self) -> None:
52
+ _check_reduce(self.reduce)
53
+
54
+ def items(self, prompt: Prompt) -> list[Item]:
55
+ r = _resolve(prompt, self.response)
56
+ return [Item(r.id, r.text, r.label, r.span, self.reduce)]
57
+
58
+ def combine(self, columns: np.ndarray, items: list[Item]) -> np.ndarray:
59
+ return columns[:, 0]
60
+
61
+
62
+ @dataclass(frozen=True)
63
+ class Pooled(ScoredTarget):
64
+ """The mean LogProb of the prompt's responses with one label."""
65
+
66
+ label: str = "negative"
67
+ reduce: str = "mean"
68
+
69
+ def __post_init__(self) -> None:
70
+ _check_reduce(self.reduce)
71
+ if self.label not in ("positive", "negative"):
72
+ raise ValueError(f"Pooled: label must be positive or negative, not {self.label!r}")
73
+
74
+ def items(self, prompt: Prompt) -> list[Item]:
75
+ return _labelled(prompt, self.label, self.reduce)
76
+
77
+ def combine(self, columns: np.ndarray, items: list[Item]) -> np.ndarray:
78
+ return columns.mean(axis=1)
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class Contrast(ScoredTarget):
83
+ """Pooled("negative") − Pooled("positive"): what moves the model from desired to undesired responses."""
84
+
85
+ reduce: str = "mean"
86
+
87
+ def __post_init__(self) -> None:
88
+ _check_reduce(self.reduce)
89
+
90
+ def items(self, prompt: Prompt) -> list[Item]:
91
+ return _labelled(prompt, "negative", self.reduce) + _labelled(prompt, "positive", self.reduce)
92
+
93
+ def combine(self, columns: np.ndarray, items: list[Item]) -> np.ndarray:
94
+ neg = np.array([i.label == "negative" for i in items])
95
+ return columns[:, neg].mean(axis=1) - columns[:, ~neg].mean(axis=1)
96
+
97
+
98
+ @dataclass(frozen=True)
99
+ class Choice(ScoredTarget):
100
+ """log p(a) − log p(b), each the total log-probability of that text as the response (SPEX B.2.4: "True" −
101
+ "False")."""
102
+
103
+ a: str
104
+ b: str
105
+
106
+ def __post_init__(self) -> None:
107
+ if not (isinstance(self.a, str) and isinstance(self.b, str) and self.a and self.b and self.a != self.b):
108
+ raise ValueError("Choice needs two different non-empty texts")
109
+
110
+ def items(self, prompt: Prompt) -> list[Item]:
111
+ return [Item("choice:a", self.a, reduce="sum"), Item("choice:b", self.b, reduce="sum")]
112
+
113
+ def combine(self, columns: np.ndarray, items: list[Item]) -> np.ndarray:
114
+ return columns[:, 0] - columns[:, 1]
115
+
116
+
117
+ def reduce_scored(scored: Scored, item: Item) -> float:
118
+ """One number from a response's token log-probabilities, as the item says."""
119
+ logprobs = scored.logprobs
120
+ if item.span is not None:
121
+ logprobs = [x for x, inside in zip(logprobs, span_tokens(scored, item)) if inside]
122
+ if not logprobs:
123
+ raise ValueError(f"no tokens to score for {item.id!r}" + (" within its span" if item.span else ""))
124
+ total = math.fsum(logprobs)
125
+ if item.reduce == "mean":
126
+ return total / len(logprobs)
127
+ if item.reduce == "sum":
128
+ return total
129
+ return logodds(total)
130
+
131
+
132
+ def logodds(logprob: float) -> float:
133
+ """log(p / (1 − p)) for p = exp(logprob)."""
134
+ if logprob >= 0:
135
+ return math.inf
136
+ return logprob - math.log(-math.expm1(logprob))
137
+
138
+
139
+ def span_tokens(scored: Scored, item: Item) -> list[bool]:
140
+ """Which tokens overlap the item's character span; the token texts must join back to the response."""
141
+ if scored.tokens is None:
142
+ raise ValueError(f"the span of {item.id!r} needs token texts, which the wrapper's score did not give")
143
+ if "".join(scored.tokens) != item.text:
144
+ raise ValueError(f"the span of {item.id!r} needs token texts that join back to the response; the "
145
+ f"wrapper's do not")
146
+ start, end = item.span
147
+ out, pos = [], 0
148
+ for t in scored.tokens:
149
+ out.append(pos < end and pos + len(t) > start)
150
+ pos += len(t)
151
+ return out
152
+
153
+
154
+ def _check_reduce(reduce: str) -> None:
155
+ if reduce not in REDUCES:
156
+ raise ValueError(f"reduce must be one of {', '.join(REDUCES)}, not {reduce!r}")
157
+
158
+
159
+ def _resolve(prompt: Prompt, response) -> Response:
160
+ if isinstance(response, Response):
161
+ return response
162
+ if response is None:
163
+ if len(prompt.responses) != 1:
164
+ raise ValueError(f"prompt {prompt.id!r} has {len(prompt.responses)} responses: name the one to explain")
165
+ return prompt.responses[0]
166
+ for r in prompt.responses:
167
+ if r.id == response:
168
+ return r
169
+ raise ValueError(f"prompt {prompt.id!r} has no response {response!r} "
170
+ f"(it has: {', '.join(r.id for r in prompt.responses) or 'none'})")
171
+
172
+
173
+ def _labelled(prompt: Prompt, label: str, reduce: str) -> list[Item]:
174
+ items = [Item(r.id, r.text, r.label, r.span, reduce) for r in prompt.responses if r.label == label]
175
+ if not items:
176
+ raise ValueError(f"prompt {prompt.id!r} has no {label} response")
177
+ return items