open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,371 @@
1
+ """Held-out windows and the arithmetic of detection, fuzzing and embedding scores.
2
+
3
+ The judge only ever sees a window's text. So each window is measured on its own, as
4
+ probes are, not inside the snippet that it was cut from. Labels always come from a new
5
+ measurement. So a task handed to a judge carries no activations, and a stored score can
6
+ be checked again (`open-interpretability reproduce`).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import bisect
12
+ import random
13
+ from collections.abc import Sequence
14
+ from dataclasses import dataclass, replace
15
+
16
+ import torch
17
+
18
+ from openinterp.core.explanation import ControlScore, ExplanationScores, JudgeScore, ScoredWindow
19
+ from openinterp.harness.hooks import NeuronHookRunner
20
+ from openinterp.harness.units import Unit
21
+
22
+ DEFAULT_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
23
+
24
+
25
+ @dataclass(frozen=True)
26
+ class Window:
27
+ """One held-out window: its text, tokens, strengths and marked tokens."""
28
+
29
+ text: str
30
+ tokens: list[str]
31
+ activations: list[float]
32
+ highlighted: tuple[int, ...] = ()
33
+ start: int = 0 # 1 for a unit with a first-token floor (protocol 0.8)
34
+
35
+ @property
36
+ def peak(self) -> float:
37
+ """Over the measured tokens."""
38
+ return max(self.activations[self.start :], default=0.0)
39
+
40
+ def hot(self, threshold: float) -> list[int]:
41
+ """Measured tokens at or above ``threshold``."""
42
+ return [i for i, a in enumerate(self.activations) if i >= self.start and a >= threshold]
43
+
44
+ def cold(self, threshold: float) -> list[int]:
45
+ """The measured tokens below ``threshold``."""
46
+ return [i for i, a in enumerate(self.activations) if i >= self.start and a < threshold]
47
+
48
+
49
+ @dataclass(frozen=True)
50
+ class HeldOutSample:
51
+ """The detection and fuzzing windows, and how many windows they were drawn from."""
52
+
53
+ detection: list[Window]
54
+ fuzzing: list[Window]
55
+ quantile_edges: list[float]
56
+ candidate_windows: int
57
+ active_windows: int
58
+
59
+
60
+ def cut_windows(runner: NeuronHookRunner, texts: Sequence[str], window_tokens: int) -> list[str]:
61
+ """Split each text into consecutive ``window_tokens``-token pieces, decoded back to text.
62
+
63
+ A trailing piece shorter than half a window is dropped, as are blank pieces.
64
+ """
65
+ tok = runner.tokenizer
66
+ pieces = []
67
+ for text in texts:
68
+ ids = tok.encode(text, add_special_tokens=False)
69
+ for start in range(0, len(ids), window_tokens):
70
+ chunk = ids[start : start + window_tokens]
71
+ if len(chunk) * 2 < window_tokens and start > 0:
72
+ continue
73
+ piece = tok.decode(chunk)
74
+ if piece.strip():
75
+ pieces.append(piece)
76
+ return pieces
77
+
78
+
79
+ @torch.no_grad()
80
+ def measure_windows(
81
+ runner: NeuronHookRunner,
82
+ texts: Sequence[str],
83
+ unit: Unit,
84
+ *,
85
+ batch_size: int = 16,
86
+ ) -> list[Window]:
87
+ """Per-token strengths of each text, run on its own (no surrounding context)."""
88
+ windows = []
89
+ for start in range(0, len(texts), batch_size):
90
+ batch = list(texts[start : start + batch_size])
91
+ encoded = runner.encode(batch)
92
+ strengths = unit.strengths(runner, encoded).float()
93
+ ids = encoded["input_ids"]
94
+ mask = encoded.get("attention_mask", torch.ones_like(ids)).bool()
95
+ for row, text in enumerate(batch):
96
+ keep = mask[row]
97
+ windows.append(
98
+ Window(
99
+ text=text,
100
+ tokens=[runner.tokenizer.decode([i]) for i in ids[row][keep].tolist()],
101
+ activations=[round(v, 5) for v in strengths[row][keep].cpu().tolist()],
102
+ start=int(unit.skip_first),
103
+ )
104
+ )
105
+ return windows
106
+
107
+
108
+ # How sample_heldout marks fuzzing decoys; recorded in each ExplanationScores.
109
+ DECOYS = "matched-shape"
110
+
111
+
112
+ def placed(rng: random.Random, shape: Sequence[int], free: Sequence[int]) -> list[int]:
113
+ """Marks with ``shape``'s offsets, moved to a random place where every mark is in ``free``.
114
+
115
+ A decoy that copies the real marks' count and spacing gives a judge no cue from them
116
+ ("Scoring Explanations Without Reading Them", arXiv 2610.37676). If the shape fits
117
+ nowhere, it falls back to as many marks drawn at random from ``free``.
118
+ """
119
+ if not shape:
120
+ return []
121
+ offsets = [i - shape[0] for i in sorted(shape)]
122
+ allowed = set(free)
123
+ starts = [s for s in sorted(allowed) if all(s + o in allowed for o in offsets)]
124
+ if starts:
125
+ start = rng.choice(starts)
126
+ return [start + o for o in offsets]
127
+ return rng.sample(list(free), min(len(free), len(shape)))
128
+
129
+
130
+ def sample_heldout(
131
+ runner: NeuronHookRunner,
132
+ texts: Sequence[str],
133
+ unit: Unit,
134
+ threshold: float,
135
+ *,
136
+ seed: int,
137
+ exclude: Sequence[str] = (),
138
+ window_tokens: int = 32,
139
+ max_windows: int = 4000,
140
+ quantiles: int = 5,
141
+ per_quantile: int = 5,
142
+ ) -> HeldOutSample:
143
+ """Draw detection and fuzzing windows from the ``texts`` containing no ``exclude``.
144
+
145
+ Active windows (peak ≥ ``threshold``) are sorted by peak and split into ``quantiles``
146
+ equal bins. Each bin gives ``per_quantile`` windows to detection and as many to
147
+ fuzzing. So the weakest activations count as much as the top ones. Detection gets as
148
+ many inactive windows as active ones. In half its active windows, fuzzing marks the
149
+ active tokens. In the other half, it marks inactive tokens in the same shape: the same
150
+ count and spacing, moved (:func:`placed`). It also adds half as many inactive windows,
151
+ each marked in the shape of a random active window's tokens. With no active window at all,
152
+ detection gets ``quantiles × per_quantile`` inactive ones and fuzzing none.
153
+ """
154
+ rng = random.Random(seed)
155
+ # A top example may be a window of a longer document: leave out the whole document.
156
+ held_out = [t for t in texts if not any(e in t for e in exclude)]
157
+ candidates = cut_windows(runner, held_out, window_tokens)
158
+ rng.shuffle(candidates)
159
+ measured = measure_windows(runner, candidates[:max_windows], unit)
160
+ active = sorted((w for w in measured if w.peak >= threshold), key=lambda w: w.peak)
161
+ inactive = [w for w in measured if w.peak < threshold]
162
+ if not measured:
163
+ raise ValueError("no held-out windows: every corpus snippet is a top example")
164
+ if not active: # the explanation can still be scored on false positives
165
+ drawn = rng.sample(inactive, min(len(inactive), quantiles * per_quantile))
166
+ return HeldOutSample(drawn, [], [], len(measured), 0)
167
+ n = len(active)
168
+ bins = [active[i * n // quantiles : (i + 1) * n // quantiles] for i in range(quantiles)]
169
+ edges = [b[0].peak for b in bins[1:] if b]
170
+
171
+ detect_active, fuzz_active = [], []
172
+ for members in bins:
173
+ drawn = rng.sample(members, min(len(members), 2 * per_quantile))
174
+ half = (len(drawn) + 1) // 2
175
+ detect_active += drawn[:half]
176
+ fuzz_active += drawn[half:]
177
+ detection = detect_active + rng.sample(inactive, min(len(inactive), len(detect_active)))
178
+
179
+ fuzzing = []
180
+ shapes = [w.hot(threshold) for w in fuzz_active]
181
+ for n, w in enumerate(fuzz_active):
182
+ hot, cold = w.hot(threshold), w.cold(threshold)
183
+ marks = hot if n % 2 == 0 or len(cold) < len(hot) else placed(rng, hot, cold)
184
+ fuzzing.append(replace(w, highlighted=tuple(sorted(marks))))
185
+ rest = [w for w in inactive if w not in detection]
186
+ for w in rng.sample(rest, min(len(rest), len(fuzz_active) // 2)):
187
+ cold = list(range(w.start, len(w.tokens)))
188
+ marks = placed(rng, rng.choice(shapes), cold)
189
+ fuzzing.append(replace(w, highlighted=tuple(sorted(marks))))
190
+ rng.shuffle(detection)
191
+ rng.shuffle(fuzzing)
192
+ return HeldOutSample(detection, fuzzing, edges, len(measured), len(active))
193
+
194
+
195
+ def label_windows(
196
+ windows: Sequence[Window],
197
+ threshold: float,
198
+ edges: Sequence[float],
199
+ *,
200
+ fuzzing: bool,
201
+ judged: Sequence[bool | None] | None = None,
202
+ similarities: Sequence[float | None] | None = None,
203
+ ) -> list[ScoredWindow]:
204
+ """Label each window as the judge should answer it, with the judge's answer if given."""
205
+ out = []
206
+ for n, w in enumerate(windows):
207
+ hot = set(w.hot(threshold))
208
+ label = bool(hot) and set(w.highlighted) == hot if fuzzing else bool(hot)
209
+ out.append(
210
+ ScoredWindow(
211
+ text=w.text,
212
+ tokens=w.tokens,
213
+ activations=w.activations,
214
+ quantile=bisect.bisect_right(edges, w.peak) if hot else None,
215
+ highlighted=list(w.highlighted),
216
+ label=label,
217
+ judged=judged[n] if judged is not None else None,
218
+ similarity=similarities[n] if similarities is not None else None,
219
+ )
220
+ )
221
+ return out
222
+
223
+
224
+ def judge_score(windows: Sequence[ScoredWindow], quantiles: int) -> JudgeScore | None:
225
+ """Balanced accuracy of the judge's answers, or ``None`` if nothing was judged."""
226
+ judged = [w for w in windows if w.judged is not None]
227
+ pos = [w.judged for w in judged if w.label]
228
+ neg = [w.judged for w in judged if not w.label]
229
+ if not judged:
230
+ return None
231
+ tpr = sum(pos) / len(pos) if pos else None
232
+ fpr = sum(neg) / len(neg) if neg else None
233
+ rates = [r for r in (tpr, None if fpr is None else 1 - fpr) if r is not None]
234
+ by_q: list[float | None] = []
235
+ for q in range(quantiles):
236
+ hits = [w.judged for w in judged if w.label and w.quantile == q]
237
+ by_q.append(sum(hits) / len(hits) if hits else None)
238
+ return JudgeScore(
239
+ balanced_accuracy=sum(rates) / len(rates),
240
+ true_positive_rate=tpr,
241
+ false_positive_rate=fpr,
242
+ recall_by_quantile=by_q,
243
+ )
244
+
245
+
246
+ def auroc(scores: Sequence[float], labels: Sequence[bool]) -> float | None:
247
+ """Area under the ROC curve (Mann–Whitney U, ties count half); ``None`` with one class."""
248
+ pos = [s for s, y in zip(scores, labels, strict=True) if y]
249
+ neg = [s for s, y in zip(scores, labels, strict=True) if not y]
250
+ if not pos or not neg:
251
+ return None
252
+ wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg)
253
+ return wins / (len(pos) * len(neg))
254
+
255
+
256
+ def _balanced(pairs: Sequence[tuple[bool, bool]]) -> float | None:
257
+ """Balanced accuracy of (predicted, label) pairs, or ``None`` with one class."""
258
+ pos = [p for p, y in pairs if y]
259
+ neg = [not p for p, y in pairs if not y]
260
+ if not pos or not neg:
261
+ return None
262
+ return (sum(pos) / len(pos) + sum(neg) / len(neg)) / 2
263
+
264
+
265
+ def _contiguous(marks: Sequence[int]) -> bool:
266
+ """True when the marked tokens form one unbroken run."""
267
+ return bool(marks) and max(marks) - min(marks) + 1 == len(marks)
268
+
269
+
270
+ def _best_cut(values: Sequence[float], labels: Sequence[bool]) -> float | None:
271
+ """The best balanced accuracy of "positive if value >= cut" over every cut."""
272
+ scores = [
273
+ _balanced([(v >= c, y) for v, y in zip(values, labels, strict=True)]) for c in set(values)
274
+ ]
275
+ return max((s for s in scores if s is not None), default=None)
276
+
277
+
278
+ def explanation_free_baseline(detection, fuzzing) -> dict[str, float]:
279
+ """Explanation-free rules' balanced accuracy on scored windows (arXiv 2610.37676).
280
+
281
+ Each cut is chosen on the same windows, so each number is an upper bound.
282
+ """
283
+ out = {
284
+ "detection_length": _best_cut(
285
+ [len(w.tokens) for w in detection], [w.label for w in detection]
286
+ ),
287
+ "fuzzing_contiguous": _balanced([(_contiguous(w.highlighted), w.label) for w in fuzzing]),
288
+ "fuzzing_count": _best_cut(
289
+ [len(w.highlighted) for w in fuzzing], [w.label for w in fuzzing]
290
+ ),
291
+ }
292
+ return {k: round(v, 6) for k, v in out.items() if v is not None}
293
+
294
+
295
+ def finalize_scores(scores: ExplanationScores) -> ExplanationScores:
296
+ """Recompute the summary scores from the stored windows, labels and answers."""
297
+ det = scores.detection_windows
298
+ sims = [w.similarity for w in det]
299
+ emb = auroc(sims, [w.label for w in det]) if det and None not in sims else None
300
+ return scores.model_copy(
301
+ update={
302
+ "detection": judge_score(det, scores.quantiles),
303
+ "fuzzing": judge_score(scores.fuzzing_windows, scores.quantiles),
304
+ "embedding_auroc": emb,
305
+ "controls": [_control_scores(c, scores) for c in scores.controls],
306
+ "rule_baseline": explanation_free_baseline(det, scores.fuzzing_windows),
307
+ }
308
+ )
309
+
310
+
311
+ def _control_scores(control: ControlScore, scores: ExplanationScores) -> ControlScore:
312
+ """Score a control's answers against the real windows' labels."""
313
+
314
+ def judged(windows: Sequence[ScoredWindow], answers: Sequence[bool]):
315
+ if len(answers) != len(windows):
316
+ return None
317
+ marked = [w.model_copy(update={"judged": a}) for w, a in zip(windows, answers, strict=True)]
318
+ return judge_score(marked, scores.quantiles)
319
+
320
+ return control.model_copy(
321
+ update={
322
+ "detection": judged(scores.detection_windows, control.detection_judged),
323
+ "fuzzing": judged(scores.fuzzing_windows, control.fuzzing_judged),
324
+ }
325
+ )
326
+
327
+
328
+ def remeasure(scores: ExplanationScores, runner: NeuronHookRunner, unit: Unit):
329
+ """Re-measure every window and re-derive labels and scores, keeping the answers."""
330
+
331
+ def redo(windows: Sequence[ScoredWindow], fuzzing: bool) -> list[ScoredWindow]:
332
+ fresh = measure_windows(runner, [w.text for w in windows], unit)
333
+ pairs = zip(fresh, windows, strict=True)
334
+ fresh = [replace(f, highlighted=tuple(w.highlighted)) for f, w in pairs]
335
+ return label_windows(
336
+ fresh,
337
+ scores.threshold,
338
+ scores.quantile_edges,
339
+ fuzzing=fuzzing,
340
+ judged=[w.judged for w in windows],
341
+ similarities=[w.similarity for w in windows],
342
+ )
343
+
344
+ return finalize_scores(
345
+ scores.model_copy(
346
+ update={
347
+ "detection_windows": redo(scores.detection_windows, False),
348
+ "fuzzing_windows": redo(scores.fuzzing_windows, True),
349
+ }
350
+ )
351
+ )
352
+
353
+
354
+ @torch.no_grad()
355
+ def embedding_similarities(model_name: str, query: str, texts: Sequence[str]) -> list[float]:
356
+ """Cosine between ``query`` and each text, mean-pooled from a sentence encoder."""
357
+ from transformers import AutoModel, AutoTokenizer
358
+
359
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
360
+ model = AutoModel.from_pretrained(model_name).eval()
361
+ vectors = []
362
+ batch = [query, *texts]
363
+ for start in range(0, len(batch), 32):
364
+ enc = tokenizer(
365
+ batch[start : start + 32], padding=True, truncation=True, return_tensors="pt"
366
+ )
367
+ hidden = model(**enc).last_hidden_state
368
+ mask = enc["attention_mask"].unsqueeze(-1).to(hidden.dtype)
369
+ vectors.append((hidden * mask).sum(1) / mask.sum(1).clamp(min=1))
370
+ emb = torch.nn.functional.normalize(torch.cat(vectors), dim=-1)
371
+ return [round(float(v), 6) for v in emb[1:] @ emb[0]]
@@ -0,0 +1,276 @@
1
+ """Scanning and patching one latent of a sparse dictionary (a transcoder or SAE).
2
+
3
+ A latent's strength is its activation (see :mod:`openinterp.harness.units`). The scan
4
+ reads the dictionary's input at its site, in one pass that stops at the feature's layer.
5
+ It counts the same tokens as a neuron scan (see :mod:`scan`). So the top examples,
6
+ histogram and confound profile mean the same for every kind of unit.
7
+
8
+ The same pass measures how well the whole dictionary reconstructs its output there. It
9
+ leaves out each piece's first token. So a record says whether its baseline was a
10
+ faithful one.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from collections.abc import Sequence
16
+ from dataclasses import dataclass
17
+
18
+ import torch
19
+
20
+ from openinterp.core.schemas import DictionaryProfile, FeatureIdentifier
21
+ from openinterp.harness.annotations import Tagger
22
+ from openinterp.harness.confounds import ConfoundAccumulator
23
+ from openinterp.harness.dictionaries import Dictionary, DictionarySpec, load_spec, spec_profile
24
+ from openinterp.harness.heads import _repeat
25
+ from openinterp.harness.hooks import Inputs, NeuronHookRunner
26
+ from openinterp.harness.patcher import Effect, Positions
27
+ from openinterp.harness.scan import (
28
+ BATCH_TOKENS,
29
+ LabelPass,
30
+ ScanResult,
31
+ TopExamples,
32
+ batches,
33
+ cut_pieces,
34
+ settle,
35
+ )
36
+ from openinterp.harness.sites import dictionary_io
37
+
38
+ ENCODE_CHUNK = 1024 # tokens encoded through the whole dictionary at once
39
+
40
+
41
+ @dataclass
42
+ class FeatureScanResult(ScanResult):
43
+ """A latent's scan result, with its :class:`DictionaryProfile`."""
44
+
45
+ profile: DictionaryProfile | None = None
46
+
47
+
48
+ class _Fit:
49
+ """Running reconstruction error, output variance and L0 of a dictionary."""
50
+
51
+ def __init__(self) -> None:
52
+ self.tokens = 0
53
+ self.error = self.sum_sq = self.l0 = 0.0
54
+ self.total: torch.Tensor | None = None
55
+ self.fired: torch.Tensor | None = None # [n] latents active on some token
56
+
57
+ def update(self, dictionary: Dictionary, x: torch.Tensor, out: torch.Tensor) -> None:
58
+ for start in range(0, x.shape[0], ENCODE_CHUNK):
59
+ xs, target = x[start : start + ENCODE_CHUNK], out[start : start + ENCODE_CHUNK]
60
+ target = target.float().to(dictionary.w_dec.device)
61
+ acts = dictionary.encode(xs)
62
+ self.error += float(((dictionary.reconstruct(xs, acts) - target) ** 2).sum())
63
+ self.l0 += float((acts > 0).sum())
64
+ fired = (acts > 0).any(0)
65
+ self.fired = fired if self.fired is None else self.fired | fired
66
+ exact = target.cpu().double() # MPS has no float64
67
+ self.sum_sq += float((exact**2).sum())
68
+ chunk = exact.sum(0)
69
+ self.total = chunk if self.total is None else self.total + chunk
70
+ self.tokens += x.shape[0]
71
+
72
+ def stats(self) -> dict[str, float]:
73
+ variance = self.sum_sq - float((self.total**2).sum()) / self.tokens
74
+ explained = 1 - self.error / variance if variance > 0 else None
75
+ return {
76
+ "explained_variance": None if explained is None else round(explained, 6),
77
+ "mean_l0": round(self.l0 / self.tokens, 4),
78
+ "dead_share": round(1 - float(self.fired.float().mean()), 6),
79
+ }
80
+
81
+
82
+ def scan_feature(
83
+ runner: NeuronHookRunner,
84
+ texts: Sequence[str],
85
+ ident: FeatureIdentifier,
86
+ spec: DictionarySpec,
87
+ **options,
88
+ ) -> FeatureScanResult:
89
+ """Scan one latent: its top examples, strengths, confounds and :class:`DictionaryProfile`.
90
+
91
+ ``options`` are :func:`scan_latent`'s.
92
+ """
93
+ dictionary = load_spec(spec, ident.layer, runner.device)
94
+
95
+ def profile(**measured) -> DictionaryProfile:
96
+ return spec_profile(spec, ident.layer, n_features=dictionary.n_features, **measured)
97
+
98
+ return scan_latent(runner, texts, ident, dictionary, spec.input_site, profile, **options)
99
+
100
+
101
+ def scan_latent(
102
+ runner: NeuronHookRunner,
103
+ texts: Sequence[str],
104
+ ident,
105
+ dictionary: Dictionary,
106
+ input_site: str,
107
+ profile,
108
+ *,
109
+ k: int = 20,
110
+ batch_tokens: int = BATCH_TOKENS,
111
+ min_count: int = 3,
112
+ window_tokens: int | None = None,
113
+ stride: int | None = None,
114
+ taggers: Sequence[Tagger] = (),
115
+ ) -> FeatureScanResult:
116
+ """Scan latent ``ident.feature_idx`` of ``dictionary``, read at ``input_site``.
117
+
118
+ ``profile(density=, tokens=, **fit)`` gives the result's profile from what the scan
119
+ measured. Each of ``taggers`` labels the counted tokens, and its confound joins the
120
+ profile's. A superposition scans this way too (:mod:`openinterp.harness.superpositions`).
121
+ """
122
+ if runner.tokenizer is None:
123
+ raise ValueError("a latent scan requires a tokenizer")
124
+ layer, f = ident.layer, ident.feature_idx
125
+ if f >= dictionary.n_features:
126
+ raise ValueError(f"{ident}: the dictionary has {dictionary.n_features} latents")
127
+ pieces = list(cut_pieces(runner.tokenizer, texts, window_tokens, stride))
128
+ labels = LabelPass(runner.tokenizer, texts, taggers)
129
+ confounds, top, fit, values, first = _feature_pass(
130
+ runner, dictionary, input_site, layer, f, pieces, batch_tokens, labels
131
+ )
132
+ if not values:
133
+ raise ValueError("the corpus has no tokens to scan")
134
+
135
+ def read(inputs):
136
+ x, _ = dictionary_io(runner, inputs, layer, input_site)
137
+ return dictionary.feature(x, f)
138
+
139
+ density = round(sum(v > 0 for v in values) / len(values), 6)
140
+ tokens = len(values) # before settle, which can leave out first tokens
141
+ examples, values, floor = settle(runner, read, values, first, top, k, labels.tokens)
142
+ return FeatureScanResult(
143
+ examples=examples,
144
+ values=values,
145
+ correlated=[],
146
+ confounds=labels.finalize(confounds.finalize(runner.tokenizer, min_count), values),
147
+ windows=len(pieces),
148
+ first_token_floor=floor,
149
+ profile=profile(density=density, tokens=tokens, **fit.stats()),
150
+ )
151
+
152
+
153
+ def _feature_pass(
154
+ runner, dictionary, input_site: str, layer: int, f: int, pieces, batch_tokens, labels
155
+ ):
156
+ """Make one pass over ``pieces``, and return what it measured.
157
+
158
+ That is the confounds (and ``labels``' counts), the top examples, and the dictionary's
159
+ fit. It is also each counted token's strength, and whether it is a piece's first token.
160
+ """
161
+ confounds = ConfoundAccumulator()
162
+ top = TopExamples()
163
+ fit = _Fit()
164
+ values: list[float] = []
165
+ first: list[bool] = []
166
+ for batch in batches(runner, pieces, batch_tokens):
167
+ x, out = dictionary_io(runner, batch.encoded, layer, input_site)
168
+ strengths = dictionary.feature(x, f).cpu()
169
+ counted = batch.counted
170
+ confounds.update(batch.ids[counted], batch.positions[counted], strengths[counted])
171
+ labels.update(batch, strengths)
172
+ values.extend(strengths[counted].tolist())
173
+ first.extend(batch.first[counted].tolist())
174
+ top.update(batch, strengths)
175
+ # Not a piece's first token. Dictionaries aren't trained on that attention sink.
176
+ # Its error can be 10^8 times the output's variance, and would swamp the fit.
177
+ fitted = (counted & ~batch.first).to(x.device)
178
+ fit.update(dictionary, x[fitted], out[fitted])
179
+ return confounds, top, fit, values, first
180
+
181
+
182
+ @torch.no_grad()
183
+ def latent_activations(
184
+ runner: NeuronHookRunner, dictionary: Dictionary, inputs: Inputs, layer: int, site: str
185
+ ) -> torch.Tensor:
186
+ """Every latent's activation on one prompt: ``[seq, n]``."""
187
+ x, _ = dictionary_io(runner, inputs, layer, site)
188
+ return dictionary.encode(x[0])
189
+
190
+
191
+ @torch.no_grad()
192
+ def write_changes(
193
+ runner: NeuronHookRunner,
194
+ dictionary: Dictionary,
195
+ ident: FeatureIdentifier,
196
+ site: str,
197
+ base: str,
198
+ cf: str,
199
+ ) -> tuple[Positions, int, torch.Tensor]:
200
+ """Return where to patch and score, and every latent's write change up to there.
201
+
202
+ The write change is ``|Δa| · |W_dec[f]|``, in units of the audited latent's activation.
203
+ So the audited latent's own entry is ``|Δa|``.
204
+
205
+ Latents' decoder norms differ. So comparing writes, as heads do, matches the null
206
+ latents to the audited one by how much each moves the output that they write to.
207
+ """
208
+ a_b, a_c = (latent_activations(runner, dictionary, t, ident.layer, site) for t in (base, cf))
209
+ norms = dictionary.w_dec.norm(dim=-1)
210
+ scale = norms / norms[ident.feature_idx].clamp(min=1e-12)
211
+ if a_b.shape != a_c.shape:
212
+ return "last", -1, (a_b[-1] - a_c[-1]).abs() * scale
213
+ position = int((a_b[:, ident.feature_idx] - a_c[:, ident.feature_idx]).abs().argmax())
214
+ return "all", position, (a_b[: position + 1] - a_c[: position + 1]).abs().amax(0) * scale
215
+
216
+
217
+ @torch.no_grad()
218
+ def patch_features(
219
+ runner: NeuronHookRunner,
220
+ dictionary: Dictionary,
221
+ site: str,
222
+ source: Inputs,
223
+ target: Inputs,
224
+ layer: int,
225
+ features: Sequence[int],
226
+ *,
227
+ effect: Effect,
228
+ positions: Positions = "all",
229
+ freeze_downstream: bool = False,
230
+ batch_size: int = 8,
231
+ ) -> list[float]:
232
+ """``effect`` of patching each latent's activation, one at a time, from source into target.
233
+
234
+ The dictionary's output (the MLP's, or the residual stream) gains
235
+ ``(a_source − a_target) · W_dec[f]``.
236
+
237
+ The dictionary counterpart of :meth:`ActivationPatcher.patch_many`. Each latent gets
238
+ its own row of a batch. Later MLP outputs can be frozen against self-repair.
239
+ """
240
+ from openinterp.harness.memory import stream_rows
241
+ from openinterp.harness.patcher import ActivationPatcher
242
+
243
+ a_src = latent_activations(runner, dictionary, source, layer, site) # [seq, n]
244
+ target_enc = runner.encode(target)
245
+ downstream = list(range(layer + 1, runner.n_layers)) if freeze_downstream else []
246
+ target_logits, _, downstream_out = ActivationPatcher(runner)._target_pass(
247
+ target_enc, downstream
248
+ )
249
+ if target_logits.shape[0] != 1:
250
+ raise ValueError("patch_features takes one source and one target prompt")
251
+ if positions == "all" and a_src.shape[0] != target_logits.shape[1]:
252
+ raise ValueError("source and target differ in length; use positions='last'")
253
+ span = slice(None) if positions == "all" else slice(-1, None)
254
+ batch_size = stream_rows(runner, target_logits.shape[1], batch_size, logits=True)
255
+
256
+ effects: list[float] = []
257
+ for start in range(0, len(features), batch_size):
258
+ chunk = torch.tensor(list(features[start : start + batch_size]))
259
+ n = len(chunk)
260
+ batch = {k: v.repeat(n, *[1] * (v.dim() - 1)) for k, v in target_enc.items()}
261
+ idx = chunk.to(dictionary.w_dec.device)
262
+
263
+ def inject(x: torch.Tensor, out: torch.Tensor, idx=idx, n=n) -> torch.Tensor:
264
+ a_tgt = dictionary.feature(x[0, span], idx) # [span, n]; every row is the target
265
+ delta = (a_src[span][:, idx] - a_tgt).T # [n, span]
266
+ out = out.clone()
267
+ write = delta.unsqueeze(-1) * dictionary.w_dec[idx].unsqueeze(1) # [n, span, d]
268
+ out[:n, span] += write.to(out.dtype)
269
+ return out
270
+
271
+ frozen = {i: _repeat(cached, n) for i, cached in downstream_out.items()}
272
+ patched = runner.run_with_interventions(
273
+ batch, None, frozen, dictionary_interventions={layer: (site, inject)}
274
+ )
275
+ effects += [effect(patched[row : row + 1], target_logits) for row in range(n)]
276
+ return effects