open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,367 @@
1
+ """Input attribution: which context tokens drive a neuron pole at its peak.
2
+
3
+ Take each top example and the pole's *signed* activation at the peak token. It is
4
+ positive for the positive pole and negated for the negative pole. So larger is more
5
+ active, and the gradient is non-zero below the clamp. Integrated gradients (Sundararajan
6
+ et al., 2017) over the input token embeddings attribute it to every token up to the peak:
7
+
8
+ attr_i = (x_i − b) · (1/m) Σ_k ∂f/∂x_i |_{b + α_k (x − b)}, α_k = (k + ½)/m
9
+
10
+ Here b is the embedding of one reference token (the pad token) at every position but the
11
+ first. The first token stays as it is. Most of the model's attention rests on it (the
12
+ attention sink), so replacing it moves the whole path off-distribution. On Qwen2.5-0.5B,
13
+ the integral then fails to converge. It credits the first few tokens of a window with
14
+ much of the activation, but replacing any one of them doesn't remove it. A peak on the
15
+ first token is attributed to it alone. For a pole with a first-token floor
16
+ (``skip_first``, protocol 0.8), the peak is over the other tokens. Position embeddings,
17
+ where a model has them, are left alone. By completeness the attributions sum to
18
+ f(x) − f(b), and each is reported as a share of that gap.
19
+
20
+ The path integral is linear in each token's contribution. So a second check replaces
21
+ one token at a time with the reference token, and measures again. It checks the peak
22
+ itself, the few tokens before it, and the most attributed tokens. Tokens after the peak
23
+ cannot affect it (attention is causal). So each example is cut at its peak, and every
24
+ forward pass stops at the audited layer.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ from collections.abc import Callable, Sequence
30
+ from contextlib import suppress
31
+ from dataclasses import dataclass
32
+
33
+ import torch
34
+
35
+ from openinterp.core.attribution import (
36
+ AttributedToken,
37
+ ContextExample,
38
+ DistanceShare,
39
+ InputAttribution,
40
+ )
41
+ from openinterp.core.schemas import ActivatingExample, Pole
42
+ from openinterp.core.stats import spearman
43
+ from openinterp.harness.hooks import NeuronHookRunner, _StopForward
44
+ from openinterp.harness.memory import recompute_below, stream_rows
45
+
46
+ # Tokens back from the peak: the token itself, the one before, then wider bands.
47
+ DISTANCE_BANDS: tuple[tuple[int, int | None], ...] = (
48
+ (0, 0),
49
+ (1, 1),
50
+ (2, 4),
51
+ (5, 16),
52
+ (17, 64),
53
+ (65, None),
54
+ )
55
+
56
+ EmbedEdit = Callable[[torch.Tensor], torch.Tensor | None]
57
+
58
+ # Padded tokens per backward pass of integrated gradients. Each interpolation row keeps
59
+ # a backward graph through every layer up to the unit's, so memory grows with rows ×
60
+ # example length. A whole Qwen2.5-1.5B document at 8 rows rose 8.3 GB. Rows are
61
+ # independent, so the attribution doesn't depend on how they are batched.
62
+ ATTRIBUTION_BATCH_TOKENS = 1024
63
+
64
+
65
+ def baseline_token_id(tokenizer) -> int:
66
+ """The reference token: the pad token, else end-of-sequence, else id 0."""
67
+ for name in ("pad_token_id", "eos_token_id"):
68
+ value = getattr(tokenizer, name, None)
69
+ if isinstance(value, int):
70
+ return value
71
+ return 0
72
+
73
+
74
+ def _pole_values(
75
+ runner: NeuronHookRunner,
76
+ ids: torch.Tensor,
77
+ layer: int,
78
+ neuron_idx: int,
79
+ pole: Pole,
80
+ edit: EmbedEdit,
81
+ ) -> torch.Tensor:
82
+ """Signed pole activation ``[batch, seq]`` with the token embeddings edited by ``edit``.
83
+
84
+ The forward pass stops at ``layer``'s MLP: later layers can't change it.
85
+ """
86
+ captured: list[torch.Tensor] = []
87
+
88
+ def capture(act: torch.Tensor) -> None:
89
+ captured.append(act[..., neuron_idx])
90
+ raise _StopForward
91
+
92
+ embed = runner.model.get_input_embeddings()
93
+ handle = embed.register_forward_hook(lambda _module, _args, out: edit(out))
94
+ try:
95
+ with suppress(_StopForward), runner.hooks({layer: capture}):
96
+ runner.forward({"input_ids": ids})
97
+ finally:
98
+ handle.remove()
99
+ value = captured[-1].float()
100
+ return value if pole == "positive" else -value
101
+
102
+
103
+ def reference(x: torch.Tensor, baseline: torch.Tensor, keep_first: bool) -> torch.Tensor:
104
+ """Return the baseline input for embeddings ``x`` ``[batch, seq, d]``.
105
+
106
+ It is ``baseline`` at every position, except the first when ``keep_first``.
107
+ """
108
+ base = baseline.to(x.dtype).expand_as(x).clone()
109
+ if keep_first:
110
+ base[:, 0] = x[:, 0]
111
+ return base
112
+
113
+
114
+ def integrated_gradients(
115
+ runner: NeuronHookRunner,
116
+ ids: torch.Tensor,
117
+ layer: int,
118
+ neuron_idx: int,
119
+ pole: Pole,
120
+ baseline: torch.Tensor,
121
+ *,
122
+ steps: int = 16,
123
+ batch_size: int = 8,
124
+ keep_first: bool = True,
125
+ ) -> torch.Tensor:
126
+ """Attribution ``[seq]`` of the last token's signed pole activation, one row ``ids``.
127
+
128
+ With ``keep_first`` the first token is not interpolated and gets no attribution.
129
+ """
130
+ alphas = (torch.arange(steps, dtype=torch.float32) + 0.5) / steps
131
+ grad_sum: torch.Tensor | None = None
132
+ diff: torch.Tensor | None = None
133
+ for chunk in alphas.split(batch_size):
134
+ points: list[torch.Tensor] = []
135
+
136
+ def interpolate(out: torch.Tensor, chunk=chunk, points=points) -> torch.Tensor:
137
+ nonlocal diff
138
+ x = out.detach()
139
+ base = reference(x, baseline, keep_first)
140
+ alpha = chunk.to(x.device, x.dtype)[:, None, None]
141
+ point = (base + alpha * (x - base)).requires_grad_(True)
142
+ diff = (x[0] - base[0]).float()
143
+ points.append(point)
144
+ return point
145
+
146
+ with torch.enable_grad(), recompute_below(runner, layer):
147
+ rows = ids.expand(len(chunk), -1)
148
+ values = _pole_values(runner, rows, layer, neuron_idx, pole, interpolate)[:, -1]
149
+ (grad,) = torch.autograd.grad(values.sum(), points[-1])
150
+ part = grad.float().sum(0)
151
+ grad_sum = part if grad_sum is None else grad_sum + part
152
+ assert grad_sum is not None and diff is not None # noqa: S101 (for the type checker)
153
+ return (grad_sum / steps * diff).sum(-1)
154
+
155
+
156
+ @torch.no_grad()
157
+ def ablate_positions(
158
+ runner: NeuronHookRunner,
159
+ ids: torch.Tensor,
160
+ layer: int,
161
+ neuron_idx: int,
162
+ pole: Pole,
163
+ baseline: torch.Tensor,
164
+ positions: Sequence[int],
165
+ *,
166
+ batch_size: int = 8,
167
+ ) -> torch.Tensor:
168
+ """The last token's signed pole activation with each of ``positions`` replaced alone."""
169
+ out: list[torch.Tensor] = []
170
+ for start in range(0, len(positions), batch_size):
171
+ chunk = torch.tensor(positions[start : start + batch_size])
172
+
173
+ def replace(emb: torch.Tensor, chunk=chunk) -> torch.Tensor:
174
+ emb = emb.clone()
175
+ emb[torch.arange(len(chunk)), chunk.to(emb.device)] = baseline.to(emb.dtype)
176
+ return emb
177
+
178
+ rows = ids.expand(len(chunk), -1)
179
+ out.append(_pole_values(runner, rows, layer, neuron_idx, pole, replace)[:, -1])
180
+ return torch.cat(out)
181
+
182
+
183
+ def _distance_shares(attr: torch.Tensor) -> tuple[list[float], float]:
184
+ """Share of |attr| per band of :data:`DISTANCE_BANDS`, and the weighted mean distance."""
185
+ weight = attr.abs()
186
+ total = float(weight.sum()) or 1.0
187
+ distance = torch.arange(len(attr) - 1, -1, -1, dtype=torch.float32)
188
+ shares = []
189
+ for low, high in DISTANCE_BANDS:
190
+ band = (distance >= low) & (distance <= (high if high is not None else len(attr)))
191
+ shares.append(float(weight[band].sum()) / total)
192
+ return shares, float((weight * distance).sum()) / total
193
+
194
+
195
+ def _converged_attribution(
196
+ runner, ids, args, baseline, gap: float, settings: _Settings
197
+ ) -> tuple[torch.Tensor, int]:
198
+ """Return the attribution as shares of ``gap``, and the steps that it took.
199
+
200
+ The steps double from ``settings.steps`` until Σ is within ``settings.tolerance`` of
201
+ 1, or ``settings.max_steps`` is reached. Each backward pass takes at most
202
+ ``settings.batch_size`` rows and :data:`ATTRIBUTION_BATCH_TOKENS` tokens, and at
203
+ least one row.
204
+ """
205
+ steps, max_steps, tolerance = settings.steps, settings.max_steps, settings.tolerance
206
+ keep_first = ids.shape[1] > 1
207
+ rows = max(1, min(settings.batch_size, ATTRIBUTION_BATCH_TOKENS // ids.shape[1]))
208
+ rows = stream_rows(runner, ids.shape[1], rows, logits=False) # more when streamed
209
+ while True:
210
+ attr = integrated_gradients(
211
+ runner, ids, *args, baseline, steps=steps, batch_size=rows, keep_first=keep_first
212
+ ).cpu()
213
+ attr = attr / gap
214
+ if abs(float(attr.sum()) - 1.0) <= tolerance or steps * 2 > max_steps:
215
+ return attr, steps
216
+ steps *= 2
217
+
218
+
219
+ def input_attribution(
220
+ runner: NeuronHookRunner,
221
+ examples: Sequence[ActivatingExample],
222
+ layer: int,
223
+ neuron_idx: int,
224
+ pole: Pole,
225
+ *,
226
+ n_examples: int = 5,
227
+ steps: int = 16,
228
+ max_steps: int = 64,
229
+ tolerance: float = 0.1,
230
+ top_k: int = 5,
231
+ near: int = 4,
232
+ batch_size: int = 8,
233
+ skip_first: bool = False,
234
+ ) -> InputAttribution:
235
+ """Integrated gradients at the peak of the first ``n_examples`` top examples.
236
+
237
+ Each example starts at ``steps`` integration steps and doubles them, up to
238
+ ``max_steps``, until the attribution sums to within ``tolerance`` of the gap. Only
239
+ examples that get there count towards the distance shares and the agreement. Each
240
+ example keeps its ``top_k`` most attributed tokens; the ablation check covers those,
241
+ the peak and the ``near`` tokens before it.
242
+ """
243
+ if runner.tokenizer is None:
244
+ raise ValueError("input_attribution requires a tokenizer")
245
+ base_id = baseline_token_id(runner.tokenizer)
246
+ baseline = runner.model.get_input_embeddings().weight[base_id].detach()
247
+ args = (layer, neuron_idx, pole)
248
+ settings = _Settings(steps, max_steps, tolerance, top_k, near, batch_size, skip_first)
249
+ results: list[ContextExample] = []
250
+ band_shares: list[list[float]] = []
251
+ distances: list[float] = []
252
+ checked_attr: list[float] = []
253
+ checked_effect: list[float] = []
254
+ for n, example in enumerate(examples[:n_examples]):
255
+ found = _attribute_example(runner, n, example, args, baseline, settings)
256
+ if found is None:
257
+ continue
258
+ result, converged = found
259
+ results.append(result)
260
+ if converged is not None:
261
+ attrs, effects, shares, mean_distance = converged
262
+ checked_attr += attrs
263
+ checked_effect += effects
264
+ band_shares.append(shares)
265
+ distances.append(mean_distance)
266
+ agreement = (
267
+ spearman(torch.tensor(checked_attr), torch.tensor(checked_effect))
268
+ if len(checked_attr) >= 3
269
+ else None
270
+ )
271
+ return InputAttribution(
272
+ baseline_token=runner.tokenizer.decode([base_id]),
273
+ tolerance=tolerance,
274
+ examples=results,
275
+ distances=_band_shares(band_shares),
276
+ mean_distance=round(sum(distances) / len(distances), 2) if distances else None,
277
+ ablation_agreement=round(agreement, 4) if agreement is not None else None,
278
+ )
279
+
280
+
281
+ @dataclass(frozen=True)
282
+ class _Settings:
283
+ """``input_attribution``'s options, as one example needs them."""
284
+
285
+ steps: int
286
+ max_steps: int
287
+ tolerance: float
288
+ top_k: int
289
+ near: int
290
+ batch_size: int
291
+ skip_first: bool
292
+
293
+
294
+ def _attribute_example(runner, n: int, example, args, baseline, settings: _Settings):
295
+ """Attribute one example, and return its ContextExample and the numbers for the shares.
296
+
297
+ If it converged, the numbers are its checked attributions, ablation effects,
298
+ distance-band shares and mean distance. ``None`` if there is no gap.
299
+ """
300
+ ids, values, peak, gap = _peak_gap(runner, example, args, baseline, settings.skip_first)
301
+ if gap <= 1e-6:
302
+ return None # the reference input activates the pole as much: nothing to attribute
303
+ tolerance = settings.tolerance
304
+ attr, used = _converged_attribution(runner, ids, args, baseline, gap, settings)
305
+ error = abs(float(attr.sum()) - 1.0)
306
+ top = attr.abs().topk(min(settings.top_k, len(attr))).indices.tolist()
307
+ checked = sorted(set(top) | set(range(max(0, peak - settings.near), peak + 1)))
308
+ ablated = ablate_positions(
309
+ runner, ids, *args, baseline, checked, batch_size=settings.batch_size
310
+ )
311
+ effect = dict(zip(checked, ((float(values[peak]) - ablated.cpu()) / gap).tolist(), strict=True))
312
+ converged = None
313
+ if error <= tolerance:
314
+ shares, mean_distance = _distance_shares(attr)
315
+ attrs = [float(attr[i]) for i in checked]
316
+ converged = (attrs, [effect[i] for i in checked], shares, mean_distance)
317
+ tokenizer, token_ids = runner.tokenizer, ids[0].tolist()
318
+ result = ContextExample(
319
+ example=n,
320
+ peak_token=tokenizer.decode([token_ids[peak]]),
321
+ tokens=peak + 1,
322
+ steps=used,
323
+ converged=error <= tolerance,
324
+ completeness_error=round(error, 4),
325
+ top=[
326
+ AttributedToken(
327
+ offset=i - peak,
328
+ token=tokenizer.decode([token_ids[i]]),
329
+ attribution=round(float(attr[i]), 4),
330
+ ablation_effect=round(effect[i], 4),
331
+ )
332
+ for i in top
333
+ ],
334
+ )
335
+ return result, converged
336
+
337
+
338
+ def _peak_gap(runner, example, args, baseline, skip_first: bool):
339
+ """Return the example cut at its peak, its pole values, the peak, and the gap.
340
+
341
+ The gap is between the peak and the reference input's activation there.
342
+ """
343
+ ids = runner.encode(example.text)["input_ids"][:1]
344
+ with torch.no_grad():
345
+ values = _pole_values(runner, ids, *args, lambda _out: None)[0]
346
+ start = int(skip_first and len(values) > 1)
347
+ peak = start + int(values[start:].argmax())
348
+ ids = ids[:, : peak + 1]
349
+ empty = _pole_values(
350
+ runner, ids, *args, lambda out, keep=peak > 0: reference(out, baseline, keep)
351
+ )
352
+ return ids, values, peak, float(values[peak] - empty[0, -1])
353
+
354
+
355
+ def _band_shares(band_shares: list[list[float]]) -> list[DistanceShare]:
356
+ """Each distance band's mean share of the attribution, over the converged examples."""
357
+ count = len(band_shares)
358
+ if not count:
359
+ return []
360
+ return [
361
+ DistanceShare(
362
+ min_distance=low,
363
+ max_distance=high,
364
+ share=round(min(1.0, sum(s[b] for s in band_shares) / count), 4),
365
+ )
366
+ for b, (low, high) in enumerate(DISTANCE_BANDS)
367
+ ]
@@ -0,0 +1,183 @@
1
+ """Jacobian lens readings of a neuron pole's output direction.
2
+
3
+ The Jacobian lens is from Gurnee et al., 2026, "Verbalizable Representations Form a
4
+ Global Workspace in Language Models" (arXiv:2607.15495). It decodes a layer-ℓ residual
5
+ vector h as follows:
6
+
7
+ lens(h) = softmax(W_U · norm(J_ℓ h)), J_ℓ = E[∂h_final,t' / ∂h_ℓ,t]
8
+
9
+ The average is over source positions t, every later position t' ≥ t, and a corpus of
10
+ prompts. The logit lens is the special case J_ℓ = I. The two agree in late layers and
11
+ diverge earlier, where the logit lens is unreliable.
12
+
13
+ For one neuron we only need J_ℓ w for its output direction w, not the full matrix.
14
+ Add ε·w to the residual at *every* position. To first order, this changes the final
15
+ residual at t' by Σ_{t ≤ t'} J_{t,t'} w. (The causal mask stops later positions acting on
16
+ earlier ones.) Sum that over t', and divide by the number of (t, t') pairs. This gives
17
+ the paper's average from one perturbed pass per sign (central differences).
18
+
19
+ The average includes pairs where t' > t contributes nothing. So its overall scale
20
+ shrinks with context length: at the last layer it is I · 2/(seq+1). Every reading
21
+ we take (top-k ranking, cosine, kurtosis) is scale-invariant.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from collections.abc import Sequence
27
+ from contextlib import suppress
28
+
29
+ import torch
30
+ from torch import nn
31
+
32
+ from openinterp.core.schemas import Pole, VocabularyLens
33
+ from openinterp.harness.hooks import NeuronHookRunner, _StopForward
34
+ from openinterp.harness.memory import is_streamed, stream_rows
35
+
36
+
37
+ def _final_residual_module(runner: NeuronHookRunner) -> nn.Module:
38
+ """The module whose input is the final residual stream (pre final norm)."""
39
+ return runner.final_norm() or runner.model.get_output_embeddings()
40
+
41
+
42
+ def _final_residual(
43
+ runner: NeuronHookRunner, module: nn.Module, encoded: dict, layer: int, shift: torch.Tensor
44
+ ) -> torch.Tensor:
45
+ """Final residual stream [b, seq, d] with ``shift`` added after ``layer``'s MLP.
46
+
47
+ The pass stops at the final norm, so it skips the logits, which nothing here reads.
48
+ """
49
+ captured: list[torch.Tensor] = []
50
+
51
+ def capture(_module, args) -> None:
52
+ captured.append(args[0].detach().float())
53
+ raise _StopForward
54
+
55
+ handle = module.register_forward_pre_hook(capture)
56
+ try:
57
+ with suppress(_StopForward):
58
+ runner.run_with_interventions(
59
+ encoded, output_interventions={layer: lambda out: out + shift.to(out.dtype)}
60
+ )
61
+ finally:
62
+ handle.remove()
63
+ return captured[-1]
64
+
65
+
66
+ @torch.no_grad()
67
+ def transported_direction(
68
+ runner: NeuronHookRunner,
69
+ prompts: Sequence[str] | torch.Tensor,
70
+ layer: int,
71
+ direction: torch.Tensor,
72
+ *,
73
+ eps: float = 1e-2,
74
+ batch_size: int = 8,
75
+ ) -> torch.Tensor:
76
+ """Estimate J_ℓ · direction, where ℓ is the residual stream just after ``layer``'s MLP."""
77
+ module = _final_residual_module(runner)
78
+ direction = direction.to(runner.device, torch.float32)
79
+ step = eps / max(float(direction.norm()), 1e-12)
80
+ total = torch.zeros(direction.shape, dtype=torch.float64) # on the CPU: MPS has no float64
81
+ pairs = 0
82
+ if not isinstance(prompts, torch.Tensor) and is_streamed(runner.model):
83
+ longest = runner.encode(list(prompts))["input_ids"].shape[1]
84
+ batch_size = stream_rows(runner, longest, batch_size, logits=False)
85
+ batches = (
86
+ [prompts]
87
+ if isinstance(prompts, torch.Tensor)
88
+ else [list(prompts[i : i + batch_size]) for i in range(0, len(prompts), batch_size)]
89
+ )
90
+ for batch in batches:
91
+ encoded = runner.encode(batch)
92
+ mask = encoded.get("attention_mask", torch.ones_like(encoded["input_ids"])).bool()
93
+ plus = _final_residual(runner, module, encoded, layer, step * direction)
94
+ minus = _final_residual(runner, module, encoded, layer, -step * direction)
95
+ delta = (plus - minus) / (2 * step) # [b, seq, d]
96
+ total += delta[mask].cpu().double().sum(0)
97
+ lengths = mask.sum(1)
98
+ pairs += int((lengths * (lengths + 1) // 2).sum())
99
+ return (total / max(pairs, 1)).float().to(runner.device)
100
+
101
+
102
+ def _through_final_norm(runner: NeuronHookRunner, vector: torch.Tensor) -> torch.Tensor:
103
+ """Apply the part of the final norm that affects a direction.
104
+
105
+ That is centring for LayerNorm, and the gain. The bias is a constant offset,
106
+ unrelated to the direction, so it is dropped.
107
+ """
108
+ norm = runner.final_norm()
109
+ v = vector.float()
110
+ if isinstance(norm, nn.LayerNorm):
111
+ v = v - v.mean()
112
+ weight = getattr(norm, "weight", None)
113
+ return v * weight.detach().float() if weight is not None else v
114
+
115
+
116
+ def _decode(runner: NeuronHookRunner, vector: torch.Tensor, k: int):
117
+ unembed = runner.model.get_output_embeddings().weight.detach().float()
118
+ scores = unembed @ _through_final_norm(runner, vector).to(unembed.device)
119
+ z = (scores - scores.mean()) / scores.std().clamp(min=1e-12)
120
+ top = scores.topk(k).indices.tolist()
121
+ bottom = (-scores).topk(k).indices.tolist()
122
+ return top, bottom, float((z**4).mean()), _rounded(z[top]), _rounded(z[bottom])
123
+
124
+
125
+ def _rounded(values: torch.Tensor) -> list[float]:
126
+ return [round(float(v), 4) for v in values]
127
+
128
+
129
+ def vocabulary_lens(
130
+ runner: NeuronHookRunner,
131
+ prompts: Sequence[str],
132
+ layer: int,
133
+ neuron_idx: int,
134
+ pole: Pole,
135
+ *,
136
+ k: int = 10,
137
+ ) -> VocabularyLens:
138
+ """Read the pole's output direction through both the logit lens and the Jacobian lens."""
139
+ direction = runner.neuron_output_direction(layer, neuron_idx).float()
140
+ if pole == "negative":
141
+ direction = -direction
142
+ return direction_lens(runner, prompts, layer, direction, k=k)
143
+
144
+
145
+ def direction_lens(
146
+ runner: NeuronHookRunner,
147
+ prompts: Sequence[str],
148
+ layer: int,
149
+ direction: torch.Tensor,
150
+ *,
151
+ k: int = 10,
152
+ ) -> VocabularyLens:
153
+ """Read any vector written after ``layer``'s MLP through both lenses.
154
+
155
+ The vector is a pole's output direction, or a dictionary latent's decoder row.
156
+ """
157
+ direction = direction.float()
158
+ transported = transported_direction(runner, prompts, layer, direction)
159
+
160
+ def words(ids: list[int]) -> list[str]:
161
+ tok = runner.tokenizer
162
+ return [tok.decode([i]) if tok is not None else str(i) for i in ids]
163
+
164
+ logit_top, logit_bottom, logit_kurt, logit_top_z, logit_bottom_z = _decode(runner, direction, k)
165
+ jac_top, jac_bottom, jac_kurt, jac_top_z, jac_bottom_z = _decode(runner, transported, k)
166
+ cosine = torch.nn.functional.cosine_similarity(
167
+ direction.to(transported.device), transported, dim=0
168
+ )
169
+ return VocabularyLens(
170
+ logit_promoted=words(logit_top),
171
+ logit_suppressed=words(logit_bottom),
172
+ jacobian_promoted=words(jac_top),
173
+ jacobian_suppressed=words(jac_bottom),
174
+ logit_promoted_z=logit_top_z,
175
+ logit_suppressed_z=logit_bottom_z,
176
+ jacobian_promoted_z=jac_top_z,
177
+ jacobian_suppressed_z=jac_bottom_z,
178
+ overlap=round(len(set(logit_top) & set(jac_top)) / k, 4),
179
+ direction_cosine=round(max(-1.0, min(1.0, float(cosine))), 6),
180
+ logit_kurtosis=round(logit_kurt, 4),
181
+ jacobian_kurtosis=round(jac_kurt, 4),
182
+ prompts=len(prompts),
183
+ )