open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,184 @@
1
+ """Confound control: how much of a pole's activation trivial explanations account for.
2
+
3
+ During the scan we accumulate, per token id and per position, the count, sum and sum
4
+ of squares of the pole strength. From that we derive:
5
+
6
+ * ``token_r2`` / ``position_r2``: variance explained by a group-mean model (between-
7
+ group sum of squares over total sum of squares);
8
+ * ``frequency_correlation``: Spearman's rho, over tokens seen ``min_count`` times or
9
+ more, between token frequency and mean strength. Rank-based, because mean strength
10
+ per token is heavily skewed and the relationship of interest is monotonic;
11
+ * trivial *baselines* for the reviewer's probes: "active if any token is a trigger
12
+ token" and "active if any position is a high-activation position".
13
+
14
+ Take a hypothesis that cannot match the best trivial baseline on the same probes. It
15
+ adds nothing beyond "this neuron fires on token X" (or "at position p").
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import statistics
21
+ from collections.abc import Sequence
22
+ from typing import Any
23
+
24
+ import torch
25
+
26
+ from openinterp.core.schemas import ConfoundProfile, TokenStat
27
+ from openinterp.core.stats import spearman
28
+
29
+ MAX_TRIGGER_TOKENS = 200
30
+
31
+
32
+ class ConfoundAccumulator:
33
+ """Per-token and per-position sums of a pole's strength over a scan."""
34
+
35
+ def __init__(self) -> None:
36
+ self.token: dict[str, torch.Tensor] = {}
37
+ self.position: dict[str, torch.Tensor] = {}
38
+ self.total = torch.zeros(3, dtype=torch.float64) # n, sum, sum of squares
39
+
40
+ @staticmethod
41
+ def _add(store: dict[str, torch.Tensor], keys: torch.Tensor, values: torch.Tensor) -> None:
42
+ if keys.numel() == 0:
43
+ return
44
+ for name, weights in (
45
+ ("n", torch.ones_like(values)),
46
+ ("s", values),
47
+ ("ss", values * values),
48
+ ):
49
+ counts = torch.bincount(keys, weights=weights)
50
+ current = store.get(name, torch.zeros(0, dtype=torch.float64))
51
+ if current.numel() < counts.numel():
52
+ pad = torch.zeros(counts.numel() - current.numel(), dtype=torch.float64)
53
+ current = torch.cat([current, pad])
54
+ current[: counts.numel()] += counts
55
+ store[name] = current
56
+
57
+ def update(self, ids: torch.Tensor, positions: torch.Tensor, strengths: torch.Tensor) -> None:
58
+ """Add a batch of token ids, positions and strengths."""
59
+ values = strengths.double()
60
+ self._add(self.token, ids.long(), values)
61
+ self._add(self.position, positions.long(), values)
62
+ self.total += torch.tensor(
63
+ [values.numel(), values.sum(), (values * values).sum()], dtype=torch.float64
64
+ )
65
+
66
+ def _r2(self, store: dict[str, torch.Tensor]) -> float:
67
+ n, s, ss = (float(x) for x in self.total)
68
+ if n == 0:
69
+ return 0.0
70
+ total = ss - s * s / n
71
+ seen = store["n"] > 0
72
+ between = float((store["s"][seen] ** 2 / store["n"][seen]).sum()) - s * s / n
73
+ return min(1.0, max(0.0, between / total)) if total > 1e-12 else 0.0
74
+
75
+ def finalize(self, tokenizer: Any, min_count: int = 3) -> ConfoundProfile:
76
+ """The confound profile of everything added (see the module docstring)."""
77
+ if not self.token:
78
+ return ConfoundProfile(
79
+ token_r2=0.0,
80
+ position_r2=0.0,
81
+ frequency_correlation=0.0,
82
+ min_count=min_count,
83
+ trigger_tokens=[],
84
+ position_means=[],
85
+ )
86
+ n, s = self.token["n"], self.token["s"]
87
+ frequent = (n >= min_count).nonzero().flatten()
88
+ means = s[frequent] / n[frequent]
89
+ order = means.argsort(descending=True)[:MAX_TRIGGER_TOKENS]
90
+ triggers = [
91
+ TokenStat(
92
+ token=tokenizer.decode([int(frequent[i])]) if tokenizer else str(int(frequent[i])),
93
+ token_id=int(frequent[i]),
94
+ mean_strength=round(max(0.0, float(means[i])), 6),
95
+ count=int(n[frequent[i]]),
96
+ )
97
+ for i in order.tolist()
98
+ ]
99
+ pn, ps = self.position["n"], self.position["s"]
100
+ position_means = [
101
+ round(float(ps[p] / pn[p]), 6) if pn[p] > 0 else 0.0 for p in range(pn.numel())
102
+ ]
103
+ return ConfoundProfile(
104
+ token_r2=round(self._r2(self.token), 6),
105
+ position_r2=round(self._r2(self.position), 6),
106
+ frequency_correlation=round(spearman(n[frequent], means), 6),
107
+ min_count=min_count,
108
+ trigger_tokens=triggers,
109
+ position_means=position_means,
110
+ )
111
+
112
+
113
+ def baseline_predictions(
114
+ profile: ConfoundProfile,
115
+ tokenizer: Any,
116
+ texts: Sequence[str],
117
+ threshold: float,
118
+ skip_first: bool = False,
119
+ ) -> dict[str, list[bool]]:
120
+ """Return the predicted "active" for each text under each trivial baseline.
121
+
122
+ Like the unit's own peak, a baseline doesn't read the first token when
123
+ ``skip_first`` (protocol 0.8).
124
+ """
125
+ triggers = {t.token_id for t in profile.trigger_tokens if t.mean_strength >= threshold}
126
+ hot_positions = {p for p, m in enumerate(profile.position_means) if m >= threshold}
127
+ start = int(skip_first)
128
+ token_pred, position_pred = [], []
129
+ for text in texts:
130
+ ids = tokenizer([text], return_tensors="pt")["input_ids"][0].tolist()
131
+ token_pred.append(any(i in triggers for i in ids[start:]))
132
+ position_pred.append(any(p in hot_positions for p in range(start, len(ids))))
133
+ return {"token identity": token_pred, "position": position_pred}
134
+
135
+
136
+ def length_predictions(
137
+ tokenizer: Any, texts: Sequence[str], skip_first: bool = False
138
+ ) -> dict[str, list[bool]]:
139
+ """Return "active if the probe has at least n tokens" for each cut n.
140
+
141
+ A unit that responds to position passes probes that differ only in length. On a
142
+ randomly initialised GPT-2, two units were VERIFIED that way (issue #3525). Tokens are
143
+ counted as the unit reads them, so the first is left out when ``skip_first``. It is a
144
+ proposed confound baseline: ``scripts/protocol_mix.py`` measures what it would change.
145
+ """
146
+ lengths = [
147
+ len(tokenizer([text], return_tensors="pt")["input_ids"][0]) - int(skip_first)
148
+ for text in texts
149
+ ]
150
+ return {f"probe length ≥ {n} tokens": [n <= k for k in lengths] for n in sorted(set(lengths))}
151
+
152
+
153
+ # A ratio of the two probe groups' median token lengths above which the reviewer is warned.
154
+ # Probes that differ this much in length can be told apart by length alone (issue #3525).
155
+ LENGTH_RATIO_WARN = 1.5
156
+
157
+
158
+ def length_ratio(probes: Sequence[Any], tokenizer, skip_first: bool) -> float | None:
159
+ """The larger median token length of the two probe groups over the smaller.
160
+
161
+ The groups are the probes expected active and those expected inactive. Tokens are
162
+ counted as the unit reads them. ``None`` when a group is empty.
163
+ """
164
+ groups: dict[bool, list[int]] = {True: [], False: []}
165
+ for probe in probes:
166
+ n = len(tokenizer.encode(probe.text)) - int(skip_first)
167
+ groups[probe.expected_active].append(max(n, 1))
168
+ if not groups[True] or not groups[False]:
169
+ return None
170
+ medians = sorted(statistics.median(g) for g in groups.values())
171
+ return medians[1] / medians[0]
172
+
173
+
174
+ def length_warning(probes: Sequence[Any], tokenizer, skip_first: bool) -> str | None:
175
+ """A warning when the two probe groups differ in length by more than LENGTH_RATIO_WARN."""
176
+ ratio = length_ratio(probes, tokenizer, skip_first)
177
+ if ratio is None or ratio <= LENGTH_RATIO_WARN:
178
+ return None
179
+ return (
180
+ f"The median token lengths of the active and inactive probes differ {ratio:.1f}×, "
181
+ f"above {LENGTH_RATIO_WARN}×. A rule that reads only probe length may then explain "
182
+ "the probes as well as the hypothesis does, and the review fails a hypothesis that "
183
+ "doesn't beat it."
184
+ )
@@ -0,0 +1,301 @@
1
+ """Covariance between neurons: correlated partners and joint ablation.
2
+
3
+ A single neuron rarely acts alone. Say that neuron A's pole co-varies with neuron B.
4
+ Then a probe that "fails" on A may be handled by B. Also, ablating A alone can show no
5
+ effect, because B compensates. So we record A's partner neuron *poles* during the scan,
6
+ and the reviewer ablates A together with them.
7
+
8
+ Pole strengths are sparse and heavy-tailed: most tokens are exactly zero. So a global
9
+ Pearson correlation mostly measures agreement on "nothing is happening", and a few
10
+ large values drive it. :class:`PartnerSearch` ranks partners by the 95% lower bound of
11
+ their co-activation lift instead. That looks only at the regime where redundancy
12
+ matters, and does not reward a handful of chance coincidences. It also reports a rank
13
+ (Spearman) correlation per partner pole. Pearson on signed
14
+ activations is kept for reference.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ from collections.abc import Callable, Sequence
20
+
21
+ import torch
22
+ import torch.nn.functional as F
23
+
24
+ from openinterp.core.schemas import CorrelatedNeuron, JointAblation, Pole
25
+ from openinterp.core.stats import spearman_columns
26
+ from openinterp.harness.hooks import Inputs, NeuronHookRunner, isolate_pole
27
+
28
+
29
+ class CovarianceAccumulator:
30
+ """Streaming Pearson correlation between one target series and every layer neuron."""
31
+
32
+ def __init__(self, width: int) -> None:
33
+ self.n = 0
34
+ self.sx = 0.0
35
+ self.sxx = 0.0
36
+ self.sy = torch.zeros(width, dtype=torch.float64)
37
+ self.syy = torch.zeros(width, dtype=torch.float64)
38
+ self.sxy = torch.zeros(width, dtype=torch.float64)
39
+
40
+ def update(self, target: torch.Tensor, layer_acts: torch.Tensor) -> None:
41
+ """``target`` is [n] pole strengths; ``layer_acts`` is [n, width] raw activations."""
42
+ x, y = target.double(), layer_acts.double()
43
+ self.n += x.numel()
44
+ self.sx += float(x.sum())
45
+ self.sxx += float((x * x).sum())
46
+ self.sy += y.sum(0)
47
+ self.syy += (y * y).sum(0)
48
+ self.sxy += (x[:, None] * y).sum(0)
49
+
50
+ def correlations(self) -> torch.Tensor:
51
+ """Pearson's r between the target and each column; 0 where either has no variance."""
52
+ n = max(self.n, 1)
53
+ cov = self.sxy / n - (self.sx / n) * (self.sy / n)
54
+ var_x = self.sxx / n - (self.sx / n) ** 2
55
+ var_y = self.syy / n - (self.sy / n) ** 2
56
+ denom = torch.sqrt(torch.clamp(var_y * var_x, min=0.0))
57
+ return torch.where(denom > 1e-12, cov / denom, torch.zeros_like(cov)).clamp(-1, 1)
58
+
59
+
60
+ class PartnerSearch:
61
+ """Stream a scan and find the neuron poles that co-activate with the target pole.
62
+
63
+ Keeps, with fixed-seed randomness so results are reproducible:
64
+
65
+ * a uniform reservoir sample of tokens (target strength + whole-layer activations),
66
+ for partner base rates, 99th-percentile "active" thresholds and Spearman;
67
+ * the ``top_k`` tokens where the target is strongest, which hold the target's
68
+ active regime. It is rare, so a uniform sample would barely contain it.
69
+ """
70
+
71
+ def __init__(self, width: int, *, reservoir: int = 8192, top_k: int = 2048, seed: int = 0):
72
+ self.pearson = CovarianceAccumulator(width)
73
+ self.reservoir, self.top_k = reservoir, top_k
74
+ self.generator = torch.Generator().manual_seed(seed)
75
+ self.seen = 0
76
+ self.res_target = torch.empty(0)
77
+ self.res_rows = torch.empty(0, width, dtype=torch.float16)
78
+ self.top_target = torch.empty(0)
79
+ self.top_rows = torch.empty(0, width, dtype=torch.float16)
80
+
81
+ def update(self, target: torch.Tensor, layer_acts: torch.Tensor) -> None:
82
+ """Add a batch to the Pearson sums, the reservoir and the top-k tokens."""
83
+ target, rows = target.float(), layer_acts.half()
84
+ self.pearson.update(target, layer_acts)
85
+
86
+ # Reservoir sampling (Algorithm R), vectorised per batch.
87
+ fill = max(0, min(self.reservoir - self.res_target.numel(), target.numel()))
88
+ self.res_target = torch.cat([self.res_target, target[:fill]])
89
+ self.res_rows = torch.cat([self.res_rows, rows[:fill]])
90
+ rest = torch.arange(fill, target.numel())
91
+ if rest.numel():
92
+ index = self.seen + rest + 1
93
+ slots = (torch.rand(rest.numel(), generator=self.generator) * index).long()
94
+ for i, slot in zip(
95
+ rest[slots < self.reservoir].tolist(),
96
+ slots[slots < self.reservoir].tolist(),
97
+ strict=True,
98
+ ):
99
+ self.res_target[slot] = target[i]
100
+ self.res_rows[slot] = rows[i]
101
+ self.seen += target.numel()
102
+
103
+ merged = torch.cat([self.top_target, target])
104
+ keep = merged.topk(min(self.top_k, merged.numel())).indices
105
+ self.top_target = merged[keep]
106
+ self.top_rows = torch.cat([self.top_rows, rows])[keep]
107
+
108
+ @staticmethod
109
+ def _poles(rows: torch.Tensor) -> torch.Tensor:
110
+ """[n, w] signed activations -> [n, 2w] pole strengths (positive, then negative)."""
111
+ rows = rows.float()
112
+ return torch.cat([rows.clamp(min=0), (-rows).clamp(min=0)], dim=1)
113
+
114
+ def partners(
115
+ self, exclude: int, active_threshold: float, *, k: int = 10, quantile: float = 0.99
116
+ ) -> list[CorrelatedNeuron]:
117
+ """The ``k`` poles with the highest lower-bound co-activation lift, besides ``exclude``."""
118
+ if self.res_target.numel() < 3:
119
+ return []
120
+ width = self.res_rows.shape[1]
121
+ sample = self._poles(self.res_rows)
122
+ thresholds = _column_quantile(sample, quantile)
123
+ base = ((sample > thresholds).sum(0).double() + 1) / (sample.shape[0] + 2)
124
+
125
+ active = self._poles(self.top_rows[self.top_target >= active_threshold])
126
+ coactive = (active > thresholds).sum(0)
127
+ n_active = active.shape[0]
128
+ if n_active:
129
+ lift = ((coactive.double() + 1) / (n_active + 2)) / base
130
+ lower = wilson_lower(coactive.double(), n_active) / base
131
+ else: # the target never fired: no co-activation evidence, rank by Spearman alone
132
+ lift = torch.ones(2 * width, dtype=torch.float64)
133
+ lower = torch.zeros(2 * width, dtype=torch.float64)
134
+ rho = spearman_columns(self.res_target, sample)
135
+ pearson = self.pearson.correlations()
136
+
137
+ candidates = [c for c in range(2 * width) if c % width != exclude] # not itself
138
+ order = sorted(
139
+ candidates,
140
+ key=lambda c: (-float(lower[c]), -abs(float(rho[c])), c),
141
+ )[:k]
142
+ return [
143
+ CorrelatedNeuron(
144
+ neuron_idx=c % width,
145
+ pole="positive" if c < width else "negative",
146
+ coactivation_lift=round(float(lift[c]), 4),
147
+ coactivation_lift_lower=round(float(lower[c]), 4),
148
+ coactive_tokens=int(coactive[c]),
149
+ spearman=round(float(rho[c]), 6),
150
+ pearson=round(float(pearson[c % width]), 6),
151
+ )
152
+ for c in order
153
+ ]
154
+
155
+
156
+ def wilson_lower(successes: torch.Tensor, n: int, z: float = 1.96) -> torch.Tensor:
157
+ """Lower bound of the Wilson score interval for a binomial proportion."""
158
+ p = successes / n
159
+ centre = p + z * z / (2 * n)
160
+ margin = z * torch.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
161
+ return ((centre - margin) / (1 + z * z / n)).clamp(min=0.0)
162
+
163
+
164
+ def _column_quantile(x: torch.Tensor, q: float, chunk: int = 1024) -> torch.Tensor:
165
+ """Per-column nearest-rank quantile; chunked because torch.quantile caps input size."""
166
+ k = max(1, min(x.shape[0], round(q * (x.shape[0] - 1)) + 1))
167
+ return torch.cat(
168
+ [x[:, i : i + chunk].kthvalue(k, dim=0).values for i in range(0, x.shape[1], chunk)]
169
+ )
170
+
171
+
172
+ @torch.no_grad()
173
+ def joint_ablation(
174
+ runner: NeuronHookRunner,
175
+ prompts: Inputs,
176
+ layer: int,
177
+ neuron_idx: int,
178
+ pole: Pole,
179
+ partners: Sequence[int],
180
+ ) -> JointAblation:
181
+ """KL(clean ‖ ablated) for the pole alone, its partners alone, and all together.
182
+
183
+ The pole is zero-ablated (its opposite pole is kept). Partners are mean-ablated over
184
+ the prompts, which stays closer to their natural distribution than zeroing.
185
+
186
+ Each prompt runs alone. A batch pads every prompt to the longest, and its logits are
187
+ prompts × longest × vocabulary. For 17 prompts, one of them 577 tokens long, that was
188
+ about 5.5 GB per copy at Qwen2.5's 151,936 tokens. Alone, a prompt costs its own length.
189
+ """
190
+ from openinterp.harness.layer_major import map_layer_major
191
+
192
+ prompts = [prompts] if isinstance(prompts, str) else list(prompts)
193
+ encoded = [runner.encode([p]) for p in prompts]
194
+ partner_idx = torch.tensor(list(partners), dtype=torch.long)
195
+ means = None
196
+ if len(partners):
197
+ rows = map_layer_major(
198
+ runner, lambda e: runner.layer_activations(e, [layer])[layer][0][:, partner_idx],
199
+ encoded, tokens=max(e["input_ids"].shape[1] for e in encoded), logits=False,
200
+ ) # fmt: skip
201
+ means = torch.cat(rows).mean(0)
202
+
203
+ def ablate(target: bool, others: bool) -> Callable:
204
+ def fn(act: torch.Tensor) -> torch.Tensor:
205
+ act = act.clone()
206
+ if target:
207
+ act[..., neuron_idx] -= isolate_pole(act[..., neuron_idx], pole)
208
+ if others and means is not None:
209
+ act[..., partner_idx] = means.to(act.dtype)
210
+ return act
211
+
212
+ return fn
213
+
214
+ fns = {"single": ablate(True, False), "joint": ablate(True, True)}
215
+ if len(partners):
216
+ fns["partners"] = ablate(False, True)
217
+ kl = _kl_means(runner, encoded, fns, layer)
218
+ kl_single, kl_partners, kl_joint = kl["single"], kl.get("partners", 0.0), kl["joint"]
219
+ ratio = (kl_joint - kl_partners) / kl_single if kl_single > 1e-9 else None
220
+ return JointAblation(
221
+ partners=list(partners),
222
+ kl_single=round(kl_single, 9),
223
+ kl_partners=round(kl_partners, 9),
224
+ kl_joint=round(kl_joint, 9),
225
+ redundancy_ratio=None if ratio is None else round(ratio, 6),
226
+ )
227
+
228
+
229
+ @torch.no_grad()
230
+ def _kl_means(
231
+ runner: NeuronHookRunner, encoded: Sequence[dict], fns: dict[str, Callable], layer: int
232
+ ) -> dict[str, float]:
233
+ """Mean KL(clean ‖ ablated) per token over every prompt, for each ablation in ``fns``.
234
+
235
+ Each prompt's clean distribution is computed once and reused for every ablation. A
236
+ streamed model runs the clean prompt and its ablations as rows of fewer passes
237
+ (:func:`~openinterp.harness.memory.stream_rows`). It also runs its prompts at once
238
+ (:func:`~openinterp.harness.layer_major.map_layer_major`). Otherwise each is its own
239
+ pass, one after another.
240
+ """
241
+ from openinterp.harness.layer_major import map_layer_major
242
+ from openinterp.harness.memory import stream_rows
243
+
244
+ variants: list[str | None] = [None, *fns] # None is the clean run, so it comes first
245
+
246
+ def prompt_kls(enc: dict) -> tuple[dict[str, float], int]:
247
+ """One prompt's summed KL per ablation, and its token count."""
248
+ rows = stream_rows(runner, enc["input_ids"].shape[1], 1, logits=True)
249
+ sums, clean, tokens = dict.fromkeys(fns, 0.0), None, 0
250
+ for start in range(0, len(variants), rows):
251
+ chunk = variants[start : start + rows]
252
+ log_probs = _variant_log_probs(runner, enc, chunk, fns, layer)
253
+ for name, row in zip(chunk, log_probs, strict=True):
254
+ if name is None:
255
+ clean, tokens = row, row.shape[0]
256
+ else:
257
+ sums[name] += _kl_sum(row, clean)
258
+ return sums, tokens
259
+
260
+ longest = max(e["input_ids"].shape[1] for e in encoded)
261
+ per_prompt = map_layer_major(runner, prompt_kls, encoded, tokens=longest)
262
+ tokens = sum(n for _, n in per_prompt)
263
+ totals = {name: sum(sums[name] for sums, _ in per_prompt) for name in fns}
264
+ return {name: max(0.0, total / max(tokens, 1)) for name, total in totals.items()}
265
+
266
+
267
+ def _variant_log_probs(runner, enc: dict, chunk, fns: dict[str, Callable], layer: int):
268
+ """One pass with a row per entry of ``chunk``: clean (``None``) or ablated by ``fns``."""
269
+ batch = {k: v.repeat(len(chunk), *[1] * (v.dim() - 1)) for k, v in enc.items()}
270
+
271
+ def per_row(act: torch.Tensor) -> torch.Tensor:
272
+ rows = [
273
+ act[i : i + 1] if name is None else fns[name](act[i : i + 1])
274
+ for i, name in enumerate(chunk)
275
+ ]
276
+ return torch.cat(rows)
277
+
278
+ ablates = any(name is not None for name in chunk)
279
+ logits = runner.run_with_interventions(batch, {layer: per_row} if ablates else None)
280
+ return list(_log_probs(logits))
281
+
282
+
283
+ # Positions per block in the log-softmax and the KL. A block of 64 is 39 MB at
284
+ # Qwen2.5's 151,936 tokens. Whole, each step made a copy of the prompt's logits.
285
+ _KL_BLOCK = 64
286
+
287
+
288
+ def _log_probs(logits: torch.Tensor) -> torch.Tensor:
289
+ """``log_softmax(logits.float(), -1)``, computed in place one block at a time."""
290
+ logits = logits.float()
291
+ for row in logits.flatten(0, -2).split(_KL_BLOCK):
292
+ row -= row.logsumexp(-1, keepdim=True)
293
+ return logits
294
+
295
+
296
+ def _kl_sum(ablated: torch.Tensor, clean: torch.Tensor) -> float:
297
+ """``KL(clean ‖ ablated)`` summed over positions, one block at a time."""
298
+ return sum(
299
+ float(F.kl_div(a, c, log_target=True, reduction="sum"))
300
+ for a, c in zip(ablated.split(_KL_BLOCK), clean.split(_KL_BLOCK), strict=True)
301
+ )
@@ -0,0 +1,195 @@
1
+ """Dictionary latents aligned with a neuron pole's weights.
2
+
3
+ A pole writes ``±w_out`` (its down-projection column, negated for the negative pole) and
4
+ reads its MLP's input with ``gate``/``up`` (or ``in``). A latent writes ``W_dec[f]`` and,
5
+ when its dictionary reads the MLP's input (a transcoder), reads it with ``W_enc[:, f]``.
6
+ Both are compared by cosine in residual-stream coordinates. Each norm is folded in as the
7
+ weight wiring folds it (:func:`openinterp.harness.wiring._fold`). Each cosine is oriented
8
+ (:func:`orientation`), so + means that the latent writes what the pole writes, or reads
9
+ what drives the pole. Such a latent is a candidate for the same feature.
10
+
11
+ Two references put a cosine in context. ``chance`` is the 99th percentile, over random
12
+ directions, of the best cosine any latent reaches. ``percentile`` is the share of the
13
+ layer's other neurons whose best match is weaker than this one's. So a pole that the
14
+ dictionary happens to have a latent for stands out from a layer that it fits well
15
+ everywhere.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import hashlib
21
+ from collections.abc import Iterable
22
+ from dataclasses import dataclass
23
+
24
+ import torch
25
+ from torch import nn
26
+
27
+ from openinterp.core.exceptions import ArchitectureNotSupportedError
28
+ from openinterp.harness.dictionaries import Dictionary, DictionarySpec, load_spec
29
+ from openinterp.harness.hooks import NeuronHookRunner
30
+ from openinterp.harness.wiring import _fold, _matrix, mlp_reads
31
+
32
+ # Sites where a dictionary's encoder reads what the MLP reads.
33
+ READS_MLP_INPUT = ("resid_mid", "mlp_in", "mlp_in_normalized")
34
+ CHANCE_DRAWS = 256
35
+ CHUNK = 512
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class Match:
40
+ """One latent aligned with a pole: its index, cosine and read channel."""
41
+
42
+ feature_idx: int
43
+ cosine: float
44
+ channel: str | None = None # the neuron's read channel, for a read
45
+
46
+
47
+ @dataclass(frozen=True)
48
+ class Side:
49
+ """One pole's best-aligned latents on one side (write or read)."""
50
+
51
+ matches: list[Match]
52
+ chance: float
53
+ percentile: float
54
+
55
+
56
+ def _unit(rows: torch.Tensor) -> torch.Tensor:
57
+ return rows / rows.norm(dim=-1, keepdim=True).clamp_min(1e-12)
58
+
59
+
60
+ def latent_writes(runner: NeuronHookRunner, spec: DictionarySpec, layer: int, d: Dictionary):
61
+ """Each latent's write ``[n, d_model]``, in the space a pole writes to."""
62
+ if runner.mlp_write(layer) is not runner.down_proj(layer):
63
+ # Gemma 2 normalises the MLP's output, and its dictionaries decode after that norm.
64
+ raise ArchitectureNotSupportedError(
65
+ "the MLP's output is normalised before the residual stream, so a neuron's write "
66
+ "and a latent's are not in the same space"
67
+ )
68
+ return d.w_dec.float()
69
+
70
+
71
+ def latent_reads(
72
+ runner: NeuronHookRunner, spec: DictionarySpec, layer: int, d: Dictionary
73
+ ) -> torch.Tensor | None:
74
+ """Return each latent's read ``[n, d_model]`` of the residual stream.
75
+
76
+ ``None`` when the dictionary doesn't read the MLP's input.
77
+ """
78
+ if spec.input_site not in READS_MLP_INPUT:
79
+ return None
80
+ read = d.w_enc.T.float()
81
+ if spec.input_site == "resid_mid":
82
+ return read
83
+ return _fold(runner.mlp_norm(layer), read, gain=spec.input_site == "mlp_in")
84
+
85
+
86
+ def _seed(name: str) -> int:
87
+ """A seed that no training run is likely to share.
88
+
89
+ A dictionary's dead latents keep their initial weights. With a small seed such as 0,
90
+ the "random" directions would be those weights. (Their cosine is 0.93 with
91
+ Qwen2.5-0.5B's layer-0 transcoder.)
92
+ """
93
+ return int.from_bytes(
94
+ hashlib.sha256(f"open-interpretability crossref {name}".encode()).digest()[:8]
95
+ )
96
+
97
+
98
+ def _null(runner: NeuronHookRunner, layer: int, latents: torch.Tensor, name: str) -> float:
99
+ """Return the 99th percentile of the best cosine between a random direction and any latent.
100
+
101
+ Directions are centred as LayerNorm centres a read, so they live where reads do.
102
+ """
103
+ g = torch.Generator().manual_seed(_seed(f"{name} {layer}"))
104
+ r = torch.randn(CHANCE_DRAWS, latents.shape[1], generator=g).to(latents.device)
105
+ if isinstance(runner.mlp_norm(layer), nn.LayerNorm):
106
+ r = r - r.mean(-1, keepdim=True)
107
+ best = (_unit(r) @ latents.T).amax(-1)
108
+ return float(best.quantile(0.99))
109
+
110
+
111
+ def orientation(channel: str, pole: str) -> float:
112
+ """Return the sign of a vector that drives ``pole`` along ``channel``.
113
+
114
+ A latent fires non-negatively, so only its own direction counts. A pole writes
115
+ ``±w_out``. A gated neuron fires when its gate opens, whatever the pole, with the sign
116
+ of ``up``. An ungated neuron fires with the sign of its pre-activation.
117
+ """
118
+ return 1.0 if channel == "gate" or pole == "positive" else -1.0
119
+
120
+
121
+ def _align(
122
+ channels: dict[str, torch.Tensor],
123
+ latents: torch.Tensor,
124
+ poles: Iterable[tuple[int, str]],
125
+ k: int,
126
+ ) -> tuple[dict[tuple[int, str], list[Match]], dict[str, torch.Tensor]]:
127
+ """Return the top ``k`` latents by oriented cosine for each ``(neuron, pole)``.
128
+
129
+ They are over all the neuron's channels. A latent appears once, at its best channel.
130
+ Also returns every neuron's best oriented cosine, by pole.
131
+ """
132
+ poles = sorted(set(poles))
133
+ width = next(iter(channels.values())).shape[0]
134
+ best = {p: torch.full((width,), -1.0) for p in ("positive", "negative")}
135
+ found: dict[tuple[int, str], list[Match]] = {p: [] for p in poles}
136
+ for channel, rows in channels.items():
137
+ for start in range(0, width, CHUNK):
138
+ cos = _unit(rows[start : start + CHUNK].float()) @ latents.T # [c, n]
139
+ highest, lowest = cos.amax(-1).cpu(), (-cos).amax(-1).cpu()
140
+ for pole, b in best.items():
141
+ reach = highest if orientation(channel, pole) > 0 else lowest
142
+ b[start : start + CHUNK] = torch.maximum(b[start : start + CHUNK], reach)
143
+ for i, pole in poles:
144
+ if start <= i < start + CHUNK:
145
+ row = orientation(channel, pole) * cos[i - start]
146
+ top = row.topk(k).indices
147
+ found[(i, pole)] += [Match(int(f), float(row[f]), channel) for f in top]
148
+ for key, matches in found.items():
149
+ by_feature: dict[int, Match] = {}
150
+ for m in sorted(matches, key=lambda m: -m.cosine):
151
+ by_feature.setdefault(m.feature_idx, m)
152
+ found[key] = list(by_feature.values())[:k]
153
+ return found, best
154
+
155
+
156
+ def align_layer(
157
+ runner: NeuronHookRunner,
158
+ spec: DictionarySpec,
159
+ layer: int,
160
+ poles: Iterable[tuple[int, str]],
161
+ *,
162
+ k: int = 5,
163
+ ) -> dict[tuple[int, str], tuple[Side, Side | None]]:
164
+ """Return ``{(neuron, pole): (write, read)}`` against ``spec``'s dictionary for ``layer``.
165
+
166
+ ``read`` is ``None`` when the dictionary doesn't read the MLP's input.
167
+ """
168
+ poles = list(poles)
169
+ device = runner.device
170
+ d = load_spec(spec, layer, device)
171
+ writes = _unit(latent_writes(runner, spec, layer, d))
172
+ w_out = _matrix(runner.down_proj(layer)).T.to(device) # [width, d_model]
173
+ write_null = _null(runner, layer, writes, "write")
174
+ by_write, write_best = _align({"out": w_out}, writes, poles, k)
175
+
176
+ reads = latent_reads(runner, spec, layer, d)
177
+ by_read = read_best = read_null = None
178
+ if reads is not None:
179
+ reads = _unit(reads)
180
+ read_null = _null(runner, layer, reads, "read")
181
+ neuron_reads = {c: r.to(device) for c, r in mlp_reads(runner, layer).items()}
182
+ by_read, read_best = _align(neuron_reads, reads, poles, k)
183
+
184
+ def share_below(best: dict[str, torch.Tensor], i: int, pole: str) -> float:
185
+ return float((best[pole] < best[pole][i]).float().mean())
186
+
187
+ out = {}
188
+ for i, pole in poles:
189
+ write_matches = [Match(m.feature_idx, m.cosine) for m in by_write[(i, pole)]]
190
+ write = Side(write_matches, write_null, share_below(write_best, i, pole))
191
+ read = None
192
+ if by_read is not None:
193
+ read = Side(by_read[(i, pole)], read_null, share_below(read_best, i, pole))
194
+ out[(i, pole)] = (write, read)
195
+ return out