open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,159 @@
1
+ """Steering samples: continue neutral prompts with a unit clamped high or ablated.
2
+
3
+ Every prompt is continued under each intervention, applied at every position of the
4
+ prompt and of the continuation:
5
+
6
+ * baseline: no intervention;
7
+ * ablated: the unit's own ablation (a neuron's pole zeroed, or a dictionary latent's
8
+ write removed), as in the causal and lookahead measurements;
9
+ * clamped: a neuron's activation is set to the pole's signed top activation
10
+ (``max_activation``) times each multiple. A latent's activation is set to that value.
11
+
12
+ All prompts and interventions decode as one left-padded batch, greedily, so a sample
13
+ is deterministic given the weights. The prompts are fixed and unrelated to any
14
+ hypothesis. So a coherent drift at 1–2× says what the unit makes the model write. Drift
15
+ only at 4× is off-distribution and weak evidence. A head writes a subspace with no
16
+ "high" setting, so heads get no samples.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from collections.abc import Mapping, Sequence
22
+ from typing import Any
23
+
24
+ import torch
25
+
26
+ from openinterp.core.steering import SteeringSample, SteeringSamples
27
+ from openinterp.harness.hooks import NeuronHookRunner, isolate_pole
28
+ from openinterp.harness.units import FeatureUnit, NeuronUnit, Unit
29
+
30
+ PROMPTS = (
31
+ "The weather today is",
32
+ "I went to the shop and bought",
33
+ "The best way to learn is",
34
+ "Once upon a time,",
35
+ "The report concluded that",
36
+ )
37
+ MULTIPLES = (1.0, 2.0, 4.0)
38
+ MAX_NEW_TOKENS = 24
39
+
40
+
41
+ def steering_interventions(
42
+ runner: NeuronHookRunner, unit: Unit, ablate: torch.Tensor, clamp: torch.Tensor
43
+ ) -> dict[str, Mapping[int, Any]] | None:
44
+ """Return ``hooks`` keyword arguments that ablate or clamp the unit, row by row.
45
+
46
+ They ablate the unit in the batch rows where ``ablate`` is true. They clamp it to
47
+ ``clamp`` (a strength) where that is not NaN. ``None`` for a head.
48
+ """
49
+ clamped = ~torch.isnan(clamp)
50
+ if isinstance(unit, NeuronUnit):
51
+ idx, pole = unit.ident.neuron_idx, unit.ident.pole
52
+ signed = clamp if pole == "positive" else -clamp
53
+
54
+ def steer(act: torch.Tensor) -> torch.Tensor:
55
+ act = act.clone()
56
+ a = act[..., idx]
57
+ a = torch.where(ablate.to(a.device)[:, None], a - isolate_pole(a, pole), a)
58
+ value = signed.to(a.device, a.dtype)[:, None]
59
+ act[..., idx] = torch.where(clamped.to(a.device)[:, None], value, a)
60
+ return act
61
+
62
+ return {"interventions": {unit.ident.layer: steer}}
63
+ if isinstance(unit, FeatureUnit):
64
+ dictionary, f = unit.dictionary(runner), unit.ident.feature_idx
65
+ direction = dictionary.w_dec[f]
66
+
67
+ def steer_latent(x: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
68
+ a = dictionary.feature(x, f)
69
+ target = torch.where(ablate.to(a.device)[:, None], torch.zeros_like(a), a)
70
+ value = clamp.to(a.device, a.dtype)[:, None]
71
+ target = torch.where(clamped.to(a.device)[:, None], value, target)
72
+ return out + ((target - a).unsqueeze(-1) * direction).to(out.dtype)
73
+
74
+ site = unit.profile.input_site
75
+ return {"dictionary_interventions": {unit.ident.layer: (site, steer_latent)}}
76
+ return None
77
+
78
+
79
+ @torch.no_grad()
80
+ def _continue(
81
+ runner: NeuronHookRunner,
82
+ rows: Sequence[list[int]],
83
+ interventions: Mapping[str, Mapping[int, Any]],
84
+ max_new_tokens: int,
85
+ ) -> list[str]:
86
+ """Return greedy continuations of every row of token ids, each cut at end-of-text.
87
+
88
+ A model with ``generate`` decodes with its cache. One without (a test double) runs
89
+ the whole sequence again at each step.
90
+ """
91
+ tok = runner.tokenizer
92
+ eos = getattr(tok, "eos_token_id", None)
93
+ pad = getattr(tok, "pad_token_id", None)
94
+ pad = (eos if pad is None else pad) or 0
95
+ width = max(map(len, rows))
96
+ ids = torch.tensor([[pad] * (width - len(r)) + r for r in rows], device=runner.device)
97
+ mask = torch.tensor([[0] * (width - len(r)) + [1] * len(r) for r in rows], device=ids.device)
98
+ with runner.hooks(**interventions):
99
+ if hasattr(runner.model, "generate"):
100
+ seq = runner.model.generate(
101
+ input_ids=ids,
102
+ attention_mask=mask,
103
+ max_new_tokens=max_new_tokens,
104
+ do_sample=False,
105
+ pad_token_id=pad,
106
+ eos_token_id=eos,
107
+ )
108
+ else:
109
+ seq = ids
110
+ for _ in range(max_new_tokens):
111
+ nxt = runner.forward({"input_ids": seq, "attention_mask": mask})[:, -1].argmax(-1)
112
+ seq = torch.cat([seq, nxt[:, None]], dim=1)
113
+ mask = torch.cat([mask, torch.ones_like(mask[:, :1])], dim=1)
114
+ texts = []
115
+ for row in seq[:, width:].tolist():
116
+ if eos in row:
117
+ row = row[: row.index(eos)]
118
+ texts.append(tok.decode(row))
119
+ return texts
120
+
121
+
122
+ def steering_samples(
123
+ runner: NeuronHookRunner,
124
+ unit: Unit,
125
+ max_activation: float,
126
+ *,
127
+ prompts: Sequence[str] = PROMPTS,
128
+ multiples: Sequence[float] = MULTIPLES,
129
+ max_new_tokens: int = MAX_NEW_TOKENS,
130
+ ) -> SteeringSamples:
131
+ """Continue every prompt at baseline, ablated, and clamped.
132
+
133
+ The clamp is at each multiple of ``max_activation``.
134
+ """
135
+ if runner.tokenizer is None:
136
+ raise ValueError("steering samples need a tokenizer")
137
+ # Each prompt's rows: baseline, ablated, then clamped at each multiple.
138
+ strengths = [float("nan"), float("nan"), *(max_activation * m for m in multiples)]
139
+ per_prompt = len(strengths)
140
+ clamp = torch.tensor(strengths * len(prompts))
141
+ ablate = torch.tensor(([False, True] + [False] * len(multiples)) * len(prompts))
142
+ interventions = steering_interventions(runner, unit, ablate, clamp)
143
+ if interventions is None:
144
+ raise ValueError(f"{type(unit).__name__} has no clamped-high setting to steer with")
145
+ encoded = [runner.tokenizer.encode(p, add_special_tokens=True) for p in prompts]
146
+ rows = [ids for ids in encoded for _ in range(per_prompt)]
147
+ texts = _continue(runner, rows, interventions, max_new_tokens)
148
+ samples = []
149
+ for i, prompt in enumerate(prompts):
150
+ baseline, ablated, *clamped = texts[i * per_prompt : (i + 1) * per_prompt]
151
+ samples.append(
152
+ SteeringSample(prompt=prompt, baseline=baseline, ablated=ablated, clamped=clamped)
153
+ )
154
+ return SteeringSamples(
155
+ clamp_unit=max_activation,
156
+ multiples=[float(m) for m in multiples],
157
+ max_new_tokens=max_new_tokens,
158
+ samples=samples,
159
+ )
@@ -0,0 +1,116 @@
1
+ """A superposition as an auditable unit (issue #3215, ``docs/PROTOCOL.md`` §1).
2
+
3
+ A superposition is a sparse direction over one layer's MLP hidden activations, the input
4
+ to the down projection where neurons live. Its strength is the positive part of the unit
5
+ direction ŵ: ``max(0, ŵ · a)``. That is one latent of a ``basis`` dictionary at
6
+ ``mlp_hidden``. So this module builds such a dictionary in memory, and the feature code
7
+ scans, measures, ablates and patches it.
8
+
9
+ The dictionary's latent 0 is the superposition. Latents 1 to :data:`NULLS` are null
10
+ directions for the causal metrics. Each has as many members as the superposition, on
11
+ neurons drawn at random. Its weights are the superposition's magnitudes, in a shuffled
12
+ order and with random signs. The draws are seeded by the superposition's id, so a record always
13
+ measures against the same nulls.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import random
19
+ from dataclasses import dataclass, field
20
+ from typing import Any
21
+
22
+ import torch
23
+
24
+ from openinterp.core.profiles import SuperpositionProfile
25
+ from openinterp.core.schemas import SuperpositionIdentifier
26
+ from openinterp.harness.dictionaries import Dictionary
27
+ from openinterp.harness.hooks import NeuronHookRunner
28
+ from openinterp.harness.units import FeatureUnit
29
+
30
+ NULLS = 64 # null directions beside the superposition, for the causal metrics
31
+
32
+
33
+ def directions(ident: SuperpositionIdentifier, profile: SuperpositionProfile, width: int):
34
+ """``[width, 1 + NULLS]``: the unit direction ŵ, then the null directions."""
35
+ rng = random.Random(int(ident.superposition_id, 16))
36
+ weights = [m.weight for m in profile.members]
37
+ columns = [list(zip(profile.neurons, weights, strict=True))]
38
+ magnitudes = [abs(w) for w in weights]
39
+ for _ in range(NULLS):
40
+ neurons = rng.sample(range(width), len(weights))
41
+ shuffled = rng.sample(magnitudes, len(magnitudes))
42
+ columns.append(
43
+ [(n, w * rng.choice((1, -1))) for n, w in zip(neurons, shuffled, strict=True)]
44
+ )
45
+ q = torch.zeros(width, len(columns))
46
+ for j, column in enumerate(columns):
47
+ for neuron, weight in column:
48
+ q[neuron, j] = weight
49
+ return q / q.norm(dim=0, keepdim=True)
50
+
51
+
52
+ def build_dictionary(runner: NeuronHookRunner, ident, profile) -> Dictionary:
53
+ """The in-memory ``basis`` dictionary whose latent 0 is the superposition.
54
+
55
+ A latent's write is its activation times ``W_down q`` for its direction q. So
56
+ ablating it removes exactly its direction's positive part from the hidden activations.
57
+ """
58
+ width = runner.mlp_width(ident.layer)
59
+ if max(profile.neurons) >= width:
60
+ raise ValueError(f"{ident}: layer {ident.layer} has {width} neurons")
61
+ q = directions(ident, profile, width)
62
+ used = q.abs().sum(dim=1).nonzero().flatten().tolist() # neurons some direction reads
63
+ out = torch.stack([runner.neuron_output_direction(ident.layer, i).float().cpu() for i in used])
64
+ w_dec = q[used].T @ out # [n, d_model]
65
+ device = runner.device
66
+ return Dictionary(
67
+ w_enc=q.to(device),
68
+ b_enc=torch.zeros(q.shape[1], device=device),
69
+ w_dec=w_dec.to(device),
70
+ b_dec=torch.zeros(w_dec.shape[1], device=device),
71
+ threshold=None,
72
+ activation="relu",
73
+ subtract_decoder_bias=False,
74
+ )
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class SuperpositionUnit(FeatureUnit):
79
+ """A superposition as an auditable unit: latent 0 of its in-memory dictionary.
80
+
81
+ It is a :class:`~openinterp.harness.units.FeatureUnit`, so every measurement that
82
+ reads a feature reads it the same way. ``profile`` is its
83
+ :class:`~openinterp.core.profiles.SuperpositionProfile`.
84
+ """
85
+
86
+ _cache: dict[Any, Dictionary] = field(default_factory=dict, compare=False, repr=False)
87
+
88
+ def dictionary(self, runner: NeuronHookRunner) -> Dictionary:
89
+ """The superposition's dictionary, built once per runner."""
90
+ key = id(runner)
91
+ if key not in self._cache:
92
+ self._cache[key] = build_dictionary(runner, self.ident, self.profile)
93
+ return self._cache[key]
94
+
95
+
96
+ def scan_superposition(runner: NeuronHookRunner, texts, ident, profile, **options):
97
+ """Scan a superposition as a latent; the result's profile is ``profile`` itself."""
98
+ from openinterp.harness.features import scan_latent
99
+
100
+ dictionary = SuperpositionUnit(ident, profile).dictionary(runner)
101
+ return scan_latent(
102
+ runner, texts, ident, dictionary, profile.input_site, lambda **_: profile, **options
103
+ )
104
+
105
+
106
+ def superposition_identifier(model_name: str, layer: int, profile) -> SuperpositionIdentifier:
107
+ """The identifier of the superposition that ``profile`` defines on ``layer``."""
108
+ from openinterp.core.profiles import superposition_id
109
+
110
+ sid = superposition_id(profile.members)
111
+ return SuperpositionIdentifier(model_name=model_name, layer=layer, superposition_id=sid)
112
+
113
+
114
+ def superposition_write(runner: NeuronHookRunner, ident, profile) -> torch.Tensor:
115
+ """The residual-stream direction that the superposition writes: ``W_down ŵ``."""
116
+ return SuperpositionUnit(ident, profile).dictionary(runner).w_dec[0]
@@ -0,0 +1,203 @@
1
+ """What "strength" means for each kind of auditable unit, so one protocol covers them all.
2
+
3
+ Every measurement in an audit reads one non-negative strength per token: the scan's
4
+ top examples, probes, counterfactual Δ, held-out windows. A unit supplies it, and the
5
+ ablation the lookahead profile uses:
6
+
7
+ * An MLP neuron pole: its pole strength. Ablation zeroes the pole.
8
+ * An attention head: ``|W_O (z_t − z̄)|``. This is the distance between its write to the
9
+ residual stream and its mean write over the scan. z̄ is stored in
10
+ :class:`~openinterp.core.schemas.HeadProfile`. Ablation replaces z with z̄ (mean
11
+ ablation). So strength is exactly how far ablating the head moves the residual.
12
+ * A superposition: the positive part of its unit direction over one layer's neurons. It is
13
+ scanned and ablated as latent 0 of an in-memory dictionary
14
+ (:mod:`openinterp.harness.superpositions`).
15
+ * A dictionary feature: its latent's activation ``a_f`` (non-negative). Ablation removes
16
+ its write ``a_f · W_dec[f]`` from the MLP output, or from the residual stream for a
17
+ residual-stream SAE. The dictionary's error and every other latent stay as they were.
18
+
19
+ A unit with a *first-token floor* (protocol 0.8) is not measured at a text's first
20
+ token. The first token has no context and is the attention sink. Some early-layer units
21
+ sit above their line there whatever the token, so every probe would fire. A unit
22
+ has the floor if it is active at the first position on at least half the vocabulary
23
+ (:func:`openinterp.harness.scan.first_position_share`). A prompt's peak is then over
24
+ its other tokens (``skip_first``). Every other unit is measured at every token, so one
25
+ that detects *which* token opens a text can still be audited.
26
+
27
+ The raw write norm ``|W_O z_t|`` would not do. Most heads write a sizeable resting
28
+ vector, often while attending to the first token. So in GPT-2 small, an induction head's
29
+ median token sits at a quarter of its maximum. Half of all text would count as "active".
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ from collections.abc import Mapping
35
+ from dataclasses import dataclass
36
+ from typing import Any
37
+
38
+ import torch
39
+
40
+ from openinterp.core.schemas import (
41
+ ComponentIdentifier,
42
+ DictionaryProfile,
43
+ FeatureIdentifier,
44
+ HeadIdentifier,
45
+ NeuronIdentifier,
46
+ SuperpositionIdentifier,
47
+ )
48
+ from openinterp.harness.dictionaries import Dictionary, load_dictionary
49
+ from openinterp.harness.hooks import Inputs, NeuronHookRunner, ablate_head, ablate_pole
50
+ from openinterp.harness.sites import dictionary_io
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class NeuronUnit:
55
+ """An MLP neuron pole as an auditable unit."""
56
+
57
+ ident: NeuronIdentifier
58
+ skip_first: bool = False
59
+
60
+ def strengths(self, runner: NeuronHookRunner, inputs: Inputs) -> torch.Tensor:
61
+ """Per-token pole strength ``[batch, seq]``."""
62
+ ident = self.ident
63
+ return runner.neuron_activations(inputs, ident.layer, ident.neuron_idx, ident.pole)
64
+
65
+ def ablation(self, runner: NeuronHookRunner) -> dict[str, Mapping[int, Any]]:
66
+ """``run_with_interventions`` keyword arguments that zero the pole."""
67
+ ident = self.ident
68
+ return {"interventions": {ident.layer: ablate_pole(ident.neuron_idx, ident.pole)}}
69
+
70
+
71
+ @dataclass(frozen=True)
72
+ class HeadUnit:
73
+ """An attention head as an auditable unit, with the mean output that defines its strength."""
74
+
75
+ ident: HeadIdentifier
76
+ mean_output: tuple[float, ...]
77
+ skip_first: bool = False
78
+
79
+ def strengths(self, runner: NeuronHookRunner, inputs: Inputs) -> torch.Tensor:
80
+ """Per-token ``|W_O (z − z̄)|`` ``[batch, seq]``."""
81
+ z = runner.head_outputs(inputs, self.ident.layer)[..., self.ident.head_idx, :]
82
+ return head_strength(runner, self.ident, z, self.mean_output)
83
+
84
+ def ablation(self, runner: NeuronHookRunner) -> dict[str, Mapping[int, Any]]:
85
+ """``run_with_interventions`` keyword arguments that mean-ablate the head."""
86
+ ident = self.ident
87
+ mean = torch.tensor(self.mean_output, device=runner.device)
88
+ fn = ablate_head(ident.head_idx, runner.head_dim(ident.layer), mean)
89
+ return {"head_interventions": {ident.layer: fn}}
90
+
91
+
92
+ @dataclass(frozen=True)
93
+ class FeatureUnit:
94
+ """A dictionary feature as an auditable unit, with its pinned dictionary."""
95
+
96
+ ident: FeatureIdentifier
97
+ profile: DictionaryProfile
98
+ skip_first: bool = False
99
+
100
+ def dictionary(self, runner: NeuronHookRunner) -> Dictionary:
101
+ """The feature's dictionary, loaded on the runner's device."""
102
+ return load_dictionary(self.profile, runner.device)
103
+
104
+ def strengths(self, runner: NeuronHookRunner, inputs: Inputs) -> torch.Tensor:
105
+ """Per-token latent activation ``[batch, seq]``."""
106
+ x, _ = dictionary_io(runner, inputs, self.ident.layer, self.profile.input_site)
107
+ return self.dictionary(runner).feature(x, self.ident.feature_idx)
108
+
109
+ def ablation(self, runner: NeuronHookRunner) -> dict[str, Mapping[int, Any]]:
110
+ """``run_with_interventions`` keyword arguments that remove the latent's write."""
111
+ dictionary, f = self.dictionary(runner), self.ident.feature_idx
112
+ direction = dictionary.w_dec[f]
113
+
114
+ def ablate(x: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
115
+ write = dictionary.feature(x, f).unsqueeze(-1) * direction
116
+ return out - write.to(out.dtype)
117
+
118
+ return {"dictionary_interventions": {self.ident.layer: (self.profile.input_site, ablate)}}
119
+
120
+
121
+ Unit = NeuronUnit | HeadUnit | FeatureUnit
122
+
123
+
124
+ def head_strength(
125
+ runner: NeuronHookRunner, ident: HeadIdentifier, z: torch.Tensor, mean_output
126
+ ) -> torch.Tensor:
127
+ """``|W_O (z − z̄)|`` over the last dimension of ``z`` (one head's outputs)."""
128
+ w_o = runner.head_output_weights(ident.layer, ident.head_idx).float()
129
+ mean = torch.as_tensor(mean_output, dtype=torch.float32, device=z.device)
130
+ return ((z.float() - mean) @ w_o.T.to(z.device)).norm(dim=-1)
131
+
132
+
133
+ def unit_of(
134
+ ident: ComponentIdentifier,
135
+ head_profile=None,
136
+ dictionary_profile=None,
137
+ skip_first=False,
138
+ superposition_profile=None,
139
+ ) -> Unit:
140
+ """Return the unit that ``ident`` names.
141
+
142
+ A head, a feature or a superposition needs the profile that defines its strength.
143
+ """
144
+ if isinstance(ident, SuperpositionIdentifier):
145
+ from openinterp.harness.superpositions import SuperpositionUnit
146
+
147
+ if superposition_profile is None:
148
+ raise ValueError(f"{ident} has no superposition profile, so its strength is undefined")
149
+ return SuperpositionUnit(ident, superposition_profile, skip_first)
150
+ if isinstance(ident, HeadIdentifier):
151
+ if head_profile is None:
152
+ raise ValueError(f"{ident} has no head profile, so its strength is undefined")
153
+ return HeadUnit(ident, tuple(head_profile.mean_output), skip_first)
154
+ if isinstance(ident, FeatureIdentifier):
155
+ if dictionary_profile is None:
156
+ raise ValueError(f"{ident} has no dictionary profile, so its strength is undefined")
157
+ return FeatureUnit(ident, dictionary_profile, skip_first)
158
+ return NeuronUnit(ident, skip_first)
159
+
160
+
161
+ def submission_unit(submission) -> Unit:
162
+ """Return the unit that a submission, scan evidence or scoring task is about.
163
+
164
+ A record from before protocol 0.8 has no ``first_token_floor``, and is measured at
165
+ every token.
166
+ """
167
+ floor = bool(getattr(submission, "first_token_floor", None))
168
+ return unit_of(
169
+ submission.identifier,
170
+ submission.head_profile,
171
+ submission.dictionary_profile,
172
+ floor,
173
+ getattr(submission, "superposition_profile", None),
174
+ )
175
+
176
+
177
+ def peak_strength(runner: NeuronHookRunner, unit: Unit, text: str) -> float | None:
178
+ """Return the peak strength of ``text`` over its measured tokens.
179
+
180
+ ``None`` when it has none: one token, for a unit that skips the first.
181
+ """
182
+ strengths = unit.strengths(runner, text)[0][int(unit.skip_first) :]
183
+ return float(strengths.max()) if strengths.numel() else None
184
+
185
+
186
+ def max_strength(runner: NeuronHookRunner, unit: Unit, text: str) -> float:
187
+ """:func:`peak_strength`, for a text that must have a measured token."""
188
+ peak = peak_strength(runner, unit, text)
189
+ if peak is None:
190
+ raise ValueError(
191
+ f"{text!r} is one token, and {unit.ident} is not measured at the first token "
192
+ "(it has a first-token floor), so use two or more"
193
+ )
194
+ return peak
195
+
196
+
197
+ def attn_implementation(ident: ComponentIdentifier) -> str | None:
198
+ """Return the attention implementation to load the model with: ``"eager"`` for a head.
199
+
200
+ SDPA and flash attention return no patterns. Every measurement of a head experiment
201
+ should run the same attention kernel.
202
+ """
203
+ return "eager" if isinstance(ident, HeadIdentifier) else None