open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Confound control: how much of a pole's activation trivial explanations account for.
|
|
2
|
+
|
|
3
|
+
During the scan we accumulate, per token id and per position, the count, sum and sum
|
|
4
|
+
of squares of the pole strength. From that we derive:
|
|
5
|
+
|
|
6
|
+
* ``token_r2`` / ``position_r2``: variance explained by a group-mean model (between-
|
|
7
|
+
group sum of squares over total sum of squares);
|
|
8
|
+
* ``frequency_correlation``: Spearman's rho, over tokens seen ``min_count`` times or
|
|
9
|
+
more, between token frequency and mean strength. Rank-based, because mean strength
|
|
10
|
+
per token is heavily skewed and the relationship of interest is monotonic;
|
|
11
|
+
* trivial *baselines* for the reviewer's probes: "active if any token is a trigger
|
|
12
|
+
token" and "active if any position is a high-activation position".
|
|
13
|
+
|
|
14
|
+
Take a hypothesis that cannot match the best trivial baseline on the same probes. It
|
|
15
|
+
adds nothing beyond "this neuron fires on token X" (or "at position p").
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import statistics
|
|
21
|
+
from collections.abc import Sequence
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
import torch
|
|
25
|
+
|
|
26
|
+
from openinterp.core.schemas import ConfoundProfile, TokenStat
|
|
27
|
+
from openinterp.core.stats import spearman
|
|
28
|
+
|
|
29
|
+
MAX_TRIGGER_TOKENS = 200
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ConfoundAccumulator:
|
|
33
|
+
"""Per-token and per-position sums of a pole's strength over a scan."""
|
|
34
|
+
|
|
35
|
+
def __init__(self) -> None:
|
|
36
|
+
self.token: dict[str, torch.Tensor] = {}
|
|
37
|
+
self.position: dict[str, torch.Tensor] = {}
|
|
38
|
+
self.total = torch.zeros(3, dtype=torch.float64) # n, sum, sum of squares
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def _add(store: dict[str, torch.Tensor], keys: torch.Tensor, values: torch.Tensor) -> None:
|
|
42
|
+
if keys.numel() == 0:
|
|
43
|
+
return
|
|
44
|
+
for name, weights in (
|
|
45
|
+
("n", torch.ones_like(values)),
|
|
46
|
+
("s", values),
|
|
47
|
+
("ss", values * values),
|
|
48
|
+
):
|
|
49
|
+
counts = torch.bincount(keys, weights=weights)
|
|
50
|
+
current = store.get(name, torch.zeros(0, dtype=torch.float64))
|
|
51
|
+
if current.numel() < counts.numel():
|
|
52
|
+
pad = torch.zeros(counts.numel() - current.numel(), dtype=torch.float64)
|
|
53
|
+
current = torch.cat([current, pad])
|
|
54
|
+
current[: counts.numel()] += counts
|
|
55
|
+
store[name] = current
|
|
56
|
+
|
|
57
|
+
def update(self, ids: torch.Tensor, positions: torch.Tensor, strengths: torch.Tensor) -> None:
|
|
58
|
+
"""Add a batch of token ids, positions and strengths."""
|
|
59
|
+
values = strengths.double()
|
|
60
|
+
self._add(self.token, ids.long(), values)
|
|
61
|
+
self._add(self.position, positions.long(), values)
|
|
62
|
+
self.total += torch.tensor(
|
|
63
|
+
[values.numel(), values.sum(), (values * values).sum()], dtype=torch.float64
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def _r2(self, store: dict[str, torch.Tensor]) -> float:
|
|
67
|
+
n, s, ss = (float(x) for x in self.total)
|
|
68
|
+
if n == 0:
|
|
69
|
+
return 0.0
|
|
70
|
+
total = ss - s * s / n
|
|
71
|
+
seen = store["n"] > 0
|
|
72
|
+
between = float((store["s"][seen] ** 2 / store["n"][seen]).sum()) - s * s / n
|
|
73
|
+
return min(1.0, max(0.0, between / total)) if total > 1e-12 else 0.0
|
|
74
|
+
|
|
75
|
+
def finalize(self, tokenizer: Any, min_count: int = 3) -> ConfoundProfile:
|
|
76
|
+
"""The confound profile of everything added (see the module docstring)."""
|
|
77
|
+
if not self.token:
|
|
78
|
+
return ConfoundProfile(
|
|
79
|
+
token_r2=0.0,
|
|
80
|
+
position_r2=0.0,
|
|
81
|
+
frequency_correlation=0.0,
|
|
82
|
+
min_count=min_count,
|
|
83
|
+
trigger_tokens=[],
|
|
84
|
+
position_means=[],
|
|
85
|
+
)
|
|
86
|
+
n, s = self.token["n"], self.token["s"]
|
|
87
|
+
frequent = (n >= min_count).nonzero().flatten()
|
|
88
|
+
means = s[frequent] / n[frequent]
|
|
89
|
+
order = means.argsort(descending=True)[:MAX_TRIGGER_TOKENS]
|
|
90
|
+
triggers = [
|
|
91
|
+
TokenStat(
|
|
92
|
+
token=tokenizer.decode([int(frequent[i])]) if tokenizer else str(int(frequent[i])),
|
|
93
|
+
token_id=int(frequent[i]),
|
|
94
|
+
mean_strength=round(max(0.0, float(means[i])), 6),
|
|
95
|
+
count=int(n[frequent[i]]),
|
|
96
|
+
)
|
|
97
|
+
for i in order.tolist()
|
|
98
|
+
]
|
|
99
|
+
pn, ps = self.position["n"], self.position["s"]
|
|
100
|
+
position_means = [
|
|
101
|
+
round(float(ps[p] / pn[p]), 6) if pn[p] > 0 else 0.0 for p in range(pn.numel())
|
|
102
|
+
]
|
|
103
|
+
return ConfoundProfile(
|
|
104
|
+
token_r2=round(self._r2(self.token), 6),
|
|
105
|
+
position_r2=round(self._r2(self.position), 6),
|
|
106
|
+
frequency_correlation=round(spearman(n[frequent], means), 6),
|
|
107
|
+
min_count=min_count,
|
|
108
|
+
trigger_tokens=triggers,
|
|
109
|
+
position_means=position_means,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def baseline_predictions(
|
|
114
|
+
profile: ConfoundProfile,
|
|
115
|
+
tokenizer: Any,
|
|
116
|
+
texts: Sequence[str],
|
|
117
|
+
threshold: float,
|
|
118
|
+
skip_first: bool = False,
|
|
119
|
+
) -> dict[str, list[bool]]:
|
|
120
|
+
"""Return the predicted "active" for each text under each trivial baseline.
|
|
121
|
+
|
|
122
|
+
Like the unit's own peak, a baseline doesn't read the first token when
|
|
123
|
+
``skip_first`` (protocol 0.8).
|
|
124
|
+
"""
|
|
125
|
+
triggers = {t.token_id for t in profile.trigger_tokens if t.mean_strength >= threshold}
|
|
126
|
+
hot_positions = {p for p, m in enumerate(profile.position_means) if m >= threshold}
|
|
127
|
+
start = int(skip_first)
|
|
128
|
+
token_pred, position_pred = [], []
|
|
129
|
+
for text in texts:
|
|
130
|
+
ids = tokenizer([text], return_tensors="pt")["input_ids"][0].tolist()
|
|
131
|
+
token_pred.append(any(i in triggers for i in ids[start:]))
|
|
132
|
+
position_pred.append(any(p in hot_positions for p in range(start, len(ids))))
|
|
133
|
+
return {"token identity": token_pred, "position": position_pred}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def length_predictions(
|
|
137
|
+
tokenizer: Any, texts: Sequence[str], skip_first: bool = False
|
|
138
|
+
) -> dict[str, list[bool]]:
|
|
139
|
+
"""Return "active if the probe has at least n tokens" for each cut n.
|
|
140
|
+
|
|
141
|
+
A unit that responds to position passes probes that differ only in length. On a
|
|
142
|
+
randomly initialised GPT-2, two units were VERIFIED that way (issue #3525). Tokens are
|
|
143
|
+
counted as the unit reads them, so the first is left out when ``skip_first``. It is a
|
|
144
|
+
proposed confound baseline: ``scripts/protocol_mix.py`` measures what it would change.
|
|
145
|
+
"""
|
|
146
|
+
lengths = [
|
|
147
|
+
len(tokenizer([text], return_tensors="pt")["input_ids"][0]) - int(skip_first)
|
|
148
|
+
for text in texts
|
|
149
|
+
]
|
|
150
|
+
return {f"probe length ≥ {n} tokens": [n <= k for k in lengths] for n in sorted(set(lengths))}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# A ratio of the two probe groups' median token lengths above which the reviewer is warned.
|
|
154
|
+
# Probes that differ this much in length can be told apart by length alone (issue #3525).
|
|
155
|
+
LENGTH_RATIO_WARN = 1.5
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def length_ratio(probes: Sequence[Any], tokenizer, skip_first: bool) -> float | None:
|
|
159
|
+
"""The larger median token length of the two probe groups over the smaller.
|
|
160
|
+
|
|
161
|
+
The groups are the probes expected active and those expected inactive. Tokens are
|
|
162
|
+
counted as the unit reads them. ``None`` when a group is empty.
|
|
163
|
+
"""
|
|
164
|
+
groups: dict[bool, list[int]] = {True: [], False: []}
|
|
165
|
+
for probe in probes:
|
|
166
|
+
n = len(tokenizer.encode(probe.text)) - int(skip_first)
|
|
167
|
+
groups[probe.expected_active].append(max(n, 1))
|
|
168
|
+
if not groups[True] or not groups[False]:
|
|
169
|
+
return None
|
|
170
|
+
medians = sorted(statistics.median(g) for g in groups.values())
|
|
171
|
+
return medians[1] / medians[0]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def length_warning(probes: Sequence[Any], tokenizer, skip_first: bool) -> str | None:
|
|
175
|
+
"""A warning when the two probe groups differ in length by more than LENGTH_RATIO_WARN."""
|
|
176
|
+
ratio = length_ratio(probes, tokenizer, skip_first)
|
|
177
|
+
if ratio is None or ratio <= LENGTH_RATIO_WARN:
|
|
178
|
+
return None
|
|
179
|
+
return (
|
|
180
|
+
f"The median token lengths of the active and inactive probes differ {ratio:.1f}×, "
|
|
181
|
+
f"above {LENGTH_RATIO_WARN}×. A rule that reads only probe length may then explain "
|
|
182
|
+
"the probes as well as the hypothesis does, and the review fails a hypothesis that "
|
|
183
|
+
"doesn't beat it."
|
|
184
|
+
)
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Covariance between neurons: correlated partners and joint ablation.
|
|
2
|
+
|
|
3
|
+
A single neuron rarely acts alone. Say that neuron A's pole co-varies with neuron B.
|
|
4
|
+
Then a probe that "fails" on A may be handled by B. Also, ablating A alone can show no
|
|
5
|
+
effect, because B compensates. So we record A's partner neuron *poles* during the scan,
|
|
6
|
+
and the reviewer ablates A together with them.
|
|
7
|
+
|
|
8
|
+
Pole strengths are sparse and heavy-tailed: most tokens are exactly zero. So a global
|
|
9
|
+
Pearson correlation mostly measures agreement on "nothing is happening", and a few
|
|
10
|
+
large values drive it. :class:`PartnerSearch` ranks partners by the 95% lower bound of
|
|
11
|
+
their co-activation lift instead. That looks only at the regime where redundancy
|
|
12
|
+
matters, and does not reward a handful of chance coincidences. It also reports a rank
|
|
13
|
+
(Spearman) correlation per partner pole. Pearson on signed
|
|
14
|
+
activations is kept for reference.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from collections.abc import Callable, Sequence
|
|
20
|
+
|
|
21
|
+
import torch
|
|
22
|
+
import torch.nn.functional as F
|
|
23
|
+
|
|
24
|
+
from openinterp.core.schemas import CorrelatedNeuron, JointAblation, Pole
|
|
25
|
+
from openinterp.core.stats import spearman_columns
|
|
26
|
+
from openinterp.harness.hooks import Inputs, NeuronHookRunner, isolate_pole
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class CovarianceAccumulator:
|
|
30
|
+
"""Streaming Pearson correlation between one target series and every layer neuron."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, width: int) -> None:
|
|
33
|
+
self.n = 0
|
|
34
|
+
self.sx = 0.0
|
|
35
|
+
self.sxx = 0.0
|
|
36
|
+
self.sy = torch.zeros(width, dtype=torch.float64)
|
|
37
|
+
self.syy = torch.zeros(width, dtype=torch.float64)
|
|
38
|
+
self.sxy = torch.zeros(width, dtype=torch.float64)
|
|
39
|
+
|
|
40
|
+
def update(self, target: torch.Tensor, layer_acts: torch.Tensor) -> None:
|
|
41
|
+
"""``target`` is [n] pole strengths; ``layer_acts`` is [n, width] raw activations."""
|
|
42
|
+
x, y = target.double(), layer_acts.double()
|
|
43
|
+
self.n += x.numel()
|
|
44
|
+
self.sx += float(x.sum())
|
|
45
|
+
self.sxx += float((x * x).sum())
|
|
46
|
+
self.sy += y.sum(0)
|
|
47
|
+
self.syy += (y * y).sum(0)
|
|
48
|
+
self.sxy += (x[:, None] * y).sum(0)
|
|
49
|
+
|
|
50
|
+
def correlations(self) -> torch.Tensor:
|
|
51
|
+
"""Pearson's r between the target and each column; 0 where either has no variance."""
|
|
52
|
+
n = max(self.n, 1)
|
|
53
|
+
cov = self.sxy / n - (self.sx / n) * (self.sy / n)
|
|
54
|
+
var_x = self.sxx / n - (self.sx / n) ** 2
|
|
55
|
+
var_y = self.syy / n - (self.sy / n) ** 2
|
|
56
|
+
denom = torch.sqrt(torch.clamp(var_y * var_x, min=0.0))
|
|
57
|
+
return torch.where(denom > 1e-12, cov / denom, torch.zeros_like(cov)).clamp(-1, 1)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class PartnerSearch:
|
|
61
|
+
"""Stream a scan and find the neuron poles that co-activate with the target pole.
|
|
62
|
+
|
|
63
|
+
Keeps, with fixed-seed randomness so results are reproducible:
|
|
64
|
+
|
|
65
|
+
* a uniform reservoir sample of tokens (target strength + whole-layer activations),
|
|
66
|
+
for partner base rates, 99th-percentile "active" thresholds and Spearman;
|
|
67
|
+
* the ``top_k`` tokens where the target is strongest, which hold the target's
|
|
68
|
+
active regime. It is rare, so a uniform sample would barely contain it.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(self, width: int, *, reservoir: int = 8192, top_k: int = 2048, seed: int = 0):
|
|
72
|
+
self.pearson = CovarianceAccumulator(width)
|
|
73
|
+
self.reservoir, self.top_k = reservoir, top_k
|
|
74
|
+
self.generator = torch.Generator().manual_seed(seed)
|
|
75
|
+
self.seen = 0
|
|
76
|
+
self.res_target = torch.empty(0)
|
|
77
|
+
self.res_rows = torch.empty(0, width, dtype=torch.float16)
|
|
78
|
+
self.top_target = torch.empty(0)
|
|
79
|
+
self.top_rows = torch.empty(0, width, dtype=torch.float16)
|
|
80
|
+
|
|
81
|
+
def update(self, target: torch.Tensor, layer_acts: torch.Tensor) -> None:
|
|
82
|
+
"""Add a batch to the Pearson sums, the reservoir and the top-k tokens."""
|
|
83
|
+
target, rows = target.float(), layer_acts.half()
|
|
84
|
+
self.pearson.update(target, layer_acts)
|
|
85
|
+
|
|
86
|
+
# Reservoir sampling (Algorithm R), vectorised per batch.
|
|
87
|
+
fill = max(0, min(self.reservoir - self.res_target.numel(), target.numel()))
|
|
88
|
+
self.res_target = torch.cat([self.res_target, target[:fill]])
|
|
89
|
+
self.res_rows = torch.cat([self.res_rows, rows[:fill]])
|
|
90
|
+
rest = torch.arange(fill, target.numel())
|
|
91
|
+
if rest.numel():
|
|
92
|
+
index = self.seen + rest + 1
|
|
93
|
+
slots = (torch.rand(rest.numel(), generator=self.generator) * index).long()
|
|
94
|
+
for i, slot in zip(
|
|
95
|
+
rest[slots < self.reservoir].tolist(),
|
|
96
|
+
slots[slots < self.reservoir].tolist(),
|
|
97
|
+
strict=True,
|
|
98
|
+
):
|
|
99
|
+
self.res_target[slot] = target[i]
|
|
100
|
+
self.res_rows[slot] = rows[i]
|
|
101
|
+
self.seen += target.numel()
|
|
102
|
+
|
|
103
|
+
merged = torch.cat([self.top_target, target])
|
|
104
|
+
keep = merged.topk(min(self.top_k, merged.numel())).indices
|
|
105
|
+
self.top_target = merged[keep]
|
|
106
|
+
self.top_rows = torch.cat([self.top_rows, rows])[keep]
|
|
107
|
+
|
|
108
|
+
@staticmethod
|
|
109
|
+
def _poles(rows: torch.Tensor) -> torch.Tensor:
|
|
110
|
+
"""[n, w] signed activations -> [n, 2w] pole strengths (positive, then negative)."""
|
|
111
|
+
rows = rows.float()
|
|
112
|
+
return torch.cat([rows.clamp(min=0), (-rows).clamp(min=0)], dim=1)
|
|
113
|
+
|
|
114
|
+
def partners(
|
|
115
|
+
self, exclude: int, active_threshold: float, *, k: int = 10, quantile: float = 0.99
|
|
116
|
+
) -> list[CorrelatedNeuron]:
|
|
117
|
+
"""The ``k`` poles with the highest lower-bound co-activation lift, besides ``exclude``."""
|
|
118
|
+
if self.res_target.numel() < 3:
|
|
119
|
+
return []
|
|
120
|
+
width = self.res_rows.shape[1]
|
|
121
|
+
sample = self._poles(self.res_rows)
|
|
122
|
+
thresholds = _column_quantile(sample, quantile)
|
|
123
|
+
base = ((sample > thresholds).sum(0).double() + 1) / (sample.shape[0] + 2)
|
|
124
|
+
|
|
125
|
+
active = self._poles(self.top_rows[self.top_target >= active_threshold])
|
|
126
|
+
coactive = (active > thresholds).sum(0)
|
|
127
|
+
n_active = active.shape[0]
|
|
128
|
+
if n_active:
|
|
129
|
+
lift = ((coactive.double() + 1) / (n_active + 2)) / base
|
|
130
|
+
lower = wilson_lower(coactive.double(), n_active) / base
|
|
131
|
+
else: # the target never fired: no co-activation evidence, rank by Spearman alone
|
|
132
|
+
lift = torch.ones(2 * width, dtype=torch.float64)
|
|
133
|
+
lower = torch.zeros(2 * width, dtype=torch.float64)
|
|
134
|
+
rho = spearman_columns(self.res_target, sample)
|
|
135
|
+
pearson = self.pearson.correlations()
|
|
136
|
+
|
|
137
|
+
candidates = [c for c in range(2 * width) if c % width != exclude] # not itself
|
|
138
|
+
order = sorted(
|
|
139
|
+
candidates,
|
|
140
|
+
key=lambda c: (-float(lower[c]), -abs(float(rho[c])), c),
|
|
141
|
+
)[:k]
|
|
142
|
+
return [
|
|
143
|
+
CorrelatedNeuron(
|
|
144
|
+
neuron_idx=c % width,
|
|
145
|
+
pole="positive" if c < width else "negative",
|
|
146
|
+
coactivation_lift=round(float(lift[c]), 4),
|
|
147
|
+
coactivation_lift_lower=round(float(lower[c]), 4),
|
|
148
|
+
coactive_tokens=int(coactive[c]),
|
|
149
|
+
spearman=round(float(rho[c]), 6),
|
|
150
|
+
pearson=round(float(pearson[c % width]), 6),
|
|
151
|
+
)
|
|
152
|
+
for c in order
|
|
153
|
+
]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def wilson_lower(successes: torch.Tensor, n: int, z: float = 1.96) -> torch.Tensor:
|
|
157
|
+
"""Lower bound of the Wilson score interval for a binomial proportion."""
|
|
158
|
+
p = successes / n
|
|
159
|
+
centre = p + z * z / (2 * n)
|
|
160
|
+
margin = z * torch.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
|
|
161
|
+
return ((centre - margin) / (1 + z * z / n)).clamp(min=0.0)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _column_quantile(x: torch.Tensor, q: float, chunk: int = 1024) -> torch.Tensor:
|
|
165
|
+
"""Per-column nearest-rank quantile; chunked because torch.quantile caps input size."""
|
|
166
|
+
k = max(1, min(x.shape[0], round(q * (x.shape[0] - 1)) + 1))
|
|
167
|
+
return torch.cat(
|
|
168
|
+
[x[:, i : i + chunk].kthvalue(k, dim=0).values for i in range(0, x.shape[1], chunk)]
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@torch.no_grad()
|
|
173
|
+
def joint_ablation(
|
|
174
|
+
runner: NeuronHookRunner,
|
|
175
|
+
prompts: Inputs,
|
|
176
|
+
layer: int,
|
|
177
|
+
neuron_idx: int,
|
|
178
|
+
pole: Pole,
|
|
179
|
+
partners: Sequence[int],
|
|
180
|
+
) -> JointAblation:
|
|
181
|
+
"""KL(clean ‖ ablated) for the pole alone, its partners alone, and all together.
|
|
182
|
+
|
|
183
|
+
The pole is zero-ablated (its opposite pole is kept). Partners are mean-ablated over
|
|
184
|
+
the prompts, which stays closer to their natural distribution than zeroing.
|
|
185
|
+
|
|
186
|
+
Each prompt runs alone. A batch pads every prompt to the longest, and its logits are
|
|
187
|
+
prompts × longest × vocabulary. For 17 prompts, one of them 577 tokens long, that was
|
|
188
|
+
about 5.5 GB per copy at Qwen2.5's 151,936 tokens. Alone, a prompt costs its own length.
|
|
189
|
+
"""
|
|
190
|
+
from openinterp.harness.layer_major import map_layer_major
|
|
191
|
+
|
|
192
|
+
prompts = [prompts] if isinstance(prompts, str) else list(prompts)
|
|
193
|
+
encoded = [runner.encode([p]) for p in prompts]
|
|
194
|
+
partner_idx = torch.tensor(list(partners), dtype=torch.long)
|
|
195
|
+
means = None
|
|
196
|
+
if len(partners):
|
|
197
|
+
rows = map_layer_major(
|
|
198
|
+
runner, lambda e: runner.layer_activations(e, [layer])[layer][0][:, partner_idx],
|
|
199
|
+
encoded, tokens=max(e["input_ids"].shape[1] for e in encoded), logits=False,
|
|
200
|
+
) # fmt: skip
|
|
201
|
+
means = torch.cat(rows).mean(0)
|
|
202
|
+
|
|
203
|
+
def ablate(target: bool, others: bool) -> Callable:
|
|
204
|
+
def fn(act: torch.Tensor) -> torch.Tensor:
|
|
205
|
+
act = act.clone()
|
|
206
|
+
if target:
|
|
207
|
+
act[..., neuron_idx] -= isolate_pole(act[..., neuron_idx], pole)
|
|
208
|
+
if others and means is not None:
|
|
209
|
+
act[..., partner_idx] = means.to(act.dtype)
|
|
210
|
+
return act
|
|
211
|
+
|
|
212
|
+
return fn
|
|
213
|
+
|
|
214
|
+
fns = {"single": ablate(True, False), "joint": ablate(True, True)}
|
|
215
|
+
if len(partners):
|
|
216
|
+
fns["partners"] = ablate(False, True)
|
|
217
|
+
kl = _kl_means(runner, encoded, fns, layer)
|
|
218
|
+
kl_single, kl_partners, kl_joint = kl["single"], kl.get("partners", 0.0), kl["joint"]
|
|
219
|
+
ratio = (kl_joint - kl_partners) / kl_single if kl_single > 1e-9 else None
|
|
220
|
+
return JointAblation(
|
|
221
|
+
partners=list(partners),
|
|
222
|
+
kl_single=round(kl_single, 9),
|
|
223
|
+
kl_partners=round(kl_partners, 9),
|
|
224
|
+
kl_joint=round(kl_joint, 9),
|
|
225
|
+
redundancy_ratio=None if ratio is None else round(ratio, 6),
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@torch.no_grad()
|
|
230
|
+
def _kl_means(
|
|
231
|
+
runner: NeuronHookRunner, encoded: Sequence[dict], fns: dict[str, Callable], layer: int
|
|
232
|
+
) -> dict[str, float]:
|
|
233
|
+
"""Mean KL(clean ‖ ablated) per token over every prompt, for each ablation in ``fns``.
|
|
234
|
+
|
|
235
|
+
Each prompt's clean distribution is computed once and reused for every ablation. A
|
|
236
|
+
streamed model runs the clean prompt and its ablations as rows of fewer passes
|
|
237
|
+
(:func:`~openinterp.harness.memory.stream_rows`). It also runs its prompts at once
|
|
238
|
+
(:func:`~openinterp.harness.layer_major.map_layer_major`). Otherwise each is its own
|
|
239
|
+
pass, one after another.
|
|
240
|
+
"""
|
|
241
|
+
from openinterp.harness.layer_major import map_layer_major
|
|
242
|
+
from openinterp.harness.memory import stream_rows
|
|
243
|
+
|
|
244
|
+
variants: list[str | None] = [None, *fns] # None is the clean run, so it comes first
|
|
245
|
+
|
|
246
|
+
def prompt_kls(enc: dict) -> tuple[dict[str, float], int]:
|
|
247
|
+
"""One prompt's summed KL per ablation, and its token count."""
|
|
248
|
+
rows = stream_rows(runner, enc["input_ids"].shape[1], 1, logits=True)
|
|
249
|
+
sums, clean, tokens = dict.fromkeys(fns, 0.0), None, 0
|
|
250
|
+
for start in range(0, len(variants), rows):
|
|
251
|
+
chunk = variants[start : start + rows]
|
|
252
|
+
log_probs = _variant_log_probs(runner, enc, chunk, fns, layer)
|
|
253
|
+
for name, row in zip(chunk, log_probs, strict=True):
|
|
254
|
+
if name is None:
|
|
255
|
+
clean, tokens = row, row.shape[0]
|
|
256
|
+
else:
|
|
257
|
+
sums[name] += _kl_sum(row, clean)
|
|
258
|
+
return sums, tokens
|
|
259
|
+
|
|
260
|
+
longest = max(e["input_ids"].shape[1] for e in encoded)
|
|
261
|
+
per_prompt = map_layer_major(runner, prompt_kls, encoded, tokens=longest)
|
|
262
|
+
tokens = sum(n for _, n in per_prompt)
|
|
263
|
+
totals = {name: sum(sums[name] for sums, _ in per_prompt) for name in fns}
|
|
264
|
+
return {name: max(0.0, total / max(tokens, 1)) for name, total in totals.items()}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _variant_log_probs(runner, enc: dict, chunk, fns: dict[str, Callable], layer: int):
|
|
268
|
+
"""One pass with a row per entry of ``chunk``: clean (``None``) or ablated by ``fns``."""
|
|
269
|
+
batch = {k: v.repeat(len(chunk), *[1] * (v.dim() - 1)) for k, v in enc.items()}
|
|
270
|
+
|
|
271
|
+
def per_row(act: torch.Tensor) -> torch.Tensor:
|
|
272
|
+
rows = [
|
|
273
|
+
act[i : i + 1] if name is None else fns[name](act[i : i + 1])
|
|
274
|
+
for i, name in enumerate(chunk)
|
|
275
|
+
]
|
|
276
|
+
return torch.cat(rows)
|
|
277
|
+
|
|
278
|
+
ablates = any(name is not None for name in chunk)
|
|
279
|
+
logits = runner.run_with_interventions(batch, {layer: per_row} if ablates else None)
|
|
280
|
+
return list(_log_probs(logits))
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# Positions per block in the log-softmax and the KL. A block of 64 is 39 MB at
|
|
284
|
+
# Qwen2.5's 151,936 tokens. Whole, each step made a copy of the prompt's logits.
|
|
285
|
+
_KL_BLOCK = 64
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _log_probs(logits: torch.Tensor) -> torch.Tensor:
|
|
289
|
+
"""``log_softmax(logits.float(), -1)``, computed in place one block at a time."""
|
|
290
|
+
logits = logits.float()
|
|
291
|
+
for row in logits.flatten(0, -2).split(_KL_BLOCK):
|
|
292
|
+
row -= row.logsumexp(-1, keepdim=True)
|
|
293
|
+
return logits
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _kl_sum(ablated: torch.Tensor, clean: torch.Tensor) -> float:
|
|
297
|
+
"""``KL(clean ‖ ablated)`` summed over positions, one block at a time."""
|
|
298
|
+
return sum(
|
|
299
|
+
float(F.kl_div(a, c, log_target=True, reduction="sum"))
|
|
300
|
+
for a, c in zip(ablated.split(_KL_BLOCK), clean.split(_KL_BLOCK), strict=True)
|
|
301
|
+
)
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Dictionary latents aligned with a neuron pole's weights.
|
|
2
|
+
|
|
3
|
+
A pole writes ``±w_out`` (its down-projection column, negated for the negative pole) and
|
|
4
|
+
reads its MLP's input with ``gate``/``up`` (or ``in``). A latent writes ``W_dec[f]`` and,
|
|
5
|
+
when its dictionary reads the MLP's input (a transcoder), reads it with ``W_enc[:, f]``.
|
|
6
|
+
Both are compared by cosine in residual-stream coordinates. Each norm is folded in as the
|
|
7
|
+
weight wiring folds it (:func:`openinterp.harness.wiring._fold`). Each cosine is oriented
|
|
8
|
+
(:func:`orientation`), so + means that the latent writes what the pole writes, or reads
|
|
9
|
+
what drives the pole. Such a latent is a candidate for the same feature.
|
|
10
|
+
|
|
11
|
+
Two references put a cosine in context. ``chance`` is the 99th percentile, over random
|
|
12
|
+
directions, of the best cosine any latent reaches. ``percentile`` is the share of the
|
|
13
|
+
layer's other neurons whose best match is weaker than this one's. So a pole that the
|
|
14
|
+
dictionary happens to have a latent for stands out from a layer that it fits well
|
|
15
|
+
everywhere.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import hashlib
|
|
21
|
+
from collections.abc import Iterable
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
|
|
24
|
+
import torch
|
|
25
|
+
from torch import nn
|
|
26
|
+
|
|
27
|
+
from openinterp.core.exceptions import ArchitectureNotSupportedError
|
|
28
|
+
from openinterp.harness.dictionaries import Dictionary, DictionarySpec, load_spec
|
|
29
|
+
from openinterp.harness.hooks import NeuronHookRunner
|
|
30
|
+
from openinterp.harness.wiring import _fold, _matrix, mlp_reads
|
|
31
|
+
|
|
32
|
+
# Sites where a dictionary's encoder reads what the MLP reads.
|
|
33
|
+
READS_MLP_INPUT = ("resid_mid", "mlp_in", "mlp_in_normalized")
|
|
34
|
+
CHANCE_DRAWS = 256
|
|
35
|
+
CHUNK = 512
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class Match:
|
|
40
|
+
"""One latent aligned with a pole: its index, cosine and read channel."""
|
|
41
|
+
|
|
42
|
+
feature_idx: int
|
|
43
|
+
cosine: float
|
|
44
|
+
channel: str | None = None # the neuron's read channel, for a read
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class Side:
|
|
49
|
+
"""One pole's best-aligned latents on one side (write or read)."""
|
|
50
|
+
|
|
51
|
+
matches: list[Match]
|
|
52
|
+
chance: float
|
|
53
|
+
percentile: float
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _unit(rows: torch.Tensor) -> torch.Tensor:
|
|
57
|
+
return rows / rows.norm(dim=-1, keepdim=True).clamp_min(1e-12)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def latent_writes(runner: NeuronHookRunner, spec: DictionarySpec, layer: int, d: Dictionary):
|
|
61
|
+
"""Each latent's write ``[n, d_model]``, in the space a pole writes to."""
|
|
62
|
+
if runner.mlp_write(layer) is not runner.down_proj(layer):
|
|
63
|
+
# Gemma 2 normalises the MLP's output, and its dictionaries decode after that norm.
|
|
64
|
+
raise ArchitectureNotSupportedError(
|
|
65
|
+
"the MLP's output is normalised before the residual stream, so a neuron's write "
|
|
66
|
+
"and a latent's are not in the same space"
|
|
67
|
+
)
|
|
68
|
+
return d.w_dec.float()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def latent_reads(
|
|
72
|
+
runner: NeuronHookRunner, spec: DictionarySpec, layer: int, d: Dictionary
|
|
73
|
+
) -> torch.Tensor | None:
|
|
74
|
+
"""Return each latent's read ``[n, d_model]`` of the residual stream.
|
|
75
|
+
|
|
76
|
+
``None`` when the dictionary doesn't read the MLP's input.
|
|
77
|
+
"""
|
|
78
|
+
if spec.input_site not in READS_MLP_INPUT:
|
|
79
|
+
return None
|
|
80
|
+
read = d.w_enc.T.float()
|
|
81
|
+
if spec.input_site == "resid_mid":
|
|
82
|
+
return read
|
|
83
|
+
return _fold(runner.mlp_norm(layer), read, gain=spec.input_site == "mlp_in")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _seed(name: str) -> int:
|
|
87
|
+
"""A seed that no training run is likely to share.
|
|
88
|
+
|
|
89
|
+
A dictionary's dead latents keep their initial weights. With a small seed such as 0,
|
|
90
|
+
the "random" directions would be those weights. (Their cosine is 0.93 with
|
|
91
|
+
Qwen2.5-0.5B's layer-0 transcoder.)
|
|
92
|
+
"""
|
|
93
|
+
return int.from_bytes(
|
|
94
|
+
hashlib.sha256(f"open-interpretability crossref {name}".encode()).digest()[:8]
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _null(runner: NeuronHookRunner, layer: int, latents: torch.Tensor, name: str) -> float:
|
|
99
|
+
"""Return the 99th percentile of the best cosine between a random direction and any latent.
|
|
100
|
+
|
|
101
|
+
Directions are centred as LayerNorm centres a read, so they live where reads do.
|
|
102
|
+
"""
|
|
103
|
+
g = torch.Generator().manual_seed(_seed(f"{name} {layer}"))
|
|
104
|
+
r = torch.randn(CHANCE_DRAWS, latents.shape[1], generator=g).to(latents.device)
|
|
105
|
+
if isinstance(runner.mlp_norm(layer), nn.LayerNorm):
|
|
106
|
+
r = r - r.mean(-1, keepdim=True)
|
|
107
|
+
best = (_unit(r) @ latents.T).amax(-1)
|
|
108
|
+
return float(best.quantile(0.99))
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def orientation(channel: str, pole: str) -> float:
|
|
112
|
+
"""Return the sign of a vector that drives ``pole`` along ``channel``.
|
|
113
|
+
|
|
114
|
+
A latent fires non-negatively, so only its own direction counts. A pole writes
|
|
115
|
+
``±w_out``. A gated neuron fires when its gate opens, whatever the pole, with the sign
|
|
116
|
+
of ``up``. An ungated neuron fires with the sign of its pre-activation.
|
|
117
|
+
"""
|
|
118
|
+
return 1.0 if channel == "gate" or pole == "positive" else -1.0
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _align(
|
|
122
|
+
channels: dict[str, torch.Tensor],
|
|
123
|
+
latents: torch.Tensor,
|
|
124
|
+
poles: Iterable[tuple[int, str]],
|
|
125
|
+
k: int,
|
|
126
|
+
) -> tuple[dict[tuple[int, str], list[Match]], dict[str, torch.Tensor]]:
|
|
127
|
+
"""Return the top ``k`` latents by oriented cosine for each ``(neuron, pole)``.
|
|
128
|
+
|
|
129
|
+
They are over all the neuron's channels. A latent appears once, at its best channel.
|
|
130
|
+
Also returns every neuron's best oriented cosine, by pole.
|
|
131
|
+
"""
|
|
132
|
+
poles = sorted(set(poles))
|
|
133
|
+
width = next(iter(channels.values())).shape[0]
|
|
134
|
+
best = {p: torch.full((width,), -1.0) for p in ("positive", "negative")}
|
|
135
|
+
found: dict[tuple[int, str], list[Match]] = {p: [] for p in poles}
|
|
136
|
+
for channel, rows in channels.items():
|
|
137
|
+
for start in range(0, width, CHUNK):
|
|
138
|
+
cos = _unit(rows[start : start + CHUNK].float()) @ latents.T # [c, n]
|
|
139
|
+
highest, lowest = cos.amax(-1).cpu(), (-cos).amax(-1).cpu()
|
|
140
|
+
for pole, b in best.items():
|
|
141
|
+
reach = highest if orientation(channel, pole) > 0 else lowest
|
|
142
|
+
b[start : start + CHUNK] = torch.maximum(b[start : start + CHUNK], reach)
|
|
143
|
+
for i, pole in poles:
|
|
144
|
+
if start <= i < start + CHUNK:
|
|
145
|
+
row = orientation(channel, pole) * cos[i - start]
|
|
146
|
+
top = row.topk(k).indices
|
|
147
|
+
found[(i, pole)] += [Match(int(f), float(row[f]), channel) for f in top]
|
|
148
|
+
for key, matches in found.items():
|
|
149
|
+
by_feature: dict[int, Match] = {}
|
|
150
|
+
for m in sorted(matches, key=lambda m: -m.cosine):
|
|
151
|
+
by_feature.setdefault(m.feature_idx, m)
|
|
152
|
+
found[key] = list(by_feature.values())[:k]
|
|
153
|
+
return found, best
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def align_layer(
|
|
157
|
+
runner: NeuronHookRunner,
|
|
158
|
+
spec: DictionarySpec,
|
|
159
|
+
layer: int,
|
|
160
|
+
poles: Iterable[tuple[int, str]],
|
|
161
|
+
*,
|
|
162
|
+
k: int = 5,
|
|
163
|
+
) -> dict[tuple[int, str], tuple[Side, Side | None]]:
|
|
164
|
+
"""Return ``{(neuron, pole): (write, read)}`` against ``spec``'s dictionary for ``layer``.
|
|
165
|
+
|
|
166
|
+
``read`` is ``None`` when the dictionary doesn't read the MLP's input.
|
|
167
|
+
"""
|
|
168
|
+
poles = list(poles)
|
|
169
|
+
device = runner.device
|
|
170
|
+
d = load_spec(spec, layer, device)
|
|
171
|
+
writes = _unit(latent_writes(runner, spec, layer, d))
|
|
172
|
+
w_out = _matrix(runner.down_proj(layer)).T.to(device) # [width, d_model]
|
|
173
|
+
write_null = _null(runner, layer, writes, "write")
|
|
174
|
+
by_write, write_best = _align({"out": w_out}, writes, poles, k)
|
|
175
|
+
|
|
176
|
+
reads = latent_reads(runner, spec, layer, d)
|
|
177
|
+
by_read = read_best = read_null = None
|
|
178
|
+
if reads is not None:
|
|
179
|
+
reads = _unit(reads)
|
|
180
|
+
read_null = _null(runner, layer, reads, "read")
|
|
181
|
+
neuron_reads = {c: r.to(device) for c, r in mlp_reads(runner, layer).items()}
|
|
182
|
+
by_read, read_best = _align(neuron_reads, reads, poles, k)
|
|
183
|
+
|
|
184
|
+
def share_below(best: dict[str, torch.Tensor], i: int, pole: str) -> float:
|
|
185
|
+
return float((best[pole] < best[pole][i]).float().mean())
|
|
186
|
+
|
|
187
|
+
out = {}
|
|
188
|
+
for i, pole in poles:
|
|
189
|
+
write_matches = [Match(m.feature_idx, m.cosine) for m in by_write[(i, pole)]]
|
|
190
|
+
write = Side(write_matches, write_null, share_below(write_best, i, pole))
|
|
191
|
+
read = None
|
|
192
|
+
if by_read is not None:
|
|
193
|
+
read = Side(by_read[(i, pole)], read_null, share_below(read_best, i, pole))
|
|
194
|
+
out[(i, pole)] = (write, read)
|
|
195
|
+
return out
|