open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""Input attribution: which context tokens drive a neuron pole at its peak.
|
|
2
|
+
|
|
3
|
+
Take each top example and the pole's *signed* activation at the peak token. It is
|
|
4
|
+
positive for the positive pole and negated for the negative pole. So larger is more
|
|
5
|
+
active, and the gradient is non-zero below the clamp. Integrated gradients (Sundararajan
|
|
6
|
+
et al., 2017) over the input token embeddings attribute it to every token up to the peak:
|
|
7
|
+
|
|
8
|
+
attr_i = (x_i − b) · (1/m) Σ_k ∂f/∂x_i |_{b + α_k (x − b)}, α_k = (k + ½)/m
|
|
9
|
+
|
|
10
|
+
Here b is the embedding of one reference token (the pad token) at every position but the
|
|
11
|
+
first. The first token stays as it is. Most of the model's attention rests on it (the
|
|
12
|
+
attention sink), so replacing it moves the whole path off-distribution. On Qwen2.5-0.5B,
|
|
13
|
+
the integral then fails to converge. It credits the first few tokens of a window with
|
|
14
|
+
much of the activation, but replacing any one of them doesn't remove it. A peak on the
|
|
15
|
+
first token is attributed to it alone. For a pole with a first-token floor
|
|
16
|
+
(``skip_first``, protocol 0.8), the peak is over the other tokens. Position embeddings,
|
|
17
|
+
where a model has them, are left alone. By completeness the attributions sum to
|
|
18
|
+
f(x) − f(b), and each is reported as a share of that gap.
|
|
19
|
+
|
|
20
|
+
The path integral is linear in each token's contribution. So a second check replaces
|
|
21
|
+
one token at a time with the reference token, and measures again. It checks the peak
|
|
22
|
+
itself, the few tokens before it, and the most attributed tokens. Tokens after the peak
|
|
23
|
+
cannot affect it (attention is causal). So each example is cut at its peak, and every
|
|
24
|
+
forward pass stops at the audited layer.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from collections.abc import Callable, Sequence
|
|
30
|
+
from contextlib import suppress
|
|
31
|
+
from dataclasses import dataclass
|
|
32
|
+
|
|
33
|
+
import torch
|
|
34
|
+
|
|
35
|
+
from openinterp.core.attribution import (
|
|
36
|
+
AttributedToken,
|
|
37
|
+
ContextExample,
|
|
38
|
+
DistanceShare,
|
|
39
|
+
InputAttribution,
|
|
40
|
+
)
|
|
41
|
+
from openinterp.core.schemas import ActivatingExample, Pole
|
|
42
|
+
from openinterp.core.stats import spearman
|
|
43
|
+
from openinterp.harness.hooks import NeuronHookRunner, _StopForward
|
|
44
|
+
from openinterp.harness.memory import recompute_below, stream_rows
|
|
45
|
+
|
|
46
|
+
# Tokens back from the peak: the token itself, the one before, then wider bands.
|
|
47
|
+
DISTANCE_BANDS: tuple[tuple[int, int | None], ...] = (
|
|
48
|
+
(0, 0),
|
|
49
|
+
(1, 1),
|
|
50
|
+
(2, 4),
|
|
51
|
+
(5, 16),
|
|
52
|
+
(17, 64),
|
|
53
|
+
(65, None),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
EmbedEdit = Callable[[torch.Tensor], torch.Tensor | None]
|
|
57
|
+
|
|
58
|
+
# Padded tokens per backward pass of integrated gradients. Each interpolation row keeps
|
|
59
|
+
# a backward graph through every layer up to the unit's, so memory grows with rows ×
|
|
60
|
+
# example length. A whole Qwen2.5-1.5B document at 8 rows rose 8.3 GB. Rows are
|
|
61
|
+
# independent, so the attribution doesn't depend on how they are batched.
|
|
62
|
+
ATTRIBUTION_BATCH_TOKENS = 1024
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def baseline_token_id(tokenizer) -> int:
|
|
66
|
+
"""The reference token: the pad token, else end-of-sequence, else id 0."""
|
|
67
|
+
for name in ("pad_token_id", "eos_token_id"):
|
|
68
|
+
value = getattr(tokenizer, name, None)
|
|
69
|
+
if isinstance(value, int):
|
|
70
|
+
return value
|
|
71
|
+
return 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _pole_values(
|
|
75
|
+
runner: NeuronHookRunner,
|
|
76
|
+
ids: torch.Tensor,
|
|
77
|
+
layer: int,
|
|
78
|
+
neuron_idx: int,
|
|
79
|
+
pole: Pole,
|
|
80
|
+
edit: EmbedEdit,
|
|
81
|
+
) -> torch.Tensor:
|
|
82
|
+
"""Signed pole activation ``[batch, seq]`` with the token embeddings edited by ``edit``.
|
|
83
|
+
|
|
84
|
+
The forward pass stops at ``layer``'s MLP: later layers can't change it.
|
|
85
|
+
"""
|
|
86
|
+
captured: list[torch.Tensor] = []
|
|
87
|
+
|
|
88
|
+
def capture(act: torch.Tensor) -> None:
|
|
89
|
+
captured.append(act[..., neuron_idx])
|
|
90
|
+
raise _StopForward
|
|
91
|
+
|
|
92
|
+
embed = runner.model.get_input_embeddings()
|
|
93
|
+
handle = embed.register_forward_hook(lambda _module, _args, out: edit(out))
|
|
94
|
+
try:
|
|
95
|
+
with suppress(_StopForward), runner.hooks({layer: capture}):
|
|
96
|
+
runner.forward({"input_ids": ids})
|
|
97
|
+
finally:
|
|
98
|
+
handle.remove()
|
|
99
|
+
value = captured[-1].float()
|
|
100
|
+
return value if pole == "positive" else -value
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def reference(x: torch.Tensor, baseline: torch.Tensor, keep_first: bool) -> torch.Tensor:
|
|
104
|
+
"""Return the baseline input for embeddings ``x`` ``[batch, seq, d]``.
|
|
105
|
+
|
|
106
|
+
It is ``baseline`` at every position, except the first when ``keep_first``.
|
|
107
|
+
"""
|
|
108
|
+
base = baseline.to(x.dtype).expand_as(x).clone()
|
|
109
|
+
if keep_first:
|
|
110
|
+
base[:, 0] = x[:, 0]
|
|
111
|
+
return base
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def integrated_gradients(
|
|
115
|
+
runner: NeuronHookRunner,
|
|
116
|
+
ids: torch.Tensor,
|
|
117
|
+
layer: int,
|
|
118
|
+
neuron_idx: int,
|
|
119
|
+
pole: Pole,
|
|
120
|
+
baseline: torch.Tensor,
|
|
121
|
+
*,
|
|
122
|
+
steps: int = 16,
|
|
123
|
+
batch_size: int = 8,
|
|
124
|
+
keep_first: bool = True,
|
|
125
|
+
) -> torch.Tensor:
|
|
126
|
+
"""Attribution ``[seq]`` of the last token's signed pole activation, one row ``ids``.
|
|
127
|
+
|
|
128
|
+
With ``keep_first`` the first token is not interpolated and gets no attribution.
|
|
129
|
+
"""
|
|
130
|
+
alphas = (torch.arange(steps, dtype=torch.float32) + 0.5) / steps
|
|
131
|
+
grad_sum: torch.Tensor | None = None
|
|
132
|
+
diff: torch.Tensor | None = None
|
|
133
|
+
for chunk in alphas.split(batch_size):
|
|
134
|
+
points: list[torch.Tensor] = []
|
|
135
|
+
|
|
136
|
+
def interpolate(out: torch.Tensor, chunk=chunk, points=points) -> torch.Tensor:
|
|
137
|
+
nonlocal diff
|
|
138
|
+
x = out.detach()
|
|
139
|
+
base = reference(x, baseline, keep_first)
|
|
140
|
+
alpha = chunk.to(x.device, x.dtype)[:, None, None]
|
|
141
|
+
point = (base + alpha * (x - base)).requires_grad_(True)
|
|
142
|
+
diff = (x[0] - base[0]).float()
|
|
143
|
+
points.append(point)
|
|
144
|
+
return point
|
|
145
|
+
|
|
146
|
+
with torch.enable_grad(), recompute_below(runner, layer):
|
|
147
|
+
rows = ids.expand(len(chunk), -1)
|
|
148
|
+
values = _pole_values(runner, rows, layer, neuron_idx, pole, interpolate)[:, -1]
|
|
149
|
+
(grad,) = torch.autograd.grad(values.sum(), points[-1])
|
|
150
|
+
part = grad.float().sum(0)
|
|
151
|
+
grad_sum = part if grad_sum is None else grad_sum + part
|
|
152
|
+
assert grad_sum is not None and diff is not None # noqa: S101 (for the type checker)
|
|
153
|
+
return (grad_sum / steps * diff).sum(-1)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@torch.no_grad()
|
|
157
|
+
def ablate_positions(
|
|
158
|
+
runner: NeuronHookRunner,
|
|
159
|
+
ids: torch.Tensor,
|
|
160
|
+
layer: int,
|
|
161
|
+
neuron_idx: int,
|
|
162
|
+
pole: Pole,
|
|
163
|
+
baseline: torch.Tensor,
|
|
164
|
+
positions: Sequence[int],
|
|
165
|
+
*,
|
|
166
|
+
batch_size: int = 8,
|
|
167
|
+
) -> torch.Tensor:
|
|
168
|
+
"""The last token's signed pole activation with each of ``positions`` replaced alone."""
|
|
169
|
+
out: list[torch.Tensor] = []
|
|
170
|
+
for start in range(0, len(positions), batch_size):
|
|
171
|
+
chunk = torch.tensor(positions[start : start + batch_size])
|
|
172
|
+
|
|
173
|
+
def replace(emb: torch.Tensor, chunk=chunk) -> torch.Tensor:
|
|
174
|
+
emb = emb.clone()
|
|
175
|
+
emb[torch.arange(len(chunk)), chunk.to(emb.device)] = baseline.to(emb.dtype)
|
|
176
|
+
return emb
|
|
177
|
+
|
|
178
|
+
rows = ids.expand(len(chunk), -1)
|
|
179
|
+
out.append(_pole_values(runner, rows, layer, neuron_idx, pole, replace)[:, -1])
|
|
180
|
+
return torch.cat(out)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _distance_shares(attr: torch.Tensor) -> tuple[list[float], float]:
|
|
184
|
+
"""Share of |attr| per band of :data:`DISTANCE_BANDS`, and the weighted mean distance."""
|
|
185
|
+
weight = attr.abs()
|
|
186
|
+
total = float(weight.sum()) or 1.0
|
|
187
|
+
distance = torch.arange(len(attr) - 1, -1, -1, dtype=torch.float32)
|
|
188
|
+
shares = []
|
|
189
|
+
for low, high in DISTANCE_BANDS:
|
|
190
|
+
band = (distance >= low) & (distance <= (high if high is not None else len(attr)))
|
|
191
|
+
shares.append(float(weight[band].sum()) / total)
|
|
192
|
+
return shares, float((weight * distance).sum()) / total
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _converged_attribution(
|
|
196
|
+
runner, ids, args, baseline, gap: float, settings: _Settings
|
|
197
|
+
) -> tuple[torch.Tensor, int]:
|
|
198
|
+
"""Return the attribution as shares of ``gap``, and the steps that it took.
|
|
199
|
+
|
|
200
|
+
The steps double from ``settings.steps`` until Σ is within ``settings.tolerance`` of
|
|
201
|
+
1, or ``settings.max_steps`` is reached. Each backward pass takes at most
|
|
202
|
+
``settings.batch_size`` rows and :data:`ATTRIBUTION_BATCH_TOKENS` tokens, and at
|
|
203
|
+
least one row.
|
|
204
|
+
"""
|
|
205
|
+
steps, max_steps, tolerance = settings.steps, settings.max_steps, settings.tolerance
|
|
206
|
+
keep_first = ids.shape[1] > 1
|
|
207
|
+
rows = max(1, min(settings.batch_size, ATTRIBUTION_BATCH_TOKENS // ids.shape[1]))
|
|
208
|
+
rows = stream_rows(runner, ids.shape[1], rows, logits=False) # more when streamed
|
|
209
|
+
while True:
|
|
210
|
+
attr = integrated_gradients(
|
|
211
|
+
runner, ids, *args, baseline, steps=steps, batch_size=rows, keep_first=keep_first
|
|
212
|
+
).cpu()
|
|
213
|
+
attr = attr / gap
|
|
214
|
+
if abs(float(attr.sum()) - 1.0) <= tolerance or steps * 2 > max_steps:
|
|
215
|
+
return attr, steps
|
|
216
|
+
steps *= 2
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def input_attribution(
|
|
220
|
+
runner: NeuronHookRunner,
|
|
221
|
+
examples: Sequence[ActivatingExample],
|
|
222
|
+
layer: int,
|
|
223
|
+
neuron_idx: int,
|
|
224
|
+
pole: Pole,
|
|
225
|
+
*,
|
|
226
|
+
n_examples: int = 5,
|
|
227
|
+
steps: int = 16,
|
|
228
|
+
max_steps: int = 64,
|
|
229
|
+
tolerance: float = 0.1,
|
|
230
|
+
top_k: int = 5,
|
|
231
|
+
near: int = 4,
|
|
232
|
+
batch_size: int = 8,
|
|
233
|
+
skip_first: bool = False,
|
|
234
|
+
) -> InputAttribution:
|
|
235
|
+
"""Integrated gradients at the peak of the first ``n_examples`` top examples.
|
|
236
|
+
|
|
237
|
+
Each example starts at ``steps`` integration steps and doubles them, up to
|
|
238
|
+
``max_steps``, until the attribution sums to within ``tolerance`` of the gap. Only
|
|
239
|
+
examples that get there count towards the distance shares and the agreement. Each
|
|
240
|
+
example keeps its ``top_k`` most attributed tokens; the ablation check covers those,
|
|
241
|
+
the peak and the ``near`` tokens before it.
|
|
242
|
+
"""
|
|
243
|
+
if runner.tokenizer is None:
|
|
244
|
+
raise ValueError("input_attribution requires a tokenizer")
|
|
245
|
+
base_id = baseline_token_id(runner.tokenizer)
|
|
246
|
+
baseline = runner.model.get_input_embeddings().weight[base_id].detach()
|
|
247
|
+
args = (layer, neuron_idx, pole)
|
|
248
|
+
settings = _Settings(steps, max_steps, tolerance, top_k, near, batch_size, skip_first)
|
|
249
|
+
results: list[ContextExample] = []
|
|
250
|
+
band_shares: list[list[float]] = []
|
|
251
|
+
distances: list[float] = []
|
|
252
|
+
checked_attr: list[float] = []
|
|
253
|
+
checked_effect: list[float] = []
|
|
254
|
+
for n, example in enumerate(examples[:n_examples]):
|
|
255
|
+
found = _attribute_example(runner, n, example, args, baseline, settings)
|
|
256
|
+
if found is None:
|
|
257
|
+
continue
|
|
258
|
+
result, converged = found
|
|
259
|
+
results.append(result)
|
|
260
|
+
if converged is not None:
|
|
261
|
+
attrs, effects, shares, mean_distance = converged
|
|
262
|
+
checked_attr += attrs
|
|
263
|
+
checked_effect += effects
|
|
264
|
+
band_shares.append(shares)
|
|
265
|
+
distances.append(mean_distance)
|
|
266
|
+
agreement = (
|
|
267
|
+
spearman(torch.tensor(checked_attr), torch.tensor(checked_effect))
|
|
268
|
+
if len(checked_attr) >= 3
|
|
269
|
+
else None
|
|
270
|
+
)
|
|
271
|
+
return InputAttribution(
|
|
272
|
+
baseline_token=runner.tokenizer.decode([base_id]),
|
|
273
|
+
tolerance=tolerance,
|
|
274
|
+
examples=results,
|
|
275
|
+
distances=_band_shares(band_shares),
|
|
276
|
+
mean_distance=round(sum(distances) / len(distances), 2) if distances else None,
|
|
277
|
+
ablation_agreement=round(agreement, 4) if agreement is not None else None,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@dataclass(frozen=True)
|
|
282
|
+
class _Settings:
|
|
283
|
+
"""``input_attribution``'s options, as one example needs them."""
|
|
284
|
+
|
|
285
|
+
steps: int
|
|
286
|
+
max_steps: int
|
|
287
|
+
tolerance: float
|
|
288
|
+
top_k: int
|
|
289
|
+
near: int
|
|
290
|
+
batch_size: int
|
|
291
|
+
skip_first: bool
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _attribute_example(runner, n: int, example, args, baseline, settings: _Settings):
|
|
295
|
+
"""Attribute one example, and return its ContextExample and the numbers for the shares.
|
|
296
|
+
|
|
297
|
+
If it converged, the numbers are its checked attributions, ablation effects,
|
|
298
|
+
distance-band shares and mean distance. ``None`` if there is no gap.
|
|
299
|
+
"""
|
|
300
|
+
ids, values, peak, gap = _peak_gap(runner, example, args, baseline, settings.skip_first)
|
|
301
|
+
if gap <= 1e-6:
|
|
302
|
+
return None # the reference input activates the pole as much: nothing to attribute
|
|
303
|
+
tolerance = settings.tolerance
|
|
304
|
+
attr, used = _converged_attribution(runner, ids, args, baseline, gap, settings)
|
|
305
|
+
error = abs(float(attr.sum()) - 1.0)
|
|
306
|
+
top = attr.abs().topk(min(settings.top_k, len(attr))).indices.tolist()
|
|
307
|
+
checked = sorted(set(top) | set(range(max(0, peak - settings.near), peak + 1)))
|
|
308
|
+
ablated = ablate_positions(
|
|
309
|
+
runner, ids, *args, baseline, checked, batch_size=settings.batch_size
|
|
310
|
+
)
|
|
311
|
+
effect = dict(zip(checked, ((float(values[peak]) - ablated.cpu()) / gap).tolist(), strict=True))
|
|
312
|
+
converged = None
|
|
313
|
+
if error <= tolerance:
|
|
314
|
+
shares, mean_distance = _distance_shares(attr)
|
|
315
|
+
attrs = [float(attr[i]) for i in checked]
|
|
316
|
+
converged = (attrs, [effect[i] for i in checked], shares, mean_distance)
|
|
317
|
+
tokenizer, token_ids = runner.tokenizer, ids[0].tolist()
|
|
318
|
+
result = ContextExample(
|
|
319
|
+
example=n,
|
|
320
|
+
peak_token=tokenizer.decode([token_ids[peak]]),
|
|
321
|
+
tokens=peak + 1,
|
|
322
|
+
steps=used,
|
|
323
|
+
converged=error <= tolerance,
|
|
324
|
+
completeness_error=round(error, 4),
|
|
325
|
+
top=[
|
|
326
|
+
AttributedToken(
|
|
327
|
+
offset=i - peak,
|
|
328
|
+
token=tokenizer.decode([token_ids[i]]),
|
|
329
|
+
attribution=round(float(attr[i]), 4),
|
|
330
|
+
ablation_effect=round(effect[i], 4),
|
|
331
|
+
)
|
|
332
|
+
for i in top
|
|
333
|
+
],
|
|
334
|
+
)
|
|
335
|
+
return result, converged
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _peak_gap(runner, example, args, baseline, skip_first: bool):
|
|
339
|
+
"""Return the example cut at its peak, its pole values, the peak, and the gap.
|
|
340
|
+
|
|
341
|
+
The gap is between the peak and the reference input's activation there.
|
|
342
|
+
"""
|
|
343
|
+
ids = runner.encode(example.text)["input_ids"][:1]
|
|
344
|
+
with torch.no_grad():
|
|
345
|
+
values = _pole_values(runner, ids, *args, lambda _out: None)[0]
|
|
346
|
+
start = int(skip_first and len(values) > 1)
|
|
347
|
+
peak = start + int(values[start:].argmax())
|
|
348
|
+
ids = ids[:, : peak + 1]
|
|
349
|
+
empty = _pole_values(
|
|
350
|
+
runner, ids, *args, lambda out, keep=peak > 0: reference(out, baseline, keep)
|
|
351
|
+
)
|
|
352
|
+
return ids, values, peak, float(values[peak] - empty[0, -1])
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _band_shares(band_shares: list[list[float]]) -> list[DistanceShare]:
|
|
356
|
+
"""Each distance band's mean share of the attribution, over the converged examples."""
|
|
357
|
+
count = len(band_shares)
|
|
358
|
+
if not count:
|
|
359
|
+
return []
|
|
360
|
+
return [
|
|
361
|
+
DistanceShare(
|
|
362
|
+
min_distance=low,
|
|
363
|
+
max_distance=high,
|
|
364
|
+
share=round(min(1.0, sum(s[b] for s in band_shares) / count), 4),
|
|
365
|
+
)
|
|
366
|
+
for b, (low, high) in enumerate(DISTANCE_BANDS)
|
|
367
|
+
]
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Jacobian lens readings of a neuron pole's output direction.
|
|
2
|
+
|
|
3
|
+
The Jacobian lens is from Gurnee et al., 2026, "Verbalizable Representations Form a
|
|
4
|
+
Global Workspace in Language Models" (arXiv:2607.15495). It decodes a layer-ℓ residual
|
|
5
|
+
vector h as follows:
|
|
6
|
+
|
|
7
|
+
lens(h) = softmax(W_U · norm(J_ℓ h)), J_ℓ = E[∂h_final,t' / ∂h_ℓ,t]
|
|
8
|
+
|
|
9
|
+
The average is over source positions t, every later position t' ≥ t, and a corpus of
|
|
10
|
+
prompts. The logit lens is the special case J_ℓ = I. The two agree in late layers and
|
|
11
|
+
diverge earlier, where the logit lens is unreliable.
|
|
12
|
+
|
|
13
|
+
For one neuron we only need J_ℓ w for its output direction w, not the full matrix.
|
|
14
|
+
Add ε·w to the residual at *every* position. To first order, this changes the final
|
|
15
|
+
residual at t' by Σ_{t ≤ t'} J_{t,t'} w. (The causal mask stops later positions acting on
|
|
16
|
+
earlier ones.) Sum that over t', and divide by the number of (t, t') pairs. This gives
|
|
17
|
+
the paper's average from one perturbed pass per sign (central differences).
|
|
18
|
+
|
|
19
|
+
The average includes pairs where t' > t contributes nothing. So its overall scale
|
|
20
|
+
shrinks with context length: at the last layer it is I · 2/(seq+1). Every reading
|
|
21
|
+
we take (top-k ranking, cosine, kurtosis) is scale-invariant.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from collections.abc import Sequence
|
|
27
|
+
from contextlib import suppress
|
|
28
|
+
|
|
29
|
+
import torch
|
|
30
|
+
from torch import nn
|
|
31
|
+
|
|
32
|
+
from openinterp.core.schemas import Pole, VocabularyLens
|
|
33
|
+
from openinterp.harness.hooks import NeuronHookRunner, _StopForward
|
|
34
|
+
from openinterp.harness.memory import is_streamed, stream_rows
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _final_residual_module(runner: NeuronHookRunner) -> nn.Module:
|
|
38
|
+
"""The module whose input is the final residual stream (pre final norm)."""
|
|
39
|
+
return runner.final_norm() or runner.model.get_output_embeddings()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _final_residual(
|
|
43
|
+
runner: NeuronHookRunner, module: nn.Module, encoded: dict, layer: int, shift: torch.Tensor
|
|
44
|
+
) -> torch.Tensor:
|
|
45
|
+
"""Final residual stream [b, seq, d] with ``shift`` added after ``layer``'s MLP.
|
|
46
|
+
|
|
47
|
+
The pass stops at the final norm, so it skips the logits, which nothing here reads.
|
|
48
|
+
"""
|
|
49
|
+
captured: list[torch.Tensor] = []
|
|
50
|
+
|
|
51
|
+
def capture(_module, args) -> None:
|
|
52
|
+
captured.append(args[0].detach().float())
|
|
53
|
+
raise _StopForward
|
|
54
|
+
|
|
55
|
+
handle = module.register_forward_pre_hook(capture)
|
|
56
|
+
try:
|
|
57
|
+
with suppress(_StopForward):
|
|
58
|
+
runner.run_with_interventions(
|
|
59
|
+
encoded, output_interventions={layer: lambda out: out + shift.to(out.dtype)}
|
|
60
|
+
)
|
|
61
|
+
finally:
|
|
62
|
+
handle.remove()
|
|
63
|
+
return captured[-1]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@torch.no_grad()
|
|
67
|
+
def transported_direction(
|
|
68
|
+
runner: NeuronHookRunner,
|
|
69
|
+
prompts: Sequence[str] | torch.Tensor,
|
|
70
|
+
layer: int,
|
|
71
|
+
direction: torch.Tensor,
|
|
72
|
+
*,
|
|
73
|
+
eps: float = 1e-2,
|
|
74
|
+
batch_size: int = 8,
|
|
75
|
+
) -> torch.Tensor:
|
|
76
|
+
"""Estimate J_ℓ · direction, where ℓ is the residual stream just after ``layer``'s MLP."""
|
|
77
|
+
module = _final_residual_module(runner)
|
|
78
|
+
direction = direction.to(runner.device, torch.float32)
|
|
79
|
+
step = eps / max(float(direction.norm()), 1e-12)
|
|
80
|
+
total = torch.zeros(direction.shape, dtype=torch.float64) # on the CPU: MPS has no float64
|
|
81
|
+
pairs = 0
|
|
82
|
+
if not isinstance(prompts, torch.Tensor) and is_streamed(runner.model):
|
|
83
|
+
longest = runner.encode(list(prompts))["input_ids"].shape[1]
|
|
84
|
+
batch_size = stream_rows(runner, longest, batch_size, logits=False)
|
|
85
|
+
batches = (
|
|
86
|
+
[prompts]
|
|
87
|
+
if isinstance(prompts, torch.Tensor)
|
|
88
|
+
else [list(prompts[i : i + batch_size]) for i in range(0, len(prompts), batch_size)]
|
|
89
|
+
)
|
|
90
|
+
for batch in batches:
|
|
91
|
+
encoded = runner.encode(batch)
|
|
92
|
+
mask = encoded.get("attention_mask", torch.ones_like(encoded["input_ids"])).bool()
|
|
93
|
+
plus = _final_residual(runner, module, encoded, layer, step * direction)
|
|
94
|
+
minus = _final_residual(runner, module, encoded, layer, -step * direction)
|
|
95
|
+
delta = (plus - minus) / (2 * step) # [b, seq, d]
|
|
96
|
+
total += delta[mask].cpu().double().sum(0)
|
|
97
|
+
lengths = mask.sum(1)
|
|
98
|
+
pairs += int((lengths * (lengths + 1) // 2).sum())
|
|
99
|
+
return (total / max(pairs, 1)).float().to(runner.device)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _through_final_norm(runner: NeuronHookRunner, vector: torch.Tensor) -> torch.Tensor:
|
|
103
|
+
"""Apply the part of the final norm that affects a direction.
|
|
104
|
+
|
|
105
|
+
That is centring for LayerNorm, and the gain. The bias is a constant offset,
|
|
106
|
+
unrelated to the direction, so it is dropped.
|
|
107
|
+
"""
|
|
108
|
+
norm = runner.final_norm()
|
|
109
|
+
v = vector.float()
|
|
110
|
+
if isinstance(norm, nn.LayerNorm):
|
|
111
|
+
v = v - v.mean()
|
|
112
|
+
weight = getattr(norm, "weight", None)
|
|
113
|
+
return v * weight.detach().float() if weight is not None else v
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _decode(runner: NeuronHookRunner, vector: torch.Tensor, k: int):
|
|
117
|
+
unembed = runner.model.get_output_embeddings().weight.detach().float()
|
|
118
|
+
scores = unembed @ _through_final_norm(runner, vector).to(unembed.device)
|
|
119
|
+
z = (scores - scores.mean()) / scores.std().clamp(min=1e-12)
|
|
120
|
+
top = scores.topk(k).indices.tolist()
|
|
121
|
+
bottom = (-scores).topk(k).indices.tolist()
|
|
122
|
+
return top, bottom, float((z**4).mean()), _rounded(z[top]), _rounded(z[bottom])
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _rounded(values: torch.Tensor) -> list[float]:
|
|
126
|
+
return [round(float(v), 4) for v in values]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def vocabulary_lens(
|
|
130
|
+
runner: NeuronHookRunner,
|
|
131
|
+
prompts: Sequence[str],
|
|
132
|
+
layer: int,
|
|
133
|
+
neuron_idx: int,
|
|
134
|
+
pole: Pole,
|
|
135
|
+
*,
|
|
136
|
+
k: int = 10,
|
|
137
|
+
) -> VocabularyLens:
|
|
138
|
+
"""Read the pole's output direction through both the logit lens and the Jacobian lens."""
|
|
139
|
+
direction = runner.neuron_output_direction(layer, neuron_idx).float()
|
|
140
|
+
if pole == "negative":
|
|
141
|
+
direction = -direction
|
|
142
|
+
return direction_lens(runner, prompts, layer, direction, k=k)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def direction_lens(
|
|
146
|
+
runner: NeuronHookRunner,
|
|
147
|
+
prompts: Sequence[str],
|
|
148
|
+
layer: int,
|
|
149
|
+
direction: torch.Tensor,
|
|
150
|
+
*,
|
|
151
|
+
k: int = 10,
|
|
152
|
+
) -> VocabularyLens:
|
|
153
|
+
"""Read any vector written after ``layer``'s MLP through both lenses.
|
|
154
|
+
|
|
155
|
+
The vector is a pole's output direction, or a dictionary latent's decoder row.
|
|
156
|
+
"""
|
|
157
|
+
direction = direction.float()
|
|
158
|
+
transported = transported_direction(runner, prompts, layer, direction)
|
|
159
|
+
|
|
160
|
+
def words(ids: list[int]) -> list[str]:
|
|
161
|
+
tok = runner.tokenizer
|
|
162
|
+
return [tok.decode([i]) if tok is not None else str(i) for i in ids]
|
|
163
|
+
|
|
164
|
+
logit_top, logit_bottom, logit_kurt, logit_top_z, logit_bottom_z = _decode(runner, direction, k)
|
|
165
|
+
jac_top, jac_bottom, jac_kurt, jac_top_z, jac_bottom_z = _decode(runner, transported, k)
|
|
166
|
+
cosine = torch.nn.functional.cosine_similarity(
|
|
167
|
+
direction.to(transported.device), transported, dim=0
|
|
168
|
+
)
|
|
169
|
+
return VocabularyLens(
|
|
170
|
+
logit_promoted=words(logit_top),
|
|
171
|
+
logit_suppressed=words(logit_bottom),
|
|
172
|
+
jacobian_promoted=words(jac_top),
|
|
173
|
+
jacobian_suppressed=words(jac_bottom),
|
|
174
|
+
logit_promoted_z=logit_top_z,
|
|
175
|
+
logit_suppressed_z=logit_bottom_z,
|
|
176
|
+
jacobian_promoted_z=jac_top_z,
|
|
177
|
+
jacobian_suppressed_z=jac_bottom_z,
|
|
178
|
+
overlap=round(len(set(logit_top) & set(jac_top)) / k, 4),
|
|
179
|
+
direction_cosine=round(max(-1.0, min(1.0, float(cosine))), 6),
|
|
180
|
+
logit_kurtosis=round(logit_kurt, 4),
|
|
181
|
+
jacobian_kurtosis=round(jac_kurt, 4),
|
|
182
|
+
prompts=len(prompts),
|
|
183
|
+
)
|