open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
"""Steering samples: continue neutral prompts with a unit clamped high or ablated.
|
|
2
|
+
|
|
3
|
+
Every prompt is continued under each intervention, applied at every position of the
|
|
4
|
+
prompt and of the continuation:
|
|
5
|
+
|
|
6
|
+
* baseline: no intervention;
|
|
7
|
+
* ablated: the unit's own ablation (a neuron's pole zeroed, or a dictionary latent's
|
|
8
|
+
write removed), as in the causal and lookahead measurements;
|
|
9
|
+
* clamped: a neuron's activation is set to the pole's signed top activation
|
|
10
|
+
(``max_activation``) times each multiple. A latent's activation is set to that value.
|
|
11
|
+
|
|
12
|
+
All prompts and interventions decode as one left-padded batch, greedily, so a sample
|
|
13
|
+
is deterministic given the weights. The prompts are fixed and unrelated to any
|
|
14
|
+
hypothesis. So a coherent drift at 1–2× says what the unit makes the model write. Drift
|
|
15
|
+
only at 4× is off-distribution and weak evidence. A head writes a subspace with no
|
|
16
|
+
"high" setting, so heads get no samples.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from collections.abc import Mapping, Sequence
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
import torch
|
|
25
|
+
|
|
26
|
+
from openinterp.core.steering import SteeringSample, SteeringSamples
|
|
27
|
+
from openinterp.harness.hooks import NeuronHookRunner, isolate_pole
|
|
28
|
+
from openinterp.harness.units import FeatureUnit, NeuronUnit, Unit
|
|
29
|
+
|
|
30
|
+
PROMPTS = (
|
|
31
|
+
"The weather today is",
|
|
32
|
+
"I went to the shop and bought",
|
|
33
|
+
"The best way to learn is",
|
|
34
|
+
"Once upon a time,",
|
|
35
|
+
"The report concluded that",
|
|
36
|
+
)
|
|
37
|
+
MULTIPLES = (1.0, 2.0, 4.0)
|
|
38
|
+
MAX_NEW_TOKENS = 24
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def steering_interventions(
|
|
42
|
+
runner: NeuronHookRunner, unit: Unit, ablate: torch.Tensor, clamp: torch.Tensor
|
|
43
|
+
) -> dict[str, Mapping[int, Any]] | None:
|
|
44
|
+
"""Return ``hooks`` keyword arguments that ablate or clamp the unit, row by row.
|
|
45
|
+
|
|
46
|
+
They ablate the unit in the batch rows where ``ablate`` is true. They clamp it to
|
|
47
|
+
``clamp`` (a strength) where that is not NaN. ``None`` for a head.
|
|
48
|
+
"""
|
|
49
|
+
clamped = ~torch.isnan(clamp)
|
|
50
|
+
if isinstance(unit, NeuronUnit):
|
|
51
|
+
idx, pole = unit.ident.neuron_idx, unit.ident.pole
|
|
52
|
+
signed = clamp if pole == "positive" else -clamp
|
|
53
|
+
|
|
54
|
+
def steer(act: torch.Tensor) -> torch.Tensor:
|
|
55
|
+
act = act.clone()
|
|
56
|
+
a = act[..., idx]
|
|
57
|
+
a = torch.where(ablate.to(a.device)[:, None], a - isolate_pole(a, pole), a)
|
|
58
|
+
value = signed.to(a.device, a.dtype)[:, None]
|
|
59
|
+
act[..., idx] = torch.where(clamped.to(a.device)[:, None], value, a)
|
|
60
|
+
return act
|
|
61
|
+
|
|
62
|
+
return {"interventions": {unit.ident.layer: steer}}
|
|
63
|
+
if isinstance(unit, FeatureUnit):
|
|
64
|
+
dictionary, f = unit.dictionary(runner), unit.ident.feature_idx
|
|
65
|
+
direction = dictionary.w_dec[f]
|
|
66
|
+
|
|
67
|
+
def steer_latent(x: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
|
|
68
|
+
a = dictionary.feature(x, f)
|
|
69
|
+
target = torch.where(ablate.to(a.device)[:, None], torch.zeros_like(a), a)
|
|
70
|
+
value = clamp.to(a.device, a.dtype)[:, None]
|
|
71
|
+
target = torch.where(clamped.to(a.device)[:, None], value, target)
|
|
72
|
+
return out + ((target - a).unsqueeze(-1) * direction).to(out.dtype)
|
|
73
|
+
|
|
74
|
+
site = unit.profile.input_site
|
|
75
|
+
return {"dictionary_interventions": {unit.ident.layer: (site, steer_latent)}}
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@torch.no_grad()
|
|
80
|
+
def _continue(
|
|
81
|
+
runner: NeuronHookRunner,
|
|
82
|
+
rows: Sequence[list[int]],
|
|
83
|
+
interventions: Mapping[str, Mapping[int, Any]],
|
|
84
|
+
max_new_tokens: int,
|
|
85
|
+
) -> list[str]:
|
|
86
|
+
"""Return greedy continuations of every row of token ids, each cut at end-of-text.
|
|
87
|
+
|
|
88
|
+
A model with ``generate`` decodes with its cache. One without (a test double) runs
|
|
89
|
+
the whole sequence again at each step.
|
|
90
|
+
"""
|
|
91
|
+
tok = runner.tokenizer
|
|
92
|
+
eos = getattr(tok, "eos_token_id", None)
|
|
93
|
+
pad = getattr(tok, "pad_token_id", None)
|
|
94
|
+
pad = (eos if pad is None else pad) or 0
|
|
95
|
+
width = max(map(len, rows))
|
|
96
|
+
ids = torch.tensor([[pad] * (width - len(r)) + r for r in rows], device=runner.device)
|
|
97
|
+
mask = torch.tensor([[0] * (width - len(r)) + [1] * len(r) for r in rows], device=ids.device)
|
|
98
|
+
with runner.hooks(**interventions):
|
|
99
|
+
if hasattr(runner.model, "generate"):
|
|
100
|
+
seq = runner.model.generate(
|
|
101
|
+
input_ids=ids,
|
|
102
|
+
attention_mask=mask,
|
|
103
|
+
max_new_tokens=max_new_tokens,
|
|
104
|
+
do_sample=False,
|
|
105
|
+
pad_token_id=pad,
|
|
106
|
+
eos_token_id=eos,
|
|
107
|
+
)
|
|
108
|
+
else:
|
|
109
|
+
seq = ids
|
|
110
|
+
for _ in range(max_new_tokens):
|
|
111
|
+
nxt = runner.forward({"input_ids": seq, "attention_mask": mask})[:, -1].argmax(-1)
|
|
112
|
+
seq = torch.cat([seq, nxt[:, None]], dim=1)
|
|
113
|
+
mask = torch.cat([mask, torch.ones_like(mask[:, :1])], dim=1)
|
|
114
|
+
texts = []
|
|
115
|
+
for row in seq[:, width:].tolist():
|
|
116
|
+
if eos in row:
|
|
117
|
+
row = row[: row.index(eos)]
|
|
118
|
+
texts.append(tok.decode(row))
|
|
119
|
+
return texts
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def steering_samples(
|
|
123
|
+
runner: NeuronHookRunner,
|
|
124
|
+
unit: Unit,
|
|
125
|
+
max_activation: float,
|
|
126
|
+
*,
|
|
127
|
+
prompts: Sequence[str] = PROMPTS,
|
|
128
|
+
multiples: Sequence[float] = MULTIPLES,
|
|
129
|
+
max_new_tokens: int = MAX_NEW_TOKENS,
|
|
130
|
+
) -> SteeringSamples:
|
|
131
|
+
"""Continue every prompt at baseline, ablated, and clamped.
|
|
132
|
+
|
|
133
|
+
The clamp is at each multiple of ``max_activation``.
|
|
134
|
+
"""
|
|
135
|
+
if runner.tokenizer is None:
|
|
136
|
+
raise ValueError("steering samples need a tokenizer")
|
|
137
|
+
# Each prompt's rows: baseline, ablated, then clamped at each multiple.
|
|
138
|
+
strengths = [float("nan"), float("nan"), *(max_activation * m for m in multiples)]
|
|
139
|
+
per_prompt = len(strengths)
|
|
140
|
+
clamp = torch.tensor(strengths * len(prompts))
|
|
141
|
+
ablate = torch.tensor(([False, True] + [False] * len(multiples)) * len(prompts))
|
|
142
|
+
interventions = steering_interventions(runner, unit, ablate, clamp)
|
|
143
|
+
if interventions is None:
|
|
144
|
+
raise ValueError(f"{type(unit).__name__} has no clamped-high setting to steer with")
|
|
145
|
+
encoded = [runner.tokenizer.encode(p, add_special_tokens=True) for p in prompts]
|
|
146
|
+
rows = [ids for ids in encoded for _ in range(per_prompt)]
|
|
147
|
+
texts = _continue(runner, rows, interventions, max_new_tokens)
|
|
148
|
+
samples = []
|
|
149
|
+
for i, prompt in enumerate(prompts):
|
|
150
|
+
baseline, ablated, *clamped = texts[i * per_prompt : (i + 1) * per_prompt]
|
|
151
|
+
samples.append(
|
|
152
|
+
SteeringSample(prompt=prompt, baseline=baseline, ablated=ablated, clamped=clamped)
|
|
153
|
+
)
|
|
154
|
+
return SteeringSamples(
|
|
155
|
+
clamp_unit=max_activation,
|
|
156
|
+
multiples=[float(m) for m in multiples],
|
|
157
|
+
max_new_tokens=max_new_tokens,
|
|
158
|
+
samples=samples,
|
|
159
|
+
)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""A superposition as an auditable unit (issue #3215, ``docs/PROTOCOL.md`` §1).
|
|
2
|
+
|
|
3
|
+
A superposition is a sparse direction over one layer's MLP hidden activations, the input
|
|
4
|
+
to the down projection where neurons live. Its strength is the positive part of the unit
|
|
5
|
+
direction ŵ: ``max(0, ŵ · a)``. That is one latent of a ``basis`` dictionary at
|
|
6
|
+
``mlp_hidden``. So this module builds such a dictionary in memory, and the feature code
|
|
7
|
+
scans, measures, ablates and patches it.
|
|
8
|
+
|
|
9
|
+
The dictionary's latent 0 is the superposition. Latents 1 to :data:`NULLS` are null
|
|
10
|
+
directions for the causal metrics. Each has as many members as the superposition, on
|
|
11
|
+
neurons drawn at random. Its weights are the superposition's magnitudes, in a shuffled
|
|
12
|
+
order and with random signs. The draws are seeded by the superposition's id, so a record always
|
|
13
|
+
measures against the same nulls.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import random
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import torch
|
|
23
|
+
|
|
24
|
+
from openinterp.core.profiles import SuperpositionProfile
|
|
25
|
+
from openinterp.core.schemas import SuperpositionIdentifier
|
|
26
|
+
from openinterp.harness.dictionaries import Dictionary
|
|
27
|
+
from openinterp.harness.hooks import NeuronHookRunner
|
|
28
|
+
from openinterp.harness.units import FeatureUnit
|
|
29
|
+
|
|
30
|
+
NULLS = 64 # null directions beside the superposition, for the causal metrics
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def directions(ident: SuperpositionIdentifier, profile: SuperpositionProfile, width: int):
|
|
34
|
+
"""``[width, 1 + NULLS]``: the unit direction ŵ, then the null directions."""
|
|
35
|
+
rng = random.Random(int(ident.superposition_id, 16))
|
|
36
|
+
weights = [m.weight for m in profile.members]
|
|
37
|
+
columns = [list(zip(profile.neurons, weights, strict=True))]
|
|
38
|
+
magnitudes = [abs(w) for w in weights]
|
|
39
|
+
for _ in range(NULLS):
|
|
40
|
+
neurons = rng.sample(range(width), len(weights))
|
|
41
|
+
shuffled = rng.sample(magnitudes, len(magnitudes))
|
|
42
|
+
columns.append(
|
|
43
|
+
[(n, w * rng.choice((1, -1))) for n, w in zip(neurons, shuffled, strict=True)]
|
|
44
|
+
)
|
|
45
|
+
q = torch.zeros(width, len(columns))
|
|
46
|
+
for j, column in enumerate(columns):
|
|
47
|
+
for neuron, weight in column:
|
|
48
|
+
q[neuron, j] = weight
|
|
49
|
+
return q / q.norm(dim=0, keepdim=True)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def build_dictionary(runner: NeuronHookRunner, ident, profile) -> Dictionary:
|
|
53
|
+
"""The in-memory ``basis`` dictionary whose latent 0 is the superposition.
|
|
54
|
+
|
|
55
|
+
A latent's write is its activation times ``W_down q`` for its direction q. So
|
|
56
|
+
ablating it removes exactly its direction's positive part from the hidden activations.
|
|
57
|
+
"""
|
|
58
|
+
width = runner.mlp_width(ident.layer)
|
|
59
|
+
if max(profile.neurons) >= width:
|
|
60
|
+
raise ValueError(f"{ident}: layer {ident.layer} has {width} neurons")
|
|
61
|
+
q = directions(ident, profile, width)
|
|
62
|
+
used = q.abs().sum(dim=1).nonzero().flatten().tolist() # neurons some direction reads
|
|
63
|
+
out = torch.stack([runner.neuron_output_direction(ident.layer, i).float().cpu() for i in used])
|
|
64
|
+
w_dec = q[used].T @ out # [n, d_model]
|
|
65
|
+
device = runner.device
|
|
66
|
+
return Dictionary(
|
|
67
|
+
w_enc=q.to(device),
|
|
68
|
+
b_enc=torch.zeros(q.shape[1], device=device),
|
|
69
|
+
w_dec=w_dec.to(device),
|
|
70
|
+
b_dec=torch.zeros(w_dec.shape[1], device=device),
|
|
71
|
+
threshold=None,
|
|
72
|
+
activation="relu",
|
|
73
|
+
subtract_decoder_bias=False,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class SuperpositionUnit(FeatureUnit):
|
|
79
|
+
"""A superposition as an auditable unit: latent 0 of its in-memory dictionary.
|
|
80
|
+
|
|
81
|
+
It is a :class:`~openinterp.harness.units.FeatureUnit`, so every measurement that
|
|
82
|
+
reads a feature reads it the same way. ``profile`` is its
|
|
83
|
+
:class:`~openinterp.core.profiles.SuperpositionProfile`.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
_cache: dict[Any, Dictionary] = field(default_factory=dict, compare=False, repr=False)
|
|
87
|
+
|
|
88
|
+
def dictionary(self, runner: NeuronHookRunner) -> Dictionary:
|
|
89
|
+
"""The superposition's dictionary, built once per runner."""
|
|
90
|
+
key = id(runner)
|
|
91
|
+
if key not in self._cache:
|
|
92
|
+
self._cache[key] = build_dictionary(runner, self.ident, self.profile)
|
|
93
|
+
return self._cache[key]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def scan_superposition(runner: NeuronHookRunner, texts, ident, profile, **options):
|
|
97
|
+
"""Scan a superposition as a latent; the result's profile is ``profile`` itself."""
|
|
98
|
+
from openinterp.harness.features import scan_latent
|
|
99
|
+
|
|
100
|
+
dictionary = SuperpositionUnit(ident, profile).dictionary(runner)
|
|
101
|
+
return scan_latent(
|
|
102
|
+
runner, texts, ident, dictionary, profile.input_site, lambda **_: profile, **options
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def superposition_identifier(model_name: str, layer: int, profile) -> SuperpositionIdentifier:
|
|
107
|
+
"""The identifier of the superposition that ``profile`` defines on ``layer``."""
|
|
108
|
+
from openinterp.core.profiles import superposition_id
|
|
109
|
+
|
|
110
|
+
sid = superposition_id(profile.members)
|
|
111
|
+
return SuperpositionIdentifier(model_name=model_name, layer=layer, superposition_id=sid)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def superposition_write(runner: NeuronHookRunner, ident, profile) -> torch.Tensor:
|
|
115
|
+
"""The residual-stream direction that the superposition writes: ``W_down ŵ``."""
|
|
116
|
+
return SuperpositionUnit(ident, profile).dictionary(runner).w_dec[0]
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""What "strength" means for each kind of auditable unit, so one protocol covers them all.
|
|
2
|
+
|
|
3
|
+
Every measurement in an audit reads one non-negative strength per token: the scan's
|
|
4
|
+
top examples, probes, counterfactual Δ, held-out windows. A unit supplies it, and the
|
|
5
|
+
ablation the lookahead profile uses:
|
|
6
|
+
|
|
7
|
+
* An MLP neuron pole: its pole strength. Ablation zeroes the pole.
|
|
8
|
+
* An attention head: ``|W_O (z_t − z̄)|``. This is the distance between its write to the
|
|
9
|
+
residual stream and its mean write over the scan. z̄ is stored in
|
|
10
|
+
:class:`~openinterp.core.schemas.HeadProfile`. Ablation replaces z with z̄ (mean
|
|
11
|
+
ablation). So strength is exactly how far ablating the head moves the residual.
|
|
12
|
+
* A superposition: the positive part of its unit direction over one layer's neurons. It is
|
|
13
|
+
scanned and ablated as latent 0 of an in-memory dictionary
|
|
14
|
+
(:mod:`openinterp.harness.superpositions`).
|
|
15
|
+
* A dictionary feature: its latent's activation ``a_f`` (non-negative). Ablation removes
|
|
16
|
+
its write ``a_f · W_dec[f]`` from the MLP output, or from the residual stream for a
|
|
17
|
+
residual-stream SAE. The dictionary's error and every other latent stay as they were.
|
|
18
|
+
|
|
19
|
+
A unit with a *first-token floor* (protocol 0.8) is not measured at a text's first
|
|
20
|
+
token. The first token has no context and is the attention sink. Some early-layer units
|
|
21
|
+
sit above their line there whatever the token, so every probe would fire. A unit
|
|
22
|
+
has the floor if it is active at the first position on at least half the vocabulary
|
|
23
|
+
(:func:`openinterp.harness.scan.first_position_share`). A prompt's peak is then over
|
|
24
|
+
its other tokens (``skip_first``). Every other unit is measured at every token, so one
|
|
25
|
+
that detects *which* token opens a text can still be audited.
|
|
26
|
+
|
|
27
|
+
The raw write norm ``|W_O z_t|`` would not do. Most heads write a sizeable resting
|
|
28
|
+
vector, often while attending to the first token. So in GPT-2 small, an induction head's
|
|
29
|
+
median token sits at a quarter of its maximum. Half of all text would count as "active".
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
from collections.abc import Mapping
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
import torch
|
|
39
|
+
|
|
40
|
+
from openinterp.core.schemas import (
|
|
41
|
+
ComponentIdentifier,
|
|
42
|
+
DictionaryProfile,
|
|
43
|
+
FeatureIdentifier,
|
|
44
|
+
HeadIdentifier,
|
|
45
|
+
NeuronIdentifier,
|
|
46
|
+
SuperpositionIdentifier,
|
|
47
|
+
)
|
|
48
|
+
from openinterp.harness.dictionaries import Dictionary, load_dictionary
|
|
49
|
+
from openinterp.harness.hooks import Inputs, NeuronHookRunner, ablate_head, ablate_pole
|
|
50
|
+
from openinterp.harness.sites import dictionary_io
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class NeuronUnit:
|
|
55
|
+
"""An MLP neuron pole as an auditable unit."""
|
|
56
|
+
|
|
57
|
+
ident: NeuronIdentifier
|
|
58
|
+
skip_first: bool = False
|
|
59
|
+
|
|
60
|
+
def strengths(self, runner: NeuronHookRunner, inputs: Inputs) -> torch.Tensor:
|
|
61
|
+
"""Per-token pole strength ``[batch, seq]``."""
|
|
62
|
+
ident = self.ident
|
|
63
|
+
return runner.neuron_activations(inputs, ident.layer, ident.neuron_idx, ident.pole)
|
|
64
|
+
|
|
65
|
+
def ablation(self, runner: NeuronHookRunner) -> dict[str, Mapping[int, Any]]:
|
|
66
|
+
"""``run_with_interventions`` keyword arguments that zero the pole."""
|
|
67
|
+
ident = self.ident
|
|
68
|
+
return {"interventions": {ident.layer: ablate_pole(ident.neuron_idx, ident.pole)}}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True)
|
|
72
|
+
class HeadUnit:
|
|
73
|
+
"""An attention head as an auditable unit, with the mean output that defines its strength."""
|
|
74
|
+
|
|
75
|
+
ident: HeadIdentifier
|
|
76
|
+
mean_output: tuple[float, ...]
|
|
77
|
+
skip_first: bool = False
|
|
78
|
+
|
|
79
|
+
def strengths(self, runner: NeuronHookRunner, inputs: Inputs) -> torch.Tensor:
|
|
80
|
+
"""Per-token ``|W_O (z − z̄)|`` ``[batch, seq]``."""
|
|
81
|
+
z = runner.head_outputs(inputs, self.ident.layer)[..., self.ident.head_idx, :]
|
|
82
|
+
return head_strength(runner, self.ident, z, self.mean_output)
|
|
83
|
+
|
|
84
|
+
def ablation(self, runner: NeuronHookRunner) -> dict[str, Mapping[int, Any]]:
|
|
85
|
+
"""``run_with_interventions`` keyword arguments that mean-ablate the head."""
|
|
86
|
+
ident = self.ident
|
|
87
|
+
mean = torch.tensor(self.mean_output, device=runner.device)
|
|
88
|
+
fn = ablate_head(ident.head_idx, runner.head_dim(ident.layer), mean)
|
|
89
|
+
return {"head_interventions": {ident.layer: fn}}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(frozen=True)
|
|
93
|
+
class FeatureUnit:
|
|
94
|
+
"""A dictionary feature as an auditable unit, with its pinned dictionary."""
|
|
95
|
+
|
|
96
|
+
ident: FeatureIdentifier
|
|
97
|
+
profile: DictionaryProfile
|
|
98
|
+
skip_first: bool = False
|
|
99
|
+
|
|
100
|
+
def dictionary(self, runner: NeuronHookRunner) -> Dictionary:
|
|
101
|
+
"""The feature's dictionary, loaded on the runner's device."""
|
|
102
|
+
return load_dictionary(self.profile, runner.device)
|
|
103
|
+
|
|
104
|
+
def strengths(self, runner: NeuronHookRunner, inputs: Inputs) -> torch.Tensor:
|
|
105
|
+
"""Per-token latent activation ``[batch, seq]``."""
|
|
106
|
+
x, _ = dictionary_io(runner, inputs, self.ident.layer, self.profile.input_site)
|
|
107
|
+
return self.dictionary(runner).feature(x, self.ident.feature_idx)
|
|
108
|
+
|
|
109
|
+
def ablation(self, runner: NeuronHookRunner) -> dict[str, Mapping[int, Any]]:
|
|
110
|
+
"""``run_with_interventions`` keyword arguments that remove the latent's write."""
|
|
111
|
+
dictionary, f = self.dictionary(runner), self.ident.feature_idx
|
|
112
|
+
direction = dictionary.w_dec[f]
|
|
113
|
+
|
|
114
|
+
def ablate(x: torch.Tensor, out: torch.Tensor) -> torch.Tensor:
|
|
115
|
+
write = dictionary.feature(x, f).unsqueeze(-1) * direction
|
|
116
|
+
return out - write.to(out.dtype)
|
|
117
|
+
|
|
118
|
+
return {"dictionary_interventions": {self.ident.layer: (self.profile.input_site, ablate)}}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
Unit = NeuronUnit | HeadUnit | FeatureUnit
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def head_strength(
|
|
125
|
+
runner: NeuronHookRunner, ident: HeadIdentifier, z: torch.Tensor, mean_output
|
|
126
|
+
) -> torch.Tensor:
|
|
127
|
+
"""``|W_O (z − z̄)|`` over the last dimension of ``z`` (one head's outputs)."""
|
|
128
|
+
w_o = runner.head_output_weights(ident.layer, ident.head_idx).float()
|
|
129
|
+
mean = torch.as_tensor(mean_output, dtype=torch.float32, device=z.device)
|
|
130
|
+
return ((z.float() - mean) @ w_o.T.to(z.device)).norm(dim=-1)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def unit_of(
|
|
134
|
+
ident: ComponentIdentifier,
|
|
135
|
+
head_profile=None,
|
|
136
|
+
dictionary_profile=None,
|
|
137
|
+
skip_first=False,
|
|
138
|
+
superposition_profile=None,
|
|
139
|
+
) -> Unit:
|
|
140
|
+
"""Return the unit that ``ident`` names.
|
|
141
|
+
|
|
142
|
+
A head, a feature or a superposition needs the profile that defines its strength.
|
|
143
|
+
"""
|
|
144
|
+
if isinstance(ident, SuperpositionIdentifier):
|
|
145
|
+
from openinterp.harness.superpositions import SuperpositionUnit
|
|
146
|
+
|
|
147
|
+
if superposition_profile is None:
|
|
148
|
+
raise ValueError(f"{ident} has no superposition profile, so its strength is undefined")
|
|
149
|
+
return SuperpositionUnit(ident, superposition_profile, skip_first)
|
|
150
|
+
if isinstance(ident, HeadIdentifier):
|
|
151
|
+
if head_profile is None:
|
|
152
|
+
raise ValueError(f"{ident} has no head profile, so its strength is undefined")
|
|
153
|
+
return HeadUnit(ident, tuple(head_profile.mean_output), skip_first)
|
|
154
|
+
if isinstance(ident, FeatureIdentifier):
|
|
155
|
+
if dictionary_profile is None:
|
|
156
|
+
raise ValueError(f"{ident} has no dictionary profile, so its strength is undefined")
|
|
157
|
+
return FeatureUnit(ident, dictionary_profile, skip_first)
|
|
158
|
+
return NeuronUnit(ident, skip_first)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def submission_unit(submission) -> Unit:
|
|
162
|
+
"""Return the unit that a submission, scan evidence or scoring task is about.
|
|
163
|
+
|
|
164
|
+
A record from before protocol 0.8 has no ``first_token_floor``, and is measured at
|
|
165
|
+
every token.
|
|
166
|
+
"""
|
|
167
|
+
floor = bool(getattr(submission, "first_token_floor", None))
|
|
168
|
+
return unit_of(
|
|
169
|
+
submission.identifier,
|
|
170
|
+
submission.head_profile,
|
|
171
|
+
submission.dictionary_profile,
|
|
172
|
+
floor,
|
|
173
|
+
getattr(submission, "superposition_profile", None),
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def peak_strength(runner: NeuronHookRunner, unit: Unit, text: str) -> float | None:
|
|
178
|
+
"""Return the peak strength of ``text`` over its measured tokens.
|
|
179
|
+
|
|
180
|
+
``None`` when it has none: one token, for a unit that skips the first.
|
|
181
|
+
"""
|
|
182
|
+
strengths = unit.strengths(runner, text)[0][int(unit.skip_first) :]
|
|
183
|
+
return float(strengths.max()) if strengths.numel() else None
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def max_strength(runner: NeuronHookRunner, unit: Unit, text: str) -> float:
|
|
187
|
+
""":func:`peak_strength`, for a text that must have a measured token."""
|
|
188
|
+
peak = peak_strength(runner, unit, text)
|
|
189
|
+
if peak is None:
|
|
190
|
+
raise ValueError(
|
|
191
|
+
f"{text!r} is one token, and {unit.ident} is not measured at the first token "
|
|
192
|
+
"(it has a first-token floor), so use two or more"
|
|
193
|
+
)
|
|
194
|
+
return peak
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def attn_implementation(ident: ComponentIdentifier) -> str | None:
|
|
198
|
+
"""Return the attention implementation to load the model with: ``"eager"`` for a head.
|
|
199
|
+
|
|
200
|
+
SDPA and flash attention return no patterns. Every measurement of a head experiment
|
|
201
|
+
should run the same attention kernel.
|
|
202
|
+
"""
|
|
203
|
+
return "eager" if isinstance(ident, HeadIdentifier) else None
|