open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""A neuron's activation at a text's first position, on every token of the vocabulary.
|
|
2
|
+
|
|
3
|
+
The first-token floor check (protocol 0.8, ``scan.has_first_token_floor``) needs it. At
|
|
4
|
+
the first position, attention sees only the token itself. So these values depend only on
|
|
5
|
+
the weights, not on the corpus. They are a pass of the whole vocabulary: 151,646 inputs
|
|
6
|
+
of one token for Qwen2.5. On a 50,000-token Qwen2.5-0.5B audit, the two poles took 25 s.
|
|
7
|
+
With the model streamed they took 123 s, because each batch reads every layer from disk.
|
|
8
|
+
|
|
9
|
+
So the raw activations are computed once for both poles, and kept on disk in
|
|
10
|
+
``$OPENINTERP_FIRST_POSITION_CACHE`` (default ``~/.cache/openinterp/first-position``, ``off``
|
|
11
|
+
to disable it). An entry is keyed by the model, its revision, the dtype, the device type,
|
|
12
|
+
the layer and the neuron, and the torch and transformers versions. One entry is about
|
|
13
|
+
600 KB. A streamed model runs as many inputs at once as its planned memory holds.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import torch
|
|
24
|
+
|
|
25
|
+
from openinterp.core.cache_dirs import cache_root
|
|
26
|
+
from openinterp.harness.hooks import pole_strength
|
|
27
|
+
|
|
28
|
+
FORMAT = 1
|
|
29
|
+
ENV = "OPENINTERP_FIRST_POSITION_CACHE"
|
|
30
|
+
BATCH = 8192 # one-token inputs in one pass of a model loaded whole
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def vocabulary(runner) -> list[int]:
|
|
34
|
+
"""Every token id but the special ones."""
|
|
35
|
+
tokenizer = runner.tokenizer
|
|
36
|
+
try:
|
|
37
|
+
size = len(tokenizer)
|
|
38
|
+
except TypeError: # a tokenizer without a length: one id per embedding row
|
|
39
|
+
size = runner.model.get_input_embeddings().num_embeddings
|
|
40
|
+
special = set(getattr(tokenizer, "all_special_ids", None) or [])
|
|
41
|
+
return [i for i in range(size) if i not in special]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def strengths_of(runner, strengths, batch_size: int = BATCH) -> torch.Tensor:
|
|
45
|
+
"""``strengths`` at the first position of each token of :func:`vocabulary`."""
|
|
46
|
+
vocab, out = vocabulary(runner), []
|
|
47
|
+
for start in range(0, len(vocab), batch_size):
|
|
48
|
+
ids = torch.tensor(vocab[start : start + batch_size], device=runner.device).unsqueeze(1)
|
|
49
|
+
out.append(strengths({"input_ids": ids, "attention_mask": torch.ones_like(ids)})[:, 0])
|
|
50
|
+
return torch.cat(out).float().cpu() if out else torch.zeros(0)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class NeuronReader:
|
|
54
|
+
"""One neuron pole's strengths, and its values at the first position, kept for both poles.
|
|
55
|
+
|
|
56
|
+
The scan calls it as ``Strengths``. ``first_position()`` gives the vocabulary's values,
|
|
57
|
+
from memory, from the disk cache, or from one pass of the vocabulary.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(self, runner, layer: int, neuron: int, pole) -> None:
|
|
61
|
+
self.runner, self.layer, self.neuron, self.pole = runner, layer, neuron, pole
|
|
62
|
+
|
|
63
|
+
def __call__(self, inputs) -> torch.Tensor:
|
|
64
|
+
"""The pole's strength at each token of ``inputs``."""
|
|
65
|
+
return self.runner.neuron_activations(inputs, self.layer, self.neuron, self.pole)
|
|
66
|
+
|
|
67
|
+
def first_position(self) -> torch.Tensor:
|
|
68
|
+
"""The pole's strength at the first position, on every token of the vocabulary."""
|
|
69
|
+
return pole_strength(self._raw(), self.pole)
|
|
70
|
+
|
|
71
|
+
def _raw(self) -> torch.Tensor:
|
|
72
|
+
fields = _fields(self.runner, self.layer, self.neuron)
|
|
73
|
+
key = json.dumps(fields, sort_keys=True)
|
|
74
|
+
memo = self.runner.__dict__.setdefault("_first_position", {}) # both poles of one run
|
|
75
|
+
if key in memo:
|
|
76
|
+
return memo[key]
|
|
77
|
+
path = _path(fields)
|
|
78
|
+
raw = _read(path)
|
|
79
|
+
if raw is None:
|
|
80
|
+
raw = strengths_of(
|
|
81
|
+
self.runner,
|
|
82
|
+
lambda inputs: self.runner.neuron_activations(inputs, self.layer, self.neuron),
|
|
83
|
+
_batch_size(self.runner),
|
|
84
|
+
)
|
|
85
|
+
_write(path, raw)
|
|
86
|
+
memo[key] = raw
|
|
87
|
+
return raw
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _batch_size(runner) -> int:
|
|
91
|
+
from openinterp.harness.memory import stream_rows
|
|
92
|
+
|
|
93
|
+
return stream_rows(runner, 1, BATCH, logits=False)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _fields(runner, layer: int, neuron: int) -> dict:
|
|
97
|
+
import transformers
|
|
98
|
+
|
|
99
|
+
config = getattr(runner.model, "config", None)
|
|
100
|
+
return {
|
|
101
|
+
"format": FORMAT,
|
|
102
|
+
"model": getattr(runner, "model_name", None) or getattr(config, "_name_or_path", None),
|
|
103
|
+
"revision": getattr(runner, "model_revision", None),
|
|
104
|
+
"dtype": str(next(runner.model.parameters()).dtype).removeprefix("torch."),
|
|
105
|
+
"device": runner.device.type,
|
|
106
|
+
"torch": torch.__version__,
|
|
107
|
+
"transformers": transformers.__version__,
|
|
108
|
+
"vocabulary": len(vocabulary(runner)),
|
|
109
|
+
"layer": layer,
|
|
110
|
+
"neuron": neuron,
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _path(fields: dict) -> Path | None:
|
|
115
|
+
root = cache_root(ENV, "first-position")
|
|
116
|
+
if root is None or not fields["model"]:
|
|
117
|
+
return None
|
|
118
|
+
digest = hashlib.sha256(json.dumps(fields, sort_keys=True).encode()).hexdigest()[:32]
|
|
119
|
+
return root / f"{digest}.pt"
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _read(path: Path | None) -> torch.Tensor | None:
|
|
123
|
+
if path is None or not path.exists():
|
|
124
|
+
return None
|
|
125
|
+
try:
|
|
126
|
+
return torch.load(path, weights_only=True)
|
|
127
|
+
except Exception: # a damaged entry: compute it again
|
|
128
|
+
return None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _write(path: Path | None, raw: torch.Tensor) -> None:
|
|
132
|
+
if path is None:
|
|
133
|
+
return
|
|
134
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
135
|
+
tmp = path.with_suffix(f".{os.getpid()}.tmp")
|
|
136
|
+
torch.save(raw.clone(), tmp)
|
|
137
|
+
tmp.replace(path) # atomic: a reader sees no entry or a whole one
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Scanning and patching one attention head.
|
|
2
|
+
|
|
3
|
+
A head's strength is ``|W_O (z_t − z̄)|`` (see :mod:`openinterp.harness.units`). So the
|
|
4
|
+
scan makes two passes over the corpus. The first averages the head's output z over every
|
|
5
|
+
counted token to get z̄. The second measures strengths against it. Each pass stops at the
|
|
6
|
+
head's layer. Both count the same tokens as a neuron scan (see :mod:`scan`). So the top
|
|
7
|
+
examples, histogram and confound profile mean the same for both kinds of unit.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from functools import partial
|
|
15
|
+
|
|
16
|
+
import torch
|
|
17
|
+
|
|
18
|
+
from openinterp.core.schemas import ActivatingExample, HeadIdentifier, HeadProfile
|
|
19
|
+
from openinterp.harness.annotations import Tagger
|
|
20
|
+
from openinterp.harness.confounds import ConfoundAccumulator
|
|
21
|
+
from openinterp.harness.hooks import Inputs, NeuronHookRunner
|
|
22
|
+
from openinterp.harness.patcher import Effect, Positions
|
|
23
|
+
from openinterp.harness.scan import (
|
|
24
|
+
BATCH_TOKENS,
|
|
25
|
+
LabelPass,
|
|
26
|
+
ScanResult,
|
|
27
|
+
TopExamples,
|
|
28
|
+
batches,
|
|
29
|
+
cut_pieces,
|
|
30
|
+
settle,
|
|
31
|
+
)
|
|
32
|
+
from openinterp.harness.units import HeadUnit, head_strength
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class HeadScanResult(ScanResult):
|
|
37
|
+
"""A head's scan result, with its :class:`HeadProfile`."""
|
|
38
|
+
|
|
39
|
+
profile: HeadProfile | None = None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@torch.no_grad()
|
|
43
|
+
def pattern_scores(
|
|
44
|
+
runner: NeuronHookRunner, layer: int, head: int, *, sequences: int, length: int, seed: int
|
|
45
|
+
) -> dict[str, float]:
|
|
46
|
+
"""Previous-token, duplicate-token and induction scores on repeated random tokens."""
|
|
47
|
+
vocab = runner.model.get_input_embeddings().num_embeddings
|
|
48
|
+
generator = torch.Generator().manual_seed(seed)
|
|
49
|
+
ids = torch.randint(0, vocab, (sequences, length), generator=generator)
|
|
50
|
+
att = runner.attention_patterns(torch.cat([ids, ids], 1), layer)[:, head].float().cpu()
|
|
51
|
+
query = torch.arange(1, 2 * length)
|
|
52
|
+
second = torch.arange(length, 2 * length)
|
|
53
|
+
return {
|
|
54
|
+
"previous_token_score": float(att[:, query, query - 1].mean()),
|
|
55
|
+
"duplicate_token_score": float(att[:, second, second - length].mean()),
|
|
56
|
+
"induction_score": float(att[:, second, second - length + 1].mean()),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@torch.no_grad()
|
|
61
|
+
def peak_attention(
|
|
62
|
+
runner: NeuronHookRunner, example: ActivatingExample, layer: int, head: int, k: int = 3
|
|
63
|
+
) -> ActivatingExample:
|
|
64
|
+
"""``example`` with the ``k`` tokens its peak token attends to most."""
|
|
65
|
+
att = runner.attention_patterns(example.text, layer)[0, head, example.max_token_index]
|
|
66
|
+
top = att[: example.max_token_index + 1].float().topk(min(k, example.max_token_index + 1))
|
|
67
|
+
return example.model_copy(
|
|
68
|
+
update={
|
|
69
|
+
"attends_to": top.indices.tolist(),
|
|
70
|
+
"attention_weights": [min(1.0, round(float(w), 4)) for w in top.values],
|
|
71
|
+
}
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def scan_head(
|
|
76
|
+
runner: NeuronHookRunner,
|
|
77
|
+
texts: Sequence[str],
|
|
78
|
+
ident: HeadIdentifier,
|
|
79
|
+
*,
|
|
80
|
+
k: int = 20,
|
|
81
|
+
batch_tokens: int = BATCH_TOKENS,
|
|
82
|
+
min_count: int = 3,
|
|
83
|
+
window_tokens: int | None = None,
|
|
84
|
+
stride: int | None = None,
|
|
85
|
+
pattern_sequences: int = 16,
|
|
86
|
+
pattern_length: int = 32,
|
|
87
|
+
pattern_seed: int = 0,
|
|
88
|
+
taggers: Sequence[Tagger] = (),
|
|
89
|
+
) -> HeadScanResult:
|
|
90
|
+
"""Scan one head: its top examples, strengths, confounds and :class:`HeadProfile`.
|
|
91
|
+
|
|
92
|
+
Each of ``taggers`` labels the counted tokens, and its confound joins the profile's.
|
|
93
|
+
|
|
94
|
+
Needs eager attention (``attn_implementation="eager"``) for the attention patterns.
|
|
95
|
+
"""
|
|
96
|
+
if runner.tokenizer is None:
|
|
97
|
+
raise ValueError("scan_head requires a tokenizer")
|
|
98
|
+
layer, head = ident.layer, ident.head_idx
|
|
99
|
+
pieces = list(cut_pieces(runner.tokenizer, texts, window_tokens, stride))
|
|
100
|
+
|
|
101
|
+
mean, count = _mean_output(runner, pieces, layer, head, batch_tokens)
|
|
102
|
+
confounds = ConfoundAccumulator()
|
|
103
|
+
labels = LabelPass(runner.tokenizer, texts, taggers)
|
|
104
|
+
top = TopExamples()
|
|
105
|
+
values: list[float] = []
|
|
106
|
+
first: list[bool] = []
|
|
107
|
+
for batch in batches(runner, pieces, batch_tokens):
|
|
108
|
+
z = runner.head_outputs(batch.encoded, layer)[..., head, :]
|
|
109
|
+
strengths = head_strength(runner, ident, z, mean).cpu()
|
|
110
|
+
counted = batch.counted
|
|
111
|
+
confounds.update(batch.ids[counted], batch.positions[counted], strengths[counted])
|
|
112
|
+
labels.update(batch, strengths)
|
|
113
|
+
values.extend(strengths[counted].tolist())
|
|
114
|
+
first.extend(batch.first[counted].tolist())
|
|
115
|
+
top.update(batch, strengths)
|
|
116
|
+
unit = HeadUnit(ident, tuple(mean))
|
|
117
|
+
read = partial(unit.strengths, runner)
|
|
118
|
+
examples, values, floor = settle(runner, read, values, first, top, k, labels.tokens)
|
|
119
|
+
|
|
120
|
+
patterns = (pattern_sequences, pattern_length, pattern_seed)
|
|
121
|
+
profile = _head_profile(runner, layer, head, mean, count, *patterns)
|
|
122
|
+
return HeadScanResult(
|
|
123
|
+
examples=[peak_attention(runner, e, layer, head) for e in examples],
|
|
124
|
+
values=values,
|
|
125
|
+
correlated=[],
|
|
126
|
+
confounds=labels.finalize(confounds.finalize(runner.tokenizer, min_count), values),
|
|
127
|
+
windows=len(pieces),
|
|
128
|
+
first_token_floor=floor,
|
|
129
|
+
profile=profile,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _mean_output(runner, pieces, layer: int, head: int, batch_tokens: int):
|
|
134
|
+
"""The head's mean output over every counted token (one pass), and how many there were."""
|
|
135
|
+
total, count = None, 0
|
|
136
|
+
for batch in batches(runner, pieces, batch_tokens):
|
|
137
|
+
z = runner.head_outputs(batch.encoded, layer)[..., head, :].float().cpu()
|
|
138
|
+
chunk = z[batch.counted].sum(0)
|
|
139
|
+
total = chunk if total is None else total + chunk
|
|
140
|
+
count += int(batch.counted.sum())
|
|
141
|
+
if not count:
|
|
142
|
+
raise ValueError("the corpus has no tokens to scan")
|
|
143
|
+
return [round(float(v), 6) for v in total / count], count
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _head_profile(runner, layer, head, mean, count, sequences, length, seed) -> HeadProfile:
|
|
147
|
+
scores = pattern_scores(runner, layer, head, sequences=sequences, length=length, seed=seed)
|
|
148
|
+
return HeadProfile(
|
|
149
|
+
mean_output=mean,
|
|
150
|
+
mean_tokens=count,
|
|
151
|
+
**{name: round(min(1.0, max(0.0, v)), 6) for name, v in scores.items()},
|
|
152
|
+
pattern_sequences=sequences,
|
|
153
|
+
pattern_length=length,
|
|
154
|
+
pattern_seed=seed,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def head_write_norms(runner: NeuronHookRunner, layer: int, z: torch.Tensor) -> torch.Tensor:
|
|
159
|
+
"""``|W_O,h z_h|`` for every head h of ``z`` ``[..., n_heads, head_dim]``."""
|
|
160
|
+
return torch.stack(
|
|
161
|
+
[
|
|
162
|
+
(z[..., h, :].float() @ runner.head_output_weights(layer, h).float().T).norm(dim=-1)
|
|
163
|
+
for h in range(runner.n_heads)
|
|
164
|
+
],
|
|
165
|
+
dim=-1,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@torch.no_grad()
|
|
170
|
+
def patch_heads(
|
|
171
|
+
runner: NeuronHookRunner,
|
|
172
|
+
source: Inputs,
|
|
173
|
+
target: Inputs,
|
|
174
|
+
layer: int,
|
|
175
|
+
heads: Sequence[int],
|
|
176
|
+
*,
|
|
177
|
+
effect: Effect,
|
|
178
|
+
positions: Positions = "all",
|
|
179
|
+
freeze_downstream: bool = False,
|
|
180
|
+
batch_size: int = 8,
|
|
181
|
+
) -> list[float]:
|
|
182
|
+
"""``effect`` of patching each head's output, one at a time, from source into target.
|
|
183
|
+
|
|
184
|
+
The attention-head counterpart of :meth:`ActivationPatcher.patch_many`. Each head gets
|
|
185
|
+
its own row of a batch. Later MLP outputs can be frozen against self-repair.
|
|
186
|
+
"""
|
|
187
|
+
from openinterp.harness.memory import stream_rows
|
|
188
|
+
from openinterp.harness.patcher import ActivationPatcher
|
|
189
|
+
|
|
190
|
+
d = runner.head_dim(layer)
|
|
191
|
+
z_src = runner.head_outputs(source, layer).flatten(-2) # [1, seq, n_heads * d]
|
|
192
|
+
target_enc = runner.encode(target)
|
|
193
|
+
downstream = list(range(layer + 1, runner.n_layers)) if freeze_downstream else []
|
|
194
|
+
target_logits, _, downstream_out = ActivationPatcher(runner)._target_pass(
|
|
195
|
+
target_enc, downstream
|
|
196
|
+
)
|
|
197
|
+
if z_src.shape[0] != 1 or target_logits.shape[0] != 1:
|
|
198
|
+
raise ValueError("patch_heads takes one source and one target prompt")
|
|
199
|
+
if positions == "all" and z_src.shape[1] != target_logits.shape[1]:
|
|
200
|
+
raise ValueError("source and target differ in length; use positions='last'")
|
|
201
|
+
span = slice(None) if positions == "all" else slice(-1, None)
|
|
202
|
+
batch_size = stream_rows(runner, target_logits.shape[1], batch_size, logits=True)
|
|
203
|
+
|
|
204
|
+
effects: list[float] = []
|
|
205
|
+
for start in range(0, len(heads), batch_size):
|
|
206
|
+
chunk = list(heads[start : start + batch_size])
|
|
207
|
+
batch = {k: v.repeat(len(chunk), *[1] * (v.dim() - 1)) for k, v in target_enc.items()}
|
|
208
|
+
|
|
209
|
+
def inject(z: torch.Tensor, chunk=chunk) -> torch.Tensor:
|
|
210
|
+
z = z.clone()
|
|
211
|
+
for row, h in enumerate(chunk):
|
|
212
|
+
cols = slice(h * d, (h + 1) * d)
|
|
213
|
+
z[row, span, cols] = z_src[0, span, cols].to(z.dtype)
|
|
214
|
+
return z
|
|
215
|
+
|
|
216
|
+
n = len(chunk)
|
|
217
|
+
frozen = {i: _repeat(cached, n) for i, cached in downstream_out.items()}
|
|
218
|
+
patched = runner.run_with_interventions(batch, None, frozen, {layer: inject})
|
|
219
|
+
effects += [effect(patched[row : row + 1], target_logits) for row in range(n)]
|
|
220
|
+
return effects
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _repeat(cached: torch.Tensor, n: int):
|
|
224
|
+
"""An output intervention returning ``cached`` for each of ``n`` batch rows."""
|
|
225
|
+
value = cached.repeat(n, 1, 1)
|
|
226
|
+
return lambda _out: value
|