open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"""Held-out windows and the arithmetic of detection, fuzzing and embedding scores.
|
|
2
|
+
|
|
3
|
+
The judge only ever sees a window's text. So each window is measured on its own, as
|
|
4
|
+
probes are, not inside the snippet that it was cut from. Labels always come from a new
|
|
5
|
+
measurement. So a task handed to a judge carries no activations, and a stored score can
|
|
6
|
+
be checked again (`open-interpretability reproduce`).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import bisect
|
|
12
|
+
import random
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from dataclasses import dataclass, replace
|
|
15
|
+
|
|
16
|
+
import torch
|
|
17
|
+
|
|
18
|
+
from openinterp.core.explanation import ControlScore, ExplanationScores, JudgeScore, ScoredWindow
|
|
19
|
+
from openinterp.harness.hooks import NeuronHookRunner
|
|
20
|
+
from openinterp.harness.units import Unit
|
|
21
|
+
|
|
22
|
+
DEFAULT_EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class Window:
|
|
27
|
+
"""One held-out window: its text, tokens, strengths and marked tokens."""
|
|
28
|
+
|
|
29
|
+
text: str
|
|
30
|
+
tokens: list[str]
|
|
31
|
+
activations: list[float]
|
|
32
|
+
highlighted: tuple[int, ...] = ()
|
|
33
|
+
start: int = 0 # 1 for a unit with a first-token floor (protocol 0.8)
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def peak(self) -> float:
|
|
37
|
+
"""Over the measured tokens."""
|
|
38
|
+
return max(self.activations[self.start :], default=0.0)
|
|
39
|
+
|
|
40
|
+
def hot(self, threshold: float) -> list[int]:
|
|
41
|
+
"""Measured tokens at or above ``threshold``."""
|
|
42
|
+
return [i for i, a in enumerate(self.activations) if i >= self.start and a >= threshold]
|
|
43
|
+
|
|
44
|
+
def cold(self, threshold: float) -> list[int]:
|
|
45
|
+
"""The measured tokens below ``threshold``."""
|
|
46
|
+
return [i for i, a in enumerate(self.activations) if i >= self.start and a < threshold]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class HeldOutSample:
|
|
51
|
+
"""The detection and fuzzing windows, and how many windows they were drawn from."""
|
|
52
|
+
|
|
53
|
+
detection: list[Window]
|
|
54
|
+
fuzzing: list[Window]
|
|
55
|
+
quantile_edges: list[float]
|
|
56
|
+
candidate_windows: int
|
|
57
|
+
active_windows: int
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def cut_windows(runner: NeuronHookRunner, texts: Sequence[str], window_tokens: int) -> list[str]:
|
|
61
|
+
"""Split each text into consecutive ``window_tokens``-token pieces, decoded back to text.
|
|
62
|
+
|
|
63
|
+
A trailing piece shorter than half a window is dropped, as are blank pieces.
|
|
64
|
+
"""
|
|
65
|
+
tok = runner.tokenizer
|
|
66
|
+
pieces = []
|
|
67
|
+
for text in texts:
|
|
68
|
+
ids = tok.encode(text, add_special_tokens=False)
|
|
69
|
+
for start in range(0, len(ids), window_tokens):
|
|
70
|
+
chunk = ids[start : start + window_tokens]
|
|
71
|
+
if len(chunk) * 2 < window_tokens and start > 0:
|
|
72
|
+
continue
|
|
73
|
+
piece = tok.decode(chunk)
|
|
74
|
+
if piece.strip():
|
|
75
|
+
pieces.append(piece)
|
|
76
|
+
return pieces
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@torch.no_grad()
|
|
80
|
+
def measure_windows(
|
|
81
|
+
runner: NeuronHookRunner,
|
|
82
|
+
texts: Sequence[str],
|
|
83
|
+
unit: Unit,
|
|
84
|
+
*,
|
|
85
|
+
batch_size: int = 16,
|
|
86
|
+
) -> list[Window]:
|
|
87
|
+
"""Per-token strengths of each text, run on its own (no surrounding context)."""
|
|
88
|
+
windows = []
|
|
89
|
+
for start in range(0, len(texts), batch_size):
|
|
90
|
+
batch = list(texts[start : start + batch_size])
|
|
91
|
+
encoded = runner.encode(batch)
|
|
92
|
+
strengths = unit.strengths(runner, encoded).float()
|
|
93
|
+
ids = encoded["input_ids"]
|
|
94
|
+
mask = encoded.get("attention_mask", torch.ones_like(ids)).bool()
|
|
95
|
+
for row, text in enumerate(batch):
|
|
96
|
+
keep = mask[row]
|
|
97
|
+
windows.append(
|
|
98
|
+
Window(
|
|
99
|
+
text=text,
|
|
100
|
+
tokens=[runner.tokenizer.decode([i]) for i in ids[row][keep].tolist()],
|
|
101
|
+
activations=[round(v, 5) for v in strengths[row][keep].cpu().tolist()],
|
|
102
|
+
start=int(unit.skip_first),
|
|
103
|
+
)
|
|
104
|
+
)
|
|
105
|
+
return windows
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# How sample_heldout marks fuzzing decoys; recorded in each ExplanationScores.
|
|
109
|
+
DECOYS = "matched-shape"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def placed(rng: random.Random, shape: Sequence[int], free: Sequence[int]) -> list[int]:
|
|
113
|
+
"""Marks with ``shape``'s offsets, moved to a random place where every mark is in ``free``.
|
|
114
|
+
|
|
115
|
+
A decoy that copies the real marks' count and spacing gives a judge no cue from them
|
|
116
|
+
("Scoring Explanations Without Reading Them", arXiv 2610.37676). If the shape fits
|
|
117
|
+
nowhere, it falls back to as many marks drawn at random from ``free``.
|
|
118
|
+
"""
|
|
119
|
+
if not shape:
|
|
120
|
+
return []
|
|
121
|
+
offsets = [i - shape[0] for i in sorted(shape)]
|
|
122
|
+
allowed = set(free)
|
|
123
|
+
starts = [s for s in sorted(allowed) if all(s + o in allowed for o in offsets)]
|
|
124
|
+
if starts:
|
|
125
|
+
start = rng.choice(starts)
|
|
126
|
+
return [start + o for o in offsets]
|
|
127
|
+
return rng.sample(list(free), min(len(free), len(shape)))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def sample_heldout(
|
|
131
|
+
runner: NeuronHookRunner,
|
|
132
|
+
texts: Sequence[str],
|
|
133
|
+
unit: Unit,
|
|
134
|
+
threshold: float,
|
|
135
|
+
*,
|
|
136
|
+
seed: int,
|
|
137
|
+
exclude: Sequence[str] = (),
|
|
138
|
+
window_tokens: int = 32,
|
|
139
|
+
max_windows: int = 4000,
|
|
140
|
+
quantiles: int = 5,
|
|
141
|
+
per_quantile: int = 5,
|
|
142
|
+
) -> HeldOutSample:
|
|
143
|
+
"""Draw detection and fuzzing windows from the ``texts`` containing no ``exclude``.
|
|
144
|
+
|
|
145
|
+
Active windows (peak ≥ ``threshold``) are sorted by peak and split into ``quantiles``
|
|
146
|
+
equal bins. Each bin gives ``per_quantile`` windows to detection and as many to
|
|
147
|
+
fuzzing. So the weakest activations count as much as the top ones. Detection gets as
|
|
148
|
+
many inactive windows as active ones. In half its active windows, fuzzing marks the
|
|
149
|
+
active tokens. In the other half, it marks inactive tokens in the same shape: the same
|
|
150
|
+
count and spacing, moved (:func:`placed`). It also adds half as many inactive windows,
|
|
151
|
+
each marked in the shape of a random active window's tokens. With no active window at all,
|
|
152
|
+
detection gets ``quantiles × per_quantile`` inactive ones and fuzzing none.
|
|
153
|
+
"""
|
|
154
|
+
rng = random.Random(seed)
|
|
155
|
+
# A top example may be a window of a longer document: leave out the whole document.
|
|
156
|
+
held_out = [t for t in texts if not any(e in t for e in exclude)]
|
|
157
|
+
candidates = cut_windows(runner, held_out, window_tokens)
|
|
158
|
+
rng.shuffle(candidates)
|
|
159
|
+
measured = measure_windows(runner, candidates[:max_windows], unit)
|
|
160
|
+
active = sorted((w for w in measured if w.peak >= threshold), key=lambda w: w.peak)
|
|
161
|
+
inactive = [w for w in measured if w.peak < threshold]
|
|
162
|
+
if not measured:
|
|
163
|
+
raise ValueError("no held-out windows: every corpus snippet is a top example")
|
|
164
|
+
if not active: # the explanation can still be scored on false positives
|
|
165
|
+
drawn = rng.sample(inactive, min(len(inactive), quantiles * per_quantile))
|
|
166
|
+
return HeldOutSample(drawn, [], [], len(measured), 0)
|
|
167
|
+
n = len(active)
|
|
168
|
+
bins = [active[i * n // quantiles : (i + 1) * n // quantiles] for i in range(quantiles)]
|
|
169
|
+
edges = [b[0].peak for b in bins[1:] if b]
|
|
170
|
+
|
|
171
|
+
detect_active, fuzz_active = [], []
|
|
172
|
+
for members in bins:
|
|
173
|
+
drawn = rng.sample(members, min(len(members), 2 * per_quantile))
|
|
174
|
+
half = (len(drawn) + 1) // 2
|
|
175
|
+
detect_active += drawn[:half]
|
|
176
|
+
fuzz_active += drawn[half:]
|
|
177
|
+
detection = detect_active + rng.sample(inactive, min(len(inactive), len(detect_active)))
|
|
178
|
+
|
|
179
|
+
fuzzing = []
|
|
180
|
+
shapes = [w.hot(threshold) for w in fuzz_active]
|
|
181
|
+
for n, w in enumerate(fuzz_active):
|
|
182
|
+
hot, cold = w.hot(threshold), w.cold(threshold)
|
|
183
|
+
marks = hot if n % 2 == 0 or len(cold) < len(hot) else placed(rng, hot, cold)
|
|
184
|
+
fuzzing.append(replace(w, highlighted=tuple(sorted(marks))))
|
|
185
|
+
rest = [w for w in inactive if w not in detection]
|
|
186
|
+
for w in rng.sample(rest, min(len(rest), len(fuzz_active) // 2)):
|
|
187
|
+
cold = list(range(w.start, len(w.tokens)))
|
|
188
|
+
marks = placed(rng, rng.choice(shapes), cold)
|
|
189
|
+
fuzzing.append(replace(w, highlighted=tuple(sorted(marks))))
|
|
190
|
+
rng.shuffle(detection)
|
|
191
|
+
rng.shuffle(fuzzing)
|
|
192
|
+
return HeldOutSample(detection, fuzzing, edges, len(measured), len(active))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def label_windows(
|
|
196
|
+
windows: Sequence[Window],
|
|
197
|
+
threshold: float,
|
|
198
|
+
edges: Sequence[float],
|
|
199
|
+
*,
|
|
200
|
+
fuzzing: bool,
|
|
201
|
+
judged: Sequence[bool | None] | None = None,
|
|
202
|
+
similarities: Sequence[float | None] | None = None,
|
|
203
|
+
) -> list[ScoredWindow]:
|
|
204
|
+
"""Label each window as the judge should answer it, with the judge's answer if given."""
|
|
205
|
+
out = []
|
|
206
|
+
for n, w in enumerate(windows):
|
|
207
|
+
hot = set(w.hot(threshold))
|
|
208
|
+
label = bool(hot) and set(w.highlighted) == hot if fuzzing else bool(hot)
|
|
209
|
+
out.append(
|
|
210
|
+
ScoredWindow(
|
|
211
|
+
text=w.text,
|
|
212
|
+
tokens=w.tokens,
|
|
213
|
+
activations=w.activations,
|
|
214
|
+
quantile=bisect.bisect_right(edges, w.peak) if hot else None,
|
|
215
|
+
highlighted=list(w.highlighted),
|
|
216
|
+
label=label,
|
|
217
|
+
judged=judged[n] if judged is not None else None,
|
|
218
|
+
similarity=similarities[n] if similarities is not None else None,
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
return out
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def judge_score(windows: Sequence[ScoredWindow], quantiles: int) -> JudgeScore | None:
|
|
225
|
+
"""Balanced accuracy of the judge's answers, or ``None`` if nothing was judged."""
|
|
226
|
+
judged = [w for w in windows if w.judged is not None]
|
|
227
|
+
pos = [w.judged for w in judged if w.label]
|
|
228
|
+
neg = [w.judged for w in judged if not w.label]
|
|
229
|
+
if not judged:
|
|
230
|
+
return None
|
|
231
|
+
tpr = sum(pos) / len(pos) if pos else None
|
|
232
|
+
fpr = sum(neg) / len(neg) if neg else None
|
|
233
|
+
rates = [r for r in (tpr, None if fpr is None else 1 - fpr) if r is not None]
|
|
234
|
+
by_q: list[float | None] = []
|
|
235
|
+
for q in range(quantiles):
|
|
236
|
+
hits = [w.judged for w in judged if w.label and w.quantile == q]
|
|
237
|
+
by_q.append(sum(hits) / len(hits) if hits else None)
|
|
238
|
+
return JudgeScore(
|
|
239
|
+
balanced_accuracy=sum(rates) / len(rates),
|
|
240
|
+
true_positive_rate=tpr,
|
|
241
|
+
false_positive_rate=fpr,
|
|
242
|
+
recall_by_quantile=by_q,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def auroc(scores: Sequence[float], labels: Sequence[bool]) -> float | None:
|
|
247
|
+
"""Area under the ROC curve (Mann–Whitney U, ties count half); ``None`` with one class."""
|
|
248
|
+
pos = [s for s, y in zip(scores, labels, strict=True) if y]
|
|
249
|
+
neg = [s for s, y in zip(scores, labels, strict=True) if not y]
|
|
250
|
+
if not pos or not neg:
|
|
251
|
+
return None
|
|
252
|
+
wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg)
|
|
253
|
+
return wins / (len(pos) * len(neg))
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _balanced(pairs: Sequence[tuple[bool, bool]]) -> float | None:
|
|
257
|
+
"""Balanced accuracy of (predicted, label) pairs, or ``None`` with one class."""
|
|
258
|
+
pos = [p for p, y in pairs if y]
|
|
259
|
+
neg = [not p for p, y in pairs if not y]
|
|
260
|
+
if not pos or not neg:
|
|
261
|
+
return None
|
|
262
|
+
return (sum(pos) / len(pos) + sum(neg) / len(neg)) / 2
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _contiguous(marks: Sequence[int]) -> bool:
|
|
266
|
+
"""True when the marked tokens form one unbroken run."""
|
|
267
|
+
return bool(marks) and max(marks) - min(marks) + 1 == len(marks)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def _best_cut(values: Sequence[float], labels: Sequence[bool]) -> float | None:
|
|
271
|
+
"""The best balanced accuracy of "positive if value >= cut" over every cut."""
|
|
272
|
+
scores = [
|
|
273
|
+
_balanced([(v >= c, y) for v, y in zip(values, labels, strict=True)]) for c in set(values)
|
|
274
|
+
]
|
|
275
|
+
return max((s for s in scores if s is not None), default=None)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def explanation_free_baseline(detection, fuzzing) -> dict[str, float]:
|
|
279
|
+
"""Explanation-free rules' balanced accuracy on scored windows (arXiv 2610.37676).
|
|
280
|
+
|
|
281
|
+
Each cut is chosen on the same windows, so each number is an upper bound.
|
|
282
|
+
"""
|
|
283
|
+
out = {
|
|
284
|
+
"detection_length": _best_cut(
|
|
285
|
+
[len(w.tokens) for w in detection], [w.label for w in detection]
|
|
286
|
+
),
|
|
287
|
+
"fuzzing_contiguous": _balanced([(_contiguous(w.highlighted), w.label) for w in fuzzing]),
|
|
288
|
+
"fuzzing_count": _best_cut(
|
|
289
|
+
[len(w.highlighted) for w in fuzzing], [w.label for w in fuzzing]
|
|
290
|
+
),
|
|
291
|
+
}
|
|
292
|
+
return {k: round(v, 6) for k, v in out.items() if v is not None}
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def finalize_scores(scores: ExplanationScores) -> ExplanationScores:
|
|
296
|
+
"""Recompute the summary scores from the stored windows, labels and answers."""
|
|
297
|
+
det = scores.detection_windows
|
|
298
|
+
sims = [w.similarity for w in det]
|
|
299
|
+
emb = auroc(sims, [w.label for w in det]) if det and None not in sims else None
|
|
300
|
+
return scores.model_copy(
|
|
301
|
+
update={
|
|
302
|
+
"detection": judge_score(det, scores.quantiles),
|
|
303
|
+
"fuzzing": judge_score(scores.fuzzing_windows, scores.quantiles),
|
|
304
|
+
"embedding_auroc": emb,
|
|
305
|
+
"controls": [_control_scores(c, scores) for c in scores.controls],
|
|
306
|
+
"rule_baseline": explanation_free_baseline(det, scores.fuzzing_windows),
|
|
307
|
+
}
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _control_scores(control: ControlScore, scores: ExplanationScores) -> ControlScore:
|
|
312
|
+
"""Score a control's answers against the real windows' labels."""
|
|
313
|
+
|
|
314
|
+
def judged(windows: Sequence[ScoredWindow], answers: Sequence[bool]):
|
|
315
|
+
if len(answers) != len(windows):
|
|
316
|
+
return None
|
|
317
|
+
marked = [w.model_copy(update={"judged": a}) for w, a in zip(windows, answers, strict=True)]
|
|
318
|
+
return judge_score(marked, scores.quantiles)
|
|
319
|
+
|
|
320
|
+
return control.model_copy(
|
|
321
|
+
update={
|
|
322
|
+
"detection": judged(scores.detection_windows, control.detection_judged),
|
|
323
|
+
"fuzzing": judged(scores.fuzzing_windows, control.fuzzing_judged),
|
|
324
|
+
}
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def remeasure(scores: ExplanationScores, runner: NeuronHookRunner, unit: Unit):
|
|
329
|
+
"""Re-measure every window and re-derive labels and scores, keeping the answers."""
|
|
330
|
+
|
|
331
|
+
def redo(windows: Sequence[ScoredWindow], fuzzing: bool) -> list[ScoredWindow]:
|
|
332
|
+
fresh = measure_windows(runner, [w.text for w in windows], unit)
|
|
333
|
+
pairs = zip(fresh, windows, strict=True)
|
|
334
|
+
fresh = [replace(f, highlighted=tuple(w.highlighted)) for f, w in pairs]
|
|
335
|
+
return label_windows(
|
|
336
|
+
fresh,
|
|
337
|
+
scores.threshold,
|
|
338
|
+
scores.quantile_edges,
|
|
339
|
+
fuzzing=fuzzing,
|
|
340
|
+
judged=[w.judged for w in windows],
|
|
341
|
+
similarities=[w.similarity for w in windows],
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
return finalize_scores(
|
|
345
|
+
scores.model_copy(
|
|
346
|
+
update={
|
|
347
|
+
"detection_windows": redo(scores.detection_windows, False),
|
|
348
|
+
"fuzzing_windows": redo(scores.fuzzing_windows, True),
|
|
349
|
+
}
|
|
350
|
+
)
|
|
351
|
+
)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@torch.no_grad()
|
|
355
|
+
def embedding_similarities(model_name: str, query: str, texts: Sequence[str]) -> list[float]:
|
|
356
|
+
"""Cosine between ``query`` and each text, mean-pooled from a sentence encoder."""
|
|
357
|
+
from transformers import AutoModel, AutoTokenizer
|
|
358
|
+
|
|
359
|
+
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
|
360
|
+
model = AutoModel.from_pretrained(model_name).eval()
|
|
361
|
+
vectors = []
|
|
362
|
+
batch = [query, *texts]
|
|
363
|
+
for start in range(0, len(batch), 32):
|
|
364
|
+
enc = tokenizer(
|
|
365
|
+
batch[start : start + 32], padding=True, truncation=True, return_tensors="pt"
|
|
366
|
+
)
|
|
367
|
+
hidden = model(**enc).last_hidden_state
|
|
368
|
+
mask = enc["attention_mask"].unsqueeze(-1).to(hidden.dtype)
|
|
369
|
+
vectors.append((hidden * mask).sum(1) / mask.sum(1).clamp(min=1))
|
|
370
|
+
emb = torch.nn.functional.normalize(torch.cat(vectors), dim=-1)
|
|
371
|
+
return [round(float(v), 6) for v in emb[1:] @ emb[0]]
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
"""Scanning and patching one latent of a sparse dictionary (a transcoder or SAE).
|
|
2
|
+
|
|
3
|
+
A latent's strength is its activation (see :mod:`openinterp.harness.units`). The scan
|
|
4
|
+
reads the dictionary's input at its site, in one pass that stops at the feature's layer.
|
|
5
|
+
It counts the same tokens as a neuron scan (see :mod:`scan`). So the top examples,
|
|
6
|
+
histogram and confound profile mean the same for every kind of unit.
|
|
7
|
+
|
|
8
|
+
The same pass measures how well the whole dictionary reconstructs its output there. It
|
|
9
|
+
leaves out each piece's first token. So a record says whether its baseline was a
|
|
10
|
+
faithful one.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
|
|
20
|
+
from openinterp.core.schemas import DictionaryProfile, FeatureIdentifier
|
|
21
|
+
from openinterp.harness.annotations import Tagger
|
|
22
|
+
from openinterp.harness.confounds import ConfoundAccumulator
|
|
23
|
+
from openinterp.harness.dictionaries import Dictionary, DictionarySpec, load_spec, spec_profile
|
|
24
|
+
from openinterp.harness.heads import _repeat
|
|
25
|
+
from openinterp.harness.hooks import Inputs, NeuronHookRunner
|
|
26
|
+
from openinterp.harness.patcher import Effect, Positions
|
|
27
|
+
from openinterp.harness.scan import (
|
|
28
|
+
BATCH_TOKENS,
|
|
29
|
+
LabelPass,
|
|
30
|
+
ScanResult,
|
|
31
|
+
TopExamples,
|
|
32
|
+
batches,
|
|
33
|
+
cut_pieces,
|
|
34
|
+
settle,
|
|
35
|
+
)
|
|
36
|
+
from openinterp.harness.sites import dictionary_io
|
|
37
|
+
|
|
38
|
+
ENCODE_CHUNK = 1024 # tokens encoded through the whole dictionary at once
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class FeatureScanResult(ScanResult):
|
|
43
|
+
"""A latent's scan result, with its :class:`DictionaryProfile`."""
|
|
44
|
+
|
|
45
|
+
profile: DictionaryProfile | None = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _Fit:
|
|
49
|
+
"""Running reconstruction error, output variance and L0 of a dictionary."""
|
|
50
|
+
|
|
51
|
+
def __init__(self) -> None:
|
|
52
|
+
self.tokens = 0
|
|
53
|
+
self.error = self.sum_sq = self.l0 = 0.0
|
|
54
|
+
self.total: torch.Tensor | None = None
|
|
55
|
+
self.fired: torch.Tensor | None = None # [n] latents active on some token
|
|
56
|
+
|
|
57
|
+
def update(self, dictionary: Dictionary, x: torch.Tensor, out: torch.Tensor) -> None:
|
|
58
|
+
for start in range(0, x.shape[0], ENCODE_CHUNK):
|
|
59
|
+
xs, target = x[start : start + ENCODE_CHUNK], out[start : start + ENCODE_CHUNK]
|
|
60
|
+
target = target.float().to(dictionary.w_dec.device)
|
|
61
|
+
acts = dictionary.encode(xs)
|
|
62
|
+
self.error += float(((dictionary.reconstruct(xs, acts) - target) ** 2).sum())
|
|
63
|
+
self.l0 += float((acts > 0).sum())
|
|
64
|
+
fired = (acts > 0).any(0)
|
|
65
|
+
self.fired = fired if self.fired is None else self.fired | fired
|
|
66
|
+
exact = target.cpu().double() # MPS has no float64
|
|
67
|
+
self.sum_sq += float((exact**2).sum())
|
|
68
|
+
chunk = exact.sum(0)
|
|
69
|
+
self.total = chunk if self.total is None else self.total + chunk
|
|
70
|
+
self.tokens += x.shape[0]
|
|
71
|
+
|
|
72
|
+
def stats(self) -> dict[str, float]:
|
|
73
|
+
variance = self.sum_sq - float((self.total**2).sum()) / self.tokens
|
|
74
|
+
explained = 1 - self.error / variance if variance > 0 else None
|
|
75
|
+
return {
|
|
76
|
+
"explained_variance": None if explained is None else round(explained, 6),
|
|
77
|
+
"mean_l0": round(self.l0 / self.tokens, 4),
|
|
78
|
+
"dead_share": round(1 - float(self.fired.float().mean()), 6),
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def scan_feature(
|
|
83
|
+
runner: NeuronHookRunner,
|
|
84
|
+
texts: Sequence[str],
|
|
85
|
+
ident: FeatureIdentifier,
|
|
86
|
+
spec: DictionarySpec,
|
|
87
|
+
**options,
|
|
88
|
+
) -> FeatureScanResult:
|
|
89
|
+
"""Scan one latent: its top examples, strengths, confounds and :class:`DictionaryProfile`.
|
|
90
|
+
|
|
91
|
+
``options`` are :func:`scan_latent`'s.
|
|
92
|
+
"""
|
|
93
|
+
dictionary = load_spec(spec, ident.layer, runner.device)
|
|
94
|
+
|
|
95
|
+
def profile(**measured) -> DictionaryProfile:
|
|
96
|
+
return spec_profile(spec, ident.layer, n_features=dictionary.n_features, **measured)
|
|
97
|
+
|
|
98
|
+
return scan_latent(runner, texts, ident, dictionary, spec.input_site, profile, **options)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def scan_latent(
|
|
102
|
+
runner: NeuronHookRunner,
|
|
103
|
+
texts: Sequence[str],
|
|
104
|
+
ident,
|
|
105
|
+
dictionary: Dictionary,
|
|
106
|
+
input_site: str,
|
|
107
|
+
profile,
|
|
108
|
+
*,
|
|
109
|
+
k: int = 20,
|
|
110
|
+
batch_tokens: int = BATCH_TOKENS,
|
|
111
|
+
min_count: int = 3,
|
|
112
|
+
window_tokens: int | None = None,
|
|
113
|
+
stride: int | None = None,
|
|
114
|
+
taggers: Sequence[Tagger] = (),
|
|
115
|
+
) -> FeatureScanResult:
|
|
116
|
+
"""Scan latent ``ident.feature_idx`` of ``dictionary``, read at ``input_site``.
|
|
117
|
+
|
|
118
|
+
``profile(density=, tokens=, **fit)`` gives the result's profile from what the scan
|
|
119
|
+
measured. Each of ``taggers`` labels the counted tokens, and its confound joins the
|
|
120
|
+
profile's. A superposition scans this way too (:mod:`openinterp.harness.superpositions`).
|
|
121
|
+
"""
|
|
122
|
+
if runner.tokenizer is None:
|
|
123
|
+
raise ValueError("a latent scan requires a tokenizer")
|
|
124
|
+
layer, f = ident.layer, ident.feature_idx
|
|
125
|
+
if f >= dictionary.n_features:
|
|
126
|
+
raise ValueError(f"{ident}: the dictionary has {dictionary.n_features} latents")
|
|
127
|
+
pieces = list(cut_pieces(runner.tokenizer, texts, window_tokens, stride))
|
|
128
|
+
labels = LabelPass(runner.tokenizer, texts, taggers)
|
|
129
|
+
confounds, top, fit, values, first = _feature_pass(
|
|
130
|
+
runner, dictionary, input_site, layer, f, pieces, batch_tokens, labels
|
|
131
|
+
)
|
|
132
|
+
if not values:
|
|
133
|
+
raise ValueError("the corpus has no tokens to scan")
|
|
134
|
+
|
|
135
|
+
def read(inputs):
|
|
136
|
+
x, _ = dictionary_io(runner, inputs, layer, input_site)
|
|
137
|
+
return dictionary.feature(x, f)
|
|
138
|
+
|
|
139
|
+
density = round(sum(v > 0 for v in values) / len(values), 6)
|
|
140
|
+
tokens = len(values) # before settle, which can leave out first tokens
|
|
141
|
+
examples, values, floor = settle(runner, read, values, first, top, k, labels.tokens)
|
|
142
|
+
return FeatureScanResult(
|
|
143
|
+
examples=examples,
|
|
144
|
+
values=values,
|
|
145
|
+
correlated=[],
|
|
146
|
+
confounds=labels.finalize(confounds.finalize(runner.tokenizer, min_count), values),
|
|
147
|
+
windows=len(pieces),
|
|
148
|
+
first_token_floor=floor,
|
|
149
|
+
profile=profile(density=density, tokens=tokens, **fit.stats()),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _feature_pass(
|
|
154
|
+
runner, dictionary, input_site: str, layer: int, f: int, pieces, batch_tokens, labels
|
|
155
|
+
):
|
|
156
|
+
"""Make one pass over ``pieces``, and return what it measured.
|
|
157
|
+
|
|
158
|
+
That is the confounds (and ``labels``' counts), the top examples, and the dictionary's
|
|
159
|
+
fit. It is also each counted token's strength, and whether it is a piece's first token.
|
|
160
|
+
"""
|
|
161
|
+
confounds = ConfoundAccumulator()
|
|
162
|
+
top = TopExamples()
|
|
163
|
+
fit = _Fit()
|
|
164
|
+
values: list[float] = []
|
|
165
|
+
first: list[bool] = []
|
|
166
|
+
for batch in batches(runner, pieces, batch_tokens):
|
|
167
|
+
x, out = dictionary_io(runner, batch.encoded, layer, input_site)
|
|
168
|
+
strengths = dictionary.feature(x, f).cpu()
|
|
169
|
+
counted = batch.counted
|
|
170
|
+
confounds.update(batch.ids[counted], batch.positions[counted], strengths[counted])
|
|
171
|
+
labels.update(batch, strengths)
|
|
172
|
+
values.extend(strengths[counted].tolist())
|
|
173
|
+
first.extend(batch.first[counted].tolist())
|
|
174
|
+
top.update(batch, strengths)
|
|
175
|
+
# Not a piece's first token. Dictionaries aren't trained on that attention sink.
|
|
176
|
+
# Its error can be 10^8 times the output's variance, and would swamp the fit.
|
|
177
|
+
fitted = (counted & ~batch.first).to(x.device)
|
|
178
|
+
fit.update(dictionary, x[fitted], out[fitted])
|
|
179
|
+
return confounds, top, fit, values, first
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@torch.no_grad()
|
|
183
|
+
def latent_activations(
|
|
184
|
+
runner: NeuronHookRunner, dictionary: Dictionary, inputs: Inputs, layer: int, site: str
|
|
185
|
+
) -> torch.Tensor:
|
|
186
|
+
"""Every latent's activation on one prompt: ``[seq, n]``."""
|
|
187
|
+
x, _ = dictionary_io(runner, inputs, layer, site)
|
|
188
|
+
return dictionary.encode(x[0])
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
@torch.no_grad()
|
|
192
|
+
def write_changes(
|
|
193
|
+
runner: NeuronHookRunner,
|
|
194
|
+
dictionary: Dictionary,
|
|
195
|
+
ident: FeatureIdentifier,
|
|
196
|
+
site: str,
|
|
197
|
+
base: str,
|
|
198
|
+
cf: str,
|
|
199
|
+
) -> tuple[Positions, int, torch.Tensor]:
|
|
200
|
+
"""Return where to patch and score, and every latent's write change up to there.
|
|
201
|
+
|
|
202
|
+
The write change is ``|Δa| · |W_dec[f]|``, in units of the audited latent's activation.
|
|
203
|
+
So the audited latent's own entry is ``|Δa|``.
|
|
204
|
+
|
|
205
|
+
Latents' decoder norms differ. So comparing writes, as heads do, matches the null
|
|
206
|
+
latents to the audited one by how much each moves the output that they write to.
|
|
207
|
+
"""
|
|
208
|
+
a_b, a_c = (latent_activations(runner, dictionary, t, ident.layer, site) for t in (base, cf))
|
|
209
|
+
norms = dictionary.w_dec.norm(dim=-1)
|
|
210
|
+
scale = norms / norms[ident.feature_idx].clamp(min=1e-12)
|
|
211
|
+
if a_b.shape != a_c.shape:
|
|
212
|
+
return "last", -1, (a_b[-1] - a_c[-1]).abs() * scale
|
|
213
|
+
position = int((a_b[:, ident.feature_idx] - a_c[:, ident.feature_idx]).abs().argmax())
|
|
214
|
+
return "all", position, (a_b[: position + 1] - a_c[: position + 1]).abs().amax(0) * scale
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@torch.no_grad()
|
|
218
|
+
def patch_features(
|
|
219
|
+
runner: NeuronHookRunner,
|
|
220
|
+
dictionary: Dictionary,
|
|
221
|
+
site: str,
|
|
222
|
+
source: Inputs,
|
|
223
|
+
target: Inputs,
|
|
224
|
+
layer: int,
|
|
225
|
+
features: Sequence[int],
|
|
226
|
+
*,
|
|
227
|
+
effect: Effect,
|
|
228
|
+
positions: Positions = "all",
|
|
229
|
+
freeze_downstream: bool = False,
|
|
230
|
+
batch_size: int = 8,
|
|
231
|
+
) -> list[float]:
|
|
232
|
+
"""``effect`` of patching each latent's activation, one at a time, from source into target.
|
|
233
|
+
|
|
234
|
+
The dictionary's output (the MLP's, or the residual stream) gains
|
|
235
|
+
``(a_source − a_target) · W_dec[f]``.
|
|
236
|
+
|
|
237
|
+
The dictionary counterpart of :meth:`ActivationPatcher.patch_many`. Each latent gets
|
|
238
|
+
its own row of a batch. Later MLP outputs can be frozen against self-repair.
|
|
239
|
+
"""
|
|
240
|
+
from openinterp.harness.memory import stream_rows
|
|
241
|
+
from openinterp.harness.patcher import ActivationPatcher
|
|
242
|
+
|
|
243
|
+
a_src = latent_activations(runner, dictionary, source, layer, site) # [seq, n]
|
|
244
|
+
target_enc = runner.encode(target)
|
|
245
|
+
downstream = list(range(layer + 1, runner.n_layers)) if freeze_downstream else []
|
|
246
|
+
target_logits, _, downstream_out = ActivationPatcher(runner)._target_pass(
|
|
247
|
+
target_enc, downstream
|
|
248
|
+
)
|
|
249
|
+
if target_logits.shape[0] != 1:
|
|
250
|
+
raise ValueError("patch_features takes one source and one target prompt")
|
|
251
|
+
if positions == "all" and a_src.shape[0] != target_logits.shape[1]:
|
|
252
|
+
raise ValueError("source and target differ in length; use positions='last'")
|
|
253
|
+
span = slice(None) if positions == "all" else slice(-1, None)
|
|
254
|
+
batch_size = stream_rows(runner, target_logits.shape[1], batch_size, logits=True)
|
|
255
|
+
|
|
256
|
+
effects: list[float] = []
|
|
257
|
+
for start in range(0, len(features), batch_size):
|
|
258
|
+
chunk = torch.tensor(list(features[start : start + batch_size]))
|
|
259
|
+
n = len(chunk)
|
|
260
|
+
batch = {k: v.repeat(n, *[1] * (v.dim() - 1)) for k, v in target_enc.items()}
|
|
261
|
+
idx = chunk.to(dictionary.w_dec.device)
|
|
262
|
+
|
|
263
|
+
def inject(x: torch.Tensor, out: torch.Tensor, idx=idx, n=n) -> torch.Tensor:
|
|
264
|
+
a_tgt = dictionary.feature(x[0, span], idx) # [span, n]; every row is the target
|
|
265
|
+
delta = (a_src[span][:, idx] - a_tgt).T # [n, span]
|
|
266
|
+
out = out.clone()
|
|
267
|
+
write = delta.unsqueeze(-1) * dictionary.w_dec[idx].unsqueeze(1) # [n, span, d]
|
|
268
|
+
out[:n, span] += write.to(out.dtype)
|
|
269
|
+
return out
|
|
270
|
+
|
|
271
|
+
frozen = {i: _repeat(cached, n) for i, cached in downstream_out.items()}
|
|
272
|
+
patched = runner.run_with_interventions(
|
|
273
|
+
batch, None, frozen, dictionary_interventions={layer: (site, inject)}
|
|
274
|
+
)
|
|
275
|
+
effects += [effect(patched[row : row + 1], target_logits) for row in range(n)]
|
|
276
|
+
return effects
|