open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
"""Per-token labels for the scan, and the confound any label set gives.
|
|
2
|
+
|
|
3
|
+
A :class:`Tagger` labels character spans of a document. A document label (its language)
|
|
4
|
+
is one span over the whole text. A word tagger (part of speech) gives one span per word.
|
|
5
|
+
The scan aligns each tagger's spans to the model's tokens once per piece, through the
|
|
6
|
+
tokenizer's character offsets. It keeps the labels next to the activations it counts.
|
|
7
|
+
|
|
8
|
+
For every label set, :class:`LabelAccumulator` reports what ``token_r2`` reports for
|
|
9
|
+
token identity, and in information units:
|
|
10
|
+
|
|
11
|
+
* ``r2``: variance explained by a per-label mean model, over labelled tokens;
|
|
12
|
+
* ``mutual_information_bits``: I(activation; label) in bits. Pole strengths are sparse
|
|
13
|
+
and heavy-tailed, so the estimate is binned, not Gaussian. Zero is a bin of its own,
|
|
14
|
+
and positive strengths fall in ``ACTIVATION_BINS`` equal-count bins. Each entropy has
|
|
15
|
+
the Miller–Madow correction. The plug-in estimate is biased upwards by about
|
|
16
|
+
(labels − 1)(bins − 1) / 2N nats when the two are independent.
|
|
17
|
+
|
|
18
|
+
Some tokens are left out of both numbers: those that a tagger leaves unlabelled. They
|
|
19
|
+
are special tokens, words that it doesn't cover, and pieces whose text is not in their
|
|
20
|
+
document.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import math
|
|
26
|
+
from bisect import bisect_left
|
|
27
|
+
from collections.abc import Callable, Sequence
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from itertools import accumulate
|
|
30
|
+
from typing import TYPE_CHECKING, Protocol
|
|
31
|
+
|
|
32
|
+
import torch
|
|
33
|
+
|
|
34
|
+
from openinterp.core.confounds import NO_MATCH
|
|
35
|
+
from openinterp.core.schemas import ConfoundProfile, LabelConfound, LabelStat, ReviewPolicy
|
|
36
|
+
|
|
37
|
+
if TYPE_CHECKING:
|
|
38
|
+
from openinterp.harness.scan import Batch, Chunk
|
|
39
|
+
|
|
40
|
+
ACTIVATION_BINS = 8
|
|
41
|
+
MAX_LEVELS = 50 # labels listed in a record, most frequent first
|
|
42
|
+
UNLABELLED = -1
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass(frozen=True)
|
|
46
|
+
class Span:
|
|
47
|
+
"""Characters [start, end) of a document carry ``label``."""
|
|
48
|
+
|
|
49
|
+
start: int
|
|
50
|
+
end: int
|
|
51
|
+
label: str
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Tagger(Protocol):
|
|
55
|
+
"""Anything that labels character spans of a document, under one label set's name."""
|
|
56
|
+
|
|
57
|
+
name: str # the label set's name in the record (``"language"``)
|
|
58
|
+
|
|
59
|
+
def spans(self, doc: int, text: str) -> list[Span]:
|
|
60
|
+
"""The labelled spans of document ``doc``, whose text is ``text``."""
|
|
61
|
+
...
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class DocumentLabels:
|
|
66
|
+
"""One label per document, known in advance (a corpus field such as ``lang``)."""
|
|
67
|
+
|
|
68
|
+
name: str
|
|
69
|
+
labels: Sequence[str | None]
|
|
70
|
+
|
|
71
|
+
def spans(self, doc: int, text: str) -> list[Span]:
|
|
72
|
+
"""One span over the whole text, or none when the document has no label."""
|
|
73
|
+
label = self.labels[doc]
|
|
74
|
+
return [] if label is None else [Span(0, len(text), label)]
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class MatchTagger:
|
|
79
|
+
"""Label what ``find`` matches in a document, and every other character ``NO_MATCH``."""
|
|
80
|
+
|
|
81
|
+
name: str
|
|
82
|
+
find: Callable[[str], Sequence[tuple[int, int, str]]]
|
|
83
|
+
|
|
84
|
+
def spans(self, doc: int, text: str) -> list[Span]:
|
|
85
|
+
"""The matches, then the gaps, so a token half in a match takes the match's label."""
|
|
86
|
+
matches = self.find(text)
|
|
87
|
+
spans = [Span(start, end, label) for start, end, label in matches]
|
|
88
|
+
position = 0
|
|
89
|
+
for start, end, _ in matches:
|
|
90
|
+
if start > position:
|
|
91
|
+
spans.append(Span(position, start, NO_MATCH))
|
|
92
|
+
position = max(position, end)
|
|
93
|
+
if position < len(text):
|
|
94
|
+
spans.append(Span(position, len(text), NO_MATCH))
|
|
95
|
+
return spans
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def align(offsets: Sequence[tuple[int, int]], spans: Sequence[Span]) -> list[str | None]:
|
|
99
|
+
"""Return each token's label: that of the span that covers most of its characters.
|
|
100
|
+
|
|
101
|
+
The first span wins a tie. ``None`` when no span covers the token. Tokens with no
|
|
102
|
+
characters (special tokens) are unlabelled.
|
|
103
|
+
"""
|
|
104
|
+
# A document has about as many syntax spans as tokens, so reading every span for every
|
|
105
|
+
# token is too slow. The spans are sorted by start. A walk back from the token's end
|
|
106
|
+
# stops when no earlier span reaches past the token's start.
|
|
107
|
+
order = sorted(range(len(spans)), key=lambda i: spans[i].start)
|
|
108
|
+
starts = [spans[i].start for i in order]
|
|
109
|
+
reach = list(accumulate((spans[i].end for i in order), max))
|
|
110
|
+
out: list[str | None] = []
|
|
111
|
+
for start, end in offsets:
|
|
112
|
+
best, most, first = None, 0, len(spans)
|
|
113
|
+
i = bisect_left(starts, end) - 1
|
|
114
|
+
while i >= 0 and reach[i] > start:
|
|
115
|
+
span = spans[order[i]]
|
|
116
|
+
overlap = min(end, span.end) - max(start, span.start)
|
|
117
|
+
if overlap > most or (overlap == most and overlap > 0 and order[i] < first):
|
|
118
|
+
best, most, first = span.label, overlap, order[i]
|
|
119
|
+
i -= 1
|
|
120
|
+
out.append(best)
|
|
121
|
+
return out
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class TokenLabels:
|
|
125
|
+
"""The labels of every piece's tokens for each tagger, aligned once per piece."""
|
|
126
|
+
|
|
127
|
+
def __init__(self, tokenizer, texts: Sequence[str], taggers: Sequence[Tagger]) -> None:
|
|
128
|
+
self.tokenizer, self.texts, self.taggers = tokenizer, texts, list(taggers)
|
|
129
|
+
self.names: dict[str, list[str]] = {t.name: [] for t in self.taggers}
|
|
130
|
+
self._index: dict[str, dict[str, int]] = {t.name: {} for t in self.taggers}
|
|
131
|
+
self._spans: dict[tuple[str, int], list[Span]] = {}
|
|
132
|
+
self._pieces: dict = {}
|
|
133
|
+
|
|
134
|
+
def of(self, piece, ids: torch.Tensor) -> dict[str, torch.Tensor]:
|
|
135
|
+
"""``{tagger: [n] label index or UNLABELLED}`` for the piece's real tokens ``ids``."""
|
|
136
|
+
if piece not in self._pieces:
|
|
137
|
+
self._pieces[piece] = {t.name: self._align(t, piece, ids) for t in self.taggers}
|
|
138
|
+
return self._pieces[piece]
|
|
139
|
+
|
|
140
|
+
def at(self, piece, ids: Sequence[int], index: int) -> dict[str, str] | None:
|
|
141
|
+
"""Return each tagger's label of token ``index`` of the piece's real tokens ``ids``.
|
|
142
|
+
|
|
143
|
+
Only the taggers that label it are included. ``None`` if none does.
|
|
144
|
+
"""
|
|
145
|
+
row = self.of(piece, torch.as_tensor(ids, dtype=torch.long))
|
|
146
|
+
found = {name: int(labels[index]) for name, labels in row.items()}
|
|
147
|
+
out = {name: self.names[name][i] for name, i in found.items() if i != UNLABELLED}
|
|
148
|
+
return out or None
|
|
149
|
+
|
|
150
|
+
def _align(self, tagger: Tagger, piece, ids: torch.Tensor) -> torch.Tensor:
|
|
151
|
+
key = (tagger.name, piece.doc)
|
|
152
|
+
if key not in self._spans:
|
|
153
|
+
self._spans[key] = tagger.spans(piece.doc, self.texts[piece.doc])
|
|
154
|
+
labels = self._piece_labels(self._spans[key], piece, ids)
|
|
155
|
+
index = self._index[tagger.name]
|
|
156
|
+
for label in labels:
|
|
157
|
+
if label is not None and label not in index:
|
|
158
|
+
index[label] = len(index)
|
|
159
|
+
self.names[tagger.name].append(label)
|
|
160
|
+
ids = [UNLABELLED if lab is None else index[lab] for lab in labels]
|
|
161
|
+
return torch.tensor(ids, dtype=torch.long)
|
|
162
|
+
|
|
163
|
+
def _piece_labels(self, spans: list[Span], piece, ids: torch.Tensor) -> list[str | None]:
|
|
164
|
+
if not spans: # no tokenising for a document with no labels (code with no grammar)
|
|
165
|
+
return [None] * len(ids)
|
|
166
|
+
special = set(getattr(self.tokenizer, "all_special_ids", None) or [])
|
|
167
|
+
text = self.texts[piece.doc]
|
|
168
|
+
if len(spans) == 1 and spans[0].start == 0 and spans[0].end >= len(text):
|
|
169
|
+
# A document label covers every token: no offsets needed.
|
|
170
|
+
return [None if int(i) in special else spans[0].label for i in ids]
|
|
171
|
+
shift = self._shift(piece, text)
|
|
172
|
+
offsets = self._offsets(piece.text) if shift is not None else None
|
|
173
|
+
if offsets is None or len(offsets) != len(ids):
|
|
174
|
+
return [None] * len(ids)
|
|
175
|
+
return align([(a + shift, b + shift) for a, b in offsets], spans)
|
|
176
|
+
|
|
177
|
+
def _shift(self, piece, text: str) -> int | None:
|
|
178
|
+
"""Return where the piece's text starts in its document.
|
|
179
|
+
|
|
180
|
+
``None`` if it isn't there. A window's decoded text need not match the document's
|
|
181
|
+
characters.
|
|
182
|
+
"""
|
|
183
|
+
if piece.start == 0:
|
|
184
|
+
return 0 if text.startswith(piece.text) else None
|
|
185
|
+
doc_offsets = self._offsets(text, special=False)
|
|
186
|
+
if doc_offsets is None or piece.start >= len(doc_offsets):
|
|
187
|
+
return None
|
|
188
|
+
shift = doc_offsets[piece.start][0]
|
|
189
|
+
return shift if text[shift : shift + len(piece.text)] == piece.text else None
|
|
190
|
+
|
|
191
|
+
def _offsets(self, text: str, special: bool = True) -> list[tuple[int, int]] | None:
|
|
192
|
+
"""Return the character offsets of ``text``'s tokens as the scan encodes them.
|
|
193
|
+
|
|
194
|
+
``None`` if the tokenizer can't give them. Only fast tokenizers can.
|
|
195
|
+
"""
|
|
196
|
+
try:
|
|
197
|
+
enc = self.tokenizer(text, return_offsets_mapping=True, add_special_tokens=special)
|
|
198
|
+
except (TypeError, NotImplementedError):
|
|
199
|
+
return None
|
|
200
|
+
return [tuple(o) for o in enc["offset_mapping"]]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class LabelAccumulator:
|
|
204
|
+
"""Pole strengths and label indices of every counted token, for one label set."""
|
|
205
|
+
|
|
206
|
+
def __init__(self) -> None:
|
|
207
|
+
self.labels: list[torch.Tensor] = []
|
|
208
|
+
self.values: list[torch.Tensor] = []
|
|
209
|
+
|
|
210
|
+
def update(self, labels: torch.Tensor, strengths: torch.Tensor) -> None:
|
|
211
|
+
"""Add a batch of label indices and strengths."""
|
|
212
|
+
self.labels.append(labels.long())
|
|
213
|
+
self.values.append(strengths.double())
|
|
214
|
+
|
|
215
|
+
def finalize(self, names: Sequence[str], line: float) -> LabelConfound:
|
|
216
|
+
"""The confound, counting a token active at or above ``line``."""
|
|
217
|
+
labels = torch.cat(self.labels) if self.labels else torch.zeros(0, dtype=torch.long)
|
|
218
|
+
values = torch.cat(self.values) if self.values else torch.zeros(0, dtype=torch.float64)
|
|
219
|
+
seen = labels != UNLABELLED
|
|
220
|
+
labels, values = labels[seen], values[seen]
|
|
221
|
+
size = len(names)
|
|
222
|
+
n = torch.bincount(labels, minlength=size).double()
|
|
223
|
+
s = torch.bincount(labels, weights=values, minlength=size)
|
|
224
|
+
active = torch.bincount(labels[values >= line], minlength=size) if line > 0 else n * 0
|
|
225
|
+
order = n.argsort(descending=True)[:MAX_LEVELS]
|
|
226
|
+
return LabelConfound(
|
|
227
|
+
r2=round(_r2(values, n, s), 6),
|
|
228
|
+
mutual_information_bits=round(mutual_information(values, labels, size), 6),
|
|
229
|
+
labelled_tokens=int(labels.numel()),
|
|
230
|
+
unlabelled_tokens=int((~seen).sum()),
|
|
231
|
+
levels=[
|
|
232
|
+
LabelStat(
|
|
233
|
+
label=names[i],
|
|
234
|
+
count=int(n[i]),
|
|
235
|
+
mean_strength=round(max(0.0, float(s[i] / n[i])), 6),
|
|
236
|
+
active=int(active[i]),
|
|
237
|
+
)
|
|
238
|
+
for i in order.tolist()
|
|
239
|
+
if n[i] > 0
|
|
240
|
+
],
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _r2(values: torch.Tensor, n: torch.Tensor, s: torch.Tensor) -> float:
|
|
245
|
+
"""Between-label sum of squares over total sum of squares."""
|
|
246
|
+
count = values.numel()
|
|
247
|
+
if count == 0:
|
|
248
|
+
return 0.0
|
|
249
|
+
total = float((values * values).sum()) - float(values.sum()) ** 2 / count
|
|
250
|
+
seen = n > 0
|
|
251
|
+
between = float((s[seen] ** 2 / n[seen]).sum()) - float(values.sum()) ** 2 / count
|
|
252
|
+
return min(1.0, max(0.0, between / total)) if total > 1e-12 else 0.0
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def activation_bins(values: torch.Tensor, bins: int = ACTIVATION_BINS) -> torch.Tensor:
|
|
256
|
+
"""Return each value's bin: 0 for zero strength, then equal-count bins of the rest.
|
|
257
|
+
|
|
258
|
+
There are ``bins`` bins of the positive strengths, or fewer where ties make edges
|
|
259
|
+
coincide.
|
|
260
|
+
"""
|
|
261
|
+
out = torch.zeros(values.numel(), dtype=torch.long)
|
|
262
|
+
positive = values > 0
|
|
263
|
+
if positive.any():
|
|
264
|
+
q = torch.linspace(0, 1, bins + 1, dtype=torch.float64)[1:-1]
|
|
265
|
+
edges = torch.quantile(values[positive].double(), q).unique()
|
|
266
|
+
out[positive] = 1 + torch.bucketize(values[positive], edges, right=True)
|
|
267
|
+
return out
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def mutual_information(values: torch.Tensor, labels: torch.Tensor, n_labels: int) -> float:
|
|
271
|
+
"""I(binned strength; label) in bits, each entropy Miller–Madow corrected; at least 0."""
|
|
272
|
+
count = values.numel()
|
|
273
|
+
if count == 0 or n_labels < 2:
|
|
274
|
+
return 0.0
|
|
275
|
+
bins = activation_bins(values)
|
|
276
|
+
joint = torch.bincount(bins * n_labels + labels).double()
|
|
277
|
+
mi = _entropy(torch.bincount(bins).double(), count)
|
|
278
|
+
mi += _entropy(torch.bincount(labels).double(), count) - _entropy(joint, count)
|
|
279
|
+
return max(0.0, mi / math.log(2))
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _entropy(counts: torch.Tensor, total: int) -> float:
|
|
283
|
+
"""Miller–Madow entropy in nats."""
|
|
284
|
+
counts = counts[counts > 0]
|
|
285
|
+
p = counts / total
|
|
286
|
+
return float(-(p * p.log()).sum()) + (counts.numel() - 1) / (2 * total)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
class LabelPass:
|
|
290
|
+
"""Each tagger's labels of a scan's counted tokens, with their strengths.
|
|
291
|
+
|
|
292
|
+
They are gathered pass by pass. They give the label confounds (see
|
|
293
|
+
``harness/annotations.py``).
|
|
294
|
+
"""
|
|
295
|
+
|
|
296
|
+
def __init__(self, tokenizer, texts: Sequence[str], taggers: Sequence[Tagger]) -> None:
|
|
297
|
+
self.tokens = TokenLabels(tokenizer, texts, taggers)
|
|
298
|
+
self.accumulators = {t.name: LabelAccumulator() for t in self.tokens.taggers}
|
|
299
|
+
|
|
300
|
+
def update_chunk(self, chunk: Chunk, strengths: torch.Tensor) -> None:
|
|
301
|
+
"""Add a chunk's counted tokens; ``strengths`` is ``[n]``, one per real token."""
|
|
302
|
+
if self.accumulators:
|
|
303
|
+
rows = [self.tokens.of(piece, chunk.ids[r]) for piece, r in chunk.rows()]
|
|
304
|
+
self._add(rows, chunk.counted, strengths)
|
|
305
|
+
|
|
306
|
+
def update(self, batch: Batch, strengths: torch.Tensor) -> None:
|
|
307
|
+
"""Add a batch's counted tokens; ``strengths`` is ``[b, seq]``, padding included."""
|
|
308
|
+
if self.accumulators:
|
|
309
|
+
mask = batch.mask
|
|
310
|
+
rows = [self.tokens.of(p, batch.ids[i][mask[i]]) for i, p in enumerate(batch.pieces)]
|
|
311
|
+
self._add(rows, batch.counted[mask], strengths.cpu()[mask])
|
|
312
|
+
|
|
313
|
+
def _add(self, rows, counted: torch.Tensor, strengths: torch.Tensor) -> None:
|
|
314
|
+
for name, accumulator in self.accumulators.items():
|
|
315
|
+
every = torch.cat([row[name] for row in rows])
|
|
316
|
+
accumulator.update(every[counted], strengths[counted])
|
|
317
|
+
|
|
318
|
+
def finalize(self, confounds: ConfoundProfile, values: Sequence[float]) -> ConfoundProfile:
|
|
319
|
+
"""Return ``confounds`` with each label set's confound added.
|
|
320
|
+
|
|
321
|
+
A token counts as active at or above the protocol's line for the scan's settled
|
|
322
|
+
``values``.
|
|
323
|
+
"""
|
|
324
|
+
if not self.accumulators:
|
|
325
|
+
return confounds
|
|
326
|
+
line = ReviewPolicy().activation_threshold * max(values, default=0.0)
|
|
327
|
+
names = self.tokens.names
|
|
328
|
+
by_label = {n: a.finalize(names[n], line) for n, a in self.accumulators.items()}
|
|
329
|
+
return confounds.model_copy(update={"labels": by_label})
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Attribution patching: a first-order approximation of activation patching.
|
|
2
|
+
|
|
3
|
+
Take every neuron in the requested layers. The effect of patching its clean activation
|
|
4
|
+
into the corrupted run is approximated as follows:
|
|
5
|
+
|
|
6
|
+
(a_clean - a_corrupted) * d metric / d a |_corrupted
|
|
7
|
+
|
|
8
|
+
This needs one clean forward pass and one corrupted forward + backward pass, not one
|
|
9
|
+
forward pass per neuron. Use it to triage candidates, then confirm
|
|
10
|
+
with exact patching in :mod:`openinterp.harness.patcher`.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from openinterp.core.exceptions import ShapeMismatchError
|
|
20
|
+
from openinterp.core.schemas import Pole
|
|
21
|
+
from openinterp.harness.hooks import Inputs, NeuronHookRunner, isolate_pole
|
|
22
|
+
from openinterp.harness.patcher import Metric, Positions, token_logit_metric
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def attribution_patching(
|
|
26
|
+
runner: NeuronHookRunner,
|
|
27
|
+
clean: Inputs,
|
|
28
|
+
corrupted: Inputs,
|
|
29
|
+
layers: Sequence[int],
|
|
30
|
+
*,
|
|
31
|
+
metric: Metric | None = None,
|
|
32
|
+
pole: Pole | None = None,
|
|
33
|
+
positions: Positions = "all",
|
|
34
|
+
) -> dict[int, torch.Tensor]:
|
|
35
|
+
"""Estimated metric change per neuron, ``{layer: tensor[d_mlp]}``.
|
|
36
|
+
|
|
37
|
+
With ``pole`` set, only that pole's activation difference is attributed. The
|
|
38
|
+
default metric is the logit of the clean run's top-1 final-position token.
|
|
39
|
+
"""
|
|
40
|
+
clean_logits, clean_cache = runner.run_with_cache(clean, layers)
|
|
41
|
+
if metric is None:
|
|
42
|
+
metric = token_logit_metric(int(clean_logits[0, -1].argmax()))
|
|
43
|
+
|
|
44
|
+
ordered, grads = _corrupted_grads(runner, corrupted, layers, metric)
|
|
45
|
+
scores: dict[int, torch.Tensor] = {}
|
|
46
|
+
for layer, corrupt_act, grad in zip(layers, ordered, grads, strict=True):
|
|
47
|
+
clean_act, corrupt_act = clean_cache[layer], corrupt_act.detach()
|
|
48
|
+
if positions == "last":
|
|
49
|
+
clean_act, corrupt_act, grad = clean_act[:, -1:], corrupt_act[:, -1:], grad[:, -1:]
|
|
50
|
+
elif clean_act.shape != corrupt_act.shape:
|
|
51
|
+
raise ShapeMismatchError(
|
|
52
|
+
f"clean {tuple(clean_act.shape)} vs corrupted {tuple(corrupt_act.shape)}; "
|
|
53
|
+
"use positions='last' or align the prompts."
|
|
54
|
+
)
|
|
55
|
+
if pole is not None:
|
|
56
|
+
clean_act, corrupt_act = isolate_pole(clean_act, pole), isolate_pole(corrupt_act, pole)
|
|
57
|
+
scores[layer] = ((clean_act - corrupt_act) * grad).sum(dim=(0, 1)).detach().cpu()
|
|
58
|
+
return scores
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _corrupted_grads(runner: NeuronHookRunner, corrupted: Inputs, layers, metric: Metric):
|
|
62
|
+
"""Each layer's MLP activations on ``corrupted`` and the metric's gradient there."""
|
|
63
|
+
captured: dict[int, torch.Tensor] = {}
|
|
64
|
+
|
|
65
|
+
def capture(layer: int):
|
|
66
|
+
def hook(act: torch.Tensor) -> None:
|
|
67
|
+
captured[layer] = act
|
|
68
|
+
|
|
69
|
+
return hook
|
|
70
|
+
|
|
71
|
+
def ensure_grad(_module, _args, output: torch.Tensor) -> torch.Tensor:
|
|
72
|
+
# Frozen weights mean nothing upstream requires grad; make the embedding
|
|
73
|
+
# output a fresh leaf so the graph through the MLPs is recorded.
|
|
74
|
+
return output if output.requires_grad else output.detach().requires_grad_(True)
|
|
75
|
+
|
|
76
|
+
encoded = runner.encode(corrupted)
|
|
77
|
+
embed_handle = runner.model.get_input_embeddings().register_forward_hook(ensure_grad)
|
|
78
|
+
try:
|
|
79
|
+
with torch.enable_grad(), runner.hooks({layer: capture(layer) for layer in layers}):
|
|
80
|
+
value = metric(runner.forward(encoded))
|
|
81
|
+
ordered = [captured[layer] for layer in layers]
|
|
82
|
+
grads = torch.autograd.grad(value, ordered)
|
|
83
|
+
finally:
|
|
84
|
+
embed_handle.remove()
|
|
85
|
+
return ordered, grads
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def top_attributed_neurons(
|
|
89
|
+
scores: dict[int, torch.Tensor], k: int = 10
|
|
90
|
+
) -> list[tuple[int, int, float]]:
|
|
91
|
+
"""Flatten attribution scores to ``(layer, neuron_idx, score)`` sorted by |score|."""
|
|
92
|
+
flat: list[tuple[int, int, float]] = []
|
|
93
|
+
for layer, vec in scores.items():
|
|
94
|
+
for idx in vec.abs().topk(min(k, vec.numel())).indices.tolist():
|
|
95
|
+
flat.append((layer, idx, float(vec[idx])))
|
|
96
|
+
return sorted(flat, key=lambda t: abs(t[2]), reverse=True)[:k]
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Causal metrics of a unit: sufficiency and necessity, ranked against comparable units.
|
|
2
|
+
|
|
3
|
+
Recorded in every verdict, never gated (docs/PROTOCOL.md §4). A neuron pole is ranked
|
|
4
|
+
against the same layer's neurons. An attention head is ranked against the same layer's
|
|
5
|
+
heads. A dictionary feature is ranked against the same dictionary's other latents.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import statistics
|
|
12
|
+
from collections.abc import Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import torch
|
|
17
|
+
|
|
18
|
+
from openinterp.core.schemas import (
|
|
19
|
+
CausalMetrics,
|
|
20
|
+
CausalPairEffect,
|
|
21
|
+
CausalTarget,
|
|
22
|
+
NeuronAuditSubmission,
|
|
23
|
+
PatchEffect,
|
|
24
|
+
)
|
|
25
|
+
from openinterp.harness.hooks import NeuronHookRunner, pole_strength
|
|
26
|
+
from openinterp.harness.layer_major import map_layer_major, prompt_tokens
|
|
27
|
+
from openinterp.harness.patcher import (
|
|
28
|
+
ActivationPatcher,
|
|
29
|
+
Positions,
|
|
30
|
+
logit_direction_metric,
|
|
31
|
+
metric_change,
|
|
32
|
+
next_token_kl,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def claimed_token_weights(
|
|
37
|
+
submission: NeuronAuditSubmission, runner: NeuronHookRunner
|
|
38
|
+
) -> tuple[torch.Tensor, list[str]] | None:
|
|
39
|
+
"""Equal vocabulary weights on a lookahead claim's single-token targets, if any resolve."""
|
|
40
|
+
tokenizer = runner.tokenizer
|
|
41
|
+
if not submission.lookahead.is_lookahead or tokenizer is None:
|
|
42
|
+
return None
|
|
43
|
+
vocab = runner.model.get_output_embeddings().weight.shape[0]
|
|
44
|
+
ids: dict[int, str] = {}
|
|
45
|
+
for token in submission.lookahead.target_upcoming_tokens:
|
|
46
|
+
encoded = tokenizer.encode(token, add_special_tokens=False)
|
|
47
|
+
if len(encoded) == 1 and encoded[0] < vocab:
|
|
48
|
+
ids.setdefault(encoded[0], token)
|
|
49
|
+
if not ids:
|
|
50
|
+
return None
|
|
51
|
+
weights = torch.zeros(vocab)
|
|
52
|
+
weights[list(ids)] = 1.0 / len(ids)
|
|
53
|
+
return weights, list(ids.values())
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _neuron_changes(runner, ident, base: str, cf: str) -> tuple[Positions, int, torch.Tensor]:
|
|
57
|
+
"""Where to patch and score, and every neuron's strength change up to there."""
|
|
58
|
+
b, c = ( # the pass stops at the layer: no later layer or logits is needed
|
|
59
|
+
pole_strength(runner.layer_activations(text, [ident.layer])[ident.layer][0], ident.pole)
|
|
60
|
+
for text in (base, cf)
|
|
61
|
+
) # [seq, d_mlp]
|
|
62
|
+
if b.shape != c.shape:
|
|
63
|
+
return "last", -1, (b[-1] - c[-1]).abs()
|
|
64
|
+
position = int((b[:, ident.neuron_idx] - c[:, ident.neuron_idx]).abs().argmax())
|
|
65
|
+
return "all", position, (b[: position + 1] - c[: position + 1]).abs().amax(0)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _head_changes(runner, ident, base: str, cf: str) -> tuple[Positions, int, torch.Tensor]:
|
|
69
|
+
"""The same for heads, with each head's change in write ``|W_O,h (z_base − z_cf)|``.
|
|
70
|
+
|
|
71
|
+
The write change needs no head's mean output, so every head in the layer compares.
|
|
72
|
+
"""
|
|
73
|
+
from openinterp.harness.heads import head_write_norms
|
|
74
|
+
|
|
75
|
+
z_b, z_c = (runner.head_outputs(text, ident.layer)[0].float() for text in (base, cf))
|
|
76
|
+
if z_b.shape != z_c.shape:
|
|
77
|
+
return "last", -1, head_write_norms(runner, ident.layer, z_b[-1] - z_c[-1])
|
|
78
|
+
changes = head_write_norms(runner, ident.layer, z_b - z_c) # [seq, n_heads]
|
|
79
|
+
position = int(changes[:, ident.head_idx].argmax())
|
|
80
|
+
return "all", position, changes[: position + 1].amax(0)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def causal_metrics(
|
|
84
|
+
submission: NeuronAuditSubmission,
|
|
85
|
+
runner: NeuronHookRunner,
|
|
86
|
+
*,
|
|
87
|
+
freeze_downstream: bool = True,
|
|
88
|
+
null_neurons: int = 32,
|
|
89
|
+
min_change: float = 0.0,
|
|
90
|
+
) -> CausalMetrics | None:
|
|
91
|
+
"""Sufficiency and necessity of the unit, each ranked against comparable units.
|
|
92
|
+
|
|
93
|
+
Every minimal pair is patched clean → corrupted (sufficiency) and corrupted → clean
|
|
94
|
+
(necessity). The patch is at the position where the unit differs most between the two
|
|
95
|
+
prompts. When their lengths differ, it is the final position, since only that can be
|
|
96
|
+
patched. A neuron pole's difference is its strength change. A head's or a dictionary
|
|
97
|
+
latent's difference is the change in its write to the residual stream. The effect is
|
|
98
|
+
scored there on a target that favours no unit:
|
|
99
|
+
|
|
100
|
+
* ``claimed_tokens``: a lookahead claim's upcoming tokens, as their mean logit minus
|
|
101
|
+
the vocabulary mean (signed);
|
|
102
|
+
* ``next_token_kl``: otherwise, KL(unpatched ‖ patched) of the next-token distribution.
|
|
103
|
+
|
|
104
|
+
The same patch is applied, one unit at a time, to ``null_neurons`` comparable units.
|
|
105
|
+
They are the units of the same kind and layer whose change between the two prompts is
|
|
106
|
+
most similar. That is neurons, the other heads, or the dictionary's other latents. A score is
|
|
107
|
+
the median over pairs of the unit's |effect| percentile among theirs. So 0.5 means
|
|
108
|
+
"no more influential than a unit that changes as much".
|
|
109
|
+
|
|
110
|
+
Pairs where the unit moves by at most ``min_change`` (controls) carry no causal
|
|
111
|
+
signal and are skipped; with none left the result is ``None``.
|
|
112
|
+
|
|
113
|
+
Two alternatives read wrongly on real models. Dividing by the whole clean/corrupted
|
|
114
|
+
gap (protocol < 0.5) reads ≈ 0 for any single neuron, because the gap sums every
|
|
115
|
+
component's contribution. Ranking against *random* neurons reads ≈ 1, because minimal
|
|
116
|
+
pairs are built to change this neuron and most others barely change at all.
|
|
117
|
+
"""
|
|
118
|
+
claimed = claimed_token_weights(submission, runner)
|
|
119
|
+
target: CausalTarget = "claimed_tokens" if claimed else "next_token_kl"
|
|
120
|
+
setup = _PairSetup(
|
|
121
|
+
submission.identifier, *_unit_patching(submission, runner), claimed,
|
|
122
|
+
min_change, null_neurons, freeze_downstream,
|
|
123
|
+
) # fmt: skip
|
|
124
|
+
items = list(enumerate(submission.counterfactuals))
|
|
125
|
+
texts = [t for _, p in items for t in (p.base_prompt, p.counterfactual_prompt)]
|
|
126
|
+
results = map_layer_major(
|
|
127
|
+
runner, setup.patched, items, tokens=prompt_tokens(runner, texts)
|
|
128
|
+
) # the pairs run at once when the model streams (harness/layer_major.py)
|
|
129
|
+
scores: dict[str, list[float]] = {"sufficiency": [], "necessity": []}
|
|
130
|
+
effects: dict[str, list[float]] = {"sufficiency": [], "necessity": []}
|
|
131
|
+
nulls: list[int] = []
|
|
132
|
+
pairs: list[CausalPairEffect] = []
|
|
133
|
+
for index, own, n_null in filter(None, results):
|
|
134
|
+
nulls.append(n_null)
|
|
135
|
+
patched: dict[str, PatchEffect] = {}
|
|
136
|
+
for kind, (unit_effect, *others) in own.items():
|
|
137
|
+
patched[kind], percentile = _patch_effect(unit_effect, others)
|
|
138
|
+
effects[kind].append(unit_effect)
|
|
139
|
+
scores[kind].append(percentile)
|
|
140
|
+
pairs.append(CausalPairEffect(pair_index=index, **patched))
|
|
141
|
+
|
|
142
|
+
if not effects["sufficiency"]:
|
|
143
|
+
return None
|
|
144
|
+
tokens = claimed[1] if claimed else []
|
|
145
|
+
return _metrics(scores, effects, nulls, pairs, target, tokens, freeze_downstream)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@dataclass(frozen=True)
|
|
149
|
+
class _PairSetup:
|
|
150
|
+
"""What patching one counterfactual pair needs (:func:`causal_metrics`)."""
|
|
151
|
+
|
|
152
|
+
ident: Any
|
|
153
|
+
changes: Any
|
|
154
|
+
patch: Any
|
|
155
|
+
claimed: Any
|
|
156
|
+
min_change: float
|
|
157
|
+
null_neurons: int
|
|
158
|
+
freeze_downstream: bool
|
|
159
|
+
|
|
160
|
+
def patched(self, item: tuple[int, Any]) -> tuple[int, dict, int] | None:
|
|
161
|
+
"""One pair's unit and null effects per direction, or None if it isn't scored."""
|
|
162
|
+
index, pair = item
|
|
163
|
+
positions, position, change = self.changes(pair.base_prompt, pair.counterfactual_prompt)
|
|
164
|
+
if float(change[self.ident.index]) <= self.min_change:
|
|
165
|
+
return None
|
|
166
|
+
null = _matched_units(change, self.ident.index, self.null_neurons)
|
|
167
|
+
effect = (
|
|
168
|
+
metric_change(logit_direction_metric(self.claimed[0], position))
|
|
169
|
+
if self.claimed
|
|
170
|
+
else next_token_kl(position)
|
|
171
|
+
)
|
|
172
|
+
runs = {
|
|
173
|
+
"sufficiency": (pair.base_prompt, pair.counterfactual_prompt),
|
|
174
|
+
"necessity": (pair.counterfactual_prompt, pair.base_prompt),
|
|
175
|
+
}
|
|
176
|
+
options = {
|
|
177
|
+
"effect": effect, "positions": positions, "freeze_downstream": self.freeze_downstream
|
|
178
|
+
} # fmt: skip
|
|
179
|
+
units = [self.ident.index, *null]
|
|
180
|
+
own = {kind: self.patch(src, dst, units, **options) for kind, (src, dst) in runs.items()}
|
|
181
|
+
return index, own, len(null)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _metrics(scores, effects, nulls, pairs, target, tokens, freeze_downstream) -> CausalMetrics:
|
|
185
|
+
"""The medians over the scored pairs."""
|
|
186
|
+
return CausalMetrics(
|
|
187
|
+
sufficiency_score=_robust_unit(scores["sufficiency"]),
|
|
188
|
+
necessity_score=_robust_unit(scores["necessity"]),
|
|
189
|
+
hydra_mitigated=freeze_downstream,
|
|
190
|
+
target=target,
|
|
191
|
+
target_tokens=tokens,
|
|
192
|
+
sufficiency_effect=round(statistics.median(effects["sufficiency"]), 9),
|
|
193
|
+
necessity_effect=round(statistics.median(effects["necessity"]), 9),
|
|
194
|
+
null_neurons=min(nulls),
|
|
195
|
+
pairs_scored=len(effects["sufficiency"]),
|
|
196
|
+
pairs=pairs,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _patch_effect(own: float, others: list[float]) -> tuple[PatchEffect, float]:
|
|
201
|
+
"""The unit's patch effect against its matched units', and its |effect| percentile."""
|
|
202
|
+
null_effects = sorted(abs(o) for o in others)
|
|
203
|
+
percentile = _percentile(abs(own), null_effects)
|
|
204
|
+
effect = PatchEffect(
|
|
205
|
+
effect=round(own, 9),
|
|
206
|
+
percentile=round(percentile, 6) if math.isfinite(percentile) else None,
|
|
207
|
+
null_median=round(statistics.median(null_effects), 9) if null_effects else 0.0,
|
|
208
|
+
null_max=round(null_effects[-1], 9) if null_effects else 0.0,
|
|
209
|
+
)
|
|
210
|
+
return effect, percentile
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _unit_patching(submission: NeuronAuditSubmission, runner: NeuronHookRunner):
|
|
214
|
+
"""Return ``changes(base, cf)`` and ``patch(source, target, units, **options)``.
|
|
215
|
+
|
|
216
|
+
They are the functions for the kind of unit that ``submission`` is about.
|
|
217
|
+
"""
|
|
218
|
+
from openinterp.harness.features import patch_features, write_changes
|
|
219
|
+
from openinterp.harness.heads import patch_heads
|
|
220
|
+
from openinterp.harness.units import FeatureUnit, submission_unit
|
|
221
|
+
|
|
222
|
+
ident, unit = submission.identifier, submission_unit(submission)
|
|
223
|
+
if isinstance(unit, FeatureUnit):
|
|
224
|
+
dictionary, site = unit.dictionary(runner), unit.profile.input_site
|
|
225
|
+
return (
|
|
226
|
+
lambda base, cf: write_changes(runner, dictionary, ident, site, base, cf),
|
|
227
|
+
lambda source, target, units, **options: patch_features(
|
|
228
|
+
runner, dictionary, site, source, target, ident.layer, units, **options
|
|
229
|
+
),
|
|
230
|
+
)
|
|
231
|
+
if ident.kind == "attn_head":
|
|
232
|
+
return (
|
|
233
|
+
lambda base, cf: _head_changes(runner, ident, base, cf),
|
|
234
|
+
lambda source, target, units, **options: patch_heads(
|
|
235
|
+
runner, source, target, ident.layer, units, **options
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
patcher = ActivationPatcher(runner)
|
|
239
|
+
return (
|
|
240
|
+
lambda base, cf: _neuron_changes(runner, ident, base, cf),
|
|
241
|
+
lambda source, target, units, **options: patcher.patch_many(
|
|
242
|
+
source, target, ident.layer, units, ident.pole, **options
|
|
243
|
+
),
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _matched_units(change: torch.Tensor, index: int, k: int) -> list[int]:
|
|
248
|
+
"""The ``k`` other units whose ``change`` is closest to unit ``index``'s."""
|
|
249
|
+
order = (change - change[index]).abs().argsort(stable=True).tolist()
|
|
250
|
+
return [n for n in order if n != index][:k]
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _percentile(value: float, null: Sequence[float]) -> float:
|
|
254
|
+
"""Fraction of ``null`` below ``value``, counting ties as half."""
|
|
255
|
+
if not null:
|
|
256
|
+
return math.nan
|
|
257
|
+
below = sum(n < value for n in null)
|
|
258
|
+
ties = sum(n == value for n in null)
|
|
259
|
+
return (below + 0.5 * ties) / len(null)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _robust_unit(values: Sequence[float]) -> float:
|
|
263
|
+
finite = [v for v in values if math.isfinite(v)]
|
|
264
|
+
return round(min(1.0, max(0.0, statistics.median(finite))), 6) if finite else 0.0
|