open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Schemas for comparing a neuron pole with linear probes for its concept.
|
|
2
|
+
|
|
3
|
+
A pole verified for a concept can be the model's whole representation of it, or only
|
|
4
|
+
part of one. Probes trained on labelled tokens show which. There are two kinds of probe.
|
|
5
|
+
|
|
6
|
+
- Logistic and mass-mean probes (Marks & Tegmark, 2023), on the layer's MLP hidden
|
|
7
|
+
activations and on the residual stream after the layer.
|
|
8
|
+
- k-sparse probes over the hidden layer's neurons (Gurnee et al., 2023).
|
|
9
|
+
|
|
10
|
+
A rule over the text gives the labels, and the rule never looks at the model. Accuracy
|
|
11
|
+
is measured on held-out documents, ideally from another source. Reported, not gated
|
|
12
|
+
(docs/PROTOCOL.md §4).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Literal
|
|
18
|
+
|
|
19
|
+
from pydantic import Field
|
|
20
|
+
|
|
21
|
+
from openinterp.core.strict import StrictModel, UnitFloat
|
|
22
|
+
|
|
23
|
+
ProbeSite = Literal["mlp_hidden", "residual"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ProbeScore(StrictModel):
|
|
27
|
+
"""How well a score separates the held-out tokens' labels."""
|
|
28
|
+
|
|
29
|
+
balanced_accuracy: UnitFloat = Field(
|
|
30
|
+
description="At the threshold that maximises balanced accuracy on the training tokens."
|
|
31
|
+
)
|
|
32
|
+
auroc: UnitFloat
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class NeuronScore(ProbeScore):
|
|
36
|
+
"""The pole's strength alone, read as a one-feature probe."""
|
|
37
|
+
|
|
38
|
+
line: float = Field(
|
|
39
|
+
gt=0.0, description="The protocol's line: activation_threshold × the record's scale."
|
|
40
|
+
)
|
|
41
|
+
balanced_accuracy_at_line: UnitFloat = Field(
|
|
42
|
+
description="Counting a token as the concept when the pole is active at the line, "
|
|
43
|
+
"as a review reads a probe: nothing is fitted."
|
|
44
|
+
)
|
|
45
|
+
rank: int = Field(
|
|
46
|
+
ge=1,
|
|
47
|
+
description="The neuron's rank among the layer's by standardised class-mean "
|
|
48
|
+
"difference on the training tokens (1 = the most separating).",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SiteProbes(StrictModel):
|
|
53
|
+
"""Dense probes on every coordinate of one site."""
|
|
54
|
+
|
|
55
|
+
site: ProbeSite
|
|
56
|
+
width: int = Field(gt=0)
|
|
57
|
+
logistic: ProbeScore
|
|
58
|
+
mass_mean: ProbeScore
|
|
59
|
+
logistic_cosine: float = Field(
|
|
60
|
+
ge=-1.0,
|
|
61
|
+
le=1.0,
|
|
62
|
+
description="Cosine between the logistic probe's direction and the pole's: its "
|
|
63
|
+
"signed basis vector on the hidden layer, its signed output weights on the residual.",
|
|
64
|
+
)
|
|
65
|
+
mass_mean_cosine: float = Field(ge=-1.0, le=1.0)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class SparsePoint(StrictModel):
|
|
69
|
+
"""A logistic probe on the k neurons that best separate the training tokens."""
|
|
70
|
+
|
|
71
|
+
k: int = Field(ge=1)
|
|
72
|
+
score: ProbeScore
|
|
73
|
+
includes_neuron: bool = Field(description="The audited neuron is among the k.")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ProbeComparison(StrictModel):
|
|
77
|
+
"""One pole against probes for its concept, on the same held-out tokens."""
|
|
78
|
+
|
|
79
|
+
slug: str
|
|
80
|
+
explanation: str
|
|
81
|
+
concept: str = Field(description="What the labels mark, as their rule states it.")
|
|
82
|
+
labels: str = Field(description="The labelled file(s) the tokens came from.")
|
|
83
|
+
split: Literal["source", "document"] = Field(
|
|
84
|
+
description="source: trained on one source, tested on another; document: held-out "
|
|
85
|
+
"documents of the same source."
|
|
86
|
+
)
|
|
87
|
+
train_source: str
|
|
88
|
+
test_source: str
|
|
89
|
+
window_tokens: int = Field(gt=0)
|
|
90
|
+
seed: int
|
|
91
|
+
l2: float = Field(ge=0.0, description="Logistic penalty on standardised features.")
|
|
92
|
+
train_positive: int = Field(ge=0)
|
|
93
|
+
train_negative: int = Field(ge=0)
|
|
94
|
+
test_positive: int = Field(ge=0)
|
|
95
|
+
test_negative: int = Field(ge=0)
|
|
96
|
+
neuron: NeuronScore
|
|
97
|
+
sites: list[SiteProbes]
|
|
98
|
+
sparse: list[SparsePoint]
|
|
99
|
+
neurons_needed: int | None = Field(
|
|
100
|
+
default=None,
|
|
101
|
+
description="Smallest k whose sparse probe comes within 0.02 balanced accuracy of "
|
|
102
|
+
"the better dense hidden-layer probe; None if no k tried does.",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class ConceptSource(StrictModel):
|
|
107
|
+
"""Where a labelled concept set came from.
|
|
108
|
+
|
|
109
|
+
``open-interpretability concepts`` writes it next to the set. It holds enough to rebuild the
|
|
110
|
+
same file. It records the dataset at an exact commit, the split's files, the label
|
|
111
|
+
column and label, the seed and the sizes. ``license`` comes from the dataset card, or for a
|
|
112
|
+
Universal Dependencies treebank, from the treebank.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
preset: str = Field(examples=["conll2003-ner"])
|
|
116
|
+
dataset: str = Field(min_length=1, examples=["eriktks/conll2003"])
|
|
117
|
+
revision: str = Field(description="Dataset commit hash.")
|
|
118
|
+
files: str = Field(description="Glob of the split's Parquet files, within the dataset.")
|
|
119
|
+
column: str = Field(description="The column labels were read from.", examples=["ner_tags"])
|
|
120
|
+
label: str = Field(
|
|
121
|
+
description="The label marked: tags equal to it, or to it with a BIO prefix or a "
|
|
122
|
+
"subtype (B-PER, nmod:poss), or the documents of that class.",
|
|
123
|
+
examples=["PER"],
|
|
124
|
+
)
|
|
125
|
+
level: Literal["token", "document"] = Field(
|
|
126
|
+
description="token: spans over tagged words; document: a span over the whole text "
|
|
127
|
+
"of each document of the class, half the documents being of it."
|
|
128
|
+
)
|
|
129
|
+
documents: int = Field(ge=0)
|
|
130
|
+
positive_documents: int = Field(ge=0, description="Documents with at least one span.")
|
|
131
|
+
spans: int = Field(ge=0)
|
|
132
|
+
skipped_sentences: int = Field(
|
|
133
|
+
default=0,
|
|
134
|
+
ge=0,
|
|
135
|
+
description="Sentences left out because their words weren't found in their text.",
|
|
136
|
+
)
|
|
137
|
+
seed: int
|
|
138
|
+
max_chars: int = Field(gt=0)
|
|
139
|
+
license: str | None = None
|
|
140
|
+
sha256: str = Field(description="SHA-256 of the concept set file this describes.")
|
|
141
|
+
|
|
142
|
+
def __str__(self) -> str:
|
|
143
|
+
return (
|
|
144
|
+
f"{self.dataset}@{self.revision[:8]} ({self.files}, {self.column} = {self.label}: "
|
|
145
|
+
f"{self.spans} spans in {self.positive_documents} of {self.documents} docs)"
|
|
146
|
+
)
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""What defines the strength of a unit that is not a neuron pole.
|
|
2
|
+
|
|
3
|
+
For an attention head it is the head's mean output. For a dictionary feature it is the
|
|
4
|
+
dictionary. For a superposition it is its members and their weights. Stored in the
|
|
5
|
+
record, so every later measurement of the experiment uses the same definition.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
from typing import Literal
|
|
13
|
+
|
|
14
|
+
from pydantic import Field, model_validator
|
|
15
|
+
|
|
16
|
+
from openinterp.core.strict import StrictModel, UnitFloat
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class HeadProfile(StrictModel):
|
|
20
|
+
"""What defines an attention head's activity, and its standard attention-pattern scores.
|
|
21
|
+
|
|
22
|
+
A head's strength at token t is ``|W_O (z_t − mean_output)|``. This is the distance
|
|
23
|
+
from its write to the residual stream to its mean write over the scan. It is also how
|
|
24
|
+
far mean-ablating the head would move the residual stream at t. The scan measures
|
|
25
|
+
``mean_output`` (z̄, one value per head dimension) once. Every later measurement of
|
|
26
|
+
the experiment uses this stored value, so a record reproduces without the corpus.
|
|
27
|
+
|
|
28
|
+
The pattern scores are mean attention weights on ``pattern_sequences`` sequences.
|
|
29
|
+
Each sequence is ``pattern_length`` uniformly random tokens, repeated twice (Olsson et
|
|
30
|
+
al., 2022). The scores are:
|
|
31
|
+
|
|
32
|
+
- ``previous_token``: attention to the token before.
|
|
33
|
+
- ``duplicate_token``: attention from the second copy to the same token in the first.
|
|
34
|
+
- ``induction``: attention from the second copy to the token *after* the same token
|
|
35
|
+
in the first.
|
|
36
|
+
|
|
37
|
+
Each score is between 0 and 1. Reported, not gated.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
mean_output: list[float] = Field(min_length=1)
|
|
41
|
+
mean_tokens: int = Field(gt=0, description="Tokens z̄ was averaged over.")
|
|
42
|
+
previous_token_score: UnitFloat
|
|
43
|
+
duplicate_token_score: UnitFloat
|
|
44
|
+
induction_score: UnitFloat
|
|
45
|
+
pattern_sequences: int = Field(gt=0)
|
|
46
|
+
pattern_length: int = Field(gt=1)
|
|
47
|
+
pattern_seed: int
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
DictionarySite = Literal[
|
|
51
|
+
"resid_mid", "mlp_in", "mlp_in_normalized", "mlp_out", "resid_post", "mlp_hidden"
|
|
52
|
+
]
|
|
53
|
+
DictionaryOutputSite = Literal["mlp_out", "resid_post"]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def output_site_of(input_site: DictionarySite) -> DictionaryOutputSite:
|
|
57
|
+
"""Return where a dictionary that reads ``input_site`` writes.
|
|
58
|
+
|
|
59
|
+
A residual-stream SAE reconstructs the residual stream it reads. Every other
|
|
60
|
+
dictionary reconstructs the MLP's output.
|
|
61
|
+
"""
|
|
62
|
+
return "resid_post" if input_site == "resid_post" else "mlp_out"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class DictionaryProfile(StrictModel):
|
|
66
|
+
"""The dictionary a feature belongs to, pinned, and how well it fits the scan corpus.
|
|
67
|
+
|
|
68
|
+
A latent's strength is its activation ``act(W_enc[:, f] · x + b_enc[f])``. The
|
|
69
|
+
harness reads x at ``input_site`` of the feature's layer:
|
|
70
|
+
|
|
71
|
+
- ``resid_mid``: the residual stream that the MLP reads, before its norm.
|
|
72
|
+
- ``mlp_in``: the MLP's input.
|
|
73
|
+
- ``mlp_in_normalized``: the norm's input, normalised without its gain and bias. This
|
|
74
|
+
is the TransformerLens ``ln2.hook_normalized`` convention, which most public
|
|
75
|
+
transcoders were trained on.
|
|
76
|
+
- ``mlp_out``: the MLP's output, for an SAE.
|
|
77
|
+
- ``resid_post``: the residual stream after the feature's layer, for a residual-stream
|
|
78
|
+
SAE (TransformerLens's ``hook_resid_post``).
|
|
79
|
+
- ``mlp_hidden``: the MLP's hidden activations, for a ``basis``. These are the down
|
|
80
|
+
projection's input, where native neurons live.
|
|
81
|
+
|
|
82
|
+
With ``subtract_decoder_bias``, the harness first subtracts ``b_dec`` from x, as SAEs
|
|
83
|
+
often do. The MLP's output is what the MLP adds to the residual stream. In Gemma 2
|
|
84
|
+
this is after the norm that follows the MLP (TransformerLens's ``hook_mlp_out``).
|
|
85
|
+
|
|
86
|
+
Each latent of a ``basis`` is the positive part of one direction in the hidden
|
|
87
|
+
activations, and writes through the down projection. With the identity basis, the
|
|
88
|
+
latents are the native positive poles. With a random orthogonal basis, they are
|
|
89
|
+
rotated units (``docs/analyses/null-calibration.md``).
|
|
90
|
+
|
|
91
|
+
An ``spd`` dictionary holds the rank-1 subcomponents of a Stochastic Parameter
|
|
92
|
+
Decomposition of the down projection (``scripts/train_spd.py``). It reads
|
|
93
|
+
``mlp_hidden``, and its activation is ``causal_importance``. Each subcomponent is
|
|
94
|
+
two latents: the positive and the negative part of ``g · (W_enc[:, f] · x)``. Here
|
|
95
|
+
``g`` in [0, 1] is the subcomponent's causal importance on that token.
|
|
96
|
+
|
|
97
|
+
The latent writes ``a_f · W_dec[f]`` to ``output_site``. That is the MLP's output, or
|
|
98
|
+
for a residual-stream SAE, the residual stream that it reads. Ablating the latent
|
|
99
|
+
removes that write. A skip transcoder's linear skip term is part of its
|
|
100
|
+
reconstruction, not of a latent's write. Every measurement reads the weights at
|
|
101
|
+
``repo``/``revision``/``path``. So a record reproduces even if its preset changes.
|
|
102
|
+
|
|
103
|
+
The harness measures two fits on the scan's tokens. ``explained_variance`` is the
|
|
104
|
+
fraction of ``output_site``'s variance that the whole dictionary reconstructs.
|
|
105
|
+
``mean_l0`` is the number of active latents per token. Reported, not gated.
|
|
106
|
+
|
|
107
|
+
A dictionary that fits poorly makes a weak baseline. So does a dictionary with a
|
|
108
|
+
high ``dead_share``, because a random draw cannot audit a dead latent. ``density`` is
|
|
109
|
+
the share of scanned tokens on which this latent is non-zero.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
kind: Literal["transcoder", "sae", "basis", "spd"]
|
|
113
|
+
repo: str = Field(min_length=1, description="Hugging Face repository of the weights.")
|
|
114
|
+
revision: str | None = Field(default=None, description="Commit hash of ``repo``.")
|
|
115
|
+
path: str = Field(min_length=1, description="Safetensors file within ``repo``.")
|
|
116
|
+
input_site: DictionarySite
|
|
117
|
+
output_site: DictionaryOutputSite = "mlp_out"
|
|
118
|
+
activation: Literal["relu", "jumprelu", "causal_importance"]
|
|
119
|
+
subtract_decoder_bias: bool = False
|
|
120
|
+
n_features: int = Field(gt=0)
|
|
121
|
+
density: UnitFloat
|
|
122
|
+
explained_variance: float | None = Field(default=None, le=1.0)
|
|
123
|
+
mean_l0: float | None = Field(default=None, ge=0.0)
|
|
124
|
+
dead_share: UnitFloat | None = Field(
|
|
125
|
+
default=None, description="Share of the latents inactive on every scanned token."
|
|
126
|
+
)
|
|
127
|
+
tokens: int = Field(
|
|
128
|
+
gt=0, description="Scanned tokens; the fit statistics leave out each piece's first."
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
@model_validator(mode="after")
|
|
132
|
+
def _sites_agree(self) -> DictionaryProfile:
|
|
133
|
+
if self.output_site != output_site_of(self.input_site):
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"a dictionary reading {self.input_site} writes to "
|
|
136
|
+
f"{output_site_of(self.input_site)}, not {self.output_site}"
|
|
137
|
+
)
|
|
138
|
+
return self
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class SuperpositionMember(StrictModel):
|
|
142
|
+
"""One neuron of a superposition, and its weight in the direction."""
|
|
143
|
+
|
|
144
|
+
neuron_idx: int = Field(ge=0)
|
|
145
|
+
weight: float
|
|
146
|
+
|
|
147
|
+
@model_validator(mode="after")
|
|
148
|
+
def _nonzero(self) -> SuperpositionMember:
|
|
149
|
+
if self.weight == 0:
|
|
150
|
+
raise ValueError("a member's weight must not be zero")
|
|
151
|
+
return self
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class SuperpositionProfile(StrictModel):
|
|
155
|
+
"""What defines a superposition (issue #3215, ``docs/PROTOCOL.md`` §1).
|
|
156
|
+
|
|
157
|
+
A superposition is the sparse direction w that ``members`` give. It is over one
|
|
158
|
+
layer's MLP hidden activations a, the down projection's input, where neurons live. Its strength
|
|
159
|
+
at a token is ``max(0, ŵ · a)``, with ŵ the unit vector of w. So it is the positive
|
|
160
|
+
part of one direction in ``mlp_hidden``, as a ``basis`` latent is. Its ablation
|
|
161
|
+
removes that part: ``a ← a − max(0, ŵ · a) ŵ``.
|
|
162
|
+
|
|
163
|
+
``method`` says how the direction was found, for example ``circuit`` or
|
|
164
|
+
``sparse-probe``. ``source`` points to where: an analysis, a script or a record.
|
|
165
|
+
"""
|
|
166
|
+
|
|
167
|
+
members: list[SuperpositionMember] = Field(min_length=2)
|
|
168
|
+
method: str = Field(min_length=1, pattern=r"^[a-z0-9-]+$")
|
|
169
|
+
source: str = Field(min_length=1)
|
|
170
|
+
|
|
171
|
+
@model_validator(mode="after")
|
|
172
|
+
def _sorted_and_distinct(self) -> SuperpositionProfile:
|
|
173
|
+
indices = [m.neuron_idx for m in self.members]
|
|
174
|
+
if indices != sorted(set(indices)):
|
|
175
|
+
raise ValueError("members must be distinct neurons, sorted by neuron_idx")
|
|
176
|
+
return self
|
|
177
|
+
|
|
178
|
+
@property
|
|
179
|
+
def input_site(self) -> str:
|
|
180
|
+
"""Where the direction reads: the MLP's hidden activations."""
|
|
181
|
+
return "mlp_hidden"
|
|
182
|
+
|
|
183
|
+
@property
|
|
184
|
+
def neurons(self) -> list[int]:
|
|
185
|
+
"""The member neurons, in order."""
|
|
186
|
+
return [m.neuron_idx for m in self.members]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def superposition_id(members: list[SuperpositionMember]) -> str:
|
|
190
|
+
"""The id of the direction that ``members`` give: 12 hex digits of a SHA-256.
|
|
191
|
+
|
|
192
|
+
The members are read in neuron order, and each weight is scaled by the largest
|
|
193
|
+
magnitude and rounded to 6 digits. So the same direction always gets the same id,
|
|
194
|
+
whatever the order or scale of its weights.
|
|
195
|
+
"""
|
|
196
|
+
ordered = sorted(members, key=lambda m: m.neuron_idx)
|
|
197
|
+
top = max(abs(m.weight) for m in ordered)
|
|
198
|
+
canon = [[m.neuron_idx, round(m.weight / top, 6)] for m in ordered]
|
|
199
|
+
return hashlib.sha256(json.dumps(canon).encode()).hexdigest()[:12]
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def require_unit_profile(model):
|
|
203
|
+
"""Require the profile that defines a non-neuron unit's strength.
|
|
204
|
+
|
|
205
|
+
A head's strength is undefined without its mean output. A dictionary feature's is
|
|
206
|
+
undefined without its dictionary's weights, and a superposition's without its members.
|
|
207
|
+
A neuron needs none of them.
|
|
208
|
+
"""
|
|
209
|
+
kind = model.identifier.kind
|
|
210
|
+
for field, owner, unit, why in (
|
|
211
|
+
("head_profile", "attn_head", "an attention head", "its mean output"),
|
|
212
|
+
("dictionary_profile", "dictionary_feature", "a dictionary feature", "its dictionary"),
|
|
213
|
+
("superposition_profile", "superposition", "a superposition", "its members"),
|
|
214
|
+
):
|
|
215
|
+
present = getattr(model, field) is not None
|
|
216
|
+
units = unit.split(" ", 1)[1] + "s"
|
|
217
|
+
if kind == owner and not present:
|
|
218
|
+
raise ValueError(f"{unit} needs a {field}: {why} defines strength")
|
|
219
|
+
if kind != owner and present:
|
|
220
|
+
raise ValueError(f"{field} is only for {units}")
|
|
221
|
+
return model
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""The project's public links, read from ``pyproject.toml``'s metadata.
|
|
2
|
+
|
|
3
|
+
``[project.urls]`` holds the site's URL (``Homepage``) and the repo's (``Repository``).
|
|
4
|
+
The builder and ``open-interpretability deploy`` read them here, so each URL is written down once.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from email.utils import parseaddr
|
|
10
|
+
from importlib.metadata import metadata
|
|
11
|
+
|
|
12
|
+
from openinterp import DISTRIBUTION
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def project_links() -> dict:
|
|
16
|
+
"""The site and repo URLs and the contact email."""
|
|
17
|
+
meta = metadata(DISTRIBUTION)
|
|
18
|
+
urls = dict(u.split(", ", 1) for u in meta.get_all("Project-URL") or [])
|
|
19
|
+
return {
|
|
20
|
+
"site_url": urls["Homepage"],
|
|
21
|
+
"repo_url": urls["Repository"],
|
|
22
|
+
"contact_email": parseaddr(meta["Author-email"])[1],
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def site_url() -> str:
|
|
27
|
+
"""The public URL of the site's root, with a final ``/``."""
|
|
28
|
+
return project_links()["site_url"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Where a measurement ran: the device, the hardware, the versions and how the device was chosen."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
|
|
7
|
+
from openinterp.core.strict import StrictModel
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RunEnvironment(StrictModel):
|
|
11
|
+
"""Where a measurement ran. Reproducing a record pins the model revision and dtype.
|
|
12
|
+
|
|
13
|
+
The rest explains drift that another setup can cause. That is the device and its
|
|
14
|
+
hardware, the operating system, the library versions, and where the scan's forward
|
|
15
|
+
pass ran. ``device_choice`` says how the device was chosen. For example, the planner
|
|
16
|
+
can send a job to the CPU because the GPU is busy. A record holds no hostname or
|
|
17
|
+
user name.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
device: str
|
|
21
|
+
dtype: str = Field(description="Dtype of the loaded weights, e.g. float32.")
|
|
22
|
+
model_revision: str | None = Field(default=None, description="Hugging Face commit hash.")
|
|
23
|
+
openinterp_version: str
|
|
24
|
+
torch_version: str
|
|
25
|
+
transformers_version: str | None = None
|
|
26
|
+
streamed: bool | None = Field(default=None, description="Layers read from disk per pass.")
|
|
27
|
+
experts_implementation: str | None = Field(default=None, description="MoE experts forward.")
|
|
28
|
+
scan_backend: str | None = Field(default=None, description="Scan's pass, if not on device.")
|
|
29
|
+
device_name: str | None = Field(
|
|
30
|
+
default=None, description="The hardware: the GPU's name, or the chip or CPU's name."
|
|
31
|
+
)
|
|
32
|
+
os: str | None = Field(default=None, description="Operating system, e.g. macOS-15.6.")
|
|
33
|
+
device_choice: str | None = Field(
|
|
34
|
+
default=None, description="How the device was chosen: forced, planned, or a fallback."
|
|
35
|
+
)
|
|
36
|
+
coremltools_version: str | None = Field(
|
|
37
|
+
default=None, description="Core ML tools, if the scan ran on the ANE."
|
|
38
|
+
)
|