open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
"""Published sparse dictionaries (transcoders, SAEs) whose latents can be audited.
|
|
2
|
+
|
|
3
|
+
This is the learned-basis baseline for the native-neuron premise. Audit randomly chosen
|
|
4
|
+
latents of a dictionary trained on the same layers, under the same protocol. Then compare
|
|
5
|
+
the share VERIFIED with native neurons' (Dunefsky et al., 2024, arXiv:2406.11944; Paulo
|
|
6
|
+
et al., 2025, arXiv:2501.18823).
|
|
7
|
+
|
|
8
|
+
A preset (:data:`DICTIONARIES`) names a dictionary family: where its per-layer weights
|
|
9
|
+
live and how to read them. A submission pins the exact file in its
|
|
10
|
+
:class:`~openinterp.core.schemas.DictionaryProfile`. A later measurement reads only
|
|
11
|
+
that, so changing a preset never changes an old record.
|
|
12
|
+
|
|
13
|
+
Weights are one safetensors or NumPy ``.npz`` file per layer. The arrays are ``W_enc``
|
|
14
|
+
``[d_in, n]``, ``b_enc`` ``[n]``, ``W_dec`` ``[n, d_out]`` and ``b_dec`` ``[d_out]``.
|
|
15
|
+
JumpReLU also has ``threshold`` ``[n]``. Matrices stored the other way round are
|
|
16
|
+
transposed on load. An OpenMOSS ``lm_sae`` checkpoint (``encoder.weight`` …, with
|
|
17
|
+
``hyperparams.json`` beside its ``checkpoints/`` directory, as Llama Scope ships) is
|
|
18
|
+
converted on load; see :func:`_from_lm_sae`. A skip transcoder also has ``W_skip``
|
|
19
|
+
``[d_out, d_in]``. This is a linear map from its input that it adds to its
|
|
20
|
+
reconstruction (circuit-tracer's convention). A latent's write doesn't include it.
|
|
21
|
+
|
|
22
|
+
A Stochastic Parameter Decomposition (``activation="causal_importance"``, from
|
|
23
|
+
``scripts/train_spd.py``) also has a causal-importance gate. A latent's activation is
|
|
24
|
+
``relu(g · pre)``, with ``g`` the gate's output clamped to [0, 1]. The gate is one of:
|
|
25
|
+
|
|
26
|
+
* one small GELU MLP per latent, reading only that latent's ``pre``: ``ci_w1``, ``ci_b1``,
|
|
27
|
+
``ci_w2`` ``[n, d_gate]`` and ``ci_b2`` ``[n]`` (see :func:`gate_logits`);
|
|
28
|
+
* one GELU MLP per layer, reading the whole input x: ``ci_in_w`` ``[d_in, d_gate]``,
|
|
29
|
+
``ci_in_b`` ``[d_gate]``, ``ci_out_w`` ``[d_gate, n]`` and ``ci_out_b`` ``[n]`` (see
|
|
30
|
+
:func:`layer_gate_logits`).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import json
|
|
36
|
+
from collections.abc import Mapping
|
|
37
|
+
from dataclasses import dataclass
|
|
38
|
+
from functools import lru_cache
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
|
|
41
|
+
import torch
|
|
42
|
+
|
|
43
|
+
from openinterp.core.profiles import output_site_of
|
|
44
|
+
from openinterp.core.schemas import DictionaryProfile, DictionarySite
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class DictionarySpec:
|
|
49
|
+
"""A family of per-layer dictionaries for one model."""
|
|
50
|
+
|
|
51
|
+
model_name: str
|
|
52
|
+
kind: str # "transcoder" | "sae" | "basis" | "spd"
|
|
53
|
+
repo: str
|
|
54
|
+
revision: str
|
|
55
|
+
path: str | Mapping[int, str] # file within ``repo``: ``{layer}`` is filled in, or by layer
|
|
56
|
+
layers: tuple[int, ...]
|
|
57
|
+
input_site: DictionarySite
|
|
58
|
+
activation: str = "relu"
|
|
59
|
+
subtract_decoder_bias: bool = False
|
|
60
|
+
# Longest context the dictionary fits; feature scans must use windows no longer.
|
|
61
|
+
context_tokens: int | None = None
|
|
62
|
+
reference: str = ""
|
|
63
|
+
# Neuronpedia's source for a layer (``{model}/{layer}-{set}``), where it hosts these
|
|
64
|
+
# same latents under the same indices. ``None`` where it doesn't, or where unchecked.
|
|
65
|
+
neuronpedia: str | None = None
|
|
66
|
+
|
|
67
|
+
def neuronpedia_source(self, layer: int) -> str | None:
|
|
68
|
+
"""Neuronpedia's source for ``layer``, or ``None`` where it hosts none."""
|
|
69
|
+
return self.neuronpedia.format(layer=layer) if self.neuronpedia else None
|
|
70
|
+
|
|
71
|
+
def file(self, layer: int) -> str:
|
|
72
|
+
"""The weights file of ``layer``; raises ValueError for a layer not covered."""
|
|
73
|
+
if layer not in self.layers:
|
|
74
|
+
raise ValueError(f"the dictionary has no layer {layer}; it covers {self.layers}")
|
|
75
|
+
if isinstance(self.path, Mapping):
|
|
76
|
+
return self.path[layer]
|
|
77
|
+
return self.path.format(layer=layer)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
DICTIONARIES: dict[str, DictionarySpec] = {
|
|
81
|
+
# Checked against openai-community/gpt2 at layer 8, on 16 corpus documents. When it
|
|
82
|
+
# reads the LayerNorm-standardised input, the dictionary explains 66% of the MLP
|
|
83
|
+
# output's variance, with ~69 active latents per token. When it reads the MLP's
|
|
84
|
+
# (affine) input, it explains nothing. That is why the site matters.
|
|
85
|
+
# Read through position 128, the fit holds (layers 0–2: 45–90% of variance), then
|
|
86
|
+
# falls. Past position ~200, it goes negative (−154% at layer 2 over 512-token
|
|
87
|
+
# windows). So scans use 128-token windows, as if the dictionaries were trained on
|
|
88
|
+
# 128-token contexts.
|
|
89
|
+
# Neuronpedia's tres-dc are these latents. We took the top example of 24 latents
|
|
90
|
+
# across layers 2–11. Each peaks on the same token here as there (activations
|
|
91
|
+
# correlate at 0.94–1.0). The peak's size differs, because Neuronpedia's snippets are
|
|
92
|
+
# cut from longer contexts.
|
|
93
|
+
"gpt2-small-transcoders": DictionarySpec(
|
|
94
|
+
model_name="openai-community/gpt2",
|
|
95
|
+
kind="transcoder",
|
|
96
|
+
repo="jacobdunefsky/gpt2small-transcoders",
|
|
97
|
+
revision="8f254a3a8cda641cb078137415b6b9503e10fc6b",
|
|
98
|
+
path="gpt2-small-dun-chl-mlp-tc{layer}/sae.safetensors",
|
|
99
|
+
layers=tuple(range(12)),
|
|
100
|
+
input_site="mlp_in_normalized",
|
|
101
|
+
context_tokens=128,
|
|
102
|
+
reference="Dunefsky, Chlenski & Nanda, 2024, arXiv:2406.11944",
|
|
103
|
+
neuronpedia="gpt2-small/{layer}-tres-dc",
|
|
104
|
+
),
|
|
105
|
+
# The checks below each read 16 corpus documents through position 1024 with the first
|
|
106
|
+
# token left out, and found the fit flat with position. Each site was the only one of
|
|
107
|
+
# resid_mid, mlp_in and mlp_in_normalized that fit; the others explain less than 0.
|
|
108
|
+
#
|
|
109
|
+
# Read at mlp_in_normalized, they explain 63% of the MLP output's variance at layer
|
|
110
|
+
# 5, with 88 active latents per token (the file says 87). At layer 12 it is 32% (L0
|
|
111
|
+
# 6). Gemma 2 normalises its MLP's output before adding it to the residual stream.
|
|
112
|
+
# The transcoders reconstruct that normalised output. Against the down projection's
|
|
113
|
+
# output, the fit is −180. These are the lowest-L0 transcoders circuit-tracer uses,
|
|
114
|
+
# layer by layer.
|
|
115
|
+
"gemma-scope-2b-transcoders": DictionarySpec(
|
|
116
|
+
model_name="google/gemma-2-2b",
|
|
117
|
+
kind="transcoder",
|
|
118
|
+
repo="google/gemma-scope-2b-pt-transcoders",
|
|
119
|
+
revision="50eec2f25c60545a9a74c1c3a26a0afdd0b4b872",
|
|
120
|
+
path={
|
|
121
|
+
layer: f"layer_{layer}/width_16k/average_l0_{l0}/params.npz"
|
|
122
|
+
for layer, l0 in enumerate(
|
|
123
|
+
(76, 65, 49, 54, 88, 87, 95, 70, 52, 72, 88, 5, 6)
|
|
124
|
+
+ (8, 8, 8, 10, 12, 13, 12, 11, 13, 15, 25, 37, 41)
|
|
125
|
+
)
|
|
126
|
+
},
|
|
127
|
+
layers=tuple(range(26)),
|
|
128
|
+
input_site="mlp_in_normalized",
|
|
129
|
+
activation="jumprelu",
|
|
130
|
+
context_tokens=1024,
|
|
131
|
+
reference="Lieberum et al., 2024, arXiv:2408.05147",
|
|
132
|
+
),
|
|
133
|
+
# An MLP-output SAE. It reads the normalised output that Gemma 2 adds to the residual
|
|
134
|
+
# stream. It explains 70% of the variance at layer 5 (L0 114), 63% at layer 12 (L0
|
|
135
|
+
# 113). Read
|
|
136
|
+
# before that norm, the fit is −14. Each layer has Gemma Scope's canonical 16k SAE,
|
|
137
|
+
# the one whose L0 is closest to 100 (Neuronpedia's gemmascope-mlp-16k).
|
|
138
|
+
"gemma-scope-2b-mlp-saes": DictionarySpec(
|
|
139
|
+
model_name="google/gemma-2-2b",
|
|
140
|
+
kind="sae",
|
|
141
|
+
repo="google/gemma-scope-2b-pt-mlp",
|
|
142
|
+
revision="4c9df613baa9660b522f4c3908b13c9aee41bcad",
|
|
143
|
+
path={
|
|
144
|
+
layer: f"layer_{layer}/width_16k/average_l0_{l0}/params.npz"
|
|
145
|
+
for layer, l0 in enumerate(
|
|
146
|
+
(119, 105, 95, 95, 85, 114, 133, 60, 136, 88, 110, 98, 108)
|
|
147
|
+
+ (112, 97, 80, 72, 68, 106, 109, 109, 113, 121, 128, 73, 126)
|
|
148
|
+
)
|
|
149
|
+
},
|
|
150
|
+
layers=tuple(range(26)),
|
|
151
|
+
input_site="mlp_out",
|
|
152
|
+
activation="jumprelu",
|
|
153
|
+
context_tokens=1024,
|
|
154
|
+
reference="Lieberum et al., 2024, arXiv:2408.05147",
|
|
155
|
+
neuronpedia="gemma-2-2b/{layer}-gemmascope-mlp-16k",
|
|
156
|
+
),
|
|
157
|
+
# A residual-stream SAE, reading and writing the residual stream after each layer
|
|
158
|
+
# (hook_resid_post). It explains 76–93% of its variance at layers 0, 5, 12, 20 and 25,
|
|
159
|
+
# with L0 65–116. Read one step earlier, before the MLP (resid_mid), the fit is −3 to −8.
|
|
160
|
+
# Gemma Scope's canonical 16k SAEs, as for the MLP SAEs (Neuronpedia's
|
|
161
|
+
# gemmascope-res-16k). A latent here isn't an MLP unit: compared with native neurons,
|
|
162
|
+
# it is a different site, not only a different basis.
|
|
163
|
+
"gemma-scope-2b-res-saes": DictionarySpec(
|
|
164
|
+
model_name="google/gemma-2-2b",
|
|
165
|
+
kind="sae",
|
|
166
|
+
repo="google/gemma-scope-2b-pt-res",
|
|
167
|
+
revision="fd571b47c1c64851e9b1989792367b9babb4af63",
|
|
168
|
+
path={
|
|
169
|
+
layer: f"layer_{layer}/width_16k/average_l0_{l0}/params.npz"
|
|
170
|
+
for layer, l0 in enumerate(
|
|
171
|
+
(105, 102, 141, 59, 124, 68, 70, 69, 71, 73, 77, 80, 82)
|
|
172
|
+
+ (84, 84, 78, 78, 77, 74, 73, 71, 70, 72, 75, 73, 116)
|
|
173
|
+
)
|
|
174
|
+
},
|
|
175
|
+
layers=tuple(range(26)),
|
|
176
|
+
input_site="resid_post",
|
|
177
|
+
activation="jumprelu",
|
|
178
|
+
context_tokens=1024,
|
|
179
|
+
reference="Lieberum et al., 2024, arXiv:2408.05147",
|
|
180
|
+
neuronpedia="gemma-2-2b/{layer}-gemmascope-res-16k",
|
|
181
|
+
),
|
|
182
|
+
# Llama Scope (He et al., 2024, arXiv:2410.20526): top-k SAEs and transcoders for
|
|
183
|
+
# Llama-3.1-8B, converted to JumpReLU. They are in OpenMOSS's lm_sae format (see
|
|
184
|
+
# _from_lm_sae). We read them as Neuronpedia reads them. On one test sentence, L15R
|
|
185
|
+
# latents 1791 and 2290 fire on the same tokens as Neuronpedia's
|
|
186
|
+
# 15-llamascope-res-32k. Their values are the same, to bf16 precision. The sentence
|
|
187
|
+
# is "The quick brown fox jumps over the lazy dog. Paris is the capital of France."
|
|
188
|
+
# At layer 15, that reading fits 4 points worse than taking θ unscaled (residual:
|
|
189
|
+
# 62.8% vs 66.5%), with L0 33 instead of 228. The scaling, measured on the corpus,
|
|
190
|
+
# matches the hyperparameters' (resid_post 10.80 vs 10.81, mlp_out 5.34 vs 5.28).
|
|
191
|
+
# We read layers 0, 8, 15, 24 and 31 through position 1024. The residual SAEs fit
|
|
192
|
+
# 54–86%, and the MLP SAEs 30–84% (weakest mid-network). The transcoders fit 44–87%,
|
|
193
|
+
# reading mlp_in_normalized as their hook_point_in says. That is the best of the three
|
|
194
|
+
# input sites at every layer. Fits fall 2–8 points from the first 256 positions to
|
|
195
|
+
# 512–1024.
|
|
196
|
+
# The 8x (32k-latent) dictionaries, which the Llama Scope README recommends.
|
|
197
|
+
"llama-scope-8b-res-saes": DictionarySpec(
|
|
198
|
+
model_name="meta-llama/Llama-3.1-8B",
|
|
199
|
+
kind="sae",
|
|
200
|
+
repo="fnlp/Llama3_1-8B-Base-LXR-8x",
|
|
201
|
+
revision="8dbc1d85edfced43081c03c38b05514dbab1368b",
|
|
202
|
+
path="Llama3_1-8B-Base-L{layer}R-8x/checkpoints/final.safetensors",
|
|
203
|
+
layers=tuple(range(32)),
|
|
204
|
+
input_site="resid_post",
|
|
205
|
+
activation="jumprelu",
|
|
206
|
+
context_tokens=1024,
|
|
207
|
+
reference="He et al., 2024, arXiv:2410.20526",
|
|
208
|
+
neuronpedia="llama3.1-8b/{layer}-llamascope-res-32k",
|
|
209
|
+
),
|
|
210
|
+
"llama-scope-8b-mlp-saes": DictionarySpec(
|
|
211
|
+
model_name="meta-llama/Llama-3.1-8B",
|
|
212
|
+
kind="sae",
|
|
213
|
+
repo="fnlp/Llama3_1-8B-Base-LXM-8x",
|
|
214
|
+
revision="69284f67f460bc42c91abe494b3166810179bbba",
|
|
215
|
+
path="Llama3_1-8B-Base-L{layer}M-8x/checkpoints/final.safetensors",
|
|
216
|
+
layers=tuple(range(32)),
|
|
217
|
+
input_site="mlp_out",
|
|
218
|
+
activation="jumprelu",
|
|
219
|
+
context_tokens=1024,
|
|
220
|
+
reference="He et al., 2024, arXiv:2410.20526",
|
|
221
|
+
),
|
|
222
|
+
"llama-scope-8b-transcoders": DictionarySpec(
|
|
223
|
+
model_name="meta-llama/Llama-3.1-8B",
|
|
224
|
+
kind="transcoder",
|
|
225
|
+
repo="fnlp/Llama3_1-8B-Base-LXTC-8x",
|
|
226
|
+
revision="a51e5499031b69022363a73f1fd1dece523668dd",
|
|
227
|
+
path="Llama3_1-8B-Base-L{layer}TC-8x/checkpoints/final.safetensors",
|
|
228
|
+
layers=tuple(range(32)),
|
|
229
|
+
input_site="mlp_in_normalized",
|
|
230
|
+
activation="jumprelu",
|
|
231
|
+
context_tokens=1024,
|
|
232
|
+
reference="He et al., 2024, arXiv:2410.20526",
|
|
233
|
+
),
|
|
234
|
+
# circuit-tracer's skip transcoders read the residual stream before the MLP's norm
|
|
235
|
+
# (hook_resid_mid): 78% of the variance at layer 5 with L0 47. The skip term carries
|
|
236
|
+
# most of it: without it, the latents alone explain 9%. Applied the other way round
|
|
237
|
+
# (x @ W_skip rather than x @ W_skip.T), it explains −0.70.
|
|
238
|
+
"llama-3.2-1b-transcoders": DictionarySpec(
|
|
239
|
+
model_name="meta-llama/Llama-3.2-1B",
|
|
240
|
+
kind="transcoder",
|
|
241
|
+
repo="mntss/transcoder-Llama-3.2-1B",
|
|
242
|
+
revision="c37a82c1ec4cea30d424850d159b17b720ce19e2",
|
|
243
|
+
path="layer_{layer}.safetensors",
|
|
244
|
+
layers=tuple(range(16)),
|
|
245
|
+
input_site="resid_mid",
|
|
246
|
+
context_tokens=1024,
|
|
247
|
+
reference="circuit-tracer, github.com/safety-research/circuit-tracer; skip "
|
|
248
|
+
"transcoders: Paulo, Shabalin & Belrose, 2025, arXiv:2501.18823",
|
|
249
|
+
),
|
|
250
|
+
# Read at mlp_in, the MLP's input after its norm's gain: 60% of the variance at
|
|
251
|
+
# layer 5 with L0 11.
|
|
252
|
+
"qwen3-0.6b-transcoders": DictionarySpec(
|
|
253
|
+
model_name="Qwen/Qwen3-0.6B",
|
|
254
|
+
kind="transcoder",
|
|
255
|
+
repo="mwhanna/qwen3-0.6b-transcoders-lowl0",
|
|
256
|
+
revision="f28e5268fa1f3f685e028a8ba64a553e862dc7c3",
|
|
257
|
+
path="layer_{layer}.safetensors",
|
|
258
|
+
layers=tuple(range(28)),
|
|
259
|
+
input_site="mlp_in",
|
|
260
|
+
context_tokens=1024,
|
|
261
|
+
reference="circuit-tracer, github.com/safety-research/circuit-tracer",
|
|
262
|
+
),
|
|
263
|
+
# No public dictionary covers Qwen2.5-0.5B, so these were trained here, with
|
|
264
|
+
# scripts/train_transcoders.py. They used 13.5M pile-10k tokens, without the scan
|
|
265
|
+
# corpus's documents. They fit 59–93% of the scan corpus's MLP output (L0 ~32);
|
|
266
|
+
# without the skip path, at most 34%. The card is in hf_models/.
|
|
267
|
+
"qwen2.5-0.5b-transcoders": DictionarySpec(
|
|
268
|
+
model_name="Qwen/Qwen2.5-0.5B",
|
|
269
|
+
kind="transcoder",
|
|
270
|
+
repo="sackfield/qwen2.5-0.5b-transcoders",
|
|
271
|
+
revision="d10da828bcd803ffac8cf3db6550f5a1dff4eb13",
|
|
272
|
+
path="layer_{layer}.safetensors",
|
|
273
|
+
layers=tuple(range(24)),
|
|
274
|
+
input_site="mlp_in",
|
|
275
|
+
activation="jumprelu",
|
|
276
|
+
context_tokens=1024,
|
|
277
|
+
reference="openinterp, scripts/train_transcoders.py",
|
|
278
|
+
),
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def dictionary_spec(name: str) -> DictionarySpec:
|
|
283
|
+
"""The preset named ``name``; raises ValueError for an unknown name."""
|
|
284
|
+
try:
|
|
285
|
+
return DICTIONARIES[name]
|
|
286
|
+
except KeyError:
|
|
287
|
+
known = ", ".join(sorted(DICTIONARIES))
|
|
288
|
+
raise ValueError(f"unknown dictionary {name!r}; known: {known}") from None
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def gate_logits(
|
|
292
|
+
pre: torch.Tensor, w1: torch.Tensor, b1: torch.Tensor, w2: torch.Tensor, b2: torch.Tensor
|
|
293
|
+
) -> torch.Tensor:
|
|
294
|
+
"""Each latent's causal importance before it is clamped to [0, 1].
|
|
295
|
+
|
|
296
|
+
``pre`` is ``[..., k]`` with gate weights ``[k, d_gate]`` and ``b2`` ``[k]``, or
|
|
297
|
+
``[...]`` for one latent's weights ``[d_gate]`` and a scalar ``b2``. Each latent's
|
|
298
|
+
gate reads only that latent's ``pre``.
|
|
299
|
+
"""
|
|
300
|
+
return (torch.nn.functional.gelu(pre.unsqueeze(-1) * w1 + b1) * w2).sum(-1) + b2
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def layer_gate_logits(
|
|
304
|
+
x: torch.Tensor,
|
|
305
|
+
w_in: torch.Tensor,
|
|
306
|
+
b_in: torch.Tensor,
|
|
307
|
+
w_out: torch.Tensor,
|
|
308
|
+
b_out: torch.Tensor,
|
|
309
|
+
) -> torch.Tensor:
|
|
310
|
+
"""Every latent's causal importance before it is clamped, from the whole input ``x``.
|
|
311
|
+
|
|
312
|
+
``w_out`` is ``[d_gate, k]`` and ``b_out`` ``[k]`` for k latents.
|
|
313
|
+
"""
|
|
314
|
+
return torch.nn.functional.gelu(x @ w_in + b_in) @ w_out + b_out
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
@dataclass(frozen=True)
|
|
318
|
+
class Dictionary:
|
|
319
|
+
"""One layer's dictionary, in float32 on one device."""
|
|
320
|
+
|
|
321
|
+
w_enc: torch.Tensor # [d_in, n]
|
|
322
|
+
b_enc: torch.Tensor # [n]
|
|
323
|
+
w_dec: torch.Tensor # [n, d_out]
|
|
324
|
+
b_dec: torch.Tensor # [d_out]
|
|
325
|
+
threshold: torch.Tensor | None
|
|
326
|
+
activation: str
|
|
327
|
+
subtract_decoder_bias: bool
|
|
328
|
+
w_skip: torch.Tensor | None = None # [d_in, d_out]
|
|
329
|
+
gates: tuple[torch.Tensor, ...] | None = None # ci_w1, ci_b1, ci_w2, ci_b2
|
|
330
|
+
layer_gate: tuple[torch.Tensor, ...] | None = None # ci_in_w, ci_in_b, ci_out_w, ci_out_b
|
|
331
|
+
|
|
332
|
+
@property
|
|
333
|
+
def n_features(self) -> int:
|
|
334
|
+
"""The number of latents."""
|
|
335
|
+
return self.w_enc.shape[1]
|
|
336
|
+
|
|
337
|
+
def _input(self, x: torch.Tensor) -> torch.Tensor:
|
|
338
|
+
x = x.float().to(self.w_enc.device)
|
|
339
|
+
return x - self.b_dec if self.subtract_decoder_bias else x
|
|
340
|
+
|
|
341
|
+
def _importance(self, x: torch.Tensor, pre: torch.Tensor, f) -> torch.Tensor:
|
|
342
|
+
"""The gate logits of latents ``f`` (``None``: all), from the input and ``pre``."""
|
|
343
|
+
if self.layer_gate is not None:
|
|
344
|
+
w_in, b_in, w_out, b_out = self.layer_gate
|
|
345
|
+
if f is not None:
|
|
346
|
+
w_out, b_out = w_out[:, f], b_out[f]
|
|
347
|
+
return layer_gate_logits(x, w_in, b_in, w_out, b_out)
|
|
348
|
+
gates = self.gates if f is None else tuple(g[f] for g in self.gates)
|
|
349
|
+
return gate_logits(pre, *gates)
|
|
350
|
+
|
|
351
|
+
def _activate(self, x: torch.Tensor, pre: torch.Tensor, f) -> torch.Tensor:
|
|
352
|
+
if self.activation == "jumprelu":
|
|
353
|
+
return pre * (pre > (self.threshold if f is None else self.threshold[f]))
|
|
354
|
+
if self.activation == "causal_importance":
|
|
355
|
+
return torch.relu(self._importance(x, pre, f).clamp(0.0, 1.0) * pre)
|
|
356
|
+
return torch.relu(pre)
|
|
357
|
+
|
|
358
|
+
def encode(self, x: torch.Tensor) -> torch.Tensor:
|
|
359
|
+
"""Every latent's activation ``[..., n]``."""
|
|
360
|
+
x = self._input(x)
|
|
361
|
+
return self._activate(x, x @ self.w_enc + self.b_enc, None)
|
|
362
|
+
|
|
363
|
+
def feature(self, x: torch.Tensor, f: int | torch.Tensor) -> torch.Tensor:
|
|
364
|
+
"""Return latent ``f``'s activation ``[...]``, without encoding the rest.
|
|
365
|
+
|
|
366
|
+
For a tensor of k indices, it is ``[..., k]``.
|
|
367
|
+
"""
|
|
368
|
+
x = self._input(x)
|
|
369
|
+
return self._activate(x, x @ self.w_enc[:, f] + self.b_enc[f], f)
|
|
370
|
+
|
|
371
|
+
def decode(self, acts: torch.Tensor) -> torch.Tensor:
|
|
372
|
+
"""The decoder's output for latent activations ``acts``, without the skip path."""
|
|
373
|
+
return acts @ self.w_dec + self.b_dec
|
|
374
|
+
|
|
375
|
+
def reconstruct(self, x: torch.Tensor, acts: torch.Tensor | None = None) -> torch.Tensor:
|
|
376
|
+
"""Return the dictionary's estimate of its output from its input ``x``.
|
|
377
|
+
|
|
378
|
+
``acts`` are the latent activations of ``x``, if already encoded. Any skip
|
|
379
|
+
connection is included.
|
|
380
|
+
"""
|
|
381
|
+
out = self.decode(self.encode(x) if acts is None else acts)
|
|
382
|
+
if self.w_skip is not None:
|
|
383
|
+
out = out + x.float().to(self.w_skip.device) @ self.w_skip
|
|
384
|
+
return out
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def load_dictionary(profile: DictionaryProfile, device: torch.device | str = "cpu") -> Dictionary:
|
|
388
|
+
"""The weights ``profile`` pins, on ``device``."""
|
|
389
|
+
reading = (profile.activation, profile.subtract_decoder_bias)
|
|
390
|
+
return _load(profile.repo, profile.revision, profile.path, str(device), *reading)
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def load_spec(spec: DictionarySpec, layer: int, device: torch.device | str = "cpu") -> Dictionary:
|
|
394
|
+
"""``spec``'s dictionary for ``layer``, on ``device``."""
|
|
395
|
+
reading = (spec.activation, spec.subtract_decoder_bias)
|
|
396
|
+
return _load(spec.repo, spec.revision, spec.file(layer), str(device), *reading)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _read(local: Path) -> dict[str, torch.Tensor]:
|
|
400
|
+
if local.suffix == ".npz":
|
|
401
|
+
import numpy as np
|
|
402
|
+
|
|
403
|
+
with np.load(local) as arrays:
|
|
404
|
+
return {k: torch.from_numpy(arrays[k]) for k in arrays.files}
|
|
405
|
+
from safetensors.torch import load_file
|
|
406
|
+
|
|
407
|
+
return load_file(str(local))
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _fetch(repo: str, revision: str | None, path: str) -> Path:
|
|
411
|
+
local = Path(repo) / path
|
|
412
|
+
if local.exists(): # a local directory in tests
|
|
413
|
+
return local
|
|
414
|
+
from huggingface_hub import hf_hub_download
|
|
415
|
+
|
|
416
|
+
return Path(hf_hub_download(repo, path, revision=revision))
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _from_lm_sae(w: dict[str, torch.Tensor], hyperparams: dict) -> dict[str, torch.Tensor]:
|
|
420
|
+
"""An OpenMOSS ``lm_sae`` checkpoint (Llama Scope's format) as plain JumpReLU weights.
|
|
421
|
+
|
|
422
|
+
lm_sae scales its input by ``s_in = sqrt(d) / dataset_average_activation_norm["in"]``
|
|
423
|
+
and its reconstruction back by ``s_out``, and has one ``jump_relu_threshold`` θ.
|
|
424
|
+
Neuronpedia (through SAELens) reads a latent as active where its pre-activation
|
|
425
|
+
``h = s_in·x·W_enc + b_enc`` exceeds ``θ·s_in``. Its strength is then
|
|
426
|
+
``h·‖W_dec[f]‖ / s_in``.
|
|
427
|
+
Folding those into the weights gives the same activations, so a latent here reads as
|
|
428
|
+
it does there.
|
|
429
|
+
"""
|
|
430
|
+
d = w["decoder.bias"].shape[0]
|
|
431
|
+
norms = hyperparams["dataset_average_activation_norm"]
|
|
432
|
+
s_in, s_out = d**0.5 / norms["in"], d**0.5 / norms["out"]
|
|
433
|
+
w_enc, w_dec = w["encoder.weight"].T, w["decoder.weight"].T # [d_in, n], [n, d_out]
|
|
434
|
+
dec_norm = w_dec.norm(dim=-1) # [n]
|
|
435
|
+
return {
|
|
436
|
+
# a = h·‖W_dec[f]‖ / s_in, so a > θ·‖W_dec[f]‖ exactly where h > θ·s_in
|
|
437
|
+
"W_enc": w_enc * dec_norm,
|
|
438
|
+
"b_enc": w["encoder.bias"] * dec_norm / s_in,
|
|
439
|
+
"threshold": hyperparams["jump_relu_threshold"] * dec_norm,
|
|
440
|
+
# h·W_dec / s_out = a·W_dec·s_in / (‖W_dec[f]‖·s_out)
|
|
441
|
+
"W_dec": w_dec * (s_in / s_out) / dec_norm[:, None],
|
|
442
|
+
"b_dec": w["decoder.bias"] / s_out,
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
@lru_cache(maxsize=2) # one layer's weights can be hundreds of MB
|
|
447
|
+
def _load(repo, revision, path, device, activation, subtract_decoder_bias) -> Dictionary:
|
|
448
|
+
local = _fetch(repo, revision, path)
|
|
449
|
+
# Copied: a tensor left on the file's memory map can be misaligned (see cli._load).
|
|
450
|
+
w = {k: v.to(device, torch.float32, copy=True) for k, v in _read(local).items()}
|
|
451
|
+
if "encoder.weight" in w: # lm_sae: its hyperparameters sit beside "checkpoints/"
|
|
452
|
+
hyperparams_path = str(Path(path).parent.parent / "hyperparams.json")
|
|
453
|
+
hyperparams = json.loads(_fetch(repo, revision, hyperparams_path).read_text())
|
|
454
|
+
w = _from_lm_sae(w, hyperparams)
|
|
455
|
+
n = w["b_enc"].shape[0]
|
|
456
|
+
w_enc = w["W_enc"] if w["W_enc"].shape[1] == n else w["W_enc"].T
|
|
457
|
+
w_dec = w["W_dec"] if w["W_dec"].shape[0] == n else w["W_dec"].T
|
|
458
|
+
if activation == "jumprelu" and "threshold" not in w:
|
|
459
|
+
raise ValueError(f"{path} is a JumpReLU dictionary without a threshold tensor")
|
|
460
|
+
gates = layer_gate = None
|
|
461
|
+
if activation == "causal_importance":
|
|
462
|
+
per_latent = ("ci_w1", "ci_b1", "ci_w2", "ci_b2")
|
|
463
|
+
per_layer = ("ci_in_w", "ci_in_b", "ci_out_w", "ci_out_b")
|
|
464
|
+
if all(name in w for name in per_layer):
|
|
465
|
+
layer_gate = tuple(w[name].contiguous() for name in per_layer)
|
|
466
|
+
elif all(name in w for name in per_latent):
|
|
467
|
+
gates = tuple(w[name].contiguous() for name in per_latent)
|
|
468
|
+
else:
|
|
469
|
+
raise ValueError(f"{path} has neither gate: {per_latent} or {per_layer}")
|
|
470
|
+
return Dictionary(
|
|
471
|
+
w_enc=w_enc.contiguous(),
|
|
472
|
+
b_enc=w["b_enc"],
|
|
473
|
+
w_dec=w_dec.contiguous(),
|
|
474
|
+
b_dec=w["b_dec"],
|
|
475
|
+
threshold=w.get("threshold") if activation == "jumprelu" else None,
|
|
476
|
+
activation=activation,
|
|
477
|
+
subtract_decoder_bias=subtract_decoder_bias,
|
|
478
|
+
w_skip=w["W_skip"].T.contiguous() if "W_skip" in w else None,
|
|
479
|
+
gates=gates,
|
|
480
|
+
layer_gate=layer_gate,
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def spec_profile(spec: DictionarySpec, layer: int, **measured) -> DictionaryProfile:
|
|
485
|
+
"""A profile pinning ``spec``'s file for ``layer``, with the scan's fit statistics."""
|
|
486
|
+
return DictionaryProfile(
|
|
487
|
+
kind=spec.kind,
|
|
488
|
+
repo=spec.repo,
|
|
489
|
+
revision=spec.revision,
|
|
490
|
+
path=spec.file(layer),
|
|
491
|
+
input_site=spec.input_site,
|
|
492
|
+
output_site=output_site_of(spec.input_site),
|
|
493
|
+
activation=spec.activation,
|
|
494
|
+
subtract_decoder_bias=spec.subtract_decoder_bias,
|
|
495
|
+
**measured,
|
|
496
|
+
)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Fluent dreaming: prompts optimised to drive a neuron pole as high as it goes (issue #3228).
|
|
2
|
+
|
|
3
|
+
This is EPO, from Thompson, Straznickas & Sklar (2024), "Fluent dreaming for language
|
|
4
|
+
models", arXiv:2402.01702, §2. A population of prompts of ``n`` tokens has one member for
|
|
5
|
+
each λ. Member i maximises
|
|
6
|
+
|
|
7
|
+
L_λ(t) = f(t) − (λ / n) · Σ_i H(m(t_≤i), t_{i+1})
|
|
8
|
+
|
|
9
|
+
where f is the pole's strength and H is the model's cross-entropy on its own prompt. So a
|
|
10
|
+
larger λ asks for more fluent text. Each step backpropagates L to a one-hot encoding of
|
|
11
|
+
each prompt, and takes the ``k`` tokens with the largest gradient at each position. Each
|
|
12
|
+
member then has ``r`` children, each with one position, chosen at random, swapped for one
|
|
13
|
+
of its top-``k`` tokens. Each λ keeps the best prompt of the pooled population and
|
|
14
|
+
children, with replacement.
|
|
15
|
+
|
|
16
|
+
f here is the pole's largest strength over the prompt, with the first token left out:
|
|
17
|
+
many units take extreme values there (``docs/PROTOCOL.md`` §1). This version has no
|
|
18
|
+
restarts, and the defaults for ``k`` and the number of steps are lower than the paper's.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import math
|
|
24
|
+
from collections.abc import Sequence
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
|
|
27
|
+
import torch
|
|
28
|
+
|
|
29
|
+
from openinterp.harness.hooks import NeuronHookRunner, pole_strength
|
|
30
|
+
|
|
31
|
+
LENGTH = 12
|
|
32
|
+
LAMBDAS = tuple(math.exp(x) for x in torch.linspace(math.log(0.1), math.log(10), 8).tolist())
|
|
33
|
+
CHILDREN = 32
|
|
34
|
+
TOP_K = 128
|
|
35
|
+
STEPS = 150
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class Dream:
|
|
40
|
+
"""One λ's final prompt: its tokens, text, pole strength and self-cross-entropy."""
|
|
41
|
+
|
|
42
|
+
lam: float
|
|
43
|
+
tokens: list[int]
|
|
44
|
+
text: str
|
|
45
|
+
strength: float
|
|
46
|
+
cross_entropy: float
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _score(runner, layer, neuron, pole, embeds: torch.Tensor, tokens: torch.Tensor):
|
|
50
|
+
"""(f, mean cross-entropy) of each prompt, from its input embeddings ``[B, n, d]``."""
|
|
51
|
+
captured = {}
|
|
52
|
+
|
|
53
|
+
def grab(_module, args):
|
|
54
|
+
captured["act"] = args[0][..., neuron]
|
|
55
|
+
|
|
56
|
+
handle = runner.down_proj(layer).register_forward_pre_hook(grab)
|
|
57
|
+
try:
|
|
58
|
+
logits = runner.model(inputs_embeds=embeds).logits
|
|
59
|
+
finally:
|
|
60
|
+
handle.remove()
|
|
61
|
+
strength = pole_strength(captured["act"], pole)[:, 1:].max(dim=1).values
|
|
62
|
+
ce = torch.nn.functional.cross_entropy(
|
|
63
|
+
logits[:, :-1].reshape(-1, logits.shape[-1]).float(), tokens[:, 1:].reshape(-1),
|
|
64
|
+
reduction="none",
|
|
65
|
+
).view(tokens.shape[0], -1).mean(dim=1) # fmt: skip
|
|
66
|
+
return strength.float(), ce
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def dream(
|
|
70
|
+
runner: NeuronHookRunner,
|
|
71
|
+
layer: int,
|
|
72
|
+
neuron: int,
|
|
73
|
+
pole: str,
|
|
74
|
+
*,
|
|
75
|
+
length: int = LENGTH,
|
|
76
|
+
lambdas: Sequence[float] = LAMBDAS,
|
|
77
|
+
children: int = CHILDREN,
|
|
78
|
+
top_k: int = TOP_K,
|
|
79
|
+
steps: int = STEPS,
|
|
80
|
+
seed: int = 0,
|
|
81
|
+
) -> list[Dream]:
|
|
82
|
+
"""Run EPO for one pole and return each λ's final prompt."""
|
|
83
|
+
gen = torch.Generator().manual_seed(seed)
|
|
84
|
+
table = runner.model.get_input_embeddings().weight
|
|
85
|
+
vocab, device = table.shape[0], table.device
|
|
86
|
+
lams = torch.tensor(lambdas, device=device)
|
|
87
|
+
pop = torch.randint(vocab, (len(lambdas), length), generator=gen).to(device)
|
|
88
|
+
for _ in range(steps):
|
|
89
|
+
onehot = torch.nn.functional.one_hot(pop, vocab).to(table.dtype).requires_grad_(True)
|
|
90
|
+
f, ce = _score(runner, layer, neuron, pole, onehot @ table, pop)
|
|
91
|
+
(f - lams * ce).sum().backward()
|
|
92
|
+
top = onehot.grad.topk(top_k, dim=-1).indices # [M, n, k]
|
|
93
|
+
member = torch.arange(len(lambdas)).repeat_interleave(children)
|
|
94
|
+
where = torch.randint(length, (len(member),), generator=gen)
|
|
95
|
+
pick = torch.randint(top_k, (len(member),), generator=gen)
|
|
96
|
+
kids = pop[member].clone()
|
|
97
|
+
kids[torch.arange(len(member)), where.to(device)] = top[
|
|
98
|
+
member.to(device), where.to(device), pick.to(device)
|
|
99
|
+
]
|
|
100
|
+
pool = torch.cat([pop, kids])
|
|
101
|
+
with torch.no_grad():
|
|
102
|
+
f_pool, ce_pool = _score(runner, layer, neuron, pole, table[pool], pool)
|
|
103
|
+
best = (f_pool[None, :] - lams[:, None] * ce_pool[None, :]).argmax(dim=1)
|
|
104
|
+
pop = pool[best]
|
|
105
|
+
with torch.no_grad():
|
|
106
|
+
f, ce = _score(runner, layer, neuron, pole, table[pop], pop)
|
|
107
|
+
return [
|
|
108
|
+
Dream(
|
|
109
|
+
round(float(lam), 4),
|
|
110
|
+
pop[i].tolist(),
|
|
111
|
+
runner.tokenizer.decode(pop[i].tolist()),
|
|
112
|
+
round(float(f[i]), 4),
|
|
113
|
+
round(float(ce[i]), 4),
|
|
114
|
+
) # fmt: skip
|
|
115
|
+
for i, lam in enumerate(lambdas)
|
|
116
|
+
]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""The environment a scan ran in, recorded with each submission."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import platform
|
|
6
|
+
import subprocess
|
|
7
|
+
from contextlib import suppress
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
|
|
13
|
+
from openinterp.core.schemas import RunEnvironment
|
|
14
|
+
from openinterp.harness import devices
|
|
15
|
+
from openinterp.harness.memory import is_streamed
|
|
16
|
+
from openinterp.harness.moe import experts_implementation
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from openinterp.harness.hooks import NeuronHookRunner
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def run_environment(runner: NeuronHookRunner) -> RunEnvironment:
|
|
23
|
+
"""Where ``runner`` measures: its device and hardware, versions, and how it was chosen."""
|
|
24
|
+
from openinterp import __version__
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
import transformers
|
|
28
|
+
|
|
29
|
+
transformers_version = transformers.__version__
|
|
30
|
+
except ImportError:
|
|
31
|
+
transformers_version = None
|
|
32
|
+
config = getattr(runner.model, "config", None)
|
|
33
|
+
return RunEnvironment(
|
|
34
|
+
device=str(runner.device),
|
|
35
|
+
dtype=str(next(runner.model.parameters()).dtype).removeprefix("torch."),
|
|
36
|
+
model_revision=getattr(runner, "model_revision", None)
|
|
37
|
+
or getattr(config, "_commit_hash", None),
|
|
38
|
+
openinterp_version=__version__,
|
|
39
|
+
torch_version=torch.__version__,
|
|
40
|
+
transformers_version=transformers_version,
|
|
41
|
+
streamed=True if is_streamed(runner.model) else None,
|
|
42
|
+
experts_implementation=experts_implementation(runner.model),
|
|
43
|
+
device_name=device_name(runner.device),
|
|
44
|
+
os=platform.platform(terse=True),
|
|
45
|
+
device_choice=devices.choice(),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def device_name(device: torch.device) -> str | None:
|
|
50
|
+
"""The hardware behind ``device``: a CUDA GPU's name, else the chip or CPU's name."""
|
|
51
|
+
if device.type == "cuda":
|
|
52
|
+
with suppress(Exception):
|
|
53
|
+
return torch.cuda.get_device_name(device)
|
|
54
|
+
if platform.system() == "Darwin": # one chip serves the CPU and MPS
|
|
55
|
+
with suppress(OSError, subprocess.SubprocessError):
|
|
56
|
+
out = subprocess.run(
|
|
57
|
+
["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True
|
|
58
|
+
)
|
|
59
|
+
return out.stdout.strip() or None
|
|
60
|
+
with suppress(OSError):
|
|
61
|
+
for line in Path("/proc/cpuinfo").read_text().splitlines():
|
|
62
|
+
if line.startswith("model name"):
|
|
63
|
+
return line.split(":", 1)[1].strip()
|
|
64
|
+
return platform.processor() or None
|