open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""The learned-basis baseline: random dictionary latents against random native neuron poles.
|
|
2
|
+
|
|
3
|
+
The native-neuron premise: a model's own MLP neurons are as explainable as the latents
|
|
4
|
+
of a sparse dictionary trained on the same layers. This compares the two under
|
|
5
|
+
one protocol (docs/PROTOCOL.md §1). For each dictionary, it takes the share VERIFIED and
|
|
6
|
+
the mean balanced accuracy among its randomly selected latents. It compares them with the
|
|
7
|
+
same among randomly selected neuron poles of the same model. These poles are on the
|
|
8
|
+
layers that the dictionary covers, scanned on the same corpus with the same window. (A
|
|
9
|
+
scan's context changes what a unit is seen to do.)
|
|
10
|
+
|
|
11
|
+
Each unit counts once, by its first reviewed experiment, and only if that experiment chose
|
|
12
|
+
it at random. Follow-ups (revisions, re-reviews) are left out so that neither arm gains
|
|
13
|
+
from being studied more. Neurons are also compared by pole. In a GELU MLP (GPT-2), the
|
|
14
|
+
negative pole is the activation's dip. The dip is at most 0.17, at a pre-activation of
|
|
15
|
+
about −0.75. It sits in the dip on most tokens. So a probe's peak nearly always clears the
|
|
16
|
+
activity line, and no hard negative passes. There the baseline's native arm is positive
|
|
17
|
+
poles (``audit --random --pole positive``), and :attr:`Comparison.p_positive` is the test
|
|
18
|
+
that matters.
|
|
19
|
+
|
|
20
|
+
Random draws that couldn't be audited are counted per arm next to it. They are in
|
|
21
|
+
``wiki/draws/``: dead, or default-on since protocol 0.8. So a share VERIFIED can be read
|
|
22
|
+
against every draw.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import math
|
|
28
|
+
from collections.abc import Sequence
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
|
|
31
|
+
import duckdb
|
|
32
|
+
|
|
33
|
+
from openinterp.harness.dictionaries import DICTIONARIES
|
|
34
|
+
|
|
35
|
+
FIRST_AUDITS = """
|
|
36
|
+
SELECT kind, model_name, layer, pole, dictionary, status, selection, balanced_accuracy,
|
|
37
|
+
corpus_sha256, window_tokens,
|
|
38
|
+
dictionary_explained_variance, dictionary_mean_l0, dictionary_dead_share
|
|
39
|
+
FROM r
|
|
40
|
+
WHERE kind IN ('mlp_neuron_pole', 'dictionary_feature') AND status IS NOT NULL
|
|
41
|
+
AND sister_of IS NULL -- a sister study tests a borrowed explanation, not a fresh audit
|
|
42
|
+
QUALIFY row_number() OVER (PARTITION BY slug ORDER BY reviewed_at, experiment_id) = 1
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class Arm:
|
|
48
|
+
"""One arm of a comparison: its units, how many were VERIFIED, and its unauditable draws."""
|
|
49
|
+
|
|
50
|
+
units: int
|
|
51
|
+
verified: int
|
|
52
|
+
mean_balanced_accuracy: float | None
|
|
53
|
+
dead: int = 0 # random draws of this arm the scan stopped on (wiki/draws/)
|
|
54
|
+
default_on: int = 0
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def share(self) -> float | None:
|
|
58
|
+
"""The share VERIFIED, or ``None`` with no units."""
|
|
59
|
+
return self.verified / self.units if self.units else None
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def interval(self) -> tuple[float, float] | None:
|
|
63
|
+
"""The Wilson interval of the share VERIFIED, or ``None`` with no units."""
|
|
64
|
+
return wilson(self.verified, self.units) if self.units else None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class Comparison:
|
|
69
|
+
"""One dictionary against native poles, on one scan setting."""
|
|
70
|
+
|
|
71
|
+
model_name: str
|
|
72
|
+
dictionary: str
|
|
73
|
+
layers: tuple[int, ...]
|
|
74
|
+
corpus_sha256: str | None
|
|
75
|
+
window_tokens: int | None # None: whole documents
|
|
76
|
+
features: Arm
|
|
77
|
+
neurons: Arm
|
|
78
|
+
positive_poles: Arm
|
|
79
|
+
negative_poles: Arm
|
|
80
|
+
explained_variance: float | None # mean over the latents' scans
|
|
81
|
+
mean_l0: float | None
|
|
82
|
+
dead_share: float | None # latents a random draw can't audit, so not in ``features``
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def p_value(self) -> float | None:
|
|
86
|
+
"""Two-sided Fisher exact test of equal shares VERIFIED.
|
|
87
|
+
|
|
88
|
+
It compares latents against every neuron pole.
|
|
89
|
+
"""
|
|
90
|
+
return _fisher(self.features, self.neurons)
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def p_positive(self) -> float | None:
|
|
94
|
+
"""The same test against positive poles only.
|
|
95
|
+
|
|
96
|
+
This is the comparison for a GELU MLP, whose negative poles no hard negative can
|
|
97
|
+
pass (docs/PROTOCOL.md §1).
|
|
98
|
+
"""
|
|
99
|
+
return _fisher(self.features, self.positive_poles)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _fisher(a: Arm, b: Arm) -> float | None:
|
|
103
|
+
if not (a.units and b.units):
|
|
104
|
+
return None
|
|
105
|
+
return fisher_exact(a.verified, a.units, b.verified, b.units)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def wilson(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
|
|
109
|
+
"""95% Wilson score interval for k successes in n trials."""
|
|
110
|
+
p = k / n
|
|
111
|
+
centre = (p + z * z / (2 * n)) / (1 + z * z / n)
|
|
112
|
+
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / (1 + z * z / n)
|
|
113
|
+
return max(0.0, centre - half), min(1.0, centre + half)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def fisher_exact(k1: int, n1: int, k2: int, n2: int) -> float:
|
|
117
|
+
"""Two-sided p of Fisher's exact test on the 2×2 table (k1, n1−k1; k2, n2−k2)."""
|
|
118
|
+
total, successes = n1 + n2, k1 + k2
|
|
119
|
+
|
|
120
|
+
def prob(k: int) -> float:
|
|
121
|
+
return math.comb(n1, k) * math.comb(n2, successes - k) / math.comb(total, successes)
|
|
122
|
+
|
|
123
|
+
observed = prob(k1)
|
|
124
|
+
lo, hi = max(0, successes - n2), min(n1, successes)
|
|
125
|
+
p = sum(q for q in map(prob, range(lo, hi + 1)) if q <= observed * (1 + 1e-9))
|
|
126
|
+
return min(1.0, p)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _arm(rows: list[dict], draws: Sequence = ()) -> Arm:
|
|
130
|
+
accuracies = [r["balanced_accuracy"] for r in rows if r["balanced_accuracy"] is not None]
|
|
131
|
+
return Arm(
|
|
132
|
+
units=len(rows),
|
|
133
|
+
verified=sum(r["status"] == "VERIFIED" for r in rows),
|
|
134
|
+
mean_balanced_accuracy=sum(accuracies) / len(accuracies) if accuracies else None,
|
|
135
|
+
dead=sum(d.outcome == "dead" for d in draws),
|
|
136
|
+
default_on=sum(d.outcome == "default_on" for d in draws),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _pole_arm(rows: list[dict], draws: Sequence, pole: str) -> Arm:
|
|
141
|
+
return _arm(
|
|
142
|
+
[r for r in rows if r["pole"] == pole], [d for d in draws if d.identifier.pole == pole]
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _scan_of(draw) -> tuple:
|
|
147
|
+
prov, ident = draw.provenance, draw.identifier
|
|
148
|
+
return (ident.model_name, prov.corpus_sha256, prov.snippet_tokens)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _mean(values) -> float | None:
|
|
152
|
+
values = [v for v in values if v is not None]
|
|
153
|
+
return sum(values) / len(values) if values else None
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def learned_baseline(con: duckdb.DuckDBPyConnection, draws: Sequence = ()) -> list[Comparison]:
|
|
157
|
+
"""Return one comparison per dictionary and scan setting (corpus, window), over view ``r``.
|
|
158
|
+
|
|
159
|
+
Only settings with a randomly selected latent are compared. Each has the unauditable
|
|
160
|
+
``draws`` of each arm.
|
|
161
|
+
"""
|
|
162
|
+
cur = con.execute(FIRST_AUDITS)
|
|
163
|
+
cols = [d[0] for d in cur.description]
|
|
164
|
+
first = [dict(zip(cols, row, strict=True)) for row in cur.fetchall()]
|
|
165
|
+
random = [r for r in first if r["selection"] == "random"]
|
|
166
|
+
features: dict[tuple, list[dict]] = {}
|
|
167
|
+
for r in random:
|
|
168
|
+
if r["kind"] == "dictionary_feature":
|
|
169
|
+
scan = (r["model_name"], r["corpus_sha256"], r["window_tokens"])
|
|
170
|
+
features.setdefault((*scan, r["dictionary"]), []).append(r)
|
|
171
|
+
return [
|
|
172
|
+
_comparison(tuple(scan), name, latents, random, draws)
|
|
173
|
+
for (*scan, name), latents in sorted(features.items(), key=lambda kv: str(kv[0]))
|
|
174
|
+
]
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _comparison(scan: tuple, name: str, latents, random, draws) -> Comparison:
|
|
178
|
+
"""A dictionary's latents against the native poles from the same scan and layers."""
|
|
179
|
+
spec = DICTIONARIES.get(name)
|
|
180
|
+
layers = spec.layers if spec else tuple(sorted({r["layer"] for r in latents}))
|
|
181
|
+
neurons = [
|
|
182
|
+
r
|
|
183
|
+
for r in random
|
|
184
|
+
if r["kind"] == "mlp_neuron_pole"
|
|
185
|
+
and (r["model_name"], r["corpus_sha256"], r["window_tokens"]) == scan
|
|
186
|
+
and r["layer"] in layers
|
|
187
|
+
]
|
|
188
|
+
same = [d for d in draws if _scan_of(d) == scan and d.identifier.layer in layers]
|
|
189
|
+
latent_draws = [d for d in same if getattr(d.identifier, "dictionary", None) == name]
|
|
190
|
+
pole_draws = [d for d in same if d.identifier.kind == "mlp_neuron_pole"]
|
|
191
|
+
return Comparison(
|
|
192
|
+
model_name=scan[0],
|
|
193
|
+
dictionary=name,
|
|
194
|
+
layers=layers,
|
|
195
|
+
corpus_sha256=scan[1],
|
|
196
|
+
window_tokens=scan[2],
|
|
197
|
+
features=_arm(latents, latent_draws),
|
|
198
|
+
neurons=_arm(neurons, pole_draws),
|
|
199
|
+
positive_poles=_pole_arm(neurons, pole_draws, "positive"),
|
|
200
|
+
negative_poles=_pole_arm(neurons, pole_draws, "negative"),
|
|
201
|
+
explained_variance=_mean(r["dictionary_explained_variance"] for r in latents),
|
|
202
|
+
mean_l0=_mean(r["dictionary_mean_l0"] for r in latents),
|
|
203
|
+
dead_share=_mean(r["dictionary_dead_share"] for r in latents),
|
|
204
|
+
)
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""What the Models page says about each supported model, read from its ``config.json``.
|
|
2
|
+
|
|
3
|
+
The config is the one source. The parameter count comes from building the model from
|
|
4
|
+
its config on the meta device, so no weights load. The Hub's safetensors count is not
|
|
5
|
+
used: for GPT-2 it includes the attention-mask buffers (137M, not 124M).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterable
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from openinterp.planner_models import config_dims, model_config
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class ModelFacts:
|
|
18
|
+
"""The architecture of one model. A field is None when the config does not give it."""
|
|
19
|
+
|
|
20
|
+
parameters: int | None
|
|
21
|
+
layers: int | None
|
|
22
|
+
mlp_width: int | None
|
|
23
|
+
activation: str | None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parameter_count(config: dict) -> int | None:
|
|
27
|
+
"""Parameters of the model that ``config`` describes, tied weights counted once.
|
|
28
|
+
|
|
29
|
+
Buffers (such as GPT-2's attention mask) are not parameters, so they are not counted.
|
|
30
|
+
Returns None when transformers cannot build the model from the config.
|
|
31
|
+
"""
|
|
32
|
+
import torch
|
|
33
|
+
from transformers import AutoConfig, AutoModelForCausalLM
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
cfg = AutoConfig.for_model(**config)
|
|
37
|
+
with torch.device("meta"):
|
|
38
|
+
model = AutoModelForCausalLM.from_config(cfg)
|
|
39
|
+
except Exception: # an architecture this transformers version does not know
|
|
40
|
+
return None
|
|
41
|
+
return sum(p.numel() for p in model.parameters())
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def facts_from_config(config: dict) -> ModelFacts:
|
|
45
|
+
"""The facts that ``config`` gives."""
|
|
46
|
+
dims = config_dims(config)
|
|
47
|
+
activation = config.get("hidden_act") or config.get("activation_function")
|
|
48
|
+
return ModelFacts(
|
|
49
|
+
parameters=parameter_count(config),
|
|
50
|
+
layers=dims[0] if dims else None,
|
|
51
|
+
mlp_width=dims[1] if dims else None,
|
|
52
|
+
activation=activation if isinstance(activation, str) else None,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def load_model_facts(models: Iterable[str], download: bool = True) -> dict[str, ModelFacts]:
|
|
57
|
+
"""The facts of each model whose ``config.json`` can be read; others are left out.
|
|
58
|
+
|
|
59
|
+
``download`` is as for :func:`openinterp.planner_models.model_config`.
|
|
60
|
+
"""
|
|
61
|
+
facts = {}
|
|
62
|
+
for model in models:
|
|
63
|
+
config = model_config(model, download)
|
|
64
|
+
if config is not None:
|
|
65
|
+
facts[model] = facts_from_config(config)
|
|
66
|
+
return facts
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def format_parameters(n: int | None) -> str:
|
|
70
|
+
"""``124M`` or ``1.54B``: three significant figures; ``—`` when unknown."""
|
|
71
|
+
if n is None:
|
|
72
|
+
return "—"
|
|
73
|
+
for size, unit in ((1e9, "B"), (1e6, "M"), (1e3, "K")):
|
|
74
|
+
if n >= size:
|
|
75
|
+
return f"{n / size:.3g}{unit}"
|
|
76
|
+
return str(n)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def placeholder_models(configured: Iterable[str], studied: Iterable, facts: dict) -> list:
|
|
80
|
+
"""An empty model node for each configured model with no record yet.
|
|
81
|
+
|
|
82
|
+
So such a model gets a page. A model whose config gives no layer count is left out:
|
|
83
|
+
its page would have no layers to show.
|
|
84
|
+
"""
|
|
85
|
+
from openinterp.builder.tree import ModelNode, model_href
|
|
86
|
+
from openinterp.core.identifiers import model_slug
|
|
87
|
+
|
|
88
|
+
have = {m.name for m in studied}
|
|
89
|
+
nodes = []
|
|
90
|
+
for name in configured:
|
|
91
|
+
f = facts.get(name)
|
|
92
|
+
if name in have or f is None or not f.layers:
|
|
93
|
+
continue
|
|
94
|
+
slug = model_slug(name)
|
|
95
|
+
nodes.append(ModelNode(name, slug, f.layers, model_href(slug), f.mlp_width))
|
|
96
|
+
return nodes
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def models_view(configured: Iterable[str], nodes: Iterable, facts: dict, licenses: dict):
|
|
100
|
+
"""One row per configured model for the Models page, in the configured order.
|
|
101
|
+
|
|
102
|
+
``nodes`` are the model nodes that have a page. Layers and MLP width fall back to
|
|
103
|
+
what the records give when the config cannot be read.
|
|
104
|
+
"""
|
|
105
|
+
by_name = {m.name: m for m in nodes}
|
|
106
|
+
rows = []
|
|
107
|
+
for name in configured:
|
|
108
|
+
node, f = by_name.get(name), facts.get(name)
|
|
109
|
+
rows.append(
|
|
110
|
+
{
|
|
111
|
+
"name": name,
|
|
112
|
+
"href": node.href if node else None,
|
|
113
|
+
"size": format_parameters(f.parameters if f else None),
|
|
114
|
+
"parameters": f.parameters if f else None,
|
|
115
|
+
"layers": (f and f.layers) or (node.n_layers if node else None),
|
|
116
|
+
"mlp_width": (f and f.mlp_width) or (node.mlp_width if node else None),
|
|
117
|
+
"activation": f.activation if f else None,
|
|
118
|
+
"terms": licenses.get(name),
|
|
119
|
+
"experiments": node.experiments if node else 0,
|
|
120
|
+
"verified": (node.verified + node.verified_heads) if node else 0,
|
|
121
|
+
}
|
|
122
|
+
)
|
|
123
|
+
return rows
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def render_models(render, models, facts: dict, render_model) -> None:
|
|
127
|
+
"""Write the Models page, and a page for each configured model with no record yet.
|
|
128
|
+
|
|
129
|
+
``render`` is the compiler's renderer and ``render_model`` writes one model's pages.
|
|
130
|
+
"""
|
|
131
|
+
from openinterp.planner import configured_models
|
|
132
|
+
|
|
133
|
+
configured = configured_models()
|
|
134
|
+
empty = placeholder_models(configured, models, facts)
|
|
135
|
+
for model in empty:
|
|
136
|
+
render_model(render, model, {}, set())
|
|
137
|
+
rows = models_view(configured, [*models, *empty], facts, render.common["model_licenses"])
|
|
138
|
+
render("models/index.html", "models.html", rows=rows)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""The registry's VERIFIED explanations in Neuronpedia's export format.
|
|
2
|
+
|
|
3
|
+
Neuronpedia publishes its explanations as gzipped JSON Lines, one directory per source:
|
|
4
|
+
``v1/<model>/<source>/explanations/batch-0.jsonl.gz``. This writes the same layout under
|
|
5
|
+
``data/neuronpedia/`` of the site, so that a self-hosted Neuronpedia (or anyone) can
|
|
6
|
+
import it. Nothing is sent to Neuronpedia.
|
|
7
|
+
|
|
8
|
+
Only the current standing of a unit counts: the lead experiment of a VERIFIED standing.
|
|
9
|
+
A dictionary latent goes to the source that Neuronpedia hosts it under, with the same
|
|
10
|
+
index. A native neuron goes to the source ``<layer>-<NEURON_SET>``, which Neuronpedia
|
|
11
|
+
does not host. Both poles of a neuron share its index, so each row also names its pole.
|
|
12
|
+
A model is exported only when Neuronpedia knows it, that is when one of its dictionary
|
|
13
|
+
presets names a Neuronpedia source. Each row keeps Neuronpedia's field names and adds
|
|
14
|
+
the verdict, the protocol version, the data licence and a link to the signed record.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import gzip
|
|
20
|
+
import json
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from collections.abc import Sequence
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from openinterp.builder.tree import Experiment
|
|
26
|
+
from openinterp.builder.wiki import archive_path
|
|
27
|
+
from openinterp.core.licenses import DATA_LICENSE
|
|
28
|
+
|
|
29
|
+
TYPE_NAME = "openinterp-verified"
|
|
30
|
+
NEURON_SET = "openinterp-neurons"
|
|
31
|
+
EXPORT_DIR = Path("data") / "neuronpedia" / "v1"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def neuronpedia_model(model_name: str) -> str | None:
|
|
35
|
+
"""Neuronpedia's id for ``model_name``, from its dictionary presets; ``None`` if unknown."""
|
|
36
|
+
from openinterp.harness.dictionaries import DICTIONARIES
|
|
37
|
+
|
|
38
|
+
for spec in DICTIONARIES.values():
|
|
39
|
+
if spec.model_name == model_name and spec.neuronpedia:
|
|
40
|
+
return spec.neuronpedia.split("/", 1)[0]
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def source_of(ident) -> str | None:
|
|
45
|
+
"""The Neuronpedia source (``<layer>-<set>``) for ``ident``; ``None`` if not exported."""
|
|
46
|
+
from openinterp.harness.dictionaries import DICTIONARIES
|
|
47
|
+
|
|
48
|
+
if ident.kind == "dictionary_feature":
|
|
49
|
+
spec = DICTIONARIES.get(ident.dictionary)
|
|
50
|
+
source = spec and spec.neuronpedia_source(ident.layer)
|
|
51
|
+
return source.split("/", 1)[1] if source else None
|
|
52
|
+
if ident.kind == "mlp_neuron_pole":
|
|
53
|
+
return f"{ident.layer}-{NEURON_SET}"
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def export_row(experiment: Experiment, repo_url: str) -> dict | None:
|
|
58
|
+
"""The experiment as one Neuronpedia explanation row; ``None`` if it is not exported."""
|
|
59
|
+
record = experiment.record
|
|
60
|
+
ident = record.submission.identifier
|
|
61
|
+
model, source = neuronpedia_model(ident.model_name), source_of(ident)
|
|
62
|
+
if model is None or source is None:
|
|
63
|
+
return None
|
|
64
|
+
policy = record.verdict.policy
|
|
65
|
+
return {
|
|
66
|
+
"modelId": model,
|
|
67
|
+
"layer": source,
|
|
68
|
+
"index": str(ident.feature_idx if ident.kind == "dictionary_feature" else ident.neuron_idx),
|
|
69
|
+
"pole": ident.pole,
|
|
70
|
+
"description": record.submission.natural_language_explanation,
|
|
71
|
+
"typeName": TYPE_NAME,
|
|
72
|
+
"explanationModelName": record.submission.submitting_agent_model,
|
|
73
|
+
"status": record.verdict.status,
|
|
74
|
+
"selfReviewed": record.self_reviewed, # no independent reviewer backs it
|
|
75
|
+
"protocolVersion": policy.protocol_version if policy else None,
|
|
76
|
+
"license": DATA_LICENSE,
|
|
77
|
+
"url": f"{repo_url}/blob/main/{archive_path(Path('wiki'), record).as_posix()}",
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def exported(experiments: Sequence[Experiment]) -> list[Experiment]:
|
|
82
|
+
"""The lead experiment of each VERIFIED standing, in a fixed order."""
|
|
83
|
+
leads = {
|
|
84
|
+
id(e): e
|
|
85
|
+
for e in experiments
|
|
86
|
+
if e.standing is not None and e.standing.verified and e.standing.lead is e
|
|
87
|
+
}
|
|
88
|
+
return sorted(leads.values(), key=lambda e: (e.record.submission.identifier.slug, e.id))
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def export_neuronpedia(experiments: Sequence[Experiment], output_dir: Path, repo_url: str) -> int:
|
|
92
|
+
"""Write the export under ``output_dir``; return how many rows it holds.
|
|
93
|
+
|
|
94
|
+
The files are byte-for-byte the same for the same records: the gzip header has no
|
|
95
|
+
time, and rows are sorted.
|
|
96
|
+
"""
|
|
97
|
+
by_source: dict[tuple[str, str], list[dict]] = defaultdict(list)
|
|
98
|
+
for experiment in exported(experiments):
|
|
99
|
+
row = export_row(experiment, repo_url)
|
|
100
|
+
if row is not None:
|
|
101
|
+
by_source[(row["modelId"], row["layer"])].append(row)
|
|
102
|
+
for (model, source), rows in sorted(by_source.items()):
|
|
103
|
+
path = Path(output_dir) / EXPORT_DIR / model / source / "explanations" / "batch-0.jsonl.gz"
|
|
104
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
body = "".join(json.dumps(r, ensure_ascii=False, sort_keys=True) + "\n" for r in rows)
|
|
106
|
+
path.write_bytes(gzip.compress(body.encode(), mtime=0))
|
|
107
|
+
return sum(len(rows) for rows in by_source.values())
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Render many pages in parallel worker processes, where the platform can fork.
|
|
2
|
+
|
|
3
|
+
Each unit page depends only on the built site tree, so pages can render in any order. A
|
|
4
|
+
worker is a fork of the build. So it has the tree already, and nothing large is copied to
|
|
5
|
+
it. Each worker renders every ``jobs``-th page and returns the paths that it wrote. The
|
|
6
|
+
build then records them in the same order as one process does. With one job, or with no
|
|
7
|
+
``fork`` on the platform, the pages render in this process, one by one.
|
|
8
|
+
|
|
9
|
+
A fork is safe while the build runs no other thread, and the build starts none. If that
|
|
10
|
+
changes, Python warns at the fork, and ``open-interpretability build --jobs 1`` avoids it.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import multiprocessing
|
|
16
|
+
import os
|
|
17
|
+
from collections.abc import Callable, Sequence
|
|
18
|
+
from concurrent.futures import ProcessPoolExecutor
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, TypeVar
|
|
21
|
+
|
|
22
|
+
T = TypeVar("T")
|
|
23
|
+
|
|
24
|
+
_WORK: dict[str, Any] = {} # set just before the workers fork, so that they inherit it
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def default_jobs() -> int:
|
|
28
|
+
"""Return the number of worker processes to use: one for each CPU."""
|
|
29
|
+
return os.cpu_count() or 1
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def render_in_parallel(render, page: Callable[..., None], items: Sequence[tuple]) -> None:
|
|
33
|
+
"""Call ``page(*item)`` for each of ``items``, in ``render.jobs`` worker processes.
|
|
34
|
+
|
|
35
|
+
``page`` renders with ``render``, which records each page that it writes.
|
|
36
|
+
"""
|
|
37
|
+
jobs = min(render.jobs, len(items))
|
|
38
|
+
if jobs <= 1 or "fork" not in multiprocessing.get_all_start_methods():
|
|
39
|
+
for item in items:
|
|
40
|
+
page(*item)
|
|
41
|
+
return
|
|
42
|
+
_WORK.update(render=render, page=page, items=items)
|
|
43
|
+
try:
|
|
44
|
+
context = multiprocessing.get_context("fork")
|
|
45
|
+
with ProcessPoolExecutor(jobs, mp_context=context) as pool:
|
|
46
|
+
shares = list(pool.map(_render_share, range(jobs), [jobs] * jobs))
|
|
47
|
+
finally:
|
|
48
|
+
_WORK.clear()
|
|
49
|
+
# Sort by item only. The sort is stable, so one item's pages keep the order it wrote them.
|
|
50
|
+
written = sorted((pair for share in shares for pair in share), key=lambda pair: pair[0])
|
|
51
|
+
render.report.pages.extend(path for _, path in written)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _render_share(share: int, jobs: int) -> list[tuple[int, Path]]:
|
|
55
|
+
"""Render every ``jobs``-th item, from ``share``, and return each page that it wrote."""
|
|
56
|
+
render, page, items = _WORK["render"], _WORK["page"], _WORK["items"]
|
|
57
|
+
written = []
|
|
58
|
+
for i in range(share, len(items), jobs):
|
|
59
|
+
before = len(render.report.pages)
|
|
60
|
+
page(*items[i])
|
|
61
|
+
written += [(i, path) for path in render.report.pages[before:]]
|
|
62
|
+
return written
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def in_background(fn: Callable[[], T], jobs: int) -> Callable[[], T]:
|
|
66
|
+
"""Start ``fn()`` in a forked process, and return a function that waits for its result.
|
|
67
|
+
|
|
68
|
+
With one job, or with no ``fork`` on the platform, call ``fn()`` now instead. The
|
|
69
|
+
fork uses a plain process and a pipe, not a pool, so the build starts no thread that a
|
|
70
|
+
later fork could copy.
|
|
71
|
+
"""
|
|
72
|
+
if jobs <= 1 or "fork" not in multiprocessing.get_all_start_methods():
|
|
73
|
+
value = fn()
|
|
74
|
+
return lambda: value
|
|
75
|
+
context = multiprocessing.get_context("fork")
|
|
76
|
+
reader, writer = context.Pipe(duplex=False)
|
|
77
|
+
process = context.Process(target=_send_result, args=(fn, writer))
|
|
78
|
+
process.start()
|
|
79
|
+
writer.close()
|
|
80
|
+
|
|
81
|
+
def result() -> T:
|
|
82
|
+
ok, value = reader.recv() # before join: a large result fills the pipe
|
|
83
|
+
process.join()
|
|
84
|
+
if not ok:
|
|
85
|
+
raise RuntimeError(f"a background part of the build failed:\n{value}")
|
|
86
|
+
return value
|
|
87
|
+
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _send_result(fn: Callable[[], Any], writer) -> None:
|
|
92
|
+
"""Run ``fn()`` in the forked process, and send its result, or the error, to the parent."""
|
|
93
|
+
import traceback
|
|
94
|
+
|
|
95
|
+
try:
|
|
96
|
+
writer.send((True, fn()))
|
|
97
|
+
except BaseException: # noqa: BLE001 (the parent re-raises it)
|
|
98
|
+
writer.send((False, traceback.format_exc()))
|
|
99
|
+
finally:
|
|
100
|
+
writer.close()
|