open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,470 @@
1
+ """Group every archived experiment into Model → Layer → Unit → Experiments.
2
+
3
+ A unit is an MLP neuron (both poles on one page), an attention head, a latent of a
4
+ sparse dictionary or a superposition. Latents are the learned-basis baseline. A
5
+ superposition is a sparse direction over one layer's neurons. Both are kept apart from
6
+ native-unit counts.
7
+
8
+ All experiments are published, whatever their verdict. Within a neuron, they are
9
+ ordered by how conclusive the result is (see :mod:`significance`). So a strong null
10
+ result sits next to a strong positive one, and is not hidden.
11
+
12
+ A pole's status is derived from its experiments by :mod:`standing`.
13
+
14
+ A neuron whose scans all stopped (dead or default-on) has no experiment. It still counts
15
+ as studied, as a stopped neuron of its layer (:mod:`openinterp.builder.stopped`).
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from collections.abc import Iterable, Mapping, Sequence
21
+ from dataclasses import dataclass, field
22
+ from typing import TYPE_CHECKING
23
+
24
+ from openinterp.builder.experiment import ( # noqa: F401 (re-exported)
25
+ LABEL_PREFIX,
26
+ NATIVE_KINDS,
27
+ NOUNS,
28
+ SIDES,
29
+ Experiment,
30
+ unit_side,
31
+ )
32
+ from openinterp.builder.hrefs import ( # noqa: F401 (re-exported: pages import them from here)
33
+ circuits_href,
34
+ experiment_href,
35
+ feature_href,
36
+ head_href,
37
+ layer_href,
38
+ model_href,
39
+ neuron_href,
40
+ unit_href,
41
+ )
42
+ from openinterp.builder.significance import significance
43
+ from openinterp.builder.standing import native_standings, pole_standing
44
+ from openinterp.builder.wiki import experiment_id
45
+ from openinterp.core.errata import Erratum, apply_errata, covering, record_ref
46
+ from openinterp.core.schemas import AuditRecord
47
+
48
+ if TYPE_CHECKING:
49
+ from openinterp.builder.standing import Standing
50
+ from openinterp.builder.stopped import StoppedNeuron
51
+ from openinterp.core.profiles import SuperpositionProfile
52
+
53
+
54
+ @dataclass
55
+ class UnitNode:
56
+ """One unit with all of its experiments.
57
+
58
+ The unit is a neuron (both poles), an attention head, a dictionary feature or a
59
+ superposition. A neuron's ``part_of`` lists the superpositions that it is a member of
60
+ (:mod:`openinterp.builder.superpositions`).
61
+ """
62
+
63
+ model: str
64
+ layer: int
65
+ idx: int
66
+ href: str
67
+ kind: str = "mlp_neuron_pole"
68
+ experiments: list[Experiment] = field(default_factory=list)
69
+ dictionary: str | None = None # a feature's dictionary
70
+ standings: dict[str, Standing] = field(default_factory=dict) # per pole (or head, feature)
71
+ superposition_id: str | None = None # a superposition's id
72
+ members: list = field(default_factory=list) # a superposition's members, by |weight|
73
+ profile: SuperpositionProfile | None = None # a superposition's, from its newest record
74
+ part_of: list = field(default_factory=list, repr=False, compare=False) # a neuron's
75
+
76
+ @property
77
+ def is_head(self) -> bool:
78
+ """True for an attention head."""
79
+ return self.kind == "attn_head"
80
+
81
+ @property
82
+ def is_neuron(self) -> bool:
83
+ """True for an MLP neuron."""
84
+ return self.kind == "mlp_neuron_pole"
85
+
86
+ @property
87
+ def is_feature(self) -> bool:
88
+ """True for a dictionary feature."""
89
+ return self.kind == "dictionary_feature"
90
+
91
+ @property
92
+ def is_superposition(self) -> bool:
93
+ """True for a superposition: a sparse direction over one layer's neurons."""
94
+ return self.kind == "superposition"
95
+
96
+ @property
97
+ def is_native(self) -> bool:
98
+ """True for a neuron or a head: a claim about the model's own units."""
99
+ return self.kind in NATIVE_KINDS
100
+
101
+ @property
102
+ def neuronpedia(self) -> str | None:
103
+ """The feature's Neuronpedia page, where Neuronpedia hosts it; ``None`` otherwise."""
104
+ from openinterp.crossref import latent_url
105
+
106
+ if not self.is_feature or self.dictionary is None:
107
+ return None
108
+ return latent_url(self.dictionary, self.layer, self.idx)
109
+
110
+ @property
111
+ def sides(self) -> tuple[str, ...]:
112
+ """The sides that the unit page groups experiments under."""
113
+ return SIDES[self.kind]
114
+
115
+ @property
116
+ def noun(self) -> str:
117
+ """The unit's kind as a noun, for page text."""
118
+ return NOUNS[self.kind]
119
+
120
+ @property
121
+ def label(self) -> str:
122
+ """The unit within its layer, for example ``N7``, ``H3`` or ``S0123456789ab``."""
123
+ return f"{LABEL_PREFIX[self.kind]}{self.superposition_id or self.idx}"
124
+
125
+ @property
126
+ def verified(self) -> bool:
127
+ """True when any side of the unit stands VERIFIED."""
128
+ return any(s.verified for s in self.standings.values())
129
+
130
+ @property
131
+ def top(self) -> Experiment:
132
+ """The most conclusive experiment."""
133
+ return self.experiments[0]
134
+
135
+
136
+ @dataclass
137
+ class LayerNode:
138
+ """One layer of a model, with its studied units."""
139
+
140
+ model: str
141
+ layer: int
142
+ href: str
143
+ units: list[UnitNode] = field(default_factory=list) # neurons, heads, features, superpositions
144
+ stopped: list[StoppedNeuron] = field(default_factory=list) # by index
145
+ members: list[UnitNode] = field(default_factory=list) # member neurons with no experiment
146
+
147
+ @property
148
+ def studied(self) -> int:
149
+ """Studied neurons: those with an experiment, and those whose scans all stopped."""
150
+ return len(self.neurons) + len(self.stopped)
151
+
152
+ @property
153
+ def neurons(self) -> list[UnitNode]:
154
+ """The layer's studied neurons."""
155
+ return [u for u in self.units if u.is_neuron]
156
+
157
+ @property
158
+ def heads(self) -> list[UnitNode]:
159
+ """The layer's studied heads."""
160
+ return [u for u in self.units if u.is_head]
161
+
162
+ @property
163
+ def features(self) -> list[UnitNode]:
164
+ """The layer's studied dictionary features."""
165
+ return [u for u in self.units if u.is_feature]
166
+
167
+ @property
168
+ def pages(self) -> list[UnitNode]:
169
+ """The layer's units with a page: its units, then its member neurons with no experiment."""
170
+ return [*self.units, *self.members]
171
+
172
+ @property
173
+ def superpositions(self) -> list[UnitNode]:
174
+ """The layer's studied superpositions."""
175
+ return [u for u in self.units if u.is_superposition]
176
+
177
+ @property
178
+ def member_only(self) -> int:
179
+ """Neurons studied only as members of a superposition: no experiment, no stopped scan."""
180
+ stopped = {s.idx for s in self.stopped}
181
+ return sum(n.idx not in stopped for n in self.members)
182
+
183
+ @property
184
+ def neurons_studied(self) -> int:
185
+ """Studied neurons, with those studied only as members of a superposition."""
186
+ return self.studied + self.member_only
187
+
188
+ @property
189
+ def experiments(self) -> int:
190
+ """Experiments on the layer's units."""
191
+ return sum(len(n.experiments) for n in self.units)
192
+
193
+ @property
194
+ def verified(self) -> int:
195
+ """Neurons with a verified pole."""
196
+ return sum(n.verified for n in self.neurons)
197
+
198
+ @property
199
+ def verified_heads(self) -> int:
200
+ """Heads with a VERIFIED standing."""
201
+ return sum(n.verified for n in self.heads)
202
+
203
+ @property
204
+ def verified_features(self) -> int:
205
+ """Dictionary features with a VERIFIED standing."""
206
+ return sum(n.verified for n in self.features)
207
+
208
+ @property
209
+ def verified_superpositions(self) -> int:
210
+ """Superpositions with a VERIFIED standing."""
211
+ return sum(n.verified for n in self.superpositions)
212
+
213
+
214
+ @dataclass
215
+ class ModelNode:
216
+ """One model, with its studied layers."""
217
+
218
+ name: str
219
+ slug: str
220
+ n_layers: int
221
+ href: str
222
+ mlp_width: int | None = None # neurons per layer; None when no record says
223
+ layers: list[LayerNode] = field(default_factory=list)
224
+
225
+ @property
226
+ def neurons(self) -> int:
227
+ """Studied neurons over every layer, including those whose scans all stopped."""
228
+ return sum(layer.studied for layer in self.layers)
229
+
230
+ @property
231
+ def heads(self) -> int:
232
+ """Studied heads over every layer."""
233
+ return sum(len(layer.heads) for layer in self.layers)
234
+
235
+ @property
236
+ def verified_heads(self) -> int:
237
+ """VERIFIED heads over every layer."""
238
+ return sum(layer.verified_heads for layer in self.layers)
239
+
240
+ @property
241
+ def features(self) -> int:
242
+ """Studied dictionary features over every layer."""
243
+ return sum(len(layer.features) for layer in self.layers)
244
+
245
+ @property
246
+ def verified_features(self) -> int:
247
+ """VERIFIED dictionary features over every layer."""
248
+ return sum(layer.verified_features for layer in self.layers)
249
+
250
+ @property
251
+ def superpositions(self) -> int:
252
+ """Studied superpositions over every layer."""
253
+ return sum(len(layer.superpositions) for layer in self.layers)
254
+
255
+ @property
256
+ def verified_superpositions(self) -> int:
257
+ """VERIFIED superpositions over every layer."""
258
+ return sum(layer.verified_superpositions for layer in self.layers)
259
+
260
+ @property
261
+ def member_only(self) -> int:
262
+ """Neurons studied only as members of a superposition, over every layer."""
263
+ return sum(layer.member_only for layer in self.layers)
264
+
265
+ @property
266
+ def neurons_studied(self) -> int:
267
+ """Studied neurons, with those studied only as members of a superposition."""
268
+ return self.neurons + self.member_only
269
+
270
+ @property
271
+ def experiments(self) -> int:
272
+ """Experiments over every layer."""
273
+ return sum(layer.experiments for layer in self.layers)
274
+
275
+ @property
276
+ def verified(self) -> int:
277
+ """Neurons with a VERIFIED pole, over every layer."""
278
+ return sum(layer.verified for layer in self.layers)
279
+
280
+ @property
281
+ def total_neurons(self) -> int | None:
282
+ """Every MLP neuron in the model, or ``None`` when no record gives the width."""
283
+ return self.n_layers * self.mlp_width if self.mlp_width else None
284
+
285
+ @property
286
+ def coverage(self) -> float | None:
287
+ """Share of the model's neurons that are studied (see :attr:`neurons`)."""
288
+ return self.neurons / self.total_neurons if self.total_neurons else None
289
+
290
+ @property
291
+ def verified_share(self) -> float | None:
292
+ """Share of the model's neurons with a verified pole."""
293
+ return self.verified / self.total_neurons if self.total_neurons else None
294
+
295
+ def layer_counts(self) -> list[int]:
296
+ """Studied neurons per layer index, including unstudied layers (0)."""
297
+ counts = {layer.layer: layer.studied for layer in self.layers}
298
+ return [counts.get(i, 0) for i in range(self.n_layers)]
299
+
300
+
301
+ # MLP width of models whose early records predate ``model_mlp_width``. Records are
302
+ # write-once, so the width can't be backfilled into them; a width on any record wins.
303
+ KNOWN_MLP_WIDTHS = {
304
+ "Qwen/Qwen2.5-0.5B": 4864,
305
+ "Qwen/Qwen2.5-1.5B": 8960,
306
+ }
307
+
308
+
309
+ def mlp_widths(records: Iterable[AuditRecord]) -> dict[str, int]:
310
+ """Return the neurons per MLP layer of each model, for records that do not give a width.
311
+
312
+ A width on any record of the model wins over ``KNOWN_MLP_WIDTHS``.
313
+ """
314
+ widths = dict(KNOWN_MLP_WIDTHS)
315
+ for r in records:
316
+ if r.submission.model_mlp_width:
317
+ widths[r.submission.identifier.model_name] = r.submission.model_mlp_width
318
+ return widths
319
+
320
+
321
+ def experiment_order(e: Experiment) -> tuple:
322
+ """The sort key of a unit's experiments: most conclusive first.
323
+
324
+ On ties, independent evidence comes before re-measurements, then newest first, then
325
+ a fixed order.
326
+ """
327
+ return (
328
+ -e.sig.conclusiveness,
329
+ not e.independent,
330
+ -e.record.verdict.timestamp.timestamp(),
331
+ e.side,
332
+ e.id,
333
+ )
334
+
335
+
336
+ def build_tree(
337
+ records: Sequence[AuditRecord],
338
+ errata: Sequence[Erratum],
339
+ users: Mapping[str, str] | None = None,
340
+ ) -> list[ModelNode]:
341
+ """Group the reviewed records into models, layers and units.
342
+
343
+ Each experiment lists the ``errata`` that cover its record (:mod:`openinterp.core.errata`),
344
+ and holds the record with their edits applied. ``errata`` has no default, so that no
345
+ caller can leave them out by mistake: pass ``load_errata(wiki_dir)``, or ``[]`` in a
346
+ test. Its ``user`` is the proposer's recorded
347
+ contributor, else the record's entry in ``users`` (:func:`.users.load_users`).
348
+ """
349
+ reviewed = [r for r in records if r.verdict is not None]
350
+ latest: dict[str, AuditRecord] = {}
351
+ for r in reviewed:
352
+ slug = r.submission.identifier.slug
353
+ if slug not in latest or r.verdict.timestamp > latest[slug].verdict.timestamp:
354
+ latest[slug] = r
355
+
356
+ models: dict[str, ModelNode] = {}
357
+ units: dict[tuple[str, int, int, str, int], UnitNode] = {} # neurons, heads, features
358
+ widths = mlp_widths(reviewed)
359
+ for r in reviewed:
360
+ found = covering(errata, r)
361
+ edited, edits = apply_errata(r, found)
362
+ is_latest = latest[r.submission.identifier.slug] is r
363
+ experiment = _add_experiment(models, units, edited, is_latest, widths)
364
+ experiment.errata, experiment.edits = found, edits
365
+ experiment.user = r.submission.contributor or (users or {}).get(record_ref(r))
366
+
367
+ for (name, layer_idx, *_), node in sorted(units.items()):
368
+ _mark_reuse(node.experiments)
369
+ _link_revisions(node.experiments)
370
+ for side in node.sides:
371
+ pole = [e for e in node.experiments if e.side == side]
372
+ if pole:
373
+ node.standings[side] = pole_standing(pole)
374
+ node.experiments.sort(key=experiment_order)
375
+ model = models[name]
376
+ if not model.layers or model.layers[-1].layer != layer_idx:
377
+ model.layers.append(LayerNode(name, layer_idx, layer_href(model.slug, layer_idx)))
378
+ model.layers[-1].units.append(node)
379
+ return [models[name] for name in sorted(models)]
380
+
381
+
382
+ def _add_experiment(
383
+ models: dict, units: dict, r: AuditRecord, is_latest: bool, widths: dict[str, int]
384
+ ) -> Experiment:
385
+ """File ``r`` under its unit, creating the model and unit nodes on first sight."""
386
+ ident = r.submission.identifier
387
+ mslug = ident.model_slug
388
+ model = models.get(ident.model_name)
389
+ if model is None:
390
+ model = models[ident.model_name] = ModelNode(
391
+ ident.model_name,
392
+ mslug,
393
+ 0,
394
+ model_href(mslug),
395
+ widths.get(ident.model_name),
396
+ )
397
+ model.n_layers = max(model.n_layers, r.submission.model_num_layers or 0, ident.layer + 1)
398
+ dictionary = getattr(ident, "dictionary", None)
399
+ sid = getattr(ident, "superposition_id", None)
400
+ order = list(SIDES).index(ident.kind)
401
+ key = (ident.model_name, ident.layer, order, dictionary or sid or "", ident.index)
402
+ if key not in units:
403
+ units[key] = UnitNode(
404
+ ident.model_name,
405
+ ident.layer,
406
+ ident.index,
407
+ unit_href(ident),
408
+ ident.kind,
409
+ dictionary=dictionary,
410
+ superposition_id=sid,
411
+ )
412
+ experiment = Experiment(
413
+ r,
414
+ significance(r),
415
+ experiment_id(r),
416
+ is_latest,
417
+ claim=r.submission.model_dump_json(exclude={"provenance"}),
418
+ )
419
+ units[key].experiments.append(experiment)
420
+ return experiment
421
+
422
+
423
+ def _mark_reuse(experiments: list[Experiment]) -> None:
424
+ first: dict[tuple, str] = {}
425
+ for e in sorted(experiments, key=lambda e: e.record.verdict.timestamp):
426
+ probes = frozenset(
427
+ (r.prompt, r.expected_active)
428
+ for r in e.record.verdict.adversarial_test_results
429
+ if r.kind in ("hard_negative", "positive_holdout")
430
+ )
431
+ key = (e.side, probes)
432
+ if probes and key in first:
433
+ e.reuses = first[key]
434
+ else:
435
+ first.setdefault(key, e.id)
436
+
437
+
438
+ def _link_revisions(experiments: list[Experiment]) -> None:
439
+ by_key = {(e.side, e.id): e for e in experiments}
440
+ for e in sorted(experiments, key=lambda e: e.record.verdict.timestamp):
441
+ target = by_key.get((e.side, e.record.submission.supersedes))
442
+ if target is not None:
443
+ e.revises = target
444
+ target.revised_by.append(e)
445
+
446
+
447
+ def false_discovery(models: Sequence[ModelNode]) -> dict:
448
+ """Return the expected false discoveries among the current VERIFIED claims.
449
+
450
+ There is one claim per native neuron pole or head. Dictionary features and
451
+ superpositions are left out. They are not claims about one of the model's own units.
452
+
453
+ Each claim is false with probability 1 − P(useful). So the expected number of false
454
+ claims is their sum. The estimated false discovery rate is that sum over the number
455
+ of claims.
456
+ """
457
+ from openinterp.core.stats import expected_false_discoveries
458
+
459
+ # A claim backed by several agreeing reviews takes its weakest one's P(useful).
460
+ claims = [
461
+ min(e.sig.p_useful for e in s.readings)
462
+ for _, _, s in native_standings(models)
463
+ if s.verified
464
+ ]
465
+ expected = expected_false_discoveries(claims)
466
+ return {
467
+ "claims": len(claims),
468
+ "expected_false": round(expected, 2),
469
+ "rate": round(expected / len(claims), 4) if claims else None,
470
+ }
@@ -0,0 +1,142 @@
1
+ """Who made each experiment that records no contributor: the author of the PR that merged it.
2
+
3
+ A record's ``contributor`` (:mod:`openinterp.core.contributors`) names its proposer when it
4
+ was given. Older records name nobody. For them, the site credits the GitHub user who opened
5
+ the pull request that merged the record.
6
+
7
+ 1. Git history gives the pull request: the first-parent commit on ``main`` that added the
8
+ record has the PR's number in its subject. A merge commit has "Merge pull request #N",
9
+ and a squash commit ends with "(#N)".
10
+ 2. GitHub's API gives the PR's author (``user.login``). Each login is kept in a cache
11
+ (``~/.cache/openinterp/pr-authors.json``, ``$OPENINTERP_PR_AUTHORS_CACHE``), so a build
12
+ asks only for new PRs. Without a network, PRs that the cache doesn't know name nobody.
13
+
14
+ A login is a public handle, never an email. Bots name nobody. Nothing is written to a
15
+ record: the build reads this again each time.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import os
22
+ import re
23
+ import subprocess
24
+ import urllib.request
25
+ from collections.abc import Iterable
26
+ from pathlib import Path
27
+
28
+ from openinterp.core.cache_dirs import cache_root
29
+ from openinterp.core.contributors import contributor_problem
30
+ from openinterp.core.paths import DEFAULT_WIKI_DIR, experiments_dir
31
+
32
+ CACHE_ENV = "OPENINTERP_PR_AUTHORS_CACHE"
33
+ API = "https://api.github.com"
34
+ PR_NUMBER = re.compile(r"^Merge pull request #(\d+) |\(#(\d+)\)$")
35
+
36
+
37
+ def merged_prs(wiki_dir: Path | str = DEFAULT_WIKI_DIR, ref: str = "HEAD") -> dict[str, int]:
38
+ """The PR that added each record to ``ref``, by the record's path under ``experiments/``.
39
+
40
+ A record added by a commit with no PR number, or not on ``ref`` yet, is absent. So is
41
+ every record in a shallow clone, whose history is incomplete.
42
+ """
43
+
44
+ def git(*args: str) -> str:
45
+ return subprocess.run(["git", *args], capture_output=True, text=True).stdout
46
+
47
+ if git("rev-parse", "--is-shallow-repository").strip() != "false":
48
+ return {}
49
+ root = experiments_dir(wiki_dir).as_posix().rstrip("/") + "/"
50
+ log = git(
51
+ "log", "--first-parent", "--diff-filter=A", "--name-only", "--format=@%s", ref,
52
+ "--", root,
53
+ ) # fmt: skip
54
+ prs: dict[str, int] = {}
55
+ number = None
56
+ for line in log.splitlines():
57
+ if line.startswith("@"):
58
+ found = PR_NUMBER.search(line[1:])
59
+ number = int(found.group(1) or found.group(2)) if found else None
60
+ elif line.endswith(".json") and line.startswith(root) and number is not None:
61
+ prs.setdefault(line.removeprefix(root), number)
62
+ return prs
63
+
64
+
65
+ def pr_authors(numbers: Iterable[int], repo: str) -> dict[int, str]:
66
+ """The login of each PR in ``numbers`` that a person opened, from the cache or GitHub.
67
+
68
+ ``repo`` is ``owner/name``. A PR that a bot opened, or that GitHub can't give, is absent.
69
+ """
70
+ path = _cache_path()
71
+ cache = _read_cache(path)
72
+ missing = {n for n in numbers if str(n) not in cache}
73
+ if missing:
74
+ cache |= _fetch(repo, missing)
75
+ _write_cache(path, cache)
76
+ return {n: cache[str(n)] for n in numbers if cache.get(str(n))}
77
+
78
+
79
+ def load_users(repo_url: str, wiki_dir: Path | str = DEFAULT_WIKI_DIR) -> dict[str, str]:
80
+ """The GitHub user who opened the PR that merged each record, by record path.
81
+
82
+ The paths are relative to ``experiments/``, as :func:`openinterp.core.errata.record_ref`
83
+ gives them. ``repo_url`` is the project's GitHub URL.
84
+ """
85
+ repo = repo_url.removeprefix("https://github.com/").strip("/")
86
+ prs = merged_prs(wiki_dir)
87
+ authors = pr_authors(set(prs.values()), repo) if prs else {}
88
+ return {ref: authors[n] for ref, n in prs.items() if n in authors}
89
+
90
+
91
+ def _fetch(repo: str, wanted: set[int]) -> dict[str, str]:
92
+ """Logins of the PRs in ``wanted``, newest first, a page of 100 at a time.
93
+
94
+ A bot's PR, or a login that isn't a valid handle, maps to "". So the cache knows it,
95
+ and doesn't ask again. Stops when it has every PR in ``wanted``, or at the last page.
96
+ """
97
+ found: dict[str, str] = {}
98
+ page = 1
99
+ while wanted - {int(n) for n in found}:
100
+ query = f"state=closed&sort=created&direction=desc&per_page=100&page={page}"
101
+ try:
102
+ pulls = _get(f"{API}/repos/{repo}/pulls?{query}")
103
+ except OSError as exc:
104
+ print(f"warning: can't read PR authors from GitHub ({exc}); some users are unknown")
105
+ break
106
+ if not pulls:
107
+ break
108
+ for pull in pulls:
109
+ user = pull.get("user") or {}
110
+ login = user.get("login", "")
111
+ human = user.get("type") == "User" and contributor_problem(login) is None
112
+ found[str(pull["number"])] = login if human else ""
113
+ page += 1
114
+ return found
115
+
116
+
117
+ def _get(url: str) -> list[dict]:
118
+ headers = {"Accept": "application/vnd.github+json", "User-Agent": "openinterp"}
119
+ token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
120
+ if token:
121
+ headers["Authorization"] = f"Bearer {token}"
122
+ request = urllib.request.Request(url, headers=headers) # noqa: S310 (https API URL)
123
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310
124
+ return json.load(response)
125
+
126
+
127
+ def _cache_path() -> Path | None:
128
+ root = cache_root(CACHE_ENV, "pr-authors")
129
+ return None if root is None else root / "pr-authors.json"
130
+
131
+
132
+ def _read_cache(path: Path | None) -> dict[str, str]:
133
+ try:
134
+ return json.loads(path.read_text(encoding="utf-8")) if path else {}
135
+ except (OSError, ValueError):
136
+ return {}
137
+
138
+
139
+ def _write_cache(path: Path | None, cache: dict[str, str]) -> None:
140
+ if path is not None:
141
+ path.parent.mkdir(parents=True, exist_ok=True)
142
+ path.write_text(json.dumps(cache, sort_keys=True), encoding="utf-8")