open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,495 @@
1
+ """Compile every archived experiment into a static site under ``dist/``.
2
+
3
+ Pages follow the registry hierarchy: the index, then a page per model, layer and unit.
4
+ A neuron's page is ``models/<model>/L<layer>/N<idx>.html``, and a head's is
5
+ ``H<idx>.html``. Each unit page lists all of its experiments, most conclusive first. Null
6
+ results are published alongside positive ones.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import math
12
+ import shutil
13
+ from collections.abc import Sequence
14
+ from dataclasses import dataclass, field
15
+ from pathlib import Path
16
+ from tempfile import TemporaryDirectory
17
+
18
+ from jinja2 import Environment, PackageLoader, StrictUndefined, select_autoescape
19
+ from pydantic import ValidationError
20
+
21
+ from openinterp import __version__
22
+ from openinterp.builder import charts, pieces, statuses
23
+ from openinterp.builder.agents import (
24
+ agent_href,
25
+ agents_view,
26
+ contributors_view,
27
+ legacy_redirects,
28
+ )
29
+ from openinterp.builder.aligned import aligned_latents
30
+ from openinterp.builder.circuit_pages import render_circuits
31
+ from openinterp.builder.circuits import circuits_and_hubs, wiring_view
32
+ from openinterp.builder.corpora import corpora_view, language_mix
33
+ from openinterp.builder.coverage import model_summary
34
+ from openinterp.builder.dashboard import dashboard
35
+ from openinterp.builder.heat import HeatToken, heat_html, token_heat, values_rows
36
+ from openinterp.builder.model_facts import ModelFacts, render_models
37
+ from openinterp.builder.neuronpedia_export import export_neuronpedia
38
+ from openinterp.builder.parallel import in_background, render_in_parallel
39
+ from openinterp.builder.parquet_export import export_draws, export_registry
40
+ from openinterp.builder.pii import save_spans, scrub, scrub_record
41
+ from openinterp.builder.registry_stats import registry_stats # noqa: F401 (re-exported)
42
+ from openinterp.builder.sections import render_reference_pages
43
+ from openinterp.builder.significance import P_USEFUL, evidence_strength
44
+ from openinterp.builder.sister_graph import sister_graphs
45
+ from openinterp.builder.standing import ROLES, contested_units, reviewer_agreement
46
+ from openinterp.builder.stopped import add_stopped, explore_focus
47
+ from openinterp.builder.stops import pageless_stops, stops_by_unit
48
+ from openinterp.builder.superpositions import add_members, count_pageless_neurons
49
+ from openinterp.builder.tree import (
50
+ Experiment,
51
+ build_tree,
52
+ circuits_href,
53
+ false_discovery,
54
+ neuron_href,
55
+ )
56
+ from openinterp.core.agent_names import canonical_agent
57
+ from openinterp.core.draws import ChosenStop
58
+ from openinterp.core.errata import Erratum
59
+ from openinterp.core.exceptions import RegistryError
60
+ from openinterp.core.project import project_links
61
+ from openinterp.core.schemas import ActivatingExample, AuditRecord
62
+ from openinterp.planner import configured_licenses
63
+ from openinterp.sensitive import is_offensive
64
+ from openinterp.sisters import by_match_slug, sister_source_href, sister_tests, sister_view
65
+ from openinterp.wiring_backfill import record_wiring
66
+
67
+
68
+ @dataclass
69
+ class BuildReport:
70
+ """What a build wrote."""
71
+
72
+ output_dir: Path
73
+ records_loaded: int
74
+ experiments_published: int
75
+ pages: list[Path] = field(default_factory=list)
76
+ parquet: Path | None = None
77
+
78
+
79
+ def load_records(audits_dir: Path) -> list[AuditRecord]:
80
+ """Load every ``*.json`` audit record under ``audits_dir`` (recursively)."""
81
+ records = []
82
+ for path in sorted(Path(audits_dir).rglob("*.json")):
83
+ try:
84
+ records.append(AuditRecord.model_validate_json(path.read_text()))
85
+ except ValidationError as exc:
86
+ raise RegistryError(f"{path}: invalid audit record\n{exc}") from exc
87
+ return records
88
+
89
+
90
+ def _environment() -> Environment:
91
+ env = Environment(
92
+ loader=PackageLoader("openinterp.builder", "templates"),
93
+ autoescape=select_autoescape(["html"]),
94
+ undefined=StrictUndefined,
95
+ trim_blocks=True,
96
+ lstrip_blocks=True,
97
+ )
98
+ env.filters["pct"] = lambda v: "—" if v is None else f"{100 * v:.0f}%"
99
+ env.filters["pct_fine"] = _pct_fine
100
+ env.filters["num"] = lambda v, d=3: "—" if v is None else f"{v:.{d}f}"
101
+ env.filters["thousands"] = lambda v: "?" if v is None else f"{v:,}"
102
+ # p-values below 0.001 would round to 0.000, so show them in scientific notation.
103
+ env.filters["pval"] = lambda v: "—" if v is None else f"{v:.3f}" if v >= 0.001 else f"{v:.1e}"
104
+ env.globals["roles"] = ROLES
105
+ env.globals["charts"] = charts # page-level charts: the layer map
106
+ env.globals["pieces"] = pieces # the pieces bars of a model and of a layer
107
+ env.globals["statuses"] = statuses # unit statuses, their order and colours
108
+ env.globals["agent_href"] = agent_href # links from agent names to their pages
109
+ env.globals["agent_name"] = canonical_agent # one name per model, however it was recorded
110
+ env.globals["evidence_strength"] = evidence_strength # strong / moderate / weak bands
111
+ env.globals["language_mix"] = language_mix # a scan's tokens per language
112
+ env.globals["heat_html"] = heat_html # token heatmaps, built in Python for speed
113
+ env.globals["values_rows"] = values_rows # and the rows of their values tables
114
+ return env
115
+
116
+
117
+ def _pct_fine(v: float | None) -> str:
118
+ """A percentage with two significant figures, so 1 neuron in 100k isn't shown as 0%."""
119
+ if v is None:
120
+ return "—"
121
+ if v == 0:
122
+ return "0%"
123
+ decimals = max(0, 1 - math.floor(math.log10(100 * v)))
124
+ return f"{100 * v:.{decimals}f}%"
125
+
126
+
127
+ def _root_for(rel_path: str) -> str:
128
+ return "../" * (len(Path(rel_path).parts) - 1)
129
+
130
+
131
+ # Detection outcomes, the explanation's mistakes first.
132
+ OUTCOMES = {
133
+ (True, False): "missed",
134
+ (False, True): "false alarm",
135
+ (True, True): "hit",
136
+ (False, False): "correct rejection",
137
+ }
138
+
139
+
140
+ def explanation_view(scores, scale: float) -> dict | None:
141
+ """Held-out windows grouped by how the judge did, ready for the token heatmap."""
142
+ if scores is None:
143
+ return None
144
+ scale = scale or 1.0
145
+
146
+ def window(w) -> dict:
147
+ peak = max(range(len(w.activations)), key=w.activations.__getitem__, default=-1)
148
+ marks = set(w.highlighted)
149
+ tokens = [
150
+ HeatToken(
151
+ tok.replace("\n", "↵"),
152
+ round(min(1.0, max(0.0, act / scale)), 3),
153
+ act or 0.0,
154
+ i == peak and act >= scores.threshold,
155
+ i in marks,
156
+ )
157
+ for i, (tok, act) in enumerate(zip(w.tokens, w.activations, strict=True))
158
+ ]
159
+ outcome = OUTCOMES[(w.label, w.judged)] if w.judged is not None else "not judged"
160
+ return {
161
+ "w": w,
162
+ "tokens": tokens,
163
+ "outcome": outcome,
164
+ "peak": max(w.activations, default=0),
165
+ "offensive": is_offensive(w.text),
166
+ }
167
+
168
+ order = list(OUTCOMES.values())
169
+ detection = sorted(
170
+ (window(w) for w in scores.detection_windows),
171
+ key=lambda r: (order.index(r["outcome"]) if r["outcome"] in order else 9, -r["peak"]),
172
+ )
173
+ fuzzing = sorted(
174
+ (window(w) for w in scores.fuzzing_windows),
175
+ key=lambda r: (r["w"].label == r["w"].judged, -r["peak"]),
176
+ )
177
+ counts = {o: sum(r["outcome"] == o for r in detection) for o in order}
178
+ return {"s": scores, "detection": detection, "fuzzing": fuzzing, "counts": counts}
179
+
180
+
181
+ # Paths the builder (and the Pagefind step after it) owns inside the output dir.
182
+ # They are removed before each build so deleted records leave no stale pages.
183
+ OWNED_PATHS = (
184
+ "index.html",
185
+ "about.html",
186
+ "learned-baseline.html",
187
+ "models",
188
+ "agents",
189
+ "users",
190
+ "neurons",
191
+ "research",
192
+ "data",
193
+ "pagefind",
194
+ )
195
+
196
+
197
+ def _clean_owned_paths(output_dir: Path) -> None:
198
+ for name in OWNED_PATHS:
199
+ path = output_dir / name
200
+ if path.is_dir():
201
+ shutil.rmtree(path)
202
+ elif path.exists():
203
+ path.unlink()
204
+
205
+
206
+ def _experiment_view(
207
+ e: Experiment, studied: set[str], rows: dict | None = None, circuit: str | None = None
208
+ ) -> dict:
209
+ """What an experiment's page shows.
210
+
211
+ ``rows`` is backfilled wiring (see :mod:`openinterp.wiring_backfill`). ``circuit`` is
212
+ the link to the unit's circuit, if any.
213
+ """
214
+ sub, verdict = e.record.submission, e.record.verdict
215
+ weights, activations, backfilled = record_wiring(e.record, rows or {})
216
+ ident = sub.identifier
217
+ correlated = []
218
+ for c in sub.correlated_neurons: # neuron poles only
219
+ href = neuron_href(ident.model_slug, ident.layer, c.neuron_idx)
220
+ correlated.append({"c": c, "href": href if href in studied else None})
221
+ return {
222
+ "e": e,
223
+ "sub": sub,
224
+ "verdict": verdict,
225
+ "ident": ident,
226
+ "c": verdict.causal_metrics,
227
+ "bars": charts.histogram_bars(sub.activation_histogram),
228
+ "probes": charts.probe_chart(verdict, sub.max_activation),
229
+ "causal": charts.causal_chart(verdict.causal_metrics, sub.counterfactuals),
230
+ "vocab": charts.vocab_chart(sub.vocabulary_lens),
231
+ "examples": [token_heat(ex, sub.max_activation or 0.0) for ex in sub.top_examples[:12]],
232
+ "offensive_examples": [i for i, x in enumerate(sub.top_examples) if is_offensive(x.text)],
233
+ "correlated": correlated,
234
+ "scores": explanation_view(verdict.explanation_scores, sub.max_activation or 0.0),
235
+ "wiring": [
236
+ view
237
+ for w in (weights, activations)
238
+ if (view := wiring_view(w, ident.model_slug, studied)) is not None
239
+ ],
240
+ "wiring_backfilled": backfilled,
241
+ "circuit": circuit,
242
+ "sister_of": sub.sister_of
243
+ and {"ref": sub.sister_of, "href": sister_source_href(sub.sister_of)},
244
+ }
245
+
246
+
247
+ def pole_scans(neuron, n: int = 3) -> dict | None:
248
+ """Both poles' top examples for a neuron page, side by side.
249
+
250
+ The newest experiment that recorded its other pole supplies both, so they come from
251
+ one scan. Without one (records from before both poles were scanned), each pole's
252
+ newest experiment supplies its own, and a pole with none has no column.
253
+ """
254
+ if not neuron.is_neuron:
255
+ return None
256
+ newest = sorted(neuron.experiments, key=lambda e: e.record.verdict.timestamp, reverse=True)
257
+ both = next((e for e in newest if e.record.submission.opposite_pole), None)
258
+ found: dict[str, tuple[list[ActivatingExample], Experiment]] = {}
259
+ if both is not None:
260
+ sub = both.record.submission
261
+ found[sub.identifier.pole] = (sub.top_examples, both)
262
+ found[sub.opposite_pole.pole] = (sub.opposite_pole.top_examples, both)
263
+ else:
264
+ for e in newest:
265
+ found.setdefault(e.side, (e.record.submission.top_examples, e))
266
+ poles = []
267
+ for pole in neuron.sides:
268
+ examples, e = found.get(pole, ([], None))
269
+ scale = max((ex.max_activation for ex in examples), default=0.0)
270
+ poles.append(
271
+ {
272
+ "pole": pole,
273
+ "max": scale,
274
+ "examples": [token_heat(ex, scale, before=20, after=5) for ex in examples[:n]],
275
+ "source": e,
276
+ }
277
+ )
278
+ if not any(p["examples"] for p in poles):
279
+ return None
280
+ return {"poles": poles, "shared": both}
281
+
282
+
283
+ def compile_site(
284
+ records: Sequence[AuditRecord],
285
+ output_dir: Path,
286
+ *,
287
+ site_title: str = "Open Interpretability",
288
+ wiring_rows: dict | None = None,
289
+ draws: Sequence = (),
290
+ corpus_sources: dict | None = None,
291
+ alignment_rows: dict | None = None,
292
+ sister_rows: dict | None = None,
293
+ papers_dir: Path | None = None,
294
+ stops: Sequence[ChosenStop] = (),
295
+ model_facts: dict[str, ModelFacts] | None = None,
296
+ jobs: int = 1,
297
+ errata: Sequence[Erratum] = (),
298
+ users: dict[str, str] | None = None,
299
+ coverage_rows: dict | None = None,
300
+ ) -> BuildReport:
301
+ """Render every reviewed experiment in ``records``, normally the wiki archive.
302
+
303
+ Rows fill in what old records lack. ``wiring_rows`` and ``corpus_sources`` come from
304
+ :func:`openinterp.wiring_backfill.load_rows` and :func:`openinterp.corpus.load_sources`.
305
+ ``alignment_rows`` (:func:`openinterp.crossref.load_rows`) are each pole's aligned
306
+ latents. ``sister_rows`` (:func:`openinterp.sisters.load_rows`) are its matches.
307
+ ``papers_dir`` holds the Research section (:mod:`openinterp.builder.research`).
308
+ ``stops`` (:func:`openinterp.core.draws.load_stops`) go on their unit's page, or the home's.
309
+ ``model_facts`` (:func:`openinterp.builder.model_facts.load_model_facts`) fill the
310
+ Models page, which lists every model in ``[tool.openinterp] models``.
311
+ ``jobs`` is the number of worker processes that render the unit pages.
312
+ ``errata`` (:func:`openinterp.core.errata.load_errata`) alter the experiments they cover:
313
+ each shows the erratum, and one set aside counts toward no standing.
314
+ ``users`` (:func:`openinterp.builder.users.load_users`) name who made the records that
315
+ record no contributor. Each person gets a page under ``users/``.
316
+ ``coverage_rows`` (:func:`openinterp.builder.coverage.load_rows`) give the home page each
317
+ model's registry coverage.
318
+ """
319
+ rows = wiring_rows or {}
320
+ output_dir = Path(output_dir)
321
+ _clean_owned_paths(output_dir)
322
+ output_dir.mkdir(parents=True, exist_ok=True)
323
+
324
+ models = build_tree(records, errata, users)
325
+ add_stopped(models, (*draws, *stops)) # neurons that only stopped scans cover
326
+ add_members(models) # the neurons of each superposition, linked back to it
327
+ experiments = [
328
+ e for m in models for layer in m.layers for n in layer.units for e in n.experiments
329
+ ]
330
+ # Statistics read the true text: build_tree counts near-duplicate probes, and the queries
331
+ # group readings by explanation. Only what the site publishes is scrubbed. The two
332
+ # exports run in the background while the pages render (builder/parallel.py).
333
+ stats_result = in_background(lambda: _true_text_stats(experiments, draws, corpus_sources), jobs)
334
+ for experiment in experiments:
335
+ experiment.record = scrub_record(experiment.record)
336
+ report = BuildReport(output_dir, len(records), len(experiments))
337
+ parquet = in_background(lambda: export_registry(experiments, output_dir, corpus_sources), jobs)
338
+ export_draws(draws, output_dir) # random draws no experiment could audit
339
+ export_neuronpedia(experiments, output_dir, project_links()["repo_url"])
340
+
341
+ common = _site_context(site_title, corpus_sources, experiments)
342
+ render = _Renderer(_environment(), output_dir, common, report, jobs)
343
+ studied = {n.href for m in models for layer in m.layers for n in layer.pages}
344
+ by_unit = stops_by_unit(stops)
345
+ for model in models:
346
+ sisters = (sister_rows or {}, sister_tests(experiments))
347
+ _render_model(render, model, rows, studied, alignment_rows or {}, sisters, by_unit)
348
+ render_models(render, models, model_facts or {}, _render_model)
349
+ agents = _render_agents(render, experiments, output_dir, report)
350
+ stats = stats_result()
351
+ count_pageless_neurons(stats["totals"], models) # stopped ones, and superposition members
352
+ stats["coverage"] = [model_summary(r) for r in (coverage_rows or {}).values() if r]
353
+ _render_index(render, models, experiments, agents, stats, (*draws, *stops), by_unit, studied)
354
+ report.parquet = parquet()
355
+ render_reference_pages(render, papers_dir, report)
356
+ _save_memos()
357
+ return report
358
+
359
+
360
+ def _save_memos() -> None:
361
+ """Keep the matches of new text, and the significance of new records, for the next build."""
362
+ save_spans()
363
+ P_USEFUL.save()
364
+
365
+
366
+ def _true_text_stats(experiments, draws, corpus_sources) -> dict:
367
+ """Return the home page's totals and leaderboards, read from the records' true text.
368
+
369
+ Only the leaderboards' text is scrubbed, because they show explanations.
370
+ """
371
+ with TemporaryDirectory() as unpublished:
372
+ true_text = export_registry(experiments, Path(unpublished), corpus_sources)
373
+ return scrub(registry_stats(true_text, draws))
374
+
375
+
376
+ def _render_agents(render, experiments, output_dir, report) -> list:
377
+ """Write the Agents and Users pages, each agent's and person's page, and redirects."""
378
+ agents = agents_view(experiments)
379
+ contributors = contributors_view(experiments)
380
+ render("agents/index.html", "agents.html", agents=agents, contributors=contributors)
381
+ render("users/index.html", "users.html", users=contributors)
382
+ for person in contributors:
383
+ render(person.href, "user.html", user=person)
384
+ for agent in agents:
385
+ render(agent.href, "agent.html", agent=agent)
386
+ for old, new in legacy_redirects(agents).items():
387
+ _write_redirect(output_dir / old, Path(new).name, report)
388
+ return agents
389
+
390
+
391
+ def _render_index(render, models, experiments, agents, stats, draws, by_unit, studied) -> None:
392
+ render(
393
+ "index.html",
394
+ "index.html",
395
+ models=models,
396
+ explore=explore_focus(models, draws), # draws and stops: proven dead counts as verified
397
+ stats=stats,
398
+ dashboard=dashboard(stats["verdicts"], models),
399
+ agents=agents,
400
+ fdr=false_discovery(models),
401
+ agreement=reviewer_agreement(models),
402
+ corpora=corpora_view(experiments),
403
+ contested=contested_units(models),
404
+ pageless_stops=pageless_stops(by_unit, studied),
405
+ )
406
+ if stats["baseline"]: # the home page links to it (#3272)
407
+ render("learned-baseline.html", "learned-baseline.html", stats=stats)
408
+
409
+
410
+ def _site_context(site_title: str, corpus_sources: dict | None, experiments) -> dict:
411
+ """What every page's template gets."""
412
+ return {
413
+ **project_links(),
414
+ "site_title": site_title,
415
+ "corpus_sources": corpus_sources or {}, # for records with no corpus_source of their own
416
+ "model_licenses": configured_licenses(),
417
+ "version": __version__,
418
+ # Derived from the data, never the clock: an unchanged registry must rebuild
419
+ # byte-for-byte so the deploy uploads (and purges) nothing.
420
+ "updated_at": max((e.record.verdict.timestamp for e in experiments), default=None),
421
+ }
422
+
423
+
424
+ def _write_redirect(out: Path, target: str, report: BuildReport) -> None:
425
+ """A page that used to live at ``out`` and now lives at ``target`` (relative to it)."""
426
+ out.parent.mkdir(parents=True, exist_ok=True)
427
+ out.write_text(
428
+ f'<!doctype html><meta charset="utf-8"><title>Moved</title>'
429
+ f'<link rel="canonical" href="{target}"><meta name="robots" content="noindex">'
430
+ f'<meta http-equiv="refresh" content="0; url={target}">'
431
+ f'<p>This page moved to <a href="{target}">{target}</a>.</p>\n'
432
+ )
433
+ report.pages.append(out)
434
+
435
+
436
+ @dataclass
437
+ class _Renderer:
438
+ """Renders a template to a path under the output dir, with the site-wide context."""
439
+
440
+ env: Environment
441
+ output_dir: Path
442
+ common: dict
443
+ report: BuildReport
444
+ jobs: int = 1 # worker processes for the unit pages (builder/parallel.py)
445
+
446
+ def __call__(self, rel: str, template: str, **context) -> None:
447
+ out = self.output_dir / rel
448
+ out.parent.mkdir(parents=True, exist_ok=True)
449
+ page = self.env.get_template(template)
450
+ url = self.common["site_url"] + rel
451
+ out.write_text(page.render(**self.common, root=_root_for(rel), canonical=url, **context))
452
+ self.report.pages.append(out)
453
+
454
+
455
+ def _render_model(
456
+ render: _Renderer,
457
+ model,
458
+ rows: dict,
459
+ studied: set,
460
+ alignment: dict | None = None,
461
+ sisters: tuple[dict, dict] = ({}, {}),
462
+ stops: dict | None = None,
463
+ ) -> None:
464
+ """A model's page, its circuits page (if any), and its layer and unit pages."""
465
+ found, found_hubs = circuits_and_hubs(model, rows)
466
+ sister_rows, tests = sisters
467
+ named_by = by_match_slug(sister_rows)
468
+ page = circuits_href(model.slug)
469
+ graphs = sister_graphs(model, sister_rows, studied)
470
+ shown = {"circuits": found, "hubs": found_hubs, "circuits_page": page, "sister_graphs": graphs}
471
+ render(model.href, "model.html", model=model, **shown)
472
+ circuit_of = render_circuits(render, model, found, found_hubs)
473
+ for layer in model.layers:
474
+ render(layer.href, "layer.html", model=model, layer=layer)
475
+
476
+ def unit_page(layer, neuron) -> None:
477
+ circuit = circuit_of.get(neuron.href)
478
+ for rank, e in enumerate(neuron.experiments, 1):
479
+ view = _experiment_view(e, studied, rows, circuit)
480
+ unit = {"model": model, "layer": layer, "neuron": neuron}
481
+ render(e.href, "experiment.html", **unit, v=view, rank=rank)
482
+ render(
483
+ neuron.href,
484
+ "neuron.html",
485
+ model=model,
486
+ layer=layer,
487
+ neuron=neuron,
488
+ poles=pole_scans(neuron),
489
+ aligned=aligned_latents(model, neuron, alignment or {}, studied),
490
+ sisters=sister_view(model.slug, neuron, sister_rows, named_by, studied, tests),
491
+ stops=(stops or {}).get(neuron.href, []),
492
+ )
493
+
494
+ units = [(layer, neuron) for layer in model.layers for neuron in layer.pages]
495
+ render_in_parallel(render, unit_page, units)
@@ -0,0 +1,98 @@
1
+ """The corpora the registry's experiments were scanned on, and each one's language mix.
2
+
3
+ A scan of a corpus whose documents carry ``lang`` counts its tokens per language
4
+ (``ConfoundProfile.labels``). So a record knows the language mix of what it scanned.
5
+ Registry statistics over ``selection = "random"`` draws depend on that mix, so the site
6
+ shows it for each corpus. Tokens are the model's, so the mix is per model. One corpus
7
+ splits into more tokens of some languages under one tokenizer than under another.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from collections.abc import Sequence
13
+ from dataclasses import dataclass, field
14
+
15
+ from openinterp.core.confounds import LANGUAGE
16
+
17
+ SHOWN_LANGUAGES = 8 # languages listed per mix; the rest are summed as "other"
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class LanguageShare:
22
+ """One language's tokens in a scan."""
23
+
24
+ label: str
25
+ tokens: int
26
+ share: float # of the scan's labelled tokens
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class LanguageMix:
31
+ """The languages of one scan's labelled tokens, most frequent first."""
32
+
33
+ model_name: str
34
+ tokens: int # labelled tokens
35
+ sampled: bool # from a --max-tokens sample of the corpus, not the whole of it
36
+ shares: list[LanguageShare]
37
+
38
+ @property
39
+ def shown(self) -> list[LanguageShare]:
40
+ """The languages that the page lists by name."""
41
+ return self.shares[:SHOWN_LANGUAGES]
42
+
43
+ @property
44
+ def other(self) -> float:
45
+ """The share of labelled tokens in languages not :attr:`shown`.
46
+
47
+ The record lists at most 50 languages, so this includes any that it left out.
48
+ """
49
+ return max(0.0, 1.0 - sum(s.share for s in self.shown))
50
+
51
+
52
+ def language_mix(submission) -> LanguageMix | None:
53
+ """The language mix of the submission's scan, or ``None`` if it had no language labels."""
54
+ confounds = submission.confounds
55
+ found = (confounds.labels or {}).get(LANGUAGE) if confounds is not None else None
56
+ if found is None or not found.labelled_tokens:
57
+ return None
58
+ total = found.labelled_tokens
59
+ shares = [LanguageShare(s.label, s.count, s.count / total) for s in found.levels]
60
+ sampled = submission.provenance is not None and submission.provenance.corpus_sample is not None
61
+ return LanguageMix(submission.identifier.model_name, total, sampled, shares)
62
+
63
+
64
+ @dataclass
65
+ class CorpusView:
66
+ """One scan corpus on the index: its experiments and language mixes."""
67
+
68
+ sha256: str | None
69
+ name: str # the provenance's description of the corpus
70
+ experiments: int = 0
71
+ mixes: list[LanguageMix] = field(default_factory=list) # one per model, by name
72
+
73
+
74
+ def corpora_view(experiments: Sequence) -> list[CorpusView]:
75
+ """Return every corpus that some experiment's provenance names, most used first.
76
+
77
+ Each has its language mix under each model that scanned it with labels. Each mix is
78
+ from the scan with the most labelled tokens, preferring whole-corpus scans to samples.
79
+ """
80
+ corpora: dict[str | None, CorpusView] = {}
81
+ best: dict[tuple[str | None, str], LanguageMix] = {}
82
+ for e in experiments:
83
+ sub = e.record.submission
84
+ p = sub.provenance
85
+ if p is None:
86
+ continue
87
+ view = corpora.setdefault(p.corpus_sha256, CorpusView(p.corpus_sha256, p.corpus))
88
+ view.experiments += 1
89
+ mix = language_mix(sub)
90
+ if mix is None:
91
+ continue
92
+ key = (p.corpus_sha256, mix.model_name)
93
+ old = best.get(key)
94
+ if old is None or (not mix.sampled, mix.tokens) > (not old.sampled, old.tokens):
95
+ best[key] = mix
96
+ for (sha, _), mix in sorted(best.items(), key=lambda kv: kv[0][1]):
97
+ corpora[sha].mixes.append(mix)
98
+ return sorted(corpora.values(), key=lambda c: (-c.experiments, c.name))
@@ -0,0 +1,46 @@
1
+ """Registry coverage for the site: each model's newest `open-interpretability coverage` run.
2
+
3
+ ``openinterp/coverage.py`` measures coverage, which needs the model. This module only
4
+ reads the rows that the command wrote, so the site build needs no model and no torch.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from pathlib import Path
11
+
12
+ DEFAULT_ROWS = Path("docs/analyses/registry-coverage.jsonl")
13
+
14
+
15
+ def load_rows(path: Path = DEFAULT_ROWS) -> dict[str, list[dict]]:
16
+ """Each model's newest run in ``path``, as ``{model: [rows by layer]}``. Empty if missing."""
17
+ path = Path(path)
18
+ if not path.exists():
19
+ return {}
20
+ newest: dict[str, str] = {}
21
+ rows = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
22
+ for row in rows:
23
+ newest[row["model"]] = max(newest.get(row["model"], ""), row["date"])
24
+ out: dict[str, list[dict]] = {}
25
+ for row in rows:
26
+ if row["date"] == newest[row["model"]]:
27
+ out.setdefault(row["model"], []).append(row)
28
+ return {m: sorted(rs, key=lambda r: r["layer"]) for m, rs in out.items()}
29
+
30
+
31
+ def model_summary(rows: list[dict]) -> dict:
32
+ """Loss-weighted shares over a model's layers, for the VERIFIED poles and for random ones."""
33
+ rise = sum(r["loss_ablated"] - r["loss_clean"] for r in rows)
34
+ kept = sum(r["loss_ablated"] - r.get("loss_kept_verified", r["loss_ablated"]) for r in rows)
35
+ rand = sum(
36
+ (r.get("recovered_random") or 0.0) * (r["loss_ablated"] - r["loss_clean"]) for r in rows
37
+ )
38
+ return {
39
+ "model": rows[0]["model"],
40
+ "date": rows[0]["date"],
41
+ "layers": len(rows),
42
+ "verified_poles": sum(r["verified_poles"] for r in rows),
43
+ "poles": sum(2 * r["width"] for r in rows),
44
+ "recovered": kept / rise if rise > 0 else None,
45
+ "recovered_random": rand / rise if rise > 0 else None,
46
+ }