open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,329 @@
1
+ """Per-token labels for the scan, and the confound any label set gives.
2
+
3
+ A :class:`Tagger` labels character spans of a document. A document label (its language)
4
+ is one span over the whole text. A word tagger (part of speech) gives one span per word.
5
+ The scan aligns each tagger's spans to the model's tokens once per piece, through the
6
+ tokenizer's character offsets. It keeps the labels next to the activations it counts.
7
+
8
+ For every label set, :class:`LabelAccumulator` reports what ``token_r2`` reports for
9
+ token identity, and in information units:
10
+
11
+ * ``r2``: variance explained by a per-label mean model, over labelled tokens;
12
+ * ``mutual_information_bits``: I(activation; label) in bits. Pole strengths are sparse
13
+ and heavy-tailed, so the estimate is binned, not Gaussian. Zero is a bin of its own,
14
+ and positive strengths fall in ``ACTIVATION_BINS`` equal-count bins. Each entropy has
15
+ the Miller–Madow correction. The plug-in estimate is biased upwards by about
16
+ (labels − 1)(bins − 1) / 2N nats when the two are independent.
17
+
18
+ Some tokens are left out of both numbers: those that a tagger leaves unlabelled. They
19
+ are special tokens, words that it doesn't cover, and pieces whose text is not in their
20
+ document.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import math
26
+ from bisect import bisect_left
27
+ from collections.abc import Callable, Sequence
28
+ from dataclasses import dataclass
29
+ from itertools import accumulate
30
+ from typing import TYPE_CHECKING, Protocol
31
+
32
+ import torch
33
+
34
+ from openinterp.core.confounds import NO_MATCH
35
+ from openinterp.core.schemas import ConfoundProfile, LabelConfound, LabelStat, ReviewPolicy
36
+
37
+ if TYPE_CHECKING:
38
+ from openinterp.harness.scan import Batch, Chunk
39
+
40
+ ACTIVATION_BINS = 8
41
+ MAX_LEVELS = 50 # labels listed in a record, most frequent first
42
+ UNLABELLED = -1
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class Span:
47
+ """Characters [start, end) of a document carry ``label``."""
48
+
49
+ start: int
50
+ end: int
51
+ label: str
52
+
53
+
54
+ class Tagger(Protocol):
55
+ """Anything that labels character spans of a document, under one label set's name."""
56
+
57
+ name: str # the label set's name in the record (``"language"``)
58
+
59
+ def spans(self, doc: int, text: str) -> list[Span]:
60
+ """The labelled spans of document ``doc``, whose text is ``text``."""
61
+ ...
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class DocumentLabels:
66
+ """One label per document, known in advance (a corpus field such as ``lang``)."""
67
+
68
+ name: str
69
+ labels: Sequence[str | None]
70
+
71
+ def spans(self, doc: int, text: str) -> list[Span]:
72
+ """One span over the whole text, or none when the document has no label."""
73
+ label = self.labels[doc]
74
+ return [] if label is None else [Span(0, len(text), label)]
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class MatchTagger:
79
+ """Label what ``find`` matches in a document, and every other character ``NO_MATCH``."""
80
+
81
+ name: str
82
+ find: Callable[[str], Sequence[tuple[int, int, str]]]
83
+
84
+ def spans(self, doc: int, text: str) -> list[Span]:
85
+ """The matches, then the gaps, so a token half in a match takes the match's label."""
86
+ matches = self.find(text)
87
+ spans = [Span(start, end, label) for start, end, label in matches]
88
+ position = 0
89
+ for start, end, _ in matches:
90
+ if start > position:
91
+ spans.append(Span(position, start, NO_MATCH))
92
+ position = max(position, end)
93
+ if position < len(text):
94
+ spans.append(Span(position, len(text), NO_MATCH))
95
+ return spans
96
+
97
+
98
+ def align(offsets: Sequence[tuple[int, int]], spans: Sequence[Span]) -> list[str | None]:
99
+ """Return each token's label: that of the span that covers most of its characters.
100
+
101
+ The first span wins a tie. ``None`` when no span covers the token. Tokens with no
102
+ characters (special tokens) are unlabelled.
103
+ """
104
+ # A document has about as many syntax spans as tokens, so reading every span for every
105
+ # token is too slow. The spans are sorted by start. A walk back from the token's end
106
+ # stops when no earlier span reaches past the token's start.
107
+ order = sorted(range(len(spans)), key=lambda i: spans[i].start)
108
+ starts = [spans[i].start for i in order]
109
+ reach = list(accumulate((spans[i].end for i in order), max))
110
+ out: list[str | None] = []
111
+ for start, end in offsets:
112
+ best, most, first = None, 0, len(spans)
113
+ i = bisect_left(starts, end) - 1
114
+ while i >= 0 and reach[i] > start:
115
+ span = spans[order[i]]
116
+ overlap = min(end, span.end) - max(start, span.start)
117
+ if overlap > most or (overlap == most and overlap > 0 and order[i] < first):
118
+ best, most, first = span.label, overlap, order[i]
119
+ i -= 1
120
+ out.append(best)
121
+ return out
122
+
123
+
124
+ class TokenLabels:
125
+ """The labels of every piece's tokens for each tagger, aligned once per piece."""
126
+
127
+ def __init__(self, tokenizer, texts: Sequence[str], taggers: Sequence[Tagger]) -> None:
128
+ self.tokenizer, self.texts, self.taggers = tokenizer, texts, list(taggers)
129
+ self.names: dict[str, list[str]] = {t.name: [] for t in self.taggers}
130
+ self._index: dict[str, dict[str, int]] = {t.name: {} for t in self.taggers}
131
+ self._spans: dict[tuple[str, int], list[Span]] = {}
132
+ self._pieces: dict = {}
133
+
134
+ def of(self, piece, ids: torch.Tensor) -> dict[str, torch.Tensor]:
135
+ """``{tagger: [n] label index or UNLABELLED}`` for the piece's real tokens ``ids``."""
136
+ if piece not in self._pieces:
137
+ self._pieces[piece] = {t.name: self._align(t, piece, ids) for t in self.taggers}
138
+ return self._pieces[piece]
139
+
140
+ def at(self, piece, ids: Sequence[int], index: int) -> dict[str, str] | None:
141
+ """Return each tagger's label of token ``index`` of the piece's real tokens ``ids``.
142
+
143
+ Only the taggers that label it are included. ``None`` if none does.
144
+ """
145
+ row = self.of(piece, torch.as_tensor(ids, dtype=torch.long))
146
+ found = {name: int(labels[index]) for name, labels in row.items()}
147
+ out = {name: self.names[name][i] for name, i in found.items() if i != UNLABELLED}
148
+ return out or None
149
+
150
+ def _align(self, tagger: Tagger, piece, ids: torch.Tensor) -> torch.Tensor:
151
+ key = (tagger.name, piece.doc)
152
+ if key not in self._spans:
153
+ self._spans[key] = tagger.spans(piece.doc, self.texts[piece.doc])
154
+ labels = self._piece_labels(self._spans[key], piece, ids)
155
+ index = self._index[tagger.name]
156
+ for label in labels:
157
+ if label is not None and label not in index:
158
+ index[label] = len(index)
159
+ self.names[tagger.name].append(label)
160
+ ids = [UNLABELLED if lab is None else index[lab] for lab in labels]
161
+ return torch.tensor(ids, dtype=torch.long)
162
+
163
+ def _piece_labels(self, spans: list[Span], piece, ids: torch.Tensor) -> list[str | None]:
164
+ if not spans: # no tokenising for a document with no labels (code with no grammar)
165
+ return [None] * len(ids)
166
+ special = set(getattr(self.tokenizer, "all_special_ids", None) or [])
167
+ text = self.texts[piece.doc]
168
+ if len(spans) == 1 and spans[0].start == 0 and spans[0].end >= len(text):
169
+ # A document label covers every token: no offsets needed.
170
+ return [None if int(i) in special else spans[0].label for i in ids]
171
+ shift = self._shift(piece, text)
172
+ offsets = self._offsets(piece.text) if shift is not None else None
173
+ if offsets is None or len(offsets) != len(ids):
174
+ return [None] * len(ids)
175
+ return align([(a + shift, b + shift) for a, b in offsets], spans)
176
+
177
+ def _shift(self, piece, text: str) -> int | None:
178
+ """Return where the piece's text starts in its document.
179
+
180
+ ``None`` if it isn't there. A window's decoded text need not match the document's
181
+ characters.
182
+ """
183
+ if piece.start == 0:
184
+ return 0 if text.startswith(piece.text) else None
185
+ doc_offsets = self._offsets(text, special=False)
186
+ if doc_offsets is None or piece.start >= len(doc_offsets):
187
+ return None
188
+ shift = doc_offsets[piece.start][0]
189
+ return shift if text[shift : shift + len(piece.text)] == piece.text else None
190
+
191
+ def _offsets(self, text: str, special: bool = True) -> list[tuple[int, int]] | None:
192
+ """Return the character offsets of ``text``'s tokens as the scan encodes them.
193
+
194
+ ``None`` if the tokenizer can't give them. Only fast tokenizers can.
195
+ """
196
+ try:
197
+ enc = self.tokenizer(text, return_offsets_mapping=True, add_special_tokens=special)
198
+ except (TypeError, NotImplementedError):
199
+ return None
200
+ return [tuple(o) for o in enc["offset_mapping"]]
201
+
202
+
203
+ class LabelAccumulator:
204
+ """Pole strengths and label indices of every counted token, for one label set."""
205
+
206
+ def __init__(self) -> None:
207
+ self.labels: list[torch.Tensor] = []
208
+ self.values: list[torch.Tensor] = []
209
+
210
+ def update(self, labels: torch.Tensor, strengths: torch.Tensor) -> None:
211
+ """Add a batch of label indices and strengths."""
212
+ self.labels.append(labels.long())
213
+ self.values.append(strengths.double())
214
+
215
+ def finalize(self, names: Sequence[str], line: float) -> LabelConfound:
216
+ """The confound, counting a token active at or above ``line``."""
217
+ labels = torch.cat(self.labels) if self.labels else torch.zeros(0, dtype=torch.long)
218
+ values = torch.cat(self.values) if self.values else torch.zeros(0, dtype=torch.float64)
219
+ seen = labels != UNLABELLED
220
+ labels, values = labels[seen], values[seen]
221
+ size = len(names)
222
+ n = torch.bincount(labels, minlength=size).double()
223
+ s = torch.bincount(labels, weights=values, minlength=size)
224
+ active = torch.bincount(labels[values >= line], minlength=size) if line > 0 else n * 0
225
+ order = n.argsort(descending=True)[:MAX_LEVELS]
226
+ return LabelConfound(
227
+ r2=round(_r2(values, n, s), 6),
228
+ mutual_information_bits=round(mutual_information(values, labels, size), 6),
229
+ labelled_tokens=int(labels.numel()),
230
+ unlabelled_tokens=int((~seen).sum()),
231
+ levels=[
232
+ LabelStat(
233
+ label=names[i],
234
+ count=int(n[i]),
235
+ mean_strength=round(max(0.0, float(s[i] / n[i])), 6),
236
+ active=int(active[i]),
237
+ )
238
+ for i in order.tolist()
239
+ if n[i] > 0
240
+ ],
241
+ )
242
+
243
+
244
+ def _r2(values: torch.Tensor, n: torch.Tensor, s: torch.Tensor) -> float:
245
+ """Between-label sum of squares over total sum of squares."""
246
+ count = values.numel()
247
+ if count == 0:
248
+ return 0.0
249
+ total = float((values * values).sum()) - float(values.sum()) ** 2 / count
250
+ seen = n > 0
251
+ between = float((s[seen] ** 2 / n[seen]).sum()) - float(values.sum()) ** 2 / count
252
+ return min(1.0, max(0.0, between / total)) if total > 1e-12 else 0.0
253
+
254
+
255
+ def activation_bins(values: torch.Tensor, bins: int = ACTIVATION_BINS) -> torch.Tensor:
256
+ """Return each value's bin: 0 for zero strength, then equal-count bins of the rest.
257
+
258
+ There are ``bins`` bins of the positive strengths, or fewer where ties make edges
259
+ coincide.
260
+ """
261
+ out = torch.zeros(values.numel(), dtype=torch.long)
262
+ positive = values > 0
263
+ if positive.any():
264
+ q = torch.linspace(0, 1, bins + 1, dtype=torch.float64)[1:-1]
265
+ edges = torch.quantile(values[positive].double(), q).unique()
266
+ out[positive] = 1 + torch.bucketize(values[positive], edges, right=True)
267
+ return out
268
+
269
+
270
+ def mutual_information(values: torch.Tensor, labels: torch.Tensor, n_labels: int) -> float:
271
+ """I(binned strength; label) in bits, each entropy Miller–Madow corrected; at least 0."""
272
+ count = values.numel()
273
+ if count == 0 or n_labels < 2:
274
+ return 0.0
275
+ bins = activation_bins(values)
276
+ joint = torch.bincount(bins * n_labels + labels).double()
277
+ mi = _entropy(torch.bincount(bins).double(), count)
278
+ mi += _entropy(torch.bincount(labels).double(), count) - _entropy(joint, count)
279
+ return max(0.0, mi / math.log(2))
280
+
281
+
282
+ def _entropy(counts: torch.Tensor, total: int) -> float:
283
+ """Miller–Madow entropy in nats."""
284
+ counts = counts[counts > 0]
285
+ p = counts / total
286
+ return float(-(p * p.log()).sum()) + (counts.numel() - 1) / (2 * total)
287
+
288
+
289
+ class LabelPass:
290
+ """Each tagger's labels of a scan's counted tokens, with their strengths.
291
+
292
+ They are gathered pass by pass. They give the label confounds (see
293
+ ``harness/annotations.py``).
294
+ """
295
+
296
+ def __init__(self, tokenizer, texts: Sequence[str], taggers: Sequence[Tagger]) -> None:
297
+ self.tokens = TokenLabels(tokenizer, texts, taggers)
298
+ self.accumulators = {t.name: LabelAccumulator() for t in self.tokens.taggers}
299
+
300
+ def update_chunk(self, chunk: Chunk, strengths: torch.Tensor) -> None:
301
+ """Add a chunk's counted tokens; ``strengths`` is ``[n]``, one per real token."""
302
+ if self.accumulators:
303
+ rows = [self.tokens.of(piece, chunk.ids[r]) for piece, r in chunk.rows()]
304
+ self._add(rows, chunk.counted, strengths)
305
+
306
+ def update(self, batch: Batch, strengths: torch.Tensor) -> None:
307
+ """Add a batch's counted tokens; ``strengths`` is ``[b, seq]``, padding included."""
308
+ if self.accumulators:
309
+ mask = batch.mask
310
+ rows = [self.tokens.of(p, batch.ids[i][mask[i]]) for i, p in enumerate(batch.pieces)]
311
+ self._add(rows, batch.counted[mask], strengths.cpu()[mask])
312
+
313
+ def _add(self, rows, counted: torch.Tensor, strengths: torch.Tensor) -> None:
314
+ for name, accumulator in self.accumulators.items():
315
+ every = torch.cat([row[name] for row in rows])
316
+ accumulator.update(every[counted], strengths[counted])
317
+
318
+ def finalize(self, confounds: ConfoundProfile, values: Sequence[float]) -> ConfoundProfile:
319
+ """Return ``confounds`` with each label set's confound added.
320
+
321
+ A token counts as active at or above the protocol's line for the scan's settled
322
+ ``values``.
323
+ """
324
+ if not self.accumulators:
325
+ return confounds
326
+ line = ReviewPolicy().activation_threshold * max(values, default=0.0)
327
+ names = self.tokens.names
328
+ by_label = {n: a.finalize(names[n], line) for n, a in self.accumulators.items()}
329
+ return confounds.model_copy(update={"labels": by_label})
@@ -0,0 +1,96 @@
1
+ """Attribution patching: a first-order approximation of activation patching.
2
+
3
+ Take every neuron in the requested layers. The effect of patching its clean activation
4
+ into the corrupted run is approximated as follows:
5
+
6
+ (a_clean - a_corrupted) * d metric / d a |_corrupted
7
+
8
+ This needs one clean forward pass and one corrupted forward + backward pass, not one
9
+ forward pass per neuron. Use it to triage candidates, then confirm
10
+ with exact patching in :mod:`openinterp.harness.patcher`.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from collections.abc import Sequence
16
+
17
+ import torch
18
+
19
+ from openinterp.core.exceptions import ShapeMismatchError
20
+ from openinterp.core.schemas import Pole
21
+ from openinterp.harness.hooks import Inputs, NeuronHookRunner, isolate_pole
22
+ from openinterp.harness.patcher import Metric, Positions, token_logit_metric
23
+
24
+
25
+ def attribution_patching(
26
+ runner: NeuronHookRunner,
27
+ clean: Inputs,
28
+ corrupted: Inputs,
29
+ layers: Sequence[int],
30
+ *,
31
+ metric: Metric | None = None,
32
+ pole: Pole | None = None,
33
+ positions: Positions = "all",
34
+ ) -> dict[int, torch.Tensor]:
35
+ """Estimated metric change per neuron, ``{layer: tensor[d_mlp]}``.
36
+
37
+ With ``pole`` set, only that pole's activation difference is attributed. The
38
+ default metric is the logit of the clean run's top-1 final-position token.
39
+ """
40
+ clean_logits, clean_cache = runner.run_with_cache(clean, layers)
41
+ if metric is None:
42
+ metric = token_logit_metric(int(clean_logits[0, -1].argmax()))
43
+
44
+ ordered, grads = _corrupted_grads(runner, corrupted, layers, metric)
45
+ scores: dict[int, torch.Tensor] = {}
46
+ for layer, corrupt_act, grad in zip(layers, ordered, grads, strict=True):
47
+ clean_act, corrupt_act = clean_cache[layer], corrupt_act.detach()
48
+ if positions == "last":
49
+ clean_act, corrupt_act, grad = clean_act[:, -1:], corrupt_act[:, -1:], grad[:, -1:]
50
+ elif clean_act.shape != corrupt_act.shape:
51
+ raise ShapeMismatchError(
52
+ f"clean {tuple(clean_act.shape)} vs corrupted {tuple(corrupt_act.shape)}; "
53
+ "use positions='last' or align the prompts."
54
+ )
55
+ if pole is not None:
56
+ clean_act, corrupt_act = isolate_pole(clean_act, pole), isolate_pole(corrupt_act, pole)
57
+ scores[layer] = ((clean_act - corrupt_act) * grad).sum(dim=(0, 1)).detach().cpu()
58
+ return scores
59
+
60
+
61
+ def _corrupted_grads(runner: NeuronHookRunner, corrupted: Inputs, layers, metric: Metric):
62
+ """Each layer's MLP activations on ``corrupted`` and the metric's gradient there."""
63
+ captured: dict[int, torch.Tensor] = {}
64
+
65
+ def capture(layer: int):
66
+ def hook(act: torch.Tensor) -> None:
67
+ captured[layer] = act
68
+
69
+ return hook
70
+
71
+ def ensure_grad(_module, _args, output: torch.Tensor) -> torch.Tensor:
72
+ # Frozen weights mean nothing upstream requires grad; make the embedding
73
+ # output a fresh leaf so the graph through the MLPs is recorded.
74
+ return output if output.requires_grad else output.detach().requires_grad_(True)
75
+
76
+ encoded = runner.encode(corrupted)
77
+ embed_handle = runner.model.get_input_embeddings().register_forward_hook(ensure_grad)
78
+ try:
79
+ with torch.enable_grad(), runner.hooks({layer: capture(layer) for layer in layers}):
80
+ value = metric(runner.forward(encoded))
81
+ ordered = [captured[layer] for layer in layers]
82
+ grads = torch.autograd.grad(value, ordered)
83
+ finally:
84
+ embed_handle.remove()
85
+ return ordered, grads
86
+
87
+
88
+ def top_attributed_neurons(
89
+ scores: dict[int, torch.Tensor], k: int = 10
90
+ ) -> list[tuple[int, int, float]]:
91
+ """Flatten attribution scores to ``(layer, neuron_idx, score)`` sorted by |score|."""
92
+ flat: list[tuple[int, int, float]] = []
93
+ for layer, vec in scores.items():
94
+ for idx in vec.abs().topk(min(k, vec.numel())).indices.tolist():
95
+ flat.append((layer, idx, float(vec[idx])))
96
+ return sorted(flat, key=lambda t: abs(t[2]), reverse=True)[:k]
@@ -0,0 +1,264 @@
1
+ """Causal metrics of a unit: sufficiency and necessity, ranked against comparable units.
2
+
3
+ Recorded in every verdict, never gated (docs/PROTOCOL.md §4). A neuron pole is ranked
4
+ against the same layer's neurons. An attention head is ranked against the same layer's
5
+ heads. A dictionary feature is ranked against the same dictionary's other latents.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import math
11
+ import statistics
12
+ from collections.abc import Sequence
13
+ from dataclasses import dataclass
14
+ from typing import Any
15
+
16
+ import torch
17
+
18
+ from openinterp.core.schemas import (
19
+ CausalMetrics,
20
+ CausalPairEffect,
21
+ CausalTarget,
22
+ NeuronAuditSubmission,
23
+ PatchEffect,
24
+ )
25
+ from openinterp.harness.hooks import NeuronHookRunner, pole_strength
26
+ from openinterp.harness.layer_major import map_layer_major, prompt_tokens
27
+ from openinterp.harness.patcher import (
28
+ ActivationPatcher,
29
+ Positions,
30
+ logit_direction_metric,
31
+ metric_change,
32
+ next_token_kl,
33
+ )
34
+
35
+
36
+ def claimed_token_weights(
37
+ submission: NeuronAuditSubmission, runner: NeuronHookRunner
38
+ ) -> tuple[torch.Tensor, list[str]] | None:
39
+ """Equal vocabulary weights on a lookahead claim's single-token targets, if any resolve."""
40
+ tokenizer = runner.tokenizer
41
+ if not submission.lookahead.is_lookahead or tokenizer is None:
42
+ return None
43
+ vocab = runner.model.get_output_embeddings().weight.shape[0]
44
+ ids: dict[int, str] = {}
45
+ for token in submission.lookahead.target_upcoming_tokens:
46
+ encoded = tokenizer.encode(token, add_special_tokens=False)
47
+ if len(encoded) == 1 and encoded[0] < vocab:
48
+ ids.setdefault(encoded[0], token)
49
+ if not ids:
50
+ return None
51
+ weights = torch.zeros(vocab)
52
+ weights[list(ids)] = 1.0 / len(ids)
53
+ return weights, list(ids.values())
54
+
55
+
56
+ def _neuron_changes(runner, ident, base: str, cf: str) -> tuple[Positions, int, torch.Tensor]:
57
+ """Where to patch and score, and every neuron's strength change up to there."""
58
+ b, c = ( # the pass stops at the layer: no later layer or logits is needed
59
+ pole_strength(runner.layer_activations(text, [ident.layer])[ident.layer][0], ident.pole)
60
+ for text in (base, cf)
61
+ ) # [seq, d_mlp]
62
+ if b.shape != c.shape:
63
+ return "last", -1, (b[-1] - c[-1]).abs()
64
+ position = int((b[:, ident.neuron_idx] - c[:, ident.neuron_idx]).abs().argmax())
65
+ return "all", position, (b[: position + 1] - c[: position + 1]).abs().amax(0)
66
+
67
+
68
+ def _head_changes(runner, ident, base: str, cf: str) -> tuple[Positions, int, torch.Tensor]:
69
+ """The same for heads, with each head's change in write ``|W_O,h (z_base − z_cf)|``.
70
+
71
+ The write change needs no head's mean output, so every head in the layer compares.
72
+ """
73
+ from openinterp.harness.heads import head_write_norms
74
+
75
+ z_b, z_c = (runner.head_outputs(text, ident.layer)[0].float() for text in (base, cf))
76
+ if z_b.shape != z_c.shape:
77
+ return "last", -1, head_write_norms(runner, ident.layer, z_b[-1] - z_c[-1])
78
+ changes = head_write_norms(runner, ident.layer, z_b - z_c) # [seq, n_heads]
79
+ position = int(changes[:, ident.head_idx].argmax())
80
+ return "all", position, changes[: position + 1].amax(0)
81
+
82
+
83
+ def causal_metrics(
84
+ submission: NeuronAuditSubmission,
85
+ runner: NeuronHookRunner,
86
+ *,
87
+ freeze_downstream: bool = True,
88
+ null_neurons: int = 32,
89
+ min_change: float = 0.0,
90
+ ) -> CausalMetrics | None:
91
+ """Sufficiency and necessity of the unit, each ranked against comparable units.
92
+
93
+ Every minimal pair is patched clean → corrupted (sufficiency) and corrupted → clean
94
+ (necessity). The patch is at the position where the unit differs most between the two
95
+ prompts. When their lengths differ, it is the final position, since only that can be
96
+ patched. A neuron pole's difference is its strength change. A head's or a dictionary
97
+ latent's difference is the change in its write to the residual stream. The effect is
98
+ scored there on a target that favours no unit:
99
+
100
+ * ``claimed_tokens``: a lookahead claim's upcoming tokens, as their mean logit minus
101
+ the vocabulary mean (signed);
102
+ * ``next_token_kl``: otherwise, KL(unpatched ‖ patched) of the next-token distribution.
103
+
104
+ The same patch is applied, one unit at a time, to ``null_neurons`` comparable units.
105
+ They are the units of the same kind and layer whose change between the two prompts is
106
+ most similar. That is neurons, the other heads, or the dictionary's other latents. A score is
107
+ the median over pairs of the unit's |effect| percentile among theirs. So 0.5 means
108
+ "no more influential than a unit that changes as much".
109
+
110
+ Pairs where the unit moves by at most ``min_change`` (controls) carry no causal
111
+ signal and are skipped; with none left the result is ``None``.
112
+
113
+ Two alternatives read wrongly on real models. Dividing by the whole clean/corrupted
114
+ gap (protocol < 0.5) reads ≈ 0 for any single neuron, because the gap sums every
115
+ component's contribution. Ranking against *random* neurons reads ≈ 1, because minimal
116
+ pairs are built to change this neuron and most others barely change at all.
117
+ """
118
+ claimed = claimed_token_weights(submission, runner)
119
+ target: CausalTarget = "claimed_tokens" if claimed else "next_token_kl"
120
+ setup = _PairSetup(
121
+ submission.identifier, *_unit_patching(submission, runner), claimed,
122
+ min_change, null_neurons, freeze_downstream,
123
+ ) # fmt: skip
124
+ items = list(enumerate(submission.counterfactuals))
125
+ texts = [t for _, p in items for t in (p.base_prompt, p.counterfactual_prompt)]
126
+ results = map_layer_major(
127
+ runner, setup.patched, items, tokens=prompt_tokens(runner, texts)
128
+ ) # the pairs run at once when the model streams (harness/layer_major.py)
129
+ scores: dict[str, list[float]] = {"sufficiency": [], "necessity": []}
130
+ effects: dict[str, list[float]] = {"sufficiency": [], "necessity": []}
131
+ nulls: list[int] = []
132
+ pairs: list[CausalPairEffect] = []
133
+ for index, own, n_null in filter(None, results):
134
+ nulls.append(n_null)
135
+ patched: dict[str, PatchEffect] = {}
136
+ for kind, (unit_effect, *others) in own.items():
137
+ patched[kind], percentile = _patch_effect(unit_effect, others)
138
+ effects[kind].append(unit_effect)
139
+ scores[kind].append(percentile)
140
+ pairs.append(CausalPairEffect(pair_index=index, **patched))
141
+
142
+ if not effects["sufficiency"]:
143
+ return None
144
+ tokens = claimed[1] if claimed else []
145
+ return _metrics(scores, effects, nulls, pairs, target, tokens, freeze_downstream)
146
+
147
+
148
+ @dataclass(frozen=True)
149
+ class _PairSetup:
150
+ """What patching one counterfactual pair needs (:func:`causal_metrics`)."""
151
+
152
+ ident: Any
153
+ changes: Any
154
+ patch: Any
155
+ claimed: Any
156
+ min_change: float
157
+ null_neurons: int
158
+ freeze_downstream: bool
159
+
160
+ def patched(self, item: tuple[int, Any]) -> tuple[int, dict, int] | None:
161
+ """One pair's unit and null effects per direction, or None if it isn't scored."""
162
+ index, pair = item
163
+ positions, position, change = self.changes(pair.base_prompt, pair.counterfactual_prompt)
164
+ if float(change[self.ident.index]) <= self.min_change:
165
+ return None
166
+ null = _matched_units(change, self.ident.index, self.null_neurons)
167
+ effect = (
168
+ metric_change(logit_direction_metric(self.claimed[0], position))
169
+ if self.claimed
170
+ else next_token_kl(position)
171
+ )
172
+ runs = {
173
+ "sufficiency": (pair.base_prompt, pair.counterfactual_prompt),
174
+ "necessity": (pair.counterfactual_prompt, pair.base_prompt),
175
+ }
176
+ options = {
177
+ "effect": effect, "positions": positions, "freeze_downstream": self.freeze_downstream
178
+ } # fmt: skip
179
+ units = [self.ident.index, *null]
180
+ own = {kind: self.patch(src, dst, units, **options) for kind, (src, dst) in runs.items()}
181
+ return index, own, len(null)
182
+
183
+
184
+ def _metrics(scores, effects, nulls, pairs, target, tokens, freeze_downstream) -> CausalMetrics:
185
+ """The medians over the scored pairs."""
186
+ return CausalMetrics(
187
+ sufficiency_score=_robust_unit(scores["sufficiency"]),
188
+ necessity_score=_robust_unit(scores["necessity"]),
189
+ hydra_mitigated=freeze_downstream,
190
+ target=target,
191
+ target_tokens=tokens,
192
+ sufficiency_effect=round(statistics.median(effects["sufficiency"]), 9),
193
+ necessity_effect=round(statistics.median(effects["necessity"]), 9),
194
+ null_neurons=min(nulls),
195
+ pairs_scored=len(effects["sufficiency"]),
196
+ pairs=pairs,
197
+ )
198
+
199
+
200
+ def _patch_effect(own: float, others: list[float]) -> tuple[PatchEffect, float]:
201
+ """The unit's patch effect against its matched units', and its |effect| percentile."""
202
+ null_effects = sorted(abs(o) for o in others)
203
+ percentile = _percentile(abs(own), null_effects)
204
+ effect = PatchEffect(
205
+ effect=round(own, 9),
206
+ percentile=round(percentile, 6) if math.isfinite(percentile) else None,
207
+ null_median=round(statistics.median(null_effects), 9) if null_effects else 0.0,
208
+ null_max=round(null_effects[-1], 9) if null_effects else 0.0,
209
+ )
210
+ return effect, percentile
211
+
212
+
213
+ def _unit_patching(submission: NeuronAuditSubmission, runner: NeuronHookRunner):
214
+ """Return ``changes(base, cf)`` and ``patch(source, target, units, **options)``.
215
+
216
+ They are the functions for the kind of unit that ``submission`` is about.
217
+ """
218
+ from openinterp.harness.features import patch_features, write_changes
219
+ from openinterp.harness.heads import patch_heads
220
+ from openinterp.harness.units import FeatureUnit, submission_unit
221
+
222
+ ident, unit = submission.identifier, submission_unit(submission)
223
+ if isinstance(unit, FeatureUnit):
224
+ dictionary, site = unit.dictionary(runner), unit.profile.input_site
225
+ return (
226
+ lambda base, cf: write_changes(runner, dictionary, ident, site, base, cf),
227
+ lambda source, target, units, **options: patch_features(
228
+ runner, dictionary, site, source, target, ident.layer, units, **options
229
+ ),
230
+ )
231
+ if ident.kind == "attn_head":
232
+ return (
233
+ lambda base, cf: _head_changes(runner, ident, base, cf),
234
+ lambda source, target, units, **options: patch_heads(
235
+ runner, source, target, ident.layer, units, **options
236
+ ),
237
+ )
238
+ patcher = ActivationPatcher(runner)
239
+ return (
240
+ lambda base, cf: _neuron_changes(runner, ident, base, cf),
241
+ lambda source, target, units, **options: patcher.patch_many(
242
+ source, target, ident.layer, units, ident.pole, **options
243
+ ),
244
+ )
245
+
246
+
247
+ def _matched_units(change: torch.Tensor, index: int, k: int) -> list[int]:
248
+ """The ``k`` other units whose ``change`` is closest to unit ``index``'s."""
249
+ order = (change - change[index]).abs().argsort(stable=True).tolist()
250
+ return [n for n in order if n != index][:k]
251
+
252
+
253
+ def _percentile(value: float, null: Sequence[float]) -> float:
254
+ """Fraction of ``null`` below ``value``, counting ties as half."""
255
+ if not null:
256
+ return math.nan
257
+ below = sum(n < value for n in null)
258
+ ties = sum(n == value for n in null)
259
+ return (below + 0.5 * ties) / len(null)
260
+
261
+
262
+ def _robust_unit(values: Sequence[float]) -> float:
263
+ finite = [v for v in values if math.isfinite(v)]
264
+ return round(min(1.0, max(0.0, statistics.median(finite))), 6) if finite else 0.0