open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
openinterp/checks.py ADDED
@@ -0,0 +1,429 @@
1
+ """Registry consistency checks, run in CI on every pull request (`open-interpretability check`).
2
+
3
+ An experiment is posted as an archived record ``<id>.json`` plus its page ``<id>.md``
4
+ in ``wiki/experiments/<slug>/``. A neuron may have any number of experiments. The
5
+ checks enforce that:
6
+
7
+ 1. Every archived record validates, has a verdict, and sits at the path that its
8
+ neuron slug and experiment id give.
9
+ 2. Every record has its page, and every page has its record. The page's front matter
10
+ matches the record. A record that ``supersedes`` another names an earlier experiment
11
+ on the same unit.
12
+ 3. The retired ``audits/`` directory holds no records. Import them with
13
+ ``open-interpretability wiki --import-dir audits``.
14
+ 4. With ``base_ref``: existing experiment pages were not deleted, or edited outside
15
+ their Discussion section, except to remove what a takedown covers. Archived records
16
+ were not touched (write-once). No generated index (``README.md``) was committed.
17
+ Records added since then name their agents in canonical spelling
18
+ (``openinterp.core.agent_names``) and have one shape of model fields
19
+ (``_check_model_fields``).
20
+ 5. The errata file (``wiki/errata.jsonl``) and the takedown file (``wiki/takedowns.jsonl``)
21
+ are valid and signed (:mod:`openinterp.errata_checks`, :mod:`openinterp.takedown_checks`).
22
+ 6. The current code recomputes each record's verdict, gates and balanced accuracy from
23
+ its stored results (:mod:`openinterp.verdict_regression`). For each record judged
24
+ under the current protocol, they must match. So a refactor of the gates can't silently
25
+ change what the registry claims.
26
+
27
+ With ``base_ref``, checks 1 and 2 read only the files that the branch adds or changes,
28
+ committed or not. So their cost doesn't grow with the registry. If the branch also
29
+ changes a tracked file outside the wiki, the code can read old records differently, so
30
+ they read every file. Without ``base_ref``, as on ``main``, they always read every file.
31
+
32
+ The indexes are rebuilt from the records when they are needed, and are gitignored.
33
+ Every experiment PR used to edit the same index rows, so parallel PRs conflicted.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import json
39
+ import subprocess
40
+ from collections.abc import Iterable, Iterator
41
+ from dataclasses import dataclass, field
42
+ from pathlib import Path
43
+
44
+ from pydantic import ValidationError
45
+
46
+ from openinterp.builder.wiki import _read_front_matter, archive_path
47
+ from openinterp.core.agent_names import spelling_problem
48
+ from openinterp.core.paths import DEFAULT_WIKI_DIR, experiments_dir
49
+ from openinterp.core.schemas import AuditRecord
50
+ from openinterp.errata_checks import check_errata
51
+ from openinterp.signing import ALLOWED_SIGNERS
52
+ from openinterp.takedown_checks import check_takedowns, page_redaction_ok
53
+ from openinterp.verdict_regression import verdict_drift
54
+
55
+ DISCUSSION_HEADING = "## Discussion"
56
+ LEGACY_AUDITS_DIR = Path("audits")
57
+
58
+
59
+ @dataclass
60
+ class Problem:
61
+ """One failed check: the file and what is wrong with it."""
62
+
63
+ path: Path
64
+ message: str
65
+
66
+ def __str__(self) -> str:
67
+ return f"{self.path}: {self.message}"
68
+
69
+
70
+ @dataclass
71
+ class CheckReport:
72
+ """What ``open-interpretability check`` found: the problems and how many files it read."""
73
+
74
+ problems: list[Problem] = field(default_factory=list)
75
+ records: int = 0
76
+ pages: int = 0
77
+ draws: int = 0
78
+ stops: int = 0
79
+ ledger: int = 0
80
+ changed_only: bool = False # the record and page checks read only the branch's changes
81
+
82
+ @property
83
+ def extra_counts(self) -> dict[str, int]:
84
+ """How many records of each optional kind the check read."""
85
+ return {"draw": self.draws, "stop": self.stops, "ledger entry": self.ledger}
86
+
87
+ @property
88
+ def ok(self) -> bool:
89
+ """True when no check failed."""
90
+ return not self.problems
91
+
92
+ def add(self, path: Path, message: str) -> None:
93
+ """Record a problem with ``path``."""
94
+ self.problems.append(Problem(Path(path), message))
95
+
96
+
97
+ def check_registry(
98
+ wiki_dir: Path = DEFAULT_WIKI_DIR,
99
+ *,
100
+ base_ref: str | None = None,
101
+ legacy_audits_dir: Path = LEGACY_AUDITS_DIR,
102
+ allowed_signers: Path = ALLOWED_SIGNERS,
103
+ ) -> CheckReport:
104
+ """Run every registry check on ``wiki_dir`` (see the module docstring)."""
105
+ report = CheckReport()
106
+ wiki_dir = Path(wiki_dir)
107
+ _check_legacy(Path(legacy_audits_dir), report)
108
+ changes = _branch_changes(wiki_dir, base_ref) if base_ref else []
109
+ scope = _scope(changes, wiki_dir) if base_ref else None
110
+ report.changed_only = scope is not None
111
+ _check_records(wiki_dir, report, scope)
112
+ _check_pages(wiki_dir, report, scope)
113
+ _check_draws(wiki_dir, report, scope)
114
+ _check_stops(wiki_dir, report, scope)
115
+ _check_ledger(wiki_dir, report, scope)
116
+ check_errata(wiki_dir, report, base_ref, changes, allowed_signers)
117
+ check_takedowns(wiki_dir, report, base_ref, changes, allowed_signers)
118
+ if base_ref:
119
+ _check_write_once(wiki_dir, base_ref, changes, report)
120
+ root = experiments_dir(wiki_dir)
121
+ changed = [
122
+ path
123
+ for status, path in changes
124
+ if status != "D" and path.suffix == ".json" and path.is_relative_to(root)
125
+ ]
126
+ _check_agent_names(changed, report)
127
+ _check_model_fields(changed, report)
128
+ return report
129
+
130
+
131
+ def _branch_changes(wiki_dir: Path, base_ref: str) -> list[tuple[str, Path]]:
132
+ """Return the status and path of each file changed since the branch left ``base_ref``.
133
+
134
+ The list includes uncommitted changes and untracked wiki files, because agents run the
135
+ check before they commit. Untracked files elsewhere are scratch files, so it omits them.
136
+ """
137
+ base = _git("merge-base", base_ref, "HEAD").strip()
138
+ changes = []
139
+ for line in _git("diff", "--name-status", "--no-renames", base).splitlines():
140
+ status, _, name = line.partition("\t")
141
+ changes.append((status, Path(name)))
142
+ untracked = _git("ls-files", "--others", "--exclude-standard", "--", str(wiki_dir))
143
+ return changes + [("A", Path(name)) for name in untracked.splitlines()]
144
+
145
+
146
+ def _scope(changes: list[tuple[str, Path]], wiki_dir: Path) -> set[Path] | None:
147
+ """Return the wiki files to check, or ``None`` for all of them.
148
+
149
+ All of them means a change outside the wiki, which can change how any old record reads.
150
+ A changed page brings in its record, because their front matter and values must agree.
151
+ """
152
+ if not all(path.is_relative_to(wiki_dir) for _, path in changes):
153
+ return None
154
+ scope = set()
155
+ for status, path in changes:
156
+ if status != "D":
157
+ scope.add(path)
158
+ if path.suffix == ".md":
159
+ scope.add(path.with_suffix(".json"))
160
+ return scope
161
+
162
+
163
+ def _chosen(paths: Iterable[Path], scope: set[Path] | None) -> Iterator[Path]:
164
+ """Yield the ``paths`` in ``scope``, or every path if ``scope`` is ``None``."""
165
+ return (path for path in paths if scope is None or path in scope)
166
+
167
+
168
+ def _check_legacy(audits_dir: Path, report: CheckReport) -> None:
169
+ if audits_dir.is_dir():
170
+ for path in sorted(audits_dir.rglob("*.json")):
171
+ report.add(
172
+ path,
173
+ "audits/ is retired; experiments live in wiki/experiments/. Run "
174
+ "`open-interpretability wiki --import-dir audits`, then delete audits/",
175
+ )
176
+
177
+
178
+ def _check_records(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
179
+ root = experiments_dir(wiki_dir)
180
+ if not root.is_dir():
181
+ return
182
+ for path in _chosen(sorted(root.rglob("*.json")), scope):
183
+ report.records += 1
184
+ record = _load(path)
185
+ if record is None:
186
+ report.add(path, "invalid experiment record")
187
+ continue
188
+ if record.verdict is None:
189
+ report.add(
190
+ path, "experiment record has no verdict; run `open-interpretability verify` first"
191
+ )
192
+ continue
193
+ expected = archive_path(wiki_dir, record)
194
+ if path != expected:
195
+ report.add(path, f"experiment record belongs at {expected}")
196
+ _check_supersedes(path, record, report)
197
+ drift = verdict_drift(record)
198
+ if drift:
199
+ report.add(path, drift)
200
+ page = path.with_suffix(".md")
201
+ if not page.exists():
202
+ report.add(path, "experiment record has no page; run `open-interpretability wiki`")
203
+ continue
204
+ meta = _read_front_matter(page)
205
+ values = {
206
+ "slug": record.submission.identifier.slug,
207
+ "status": record.verdict.status,
208
+ "balanced_accuracy": record.verdict.confidence_score,
209
+ "timestamp": record.verdict.timestamp.isoformat(),
210
+ "supersedes": record.submission.supersedes,
211
+ }
212
+ for key, value in values.items():
213
+ if meta.get(key) != value:
214
+ report.add(page, f"front matter {key}={meta.get(key)!r} but record has {value!r}")
215
+
216
+
217
+ def _check_draws(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
218
+ """Recorded unauditable draws (wiki/draws/) parse and sit where they belong."""
219
+ from pydantic import ValidationError
220
+
221
+ from openinterp.core.draws import UnauditableDraw, draw_files, draw_path
222
+
223
+ for path in _chosen(draw_files(wiki_dir), scope):
224
+ report.draws += 1
225
+ try:
226
+ draw = UnauditableDraw.model_validate_json(path.read_text(encoding="utf-8"))
227
+ except ValidationError:
228
+ report.add(path, "invalid draw record")
229
+ continue
230
+ if draw.provenance.selection != "random":
231
+ report.add(path, "only random draws are recorded")
232
+ if path != draw_path(wiki_dir, draw):
233
+ report.add(path, f"draw record belongs at {draw_path(wiki_dir, draw)}")
234
+
235
+
236
+ def _check_ledger(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
237
+ """Draw-ledger entries (wiki/ledger/) parse and sit where they belong."""
238
+ from pydantic import ValidationError
239
+
240
+ from openinterp.core.ledger import DrawEntry, ledger_files, ledger_path
241
+
242
+ for path in _chosen(ledger_files(wiki_dir), scope):
243
+ report.ledger += 1
244
+ try:
245
+ entry = DrawEntry.model_validate_json(path.read_text(encoding="utf-8"))
246
+ except ValidationError:
247
+ report.add(path, "invalid ledger entry")
248
+ continue
249
+ if path != ledger_path(wiki_dir, entry):
250
+ report.add(path, f"ledger entry belongs at {ledger_path(wiki_dir, entry)}")
251
+
252
+
253
+ def _check_stops(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
254
+ """Recorded stops of chosen units (wiki/stops/) parse, sit where they belong and have a page."""
255
+ from pydantic import ValidationError
256
+
257
+ from openinterp.core.draws import ChosenStop, stop_files, stop_path
258
+
259
+ for path in _chosen(stop_files(wiki_dir), scope):
260
+ report.stops += 1
261
+ try:
262
+ stop = ChosenStop.model_validate_json(path.read_text(encoding="utf-8"))
263
+ except ValidationError:
264
+ report.add(path, "invalid stop record")
265
+ continue
266
+ if stop.provenance.selection == "random":
267
+ report.add(path, "a random draw belongs in wiki/draws/")
268
+ if path != stop_path(wiki_dir, stop):
269
+ report.add(path, f"stop record belongs at {stop_path(wiki_dir, stop)}")
270
+ if not path.with_suffix(".md").exists():
271
+ report.add(path, "stop record has no page")
272
+
273
+
274
+ def _check_supersedes(path: Path, record: AuditRecord, report: CheckReport) -> None:
275
+ revises = record.submission.supersedes
276
+ if revises is None:
277
+ return
278
+ if revises >= path.stem: # ids are UTC timestamps, so they sort chronologically
279
+ report.add(path, f"supersedes {revises}, which is not an earlier experiment")
280
+ elif not path.with_name(f"{revises}.json").exists():
281
+ report.add(path, f"supersedes {revises}, but this unit has no such experiment")
282
+
283
+
284
+ def _check_pages(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
285
+ root = experiments_dir(wiki_dir)
286
+ if not root.is_dir():
287
+ return
288
+ for page in _chosen(sorted(root.rglob("*.md")), scope):
289
+ if page.name == "README.md":
290
+ continue
291
+ report.pages += 1
292
+ try:
293
+ meta = _read_front_matter(page)
294
+ except ValueError:
295
+ meta = {}
296
+ if not meta:
297
+ report.add(page, "experiment page has no valid front matter")
298
+ if not page.with_suffix(".json").exists():
299
+ report.add(page, f"missing experiment record {page.with_suffix('.json').name}")
300
+ for sig in _chosen(sorted(root.rglob("*.json.sig")), scope):
301
+ if not sig.with_suffix("").exists():
302
+ report.add(sig, f"signature for a missing record {sig.with_suffix('').name}")
303
+
304
+
305
+ def _agent_names(record: AuditRecord) -> dict[str, str | None]:
306
+ """The agent model names in ``record``, by the name of their field."""
307
+ scores = record.verdict.explanation_scores if record.verdict else None
308
+ return {
309
+ "submitting_agent_model": record.submission.submitting_agent_model,
310
+ "reviewer_agent_model": record.verdict.reviewer_agent_model if record.verdict else None,
311
+ "judge_model": scores.judge_model if scores else None,
312
+ }
313
+
314
+
315
+ def _check_agent_names(paths: list[Path], report: CheckReport) -> None:
316
+ """Require new records to spell their agents one way.
317
+
318
+ Then the site shows one model as one agent. Older records keep their spellings,
319
+ because they are signed. The build maps them. A follow-up such as ``reevaluate`` copies
320
+ names from an older record of its unit, so a name that the unit already has passes.
321
+ """
322
+ new = set(paths)
323
+ for path in paths:
324
+ record = _load(path)
325
+ if record is None or record.verdict is None:
326
+ continue # _check_records reports it
327
+ older = [
328
+ _agent_names(loaded)
329
+ for other in path.parent.glob("*.json")
330
+ if other not in new and (loaded := _load(other))
331
+ ]
332
+ for field_name, model in _agent_names(record).items():
333
+ inherited = any(names[field_name] == model for names in older)
334
+ if model and not inherited and (problem := spelling_problem(model)):
335
+ report.add(path, f"{field_name}={model!r}: {problem}")
336
+
337
+
338
+ def _check_model_fields(paths: list[Path], report: CheckReport) -> None:
339
+ """Require new records to have one shape of model fields.
340
+
341
+ Each field must be in the file. Layers and width must have a value. Heads per layer
342
+ must have a value on a head record and be null on other records.
343
+
344
+ Older records keep their shape, because they are signed. The build fills in their
345
+ widths. A re-review copies the submission of an older record, so a null value that
346
+ the unit already has passes.
347
+ """
348
+ new = set(paths)
349
+ for path in paths:
350
+ record = _load(path)
351
+ if record is None or record.verdict is None:
352
+ continue # _check_records reports it
353
+ written = json.loads(path.read_text(encoding="utf-8"))["submission"]
354
+ older = [
355
+ loaded.submission
356
+ for other in path.parent.glob("*.json")
357
+ if other not in new and (loaded := _load(other))
358
+ ]
359
+ is_head = record.submission.identifier.kind == "attn_head"
360
+ for name, required in (
361
+ ("model_num_layers", True),
362
+ ("model_mlp_width", True),
363
+ ("model_n_heads", is_head),
364
+ ):
365
+ if name not in written:
366
+ report.add(path, f"{name} is missing; record it with `open-interpretability audit`")
367
+ elif not required and written[name] is not None:
368
+ report.add(path, f"{name} must be null on a {record.submission.identifier.kind}")
369
+ elif (
370
+ required
371
+ and written[name] is None
372
+ and not any(getattr(sub, name) is None for sub in older)
373
+ ):
374
+ report.add(path, f"{name} is null; record it with `open-interpretability audit`")
375
+
376
+
377
+ def _load(path: Path) -> AuditRecord | None:
378
+ try:
379
+ return AuditRecord.model_validate_json(path.read_text(encoding="utf-8"))
380
+ except ValidationError:
381
+ return None
382
+
383
+
384
+ def _git(*args: str) -> str:
385
+ return subprocess.run(
386
+ ["git", *args], check=True, capture_output=True, text=True, encoding="utf-8"
387
+ ).stdout
388
+
389
+
390
+ def _check_write_once(
391
+ wiki_dir: Path, base_ref: str, changes: list[tuple[str, Path]], report: CheckReport
392
+ ) -> None:
393
+ for status, path in changes:
394
+ if not path.is_relative_to(wiki_dir):
395
+ continue
396
+ if path.suffix == ".json" and status in {"D", "M"}:
397
+ report.add(path, "archived experiment records are write-once")
398
+ continue
399
+ if path.name == "README.md":
400
+ if status == "A":
401
+ report.add(path, "generated wiki indexes are not committed; `git rm --cached` it")
402
+ continue
403
+ if path.suffix != ".md":
404
+ continue
405
+ if status == "D":
406
+ report.add(path, "experiment pages are write-once and must not be deleted")
407
+ elif status == "M":
408
+ before = _git("show", f"{base_ref}:{path.as_posix()}")
409
+ after = path.read_text(encoding="utf-8")
410
+ head, new_head = before.split(DISCUSSION_HEADING)[0], after.split(DISCUSSION_HEADING)[0]
411
+ if head != new_head and not page_redaction_ok(head, new_head, wiki_dir):
412
+ report.add(
413
+ path,
414
+ "experiment pages are write-once; only the Discussion section may be edited",
415
+ )
416
+
417
+
418
+ def changed_records(wiki_dir: Path, base_ref: str) -> list[Path]:
419
+ """Experiment records added or modified since ``base_ref``."""
420
+ diff = _git(
421
+ "diff",
422
+ "--name-only",
423
+ "--no-renames",
424
+ "--diff-filter=AM",
425
+ f"{base_ref}...HEAD",
426
+ "--",
427
+ str(experiments_dir(wiki_dir)),
428
+ )
429
+ return [Path(name) for name in diff.splitlines() if name.endswith(".json")]