open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
openinterp/checks.py
ADDED
|
@@ -0,0 +1,429 @@
|
|
|
1
|
+
"""Registry consistency checks, run in CI on every pull request (`open-interpretability check`).
|
|
2
|
+
|
|
3
|
+
An experiment is posted as an archived record ``<id>.json`` plus its page ``<id>.md``
|
|
4
|
+
in ``wiki/experiments/<slug>/``. A neuron may have any number of experiments. The
|
|
5
|
+
checks enforce that:
|
|
6
|
+
|
|
7
|
+
1. Every archived record validates, has a verdict, and sits at the path that its
|
|
8
|
+
neuron slug and experiment id give.
|
|
9
|
+
2. Every record has its page, and every page has its record. The page's front matter
|
|
10
|
+
matches the record. A record that ``supersedes`` another names an earlier experiment
|
|
11
|
+
on the same unit.
|
|
12
|
+
3. The retired ``audits/`` directory holds no records. Import them with
|
|
13
|
+
``open-interpretability wiki --import-dir audits``.
|
|
14
|
+
4. With ``base_ref``: existing experiment pages were not deleted, or edited outside
|
|
15
|
+
their Discussion section, except to remove what a takedown covers. Archived records
|
|
16
|
+
were not touched (write-once). No generated index (``README.md``) was committed.
|
|
17
|
+
Records added since then name their agents in canonical spelling
|
|
18
|
+
(``openinterp.core.agent_names``) and have one shape of model fields
|
|
19
|
+
(``_check_model_fields``).
|
|
20
|
+
5. The errata file (``wiki/errata.jsonl``) and the takedown file (``wiki/takedowns.jsonl``)
|
|
21
|
+
are valid and signed (:mod:`openinterp.errata_checks`, :mod:`openinterp.takedown_checks`).
|
|
22
|
+
6. The current code recomputes each record's verdict, gates and balanced accuracy from
|
|
23
|
+
its stored results (:mod:`openinterp.verdict_regression`). For each record judged
|
|
24
|
+
under the current protocol, they must match. So a refactor of the gates can't silently
|
|
25
|
+
change what the registry claims.
|
|
26
|
+
|
|
27
|
+
With ``base_ref``, checks 1 and 2 read only the files that the branch adds or changes,
|
|
28
|
+
committed or not. So their cost doesn't grow with the registry. If the branch also
|
|
29
|
+
changes a tracked file outside the wiki, the code can read old records differently, so
|
|
30
|
+
they read every file. Without ``base_ref``, as on ``main``, they always read every file.
|
|
31
|
+
|
|
32
|
+
The indexes are rebuilt from the records when they are needed, and are gitignored.
|
|
33
|
+
Every experiment PR used to edit the same index rows, so parallel PRs conflicted.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import json
|
|
39
|
+
import subprocess
|
|
40
|
+
from collections.abc import Iterable, Iterator
|
|
41
|
+
from dataclasses import dataclass, field
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
|
|
44
|
+
from pydantic import ValidationError
|
|
45
|
+
|
|
46
|
+
from openinterp.builder.wiki import _read_front_matter, archive_path
|
|
47
|
+
from openinterp.core.agent_names import spelling_problem
|
|
48
|
+
from openinterp.core.paths import DEFAULT_WIKI_DIR, experiments_dir
|
|
49
|
+
from openinterp.core.schemas import AuditRecord
|
|
50
|
+
from openinterp.errata_checks import check_errata
|
|
51
|
+
from openinterp.signing import ALLOWED_SIGNERS
|
|
52
|
+
from openinterp.takedown_checks import check_takedowns, page_redaction_ok
|
|
53
|
+
from openinterp.verdict_regression import verdict_drift
|
|
54
|
+
|
|
55
|
+
DISCUSSION_HEADING = "## Discussion"
|
|
56
|
+
LEGACY_AUDITS_DIR = Path("audits")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class Problem:
|
|
61
|
+
"""One failed check: the file and what is wrong with it."""
|
|
62
|
+
|
|
63
|
+
path: Path
|
|
64
|
+
message: str
|
|
65
|
+
|
|
66
|
+
def __str__(self) -> str:
|
|
67
|
+
return f"{self.path}: {self.message}"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class CheckReport:
|
|
72
|
+
"""What ``open-interpretability check`` found: the problems and how many files it read."""
|
|
73
|
+
|
|
74
|
+
problems: list[Problem] = field(default_factory=list)
|
|
75
|
+
records: int = 0
|
|
76
|
+
pages: int = 0
|
|
77
|
+
draws: int = 0
|
|
78
|
+
stops: int = 0
|
|
79
|
+
ledger: int = 0
|
|
80
|
+
changed_only: bool = False # the record and page checks read only the branch's changes
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def extra_counts(self) -> dict[str, int]:
|
|
84
|
+
"""How many records of each optional kind the check read."""
|
|
85
|
+
return {"draw": self.draws, "stop": self.stops, "ledger entry": self.ledger}
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def ok(self) -> bool:
|
|
89
|
+
"""True when no check failed."""
|
|
90
|
+
return not self.problems
|
|
91
|
+
|
|
92
|
+
def add(self, path: Path, message: str) -> None:
|
|
93
|
+
"""Record a problem with ``path``."""
|
|
94
|
+
self.problems.append(Problem(Path(path), message))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def check_registry(
|
|
98
|
+
wiki_dir: Path = DEFAULT_WIKI_DIR,
|
|
99
|
+
*,
|
|
100
|
+
base_ref: str | None = None,
|
|
101
|
+
legacy_audits_dir: Path = LEGACY_AUDITS_DIR,
|
|
102
|
+
allowed_signers: Path = ALLOWED_SIGNERS,
|
|
103
|
+
) -> CheckReport:
|
|
104
|
+
"""Run every registry check on ``wiki_dir`` (see the module docstring)."""
|
|
105
|
+
report = CheckReport()
|
|
106
|
+
wiki_dir = Path(wiki_dir)
|
|
107
|
+
_check_legacy(Path(legacy_audits_dir), report)
|
|
108
|
+
changes = _branch_changes(wiki_dir, base_ref) if base_ref else []
|
|
109
|
+
scope = _scope(changes, wiki_dir) if base_ref else None
|
|
110
|
+
report.changed_only = scope is not None
|
|
111
|
+
_check_records(wiki_dir, report, scope)
|
|
112
|
+
_check_pages(wiki_dir, report, scope)
|
|
113
|
+
_check_draws(wiki_dir, report, scope)
|
|
114
|
+
_check_stops(wiki_dir, report, scope)
|
|
115
|
+
_check_ledger(wiki_dir, report, scope)
|
|
116
|
+
check_errata(wiki_dir, report, base_ref, changes, allowed_signers)
|
|
117
|
+
check_takedowns(wiki_dir, report, base_ref, changes, allowed_signers)
|
|
118
|
+
if base_ref:
|
|
119
|
+
_check_write_once(wiki_dir, base_ref, changes, report)
|
|
120
|
+
root = experiments_dir(wiki_dir)
|
|
121
|
+
changed = [
|
|
122
|
+
path
|
|
123
|
+
for status, path in changes
|
|
124
|
+
if status != "D" and path.suffix == ".json" and path.is_relative_to(root)
|
|
125
|
+
]
|
|
126
|
+
_check_agent_names(changed, report)
|
|
127
|
+
_check_model_fields(changed, report)
|
|
128
|
+
return report
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _branch_changes(wiki_dir: Path, base_ref: str) -> list[tuple[str, Path]]:
|
|
132
|
+
"""Return the status and path of each file changed since the branch left ``base_ref``.
|
|
133
|
+
|
|
134
|
+
The list includes uncommitted changes and untracked wiki files, because agents run the
|
|
135
|
+
check before they commit. Untracked files elsewhere are scratch files, so it omits them.
|
|
136
|
+
"""
|
|
137
|
+
base = _git("merge-base", base_ref, "HEAD").strip()
|
|
138
|
+
changes = []
|
|
139
|
+
for line in _git("diff", "--name-status", "--no-renames", base).splitlines():
|
|
140
|
+
status, _, name = line.partition("\t")
|
|
141
|
+
changes.append((status, Path(name)))
|
|
142
|
+
untracked = _git("ls-files", "--others", "--exclude-standard", "--", str(wiki_dir))
|
|
143
|
+
return changes + [("A", Path(name)) for name in untracked.splitlines()]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _scope(changes: list[tuple[str, Path]], wiki_dir: Path) -> set[Path] | None:
|
|
147
|
+
"""Return the wiki files to check, or ``None`` for all of them.
|
|
148
|
+
|
|
149
|
+
All of them means a change outside the wiki, which can change how any old record reads.
|
|
150
|
+
A changed page brings in its record, because their front matter and values must agree.
|
|
151
|
+
"""
|
|
152
|
+
if not all(path.is_relative_to(wiki_dir) for _, path in changes):
|
|
153
|
+
return None
|
|
154
|
+
scope = set()
|
|
155
|
+
for status, path in changes:
|
|
156
|
+
if status != "D":
|
|
157
|
+
scope.add(path)
|
|
158
|
+
if path.suffix == ".md":
|
|
159
|
+
scope.add(path.with_suffix(".json"))
|
|
160
|
+
return scope
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _chosen(paths: Iterable[Path], scope: set[Path] | None) -> Iterator[Path]:
|
|
164
|
+
"""Yield the ``paths`` in ``scope``, or every path if ``scope`` is ``None``."""
|
|
165
|
+
return (path for path in paths if scope is None or path in scope)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _check_legacy(audits_dir: Path, report: CheckReport) -> None:
|
|
169
|
+
if audits_dir.is_dir():
|
|
170
|
+
for path in sorted(audits_dir.rglob("*.json")):
|
|
171
|
+
report.add(
|
|
172
|
+
path,
|
|
173
|
+
"audits/ is retired; experiments live in wiki/experiments/. Run "
|
|
174
|
+
"`open-interpretability wiki --import-dir audits`, then delete audits/",
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _check_records(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
|
|
179
|
+
root = experiments_dir(wiki_dir)
|
|
180
|
+
if not root.is_dir():
|
|
181
|
+
return
|
|
182
|
+
for path in _chosen(sorted(root.rglob("*.json")), scope):
|
|
183
|
+
report.records += 1
|
|
184
|
+
record = _load(path)
|
|
185
|
+
if record is None:
|
|
186
|
+
report.add(path, "invalid experiment record")
|
|
187
|
+
continue
|
|
188
|
+
if record.verdict is None:
|
|
189
|
+
report.add(
|
|
190
|
+
path, "experiment record has no verdict; run `open-interpretability verify` first"
|
|
191
|
+
)
|
|
192
|
+
continue
|
|
193
|
+
expected = archive_path(wiki_dir, record)
|
|
194
|
+
if path != expected:
|
|
195
|
+
report.add(path, f"experiment record belongs at {expected}")
|
|
196
|
+
_check_supersedes(path, record, report)
|
|
197
|
+
drift = verdict_drift(record)
|
|
198
|
+
if drift:
|
|
199
|
+
report.add(path, drift)
|
|
200
|
+
page = path.with_suffix(".md")
|
|
201
|
+
if not page.exists():
|
|
202
|
+
report.add(path, "experiment record has no page; run `open-interpretability wiki`")
|
|
203
|
+
continue
|
|
204
|
+
meta = _read_front_matter(page)
|
|
205
|
+
values = {
|
|
206
|
+
"slug": record.submission.identifier.slug,
|
|
207
|
+
"status": record.verdict.status,
|
|
208
|
+
"balanced_accuracy": record.verdict.confidence_score,
|
|
209
|
+
"timestamp": record.verdict.timestamp.isoformat(),
|
|
210
|
+
"supersedes": record.submission.supersedes,
|
|
211
|
+
}
|
|
212
|
+
for key, value in values.items():
|
|
213
|
+
if meta.get(key) != value:
|
|
214
|
+
report.add(page, f"front matter {key}={meta.get(key)!r} but record has {value!r}")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _check_draws(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
|
|
218
|
+
"""Recorded unauditable draws (wiki/draws/) parse and sit where they belong."""
|
|
219
|
+
from pydantic import ValidationError
|
|
220
|
+
|
|
221
|
+
from openinterp.core.draws import UnauditableDraw, draw_files, draw_path
|
|
222
|
+
|
|
223
|
+
for path in _chosen(draw_files(wiki_dir), scope):
|
|
224
|
+
report.draws += 1
|
|
225
|
+
try:
|
|
226
|
+
draw = UnauditableDraw.model_validate_json(path.read_text(encoding="utf-8"))
|
|
227
|
+
except ValidationError:
|
|
228
|
+
report.add(path, "invalid draw record")
|
|
229
|
+
continue
|
|
230
|
+
if draw.provenance.selection != "random":
|
|
231
|
+
report.add(path, "only random draws are recorded")
|
|
232
|
+
if path != draw_path(wiki_dir, draw):
|
|
233
|
+
report.add(path, f"draw record belongs at {draw_path(wiki_dir, draw)}")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _check_ledger(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
|
|
237
|
+
"""Draw-ledger entries (wiki/ledger/) parse and sit where they belong."""
|
|
238
|
+
from pydantic import ValidationError
|
|
239
|
+
|
|
240
|
+
from openinterp.core.ledger import DrawEntry, ledger_files, ledger_path
|
|
241
|
+
|
|
242
|
+
for path in _chosen(ledger_files(wiki_dir), scope):
|
|
243
|
+
report.ledger += 1
|
|
244
|
+
try:
|
|
245
|
+
entry = DrawEntry.model_validate_json(path.read_text(encoding="utf-8"))
|
|
246
|
+
except ValidationError:
|
|
247
|
+
report.add(path, "invalid ledger entry")
|
|
248
|
+
continue
|
|
249
|
+
if path != ledger_path(wiki_dir, entry):
|
|
250
|
+
report.add(path, f"ledger entry belongs at {ledger_path(wiki_dir, entry)}")
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _check_stops(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
|
|
254
|
+
"""Recorded stops of chosen units (wiki/stops/) parse, sit where they belong and have a page."""
|
|
255
|
+
from pydantic import ValidationError
|
|
256
|
+
|
|
257
|
+
from openinterp.core.draws import ChosenStop, stop_files, stop_path
|
|
258
|
+
|
|
259
|
+
for path in _chosen(stop_files(wiki_dir), scope):
|
|
260
|
+
report.stops += 1
|
|
261
|
+
try:
|
|
262
|
+
stop = ChosenStop.model_validate_json(path.read_text(encoding="utf-8"))
|
|
263
|
+
except ValidationError:
|
|
264
|
+
report.add(path, "invalid stop record")
|
|
265
|
+
continue
|
|
266
|
+
if stop.provenance.selection == "random":
|
|
267
|
+
report.add(path, "a random draw belongs in wiki/draws/")
|
|
268
|
+
if path != stop_path(wiki_dir, stop):
|
|
269
|
+
report.add(path, f"stop record belongs at {stop_path(wiki_dir, stop)}")
|
|
270
|
+
if not path.with_suffix(".md").exists():
|
|
271
|
+
report.add(path, "stop record has no page")
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _check_supersedes(path: Path, record: AuditRecord, report: CheckReport) -> None:
|
|
275
|
+
revises = record.submission.supersedes
|
|
276
|
+
if revises is None:
|
|
277
|
+
return
|
|
278
|
+
if revises >= path.stem: # ids are UTC timestamps, so they sort chronologically
|
|
279
|
+
report.add(path, f"supersedes {revises}, which is not an earlier experiment")
|
|
280
|
+
elif not path.with_name(f"{revises}.json").exists():
|
|
281
|
+
report.add(path, f"supersedes {revises}, but this unit has no such experiment")
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _check_pages(wiki_dir: Path, report: CheckReport, scope: set[Path] | None = None) -> None:
|
|
285
|
+
root = experiments_dir(wiki_dir)
|
|
286
|
+
if not root.is_dir():
|
|
287
|
+
return
|
|
288
|
+
for page in _chosen(sorted(root.rglob("*.md")), scope):
|
|
289
|
+
if page.name == "README.md":
|
|
290
|
+
continue
|
|
291
|
+
report.pages += 1
|
|
292
|
+
try:
|
|
293
|
+
meta = _read_front_matter(page)
|
|
294
|
+
except ValueError:
|
|
295
|
+
meta = {}
|
|
296
|
+
if not meta:
|
|
297
|
+
report.add(page, "experiment page has no valid front matter")
|
|
298
|
+
if not page.with_suffix(".json").exists():
|
|
299
|
+
report.add(page, f"missing experiment record {page.with_suffix('.json').name}")
|
|
300
|
+
for sig in _chosen(sorted(root.rglob("*.json.sig")), scope):
|
|
301
|
+
if not sig.with_suffix("").exists():
|
|
302
|
+
report.add(sig, f"signature for a missing record {sig.with_suffix('').name}")
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _agent_names(record: AuditRecord) -> dict[str, str | None]:
|
|
306
|
+
"""The agent model names in ``record``, by the name of their field."""
|
|
307
|
+
scores = record.verdict.explanation_scores if record.verdict else None
|
|
308
|
+
return {
|
|
309
|
+
"submitting_agent_model": record.submission.submitting_agent_model,
|
|
310
|
+
"reviewer_agent_model": record.verdict.reviewer_agent_model if record.verdict else None,
|
|
311
|
+
"judge_model": scores.judge_model if scores else None,
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _check_agent_names(paths: list[Path], report: CheckReport) -> None:
|
|
316
|
+
"""Require new records to spell their agents one way.
|
|
317
|
+
|
|
318
|
+
Then the site shows one model as one agent. Older records keep their spellings,
|
|
319
|
+
because they are signed. The build maps them. A follow-up such as ``reevaluate`` copies
|
|
320
|
+
names from an older record of its unit, so a name that the unit already has passes.
|
|
321
|
+
"""
|
|
322
|
+
new = set(paths)
|
|
323
|
+
for path in paths:
|
|
324
|
+
record = _load(path)
|
|
325
|
+
if record is None or record.verdict is None:
|
|
326
|
+
continue # _check_records reports it
|
|
327
|
+
older = [
|
|
328
|
+
_agent_names(loaded)
|
|
329
|
+
for other in path.parent.glob("*.json")
|
|
330
|
+
if other not in new and (loaded := _load(other))
|
|
331
|
+
]
|
|
332
|
+
for field_name, model in _agent_names(record).items():
|
|
333
|
+
inherited = any(names[field_name] == model for names in older)
|
|
334
|
+
if model and not inherited and (problem := spelling_problem(model)):
|
|
335
|
+
report.add(path, f"{field_name}={model!r}: {problem}")
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _check_model_fields(paths: list[Path], report: CheckReport) -> None:
|
|
339
|
+
"""Require new records to have one shape of model fields.
|
|
340
|
+
|
|
341
|
+
Each field must be in the file. Layers and width must have a value. Heads per layer
|
|
342
|
+
must have a value on a head record and be null on other records.
|
|
343
|
+
|
|
344
|
+
Older records keep their shape, because they are signed. The build fills in their
|
|
345
|
+
widths. A re-review copies the submission of an older record, so a null value that
|
|
346
|
+
the unit already has passes.
|
|
347
|
+
"""
|
|
348
|
+
new = set(paths)
|
|
349
|
+
for path in paths:
|
|
350
|
+
record = _load(path)
|
|
351
|
+
if record is None or record.verdict is None:
|
|
352
|
+
continue # _check_records reports it
|
|
353
|
+
written = json.loads(path.read_text(encoding="utf-8"))["submission"]
|
|
354
|
+
older = [
|
|
355
|
+
loaded.submission
|
|
356
|
+
for other in path.parent.glob("*.json")
|
|
357
|
+
if other not in new and (loaded := _load(other))
|
|
358
|
+
]
|
|
359
|
+
is_head = record.submission.identifier.kind == "attn_head"
|
|
360
|
+
for name, required in (
|
|
361
|
+
("model_num_layers", True),
|
|
362
|
+
("model_mlp_width", True),
|
|
363
|
+
("model_n_heads", is_head),
|
|
364
|
+
):
|
|
365
|
+
if name not in written:
|
|
366
|
+
report.add(path, f"{name} is missing; record it with `open-interpretability audit`")
|
|
367
|
+
elif not required and written[name] is not None:
|
|
368
|
+
report.add(path, f"{name} must be null on a {record.submission.identifier.kind}")
|
|
369
|
+
elif (
|
|
370
|
+
required
|
|
371
|
+
and written[name] is None
|
|
372
|
+
and not any(getattr(sub, name) is None for sub in older)
|
|
373
|
+
):
|
|
374
|
+
report.add(path, f"{name} is null; record it with `open-interpretability audit`")
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _load(path: Path) -> AuditRecord | None:
|
|
378
|
+
try:
|
|
379
|
+
return AuditRecord.model_validate_json(path.read_text(encoding="utf-8"))
|
|
380
|
+
except ValidationError:
|
|
381
|
+
return None
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _git(*args: str) -> str:
|
|
385
|
+
return subprocess.run(
|
|
386
|
+
["git", *args], check=True, capture_output=True, text=True, encoding="utf-8"
|
|
387
|
+
).stdout
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _check_write_once(
|
|
391
|
+
wiki_dir: Path, base_ref: str, changes: list[tuple[str, Path]], report: CheckReport
|
|
392
|
+
) -> None:
|
|
393
|
+
for status, path in changes:
|
|
394
|
+
if not path.is_relative_to(wiki_dir):
|
|
395
|
+
continue
|
|
396
|
+
if path.suffix == ".json" and status in {"D", "M"}:
|
|
397
|
+
report.add(path, "archived experiment records are write-once")
|
|
398
|
+
continue
|
|
399
|
+
if path.name == "README.md":
|
|
400
|
+
if status == "A":
|
|
401
|
+
report.add(path, "generated wiki indexes are not committed; `git rm --cached` it")
|
|
402
|
+
continue
|
|
403
|
+
if path.suffix != ".md":
|
|
404
|
+
continue
|
|
405
|
+
if status == "D":
|
|
406
|
+
report.add(path, "experiment pages are write-once and must not be deleted")
|
|
407
|
+
elif status == "M":
|
|
408
|
+
before = _git("show", f"{base_ref}:{path.as_posix()}")
|
|
409
|
+
after = path.read_text(encoding="utf-8")
|
|
410
|
+
head, new_head = before.split(DISCUSSION_HEADING)[0], after.split(DISCUSSION_HEADING)[0]
|
|
411
|
+
if head != new_head and not page_redaction_ok(head, new_head, wiki_dir):
|
|
412
|
+
report.add(
|
|
413
|
+
path,
|
|
414
|
+
"experiment pages are write-once; only the Discussion section may be edited",
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def changed_records(wiki_dir: Path, base_ref: str) -> list[Path]:
|
|
419
|
+
"""Experiment records added or modified since ``base_ref``."""
|
|
420
|
+
diff = _git(
|
|
421
|
+
"diff",
|
|
422
|
+
"--name-only",
|
|
423
|
+
"--no-renames",
|
|
424
|
+
"--diff-filter=AM",
|
|
425
|
+
f"{base_ref}...HEAD",
|
|
426
|
+
"--",
|
|
427
|
+
str(experiments_dir(wiki_dir)),
|
|
428
|
+
)
|
|
429
|
+
return [Path(name) for name in diff.splitlines() if name.endswith(".json")]
|