open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
"""Experiment registry and wiki: one record and one Markdown page per experiment.
|
|
2
|
+
|
|
3
|
+
Layout under ``wiki/``::
|
|
4
|
+
|
|
5
|
+
README.md home: every experiment, newest first
|
|
6
|
+
experiments/<model>/L<l>/N<n>-<pole>/README.md neuron page: its experiment history
|
|
7
|
+
experiments/<model>/L<l>/<dictionary>/F<f>/README.md dictionary feature page
|
|
8
|
+
experiments/<model>/L<l>/H<h>/... the same for an attention head
|
|
9
|
+
experiments/<model>/L<l>/N<n>-<pole>/<id>.md one experiment (write-once)
|
|
10
|
+
experiments/<model>/L<l>/N<n>-<pole>/<id>.json its full audit record (write-once)
|
|
11
|
+
|
|
12
|
+
The experiment, not the neuron, is the unit of storage. The ``<id>.json`` archive is
|
|
13
|
+
the registry's only source of truth. A neuron can have any number of experiments. Each
|
|
14
|
+
adds new files, and none edits another's, so parallel experiments on one neuron never
|
|
15
|
+
conflict. A neuron's standing (latest or most conclusive verdict) is always derived from
|
|
16
|
+
its experiments, never stored. Each page's "Discussion" section is left for people and
|
|
17
|
+
agents to edit.
|
|
18
|
+
|
|
19
|
+
Indexes (the ``README.md`` files) are regenerated from the pages' front matter. They are
|
|
20
|
+
gitignored, never committed. Otherwise every experiment would edit the same index rows,
|
|
21
|
+
and parallel PRs would conflict.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
from collections.abc import Sequence
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Literal
|
|
31
|
+
|
|
32
|
+
from jinja2 import Environment, PackageLoader, StrictUndefined
|
|
33
|
+
|
|
34
|
+
from openinterp.core.draws import ChosenStop, UnauditableDraw, load_stops, stop_path
|
|
35
|
+
from openinterp.core.paths import experiments_dir
|
|
36
|
+
from openinterp.core.schemas import ActivatingExample, AuditRecord, ReviewPolicy
|
|
37
|
+
|
|
38
|
+
ExperimentKind = Literal["review", "reevaluation", "backfill"]
|
|
39
|
+
|
|
40
|
+
# Advice for each failed gate: what to change before the next submission.
|
|
41
|
+
GATE_ADVICE = {
|
|
42
|
+
"enough_hard_negatives": "Re-run the review with more hard negatives (protocol minimum: "
|
|
43
|
+
"{min_hard_negatives}).",
|
|
44
|
+
"enough_holdouts": "Re-run the review with more positive holdouts (protocol minimum: "
|
|
45
|
+
"{min_holdouts}).",
|
|
46
|
+
"specificity": "The hypothesis is too broad: the hard negatives listed below fired. Narrow "
|
|
47
|
+
"the claim to exclude them (skill: `interpretability-illusions-review`).",
|
|
48
|
+
"sensitivity": "The hypothesis does not generalise: the holdouts listed below did not fire. "
|
|
49
|
+
"Narrow the claim to what actually fires, or rescan with longer context and broader data "
|
|
50
|
+
"(skill: `native-neuron-audit`).",
|
|
51
|
+
"counterfactual_reproduction": "Predicted Δ did not reproduce. Re-measure the pairs and "
|
|
52
|
+
"recalibrate the predictions, or replace pairs that test the wrong feature.",
|
|
53
|
+
"lookahead_consistent": "The measured mechanism contradicts the claim. Revisit "
|
|
54
|
+
"retrospective vs lookahead (skill: `vocab-projection-lookahead`).",
|
|
55
|
+
"independent_review": "The proposer and reviewer used the same model, and this policy "
|
|
56
|
+
"requires them to differ. Re-review with a different reviewer model.",
|
|
57
|
+
"beats_confound_baseline": "A trivial baseline (token identity, position or 'never "
|
|
58
|
+
"active') explains the probes at least as well as the hypothesis, or the scan predates "
|
|
59
|
+
"confound profiling. Rescan with `open-interpretability audit` and sharpen the claim "
|
|
60
|
+
"beyond the trigger tokens (skill: `interpretability-illusions-review`).",
|
|
61
|
+
"not_default_on": "The unit is active on at least half of its scan's tokens, so no hard "
|
|
62
|
+
"negative of a few words can stay inactive. It can't be audited as a detector; record "
|
|
63
|
+
"it as default-on (docs/PROTOCOL.md §1).",
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class WikiEntry:
|
|
69
|
+
"""One experiment page and its front matter."""
|
|
70
|
+
|
|
71
|
+
path: Path
|
|
72
|
+
meta: dict
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def experiment_id(record: AuditRecord) -> str:
|
|
76
|
+
"""The experiment's id: its verdict's UTC timestamp."""
|
|
77
|
+
assert record.verdict is not None # noqa: S101 (for the type checker)
|
|
78
|
+
return record.verdict.timestamp.strftime("%Y%m%dT%H%M%SZ")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def unit_label(meta: dict) -> str:
|
|
82
|
+
"""``L3/N7+``, ``L5/H1``, ``L8/F40 (dictionary)`` or ``L2/S…``, from front matter."""
|
|
83
|
+
if "head" in meta:
|
|
84
|
+
return f"L{meta['layer']}/H{meta['head']}"
|
|
85
|
+
if "feature" in meta:
|
|
86
|
+
return f"L{meta['layer']}/F{meta['feature']} ({meta['dictionary']})"
|
|
87
|
+
if "superposition" in meta:
|
|
88
|
+
return f"L{meta['layer']}/S{meta['superposition']}"
|
|
89
|
+
return f"L{meta['layer']}/N{meta['neuron']}{'+' if meta['pole'] == 'positive' else '−'}"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def neuron_dir(wiki_dir: Path, record: AuditRecord) -> Path:
|
|
93
|
+
"""The directory of the record's unit."""
|
|
94
|
+
return experiments_dir(wiki_dir) / record.submission.identifier.slug
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def home_link(slug: str) -> str:
|
|
98
|
+
"""The wiki home's README from a unit's directory, ``experiments/<slug>``."""
|
|
99
|
+
return "../" * (len(slug.split("/")) + 1) + "README.md"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _environment() -> Environment:
|
|
103
|
+
# The wiki is Markdown, so HTML escaping would corrupt it. _cell makes values safe in
|
|
104
|
+
# tables. The site's HTML templates escape (compiler._environment).
|
|
105
|
+
env = Environment( # noqa: S701
|
|
106
|
+
loader=PackageLoader("openinterp.builder", "templates/wiki"),
|
|
107
|
+
undefined=StrictUndefined,
|
|
108
|
+
trim_blocks=True,
|
|
109
|
+
lstrip_blocks=True,
|
|
110
|
+
keep_trailing_newline=True,
|
|
111
|
+
)
|
|
112
|
+
env.filters["cell"] = _cell
|
|
113
|
+
env.filters["num"] = lambda v, d=3: "—" if v is None else f"{v:.{d}f}"
|
|
114
|
+
env.filters["pct"] = lambda v: "—" if v is None else f"{100 * v:.0f}%"
|
|
115
|
+
return env
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _cell(value: object) -> str:
|
|
119
|
+
"""Make a value safe inside a Markdown table cell."""
|
|
120
|
+
return str(value).replace("\\", "\\\\").replace("|", "\\|").replace("\n", " ⏎ ")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _context(example: ActivatingExample, before: int = 12, after: int = 4) -> str:
|
|
124
|
+
i = example.max_token_index
|
|
125
|
+
text = (
|
|
126
|
+
"".join(example.tokens[max(0, i - before) : i])
|
|
127
|
+
+ f"⟦{example.tokens[i]}⟧"
|
|
128
|
+
+ "".join(example.tokens[i + 1 : i + 1 + after])
|
|
129
|
+
)
|
|
130
|
+
return text.replace("```", "ʼʼʼ").replace("\n", "⏎")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _front_matter(meta: dict) -> str:
|
|
134
|
+
# JSON values are valid YAML, so the block renders on GitHub and parses without PyYAML.
|
|
135
|
+
return "---\n" + "".join(f"{k}: {json.dumps(v)}\n" for k, v in meta.items()) + "---\n"
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _read_front_matter(path: Path) -> dict:
|
|
139
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
140
|
+
if not lines or lines[0] != "---":
|
|
141
|
+
return {}
|
|
142
|
+
meta = {}
|
|
143
|
+
for line in lines[1:]:
|
|
144
|
+
if line == "---":
|
|
145
|
+
break
|
|
146
|
+
key, _, value = line.partition(": ")
|
|
147
|
+
meta[key] = json.loads(value)
|
|
148
|
+
return meta
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def write_experiment_page(
|
|
152
|
+
record: AuditRecord,
|
|
153
|
+
wiki_dir: Path,
|
|
154
|
+
*,
|
|
155
|
+
kind: ExperimentKind = "review",
|
|
156
|
+
previous_status: str | None = None,
|
|
157
|
+
) -> Path | None:
|
|
158
|
+
"""Write the page for a completed experiment and refresh the indexes.
|
|
159
|
+
|
|
160
|
+
Returns the page path, or ``None`` if the page already exists (pages are
|
|
161
|
+
write-once) or the record has no verdict yet.
|
|
162
|
+
"""
|
|
163
|
+
if record.verdict is None:
|
|
164
|
+
return None
|
|
165
|
+
page = neuron_dir(wiki_dir, record) / f"{experiment_id(record)}.md"
|
|
166
|
+
archive_record(record, wiki_dir)
|
|
167
|
+
if page.exists():
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
meta = _page_meta(record, kind)
|
|
171
|
+
body = _page_body(record, wiki_dir, meta, kind, previous_status)
|
|
172
|
+
page.parent.mkdir(parents=True, exist_ok=True)
|
|
173
|
+
page.write_text(_front_matter(meta) + "\n" + body, encoding="utf-8")
|
|
174
|
+
rebuild_indexes(wiki_dir)
|
|
175
|
+
return page
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _unit_meta(ident) -> dict:
|
|
179
|
+
"""The front matter that names the unit within its layer."""
|
|
180
|
+
if ident.kind == "attn_head":
|
|
181
|
+
return {"head": ident.head_idx}
|
|
182
|
+
if ident.kind == "dictionary_feature":
|
|
183
|
+
return {"dictionary": ident.dictionary, "feature": ident.feature_idx}
|
|
184
|
+
if ident.kind == "superposition":
|
|
185
|
+
return {"superposition": ident.superposition_id}
|
|
186
|
+
return {"neuron": ident.neuron_idx, "pole": ident.pole}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _page_meta(record: AuditRecord, kind: ExperimentKind) -> dict:
|
|
190
|
+
"""The page's front matter."""
|
|
191
|
+
sub, verdict = record.submission, record.verdict
|
|
192
|
+
ident, policy = sub.identifier, verdict.policy
|
|
193
|
+
unit = _unit_meta(ident)
|
|
194
|
+
meta = {
|
|
195
|
+
"experiment_id": experiment_id(record),
|
|
196
|
+
"kind": kind,
|
|
197
|
+
"slug": ident.slug,
|
|
198
|
+
"model": ident.model_name,
|
|
199
|
+
"layer": ident.layer,
|
|
200
|
+
**unit,
|
|
201
|
+
"status": verdict.status,
|
|
202
|
+
"balanced_accuracy": verdict.confidence_score,
|
|
203
|
+
"protocol_version": policy.protocol_version if policy else None,
|
|
204
|
+
"timestamp": verdict.timestamp.isoformat(),
|
|
205
|
+
"hypothesis": sub.natural_language_explanation,
|
|
206
|
+
"proposer": sub.submitting_agent_model,
|
|
207
|
+
"reviewer": verdict.reviewer_agent_model,
|
|
208
|
+
}
|
|
209
|
+
if sub.contributor:
|
|
210
|
+
meta["proposer_contributor"] = sub.contributor
|
|
211
|
+
if verdict.contributor:
|
|
212
|
+
meta["reviewer_contributor"] = verdict.contributor
|
|
213
|
+
if sub.supersedes:
|
|
214
|
+
meta["supersedes"] = sub.supersedes
|
|
215
|
+
if sub.sister_of:
|
|
216
|
+
meta["sister_of"] = sub.sister_of
|
|
217
|
+
return meta
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _page_body(record: AuditRecord, wiki_dir: Path, meta: dict, kind, previous_status) -> str:
|
|
221
|
+
sub, verdict = record.submission, record.verdict
|
|
222
|
+
metrics, policy = verdict.metrics, verdict.policy
|
|
223
|
+
failed = [g for g, ok in (metrics.gates.items() if metrics else []) if not ok]
|
|
224
|
+
advice = [GATE_ADVICE[g].format(**(policy.model_dump() if policy else {})) for g in failed]
|
|
225
|
+
probe_kinds = {"hard_negative", "positive_holdout"}
|
|
226
|
+
return (
|
|
227
|
+
_environment()
|
|
228
|
+
.get_template("experiment.md")
|
|
229
|
+
.render(
|
|
230
|
+
meta=meta,
|
|
231
|
+
label=unit_label(meta),
|
|
232
|
+
sub=sub,
|
|
233
|
+
verdict=verdict,
|
|
234
|
+
metrics=metrics,
|
|
235
|
+
policy=policy,
|
|
236
|
+
failed=failed,
|
|
237
|
+
advice=advice,
|
|
238
|
+
failed_probes=[
|
|
239
|
+
t
|
|
240
|
+
for t in verdict.adversarial_test_results
|
|
241
|
+
if not t.passed and t.measured and t.kind in probe_kinds
|
|
242
|
+
],
|
|
243
|
+
examples=[(e.max_activation, _context(e)) for e in sub.top_examples[:10]],
|
|
244
|
+
kind=kind,
|
|
245
|
+
previous_status=previous_status,
|
|
246
|
+
record_link=archive_path(wiki_dir, record).name,
|
|
247
|
+
self_reviewed=record.self_reviewed,
|
|
248
|
+
home=home_link(sub.identifier.slug),
|
|
249
|
+
)
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def newest_record(wiki_dir: Path, ident) -> str | None:
|
|
254
|
+
"""The id of the unit's newest experiment; ``None`` if the unit has no experiment."""
|
|
255
|
+
records = sorted((experiments_dir(wiki_dir) / ident.slug).glob("*.json"))
|
|
256
|
+
return records[-1].stem if records else None
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def write_stop_page(draw: UnauditableDraw, wiki_dir: Path) -> Path:
|
|
260
|
+
"""Keep a scan's stop on a chosen unit as a record and a page, and refresh the indexes.
|
|
261
|
+
|
|
262
|
+
Returns the page path. Records and pages are write-once, so a stop in the same second
|
|
263
|
+
as an earlier one leaves that one as it is.
|
|
264
|
+
"""
|
|
265
|
+
stop = ChosenStop(**dict(draw), latest_record=newest_record(wiki_dir, draw.identifier))
|
|
266
|
+
path = stop_path(wiki_dir, stop)
|
|
267
|
+
page = path.with_suffix(".md")
|
|
268
|
+
if not page.exists():
|
|
269
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
270
|
+
path.write_text(stop.model_dump_json() + "\n", encoding="utf-8")
|
|
271
|
+
meta = _stop_meta(stop)
|
|
272
|
+
template = _environment().get_template("stop.md")
|
|
273
|
+
body = template.render(
|
|
274
|
+
stop=stop,
|
|
275
|
+
meta=meta,
|
|
276
|
+
label=unit_label(meta),
|
|
277
|
+
policy=ReviewPolicy(),
|
|
278
|
+
root=home_link(stop.identifier.slug).removesuffix("README.md"),
|
|
279
|
+
)
|
|
280
|
+
page.write_text(_front_matter(meta) + "\n" + body, encoding="utf-8")
|
|
281
|
+
rebuild_indexes(wiki_dir)
|
|
282
|
+
return page
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def stop_page(wiki_dir: Path | str, stop: ChosenStop) -> Path:
|
|
286
|
+
"""The page next to ``stop``'s record."""
|
|
287
|
+
return stop_path(Path(wiki_dir), stop).with_suffix(".md")
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def stop_label(stop: ChosenStop) -> str:
|
|
291
|
+
"""The stopped unit within its layer, for example ``L3/N7+``."""
|
|
292
|
+
return unit_label(_stop_meta(stop))
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _stop_meta(stop: ChosenStop) -> dict:
|
|
296
|
+
"""The stop page's front matter."""
|
|
297
|
+
ident = stop.identifier
|
|
298
|
+
return {
|
|
299
|
+
"slug": ident.slug,
|
|
300
|
+
"model": ident.model_name,
|
|
301
|
+
"layer": ident.layer,
|
|
302
|
+
**_unit_meta(ident),
|
|
303
|
+
"outcome": stop.outcome,
|
|
304
|
+
"active_share": stop.active_share,
|
|
305
|
+
"protocol_version": stop.protocol_version,
|
|
306
|
+
"timestamp": stop.timestamp.isoformat(),
|
|
307
|
+
"latest_record": stop.latest_record,
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def archive_path(wiki_dir: Path, record: AuditRecord) -> Path:
|
|
312
|
+
"""Where the record's archived JSON goes."""
|
|
313
|
+
return neuron_dir(wiki_dir, record) / f"{experiment_id(record)}.json"
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def archive_record(record: AuditRecord, wiki_dir: Path) -> Path:
|
|
317
|
+
"""Store the record of a completed experiment next to its page (write-once).
|
|
318
|
+
|
|
319
|
+
The record is slim unless ``OPENINTERP_OFFLOAD_EVIDENCE=0``: its bulky evidence goes to
|
|
320
|
+
the evidence store, and the file holds references (:mod:`openinterp.core.evidence`).
|
|
321
|
+
"""
|
|
322
|
+
from openinterp.core import evidence
|
|
323
|
+
|
|
324
|
+
path = archive_path(wiki_dir, record)
|
|
325
|
+
if not path.exists():
|
|
326
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
327
|
+
text = record.model_dump_json()
|
|
328
|
+
if evidence.offload_enabled():
|
|
329
|
+
text = json.dumps(evidence.split(json.loads(text)))
|
|
330
|
+
path.write_text(text + "\n", encoding="utf-8")
|
|
331
|
+
return path
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def load_archive_paths(wiki_dir: Path) -> list[tuple[Path, AuditRecord]]:
|
|
335
|
+
"""Every archived experiment record with its path, oldest first."""
|
|
336
|
+
root = experiments_dir(wiki_dir)
|
|
337
|
+
pairs = [
|
|
338
|
+
(p, AuditRecord.model_validate_json(p.read_text(encoding="utf-8")))
|
|
339
|
+
for p in sorted(root.rglob("*.json"))
|
|
340
|
+
]
|
|
341
|
+
return sorted(pairs, key=lambda pr: (pr[1].verdict.timestamp, pr[1].submission.identifier.slug))
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def load_archive(wiki_dir: Path) -> list[AuditRecord]:
|
|
345
|
+
"""Every archived experiment record, oldest first."""
|
|
346
|
+
return [record for _, record in load_archive_paths(wiki_dir)]
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def load_entries(wiki_dir: Path) -> list[WikiEntry]:
|
|
350
|
+
"""Every experiment page with front matter, newest first."""
|
|
351
|
+
root = experiments_dir(wiki_dir)
|
|
352
|
+
entries = [
|
|
353
|
+
WikiEntry(path, _read_front_matter(path))
|
|
354
|
+
for path in root.rglob("*.md")
|
|
355
|
+
if path.name != "README.md"
|
|
356
|
+
]
|
|
357
|
+
return sorted((e for e in entries if e.meta), key=lambda e: e.meta["timestamp"], reverse=True)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _neuron_index(env: Environment, slug: str, history: list, stops: list) -> str:
|
|
361
|
+
latest = history[0].meta
|
|
362
|
+
return env.get_template("neuron.md").render(
|
|
363
|
+
slug=slug,
|
|
364
|
+
latest=latest,
|
|
365
|
+
label=unit_label(latest),
|
|
366
|
+
history=history,
|
|
367
|
+
stops=stops,
|
|
368
|
+
home=home_link(slug),
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _unindexed_stops(stops: dict[str, list[ChosenStop]], indexed) -> list[dict]:
|
|
373
|
+
"""Stops on units with no index of their own, newest first: only the home can link them."""
|
|
374
|
+
rows = [
|
|
375
|
+
{"stop": stop, "label": stop_label(stop), "link": stop_page(".", stop).as_posix()}
|
|
376
|
+
for slug, unit_stops in stops.items()
|
|
377
|
+
if slug not in indexed
|
|
378
|
+
for stop in unit_stops
|
|
379
|
+
]
|
|
380
|
+
return sorted(rows, key=lambda row: row["stop"].timestamp, reverse=True)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def rebuild_indexes(wiki_dir: Path) -> None:
|
|
384
|
+
"""Regenerate the gitignored README indexes from the pages' front matter."""
|
|
385
|
+
wiki_dir = Path(wiki_dir)
|
|
386
|
+
wiki_dir.mkdir(parents=True, exist_ok=True)
|
|
387
|
+
entries = load_entries(wiki_dir)
|
|
388
|
+
env = _environment()
|
|
389
|
+
stops: dict[str, list[ChosenStop]] = {}
|
|
390
|
+
for stop in load_stops(wiki_dir):
|
|
391
|
+
stops.setdefault(stop.identifier.slug, []).append(stop)
|
|
392
|
+
|
|
393
|
+
by_neuron: dict[str, list[WikiEntry]] = {}
|
|
394
|
+
for entry in entries:
|
|
395
|
+
by_neuron.setdefault(entry.meta["slug"], []).append(entry)
|
|
396
|
+
for slug, history in by_neuron.items():
|
|
397
|
+
index = experiments_dir(wiki_dir) / slug / "README.md"
|
|
398
|
+
index.write_text(_neuron_index(env, slug, history, stops.get(slug, [])), encoding="utf-8")
|
|
399
|
+
|
|
400
|
+
counts: dict[str, int] = {}
|
|
401
|
+
for entry in entries:
|
|
402
|
+
counts[entry.meta["status"]] = counts.get(entry.meta["status"], 0) + 1
|
|
403
|
+
rows = [
|
|
404
|
+
{
|
|
405
|
+
**e.meta,
|
|
406
|
+
"label": unit_label(e.meta),
|
|
407
|
+
"link": e.path.relative_to(wiki_dir).as_posix(),
|
|
408
|
+
"neuron_link": e.path.parent.relative_to(wiki_dir).as_posix() + "/README.md",
|
|
409
|
+
}
|
|
410
|
+
for e in entries
|
|
411
|
+
]
|
|
412
|
+
(wiki_dir / "README.md").write_text(
|
|
413
|
+
env.get_template("index.md").render(
|
|
414
|
+
rows=rows,
|
|
415
|
+
counts=counts,
|
|
416
|
+
neurons=len(by_neuron),
|
|
417
|
+
unindexed_stops=_unindexed_stops(stops, by_neuron),
|
|
418
|
+
),
|
|
419
|
+
encoding="utf-8",
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def backfill(records: Sequence[AuditRecord], wiki_dir: Path) -> list[Path]:
|
|
424
|
+
"""Import reviewed records as experiments (record + page) if not yet present (idempotent)."""
|
|
425
|
+
written = [write_experiment_page(record, wiki_dir, kind="backfill") for record in records]
|
|
426
|
+
rebuild_indexes(wiki_dir)
|
|
427
|
+
return [p for p in written if p is not None]
|