open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
openinterp/cli_review.py
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
"""``open-interpretability review``: adversarially review a submission and post the experiment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
|
|
10
|
+
from openinterp.cli import (
|
|
11
|
+
cli,
|
|
12
|
+
contributor_option,
|
|
13
|
+
explanation_scores_option,
|
|
14
|
+
recorded_model_option,
|
|
15
|
+
wiki_option,
|
|
16
|
+
with_model_options,
|
|
17
|
+
)
|
|
18
|
+
from openinterp.core.schemas import MAX_PROBES
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@cli.command()
|
|
22
|
+
@click.option("--audit-file", required=True, type=click.Path(exists=True, path_type=Path))
|
|
23
|
+
@click.option(
|
|
24
|
+
"--agent-model",
|
|
25
|
+
default=None,
|
|
26
|
+
callback=recorded_model_option,
|
|
27
|
+
help="Reviewer. LiteLLM model string, or external/<name> to answer a prompt file. "
|
|
28
|
+
"With --probes, the name recorded for whoever wrote them.",
|
|
29
|
+
)
|
|
30
|
+
@contributor_option("reviewer")
|
|
31
|
+
@click.option(
|
|
32
|
+
"--n-probes", default=MAX_PROBES, show_default=True, type=click.IntRange(10, MAX_PROBES)
|
|
33
|
+
)
|
|
34
|
+
@click.option(
|
|
35
|
+
"--emit-prompt",
|
|
36
|
+
type=click.Path(dir_okay=False, path_type=Path),
|
|
37
|
+
help="Write the reviewer's prompt here, e.g. review.prompt.md (answer in "
|
|
38
|
+
"review.probes.json), and exit. "
|
|
39
|
+
"Loads no model, so the reviewer sees no probe activations.",
|
|
40
|
+
)
|
|
41
|
+
@click.option(
|
|
42
|
+
"--probes",
|
|
43
|
+
"probes_file",
|
|
44
|
+
type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
45
|
+
help="ProbeSet JSON answering --emit-prompt. No LLM is called.",
|
|
46
|
+
)
|
|
47
|
+
@click.option(
|
|
48
|
+
"--check-probes",
|
|
49
|
+
type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
50
|
+
help="ProbeSet JSON to show its reviewer as the model's tokens (and, for a head, the "
|
|
51
|
+
"token runs each probe repeats): writes <name>.check.prompt.md, answered by "
|
|
52
|
+
"rewriting the same file, and exits. Loads only the tokenizer.",
|
|
53
|
+
)
|
|
54
|
+
@click.option(
|
|
55
|
+
"--full-snippets",
|
|
56
|
+
is_flag=True,
|
|
57
|
+
help="Show the reviewer whole scanned documents, without windowing or dropping "
|
|
58
|
+
"near-duplicates: the prompt before trimming, for A/B comparison.",
|
|
59
|
+
)
|
|
60
|
+
@click.option(
|
|
61
|
+
"--dictionary-latents",
|
|
62
|
+
is_flag=True,
|
|
63
|
+
help="Show the reviewer the published dictionary latents aligned with the pole "
|
|
64
|
+
"(`open-interpretability crossref`) and their descriptions, fetched from Neuronpedia.",
|
|
65
|
+
)
|
|
66
|
+
@click.option(
|
|
67
|
+
"--sisters/--no-sisters",
|
|
68
|
+
default=True,
|
|
69
|
+
show_default=True,
|
|
70
|
+
help="Show the reviewer the pole's sister neurons in other models that this registry "
|
|
71
|
+
"has studied (`open-interpretability sisters`), with their explanations.",
|
|
72
|
+
)
|
|
73
|
+
@click.option("--out", type=click.Path(path_type=Path), help="Also write the record here.")
|
|
74
|
+
@explanation_scores_option
|
|
75
|
+
@click.option(
|
|
76
|
+
"--corpus",
|
|
77
|
+
type=click.Path(exists=True, path_type=Path),
|
|
78
|
+
help="Without --explanation-scores: the scan's corpus, to score the explanation on.",
|
|
79
|
+
)
|
|
80
|
+
@click.option(
|
|
81
|
+
"--judge-model",
|
|
82
|
+
callback=recorded_model_option,
|
|
83
|
+
help="Without --explanation-scores: the judge (default: the reviewer).",
|
|
84
|
+
)
|
|
85
|
+
@wiki_option
|
|
86
|
+
@with_model_options
|
|
87
|
+
def verify(
|
|
88
|
+
audit_file,
|
|
89
|
+
agent_model,
|
|
90
|
+
contributor,
|
|
91
|
+
n_probes,
|
|
92
|
+
emit_prompt,
|
|
93
|
+
probes_file,
|
|
94
|
+
check_probes,
|
|
95
|
+
full_snippets,
|
|
96
|
+
dictionary_latents,
|
|
97
|
+
sisters,
|
|
98
|
+
out,
|
|
99
|
+
scores_file,
|
|
100
|
+
corpus,
|
|
101
|
+
judge_model,
|
|
102
|
+
wiki_dir,
|
|
103
|
+
device,
|
|
104
|
+
dtype,
|
|
105
|
+
):
|
|
106
|
+
"""Adversarially review a submission and post it as a new experiment.
|
|
107
|
+
|
|
108
|
+
Every review records explanation scores on held-out text (`open-interpretability score`). Pass
|
|
109
|
+
them with --explanation-scores; in one step (no --probes), they are measured here.
|
|
110
|
+
"""
|
|
111
|
+
from openinterp.agents.base import DEFAULT_MODEL
|
|
112
|
+
from openinterp.agents.reviewer import ReviewerAgent
|
|
113
|
+
from openinterp.cli import _add_costs, _load_unit_runner, _write_copy, _write_wiki
|
|
114
|
+
from openinterp.cli_score import load_scores, score_submission
|
|
115
|
+
from openinterp.core.schemas import AuditRecord
|
|
116
|
+
|
|
117
|
+
submission = load_submission(audit_file)
|
|
118
|
+
latents = aligned_latents_text(submission, wiki_dir) if dictionary_latents else ""
|
|
119
|
+
tested = [submission.sister_of.rsplit("/", 1)[0]] if submission.sister_of else []
|
|
120
|
+
latents += sisters_text(submission.identifier, wiki_dir, "reviewer", tested) if sisters else ""
|
|
121
|
+
if emit_prompt is not None:
|
|
122
|
+
return write_review_prompt(submission, emit_prompt, n_probes, full_snippets, latents)
|
|
123
|
+
if check_probes is not None:
|
|
124
|
+
return write_token_check(submission, check_probes)
|
|
125
|
+
check_review_options(wiki_dir, out, probes_file, agent_model, scores_file, corpus)
|
|
126
|
+
scores = load_scores(scores_file, submission)
|
|
127
|
+
reviewer = ReviewerAgent(agent_model or DEFAULT_MODEL)
|
|
128
|
+
warn_self_review(reviewer.model, submission)
|
|
129
|
+
probes = review_probes(reviewer, submission, probes_file, n_probes, full_snippets, latents)
|
|
130
|
+
runner = _load_unit_runner(submission.identifier, device, dtype)
|
|
131
|
+
warn_unequal_lengths(probes, runner, submission)
|
|
132
|
+
if scores is None: # one step: score here, on the scan's corpus (checked above)
|
|
133
|
+
scores = score_submission(runner, submission, corpus, judge_model or reviewer.model)
|
|
134
|
+
verdict = reviewer.review(submission, runner, probes=probes)
|
|
135
|
+
# The scores' costs move into the record's provenance with everything else.
|
|
136
|
+
scored = scores and scores.costs
|
|
137
|
+
scores = scores and scores.model_copy(update={"costs": None})
|
|
138
|
+
verdict = verdict.model_copy(update={"explanation_scores": scores, "contributor": contributor})
|
|
139
|
+
|
|
140
|
+
record = AuditRecord(submission=_add_costs(submission, scored), verdict=verdict)
|
|
141
|
+
click.echo(f"{verdict.status} (confidence {verdict.confidence_score:.2f}) — {verdict.notes}")
|
|
142
|
+
_write_copy(record, out)
|
|
143
|
+
_write_wiki(record, wiki_dir, kind="review")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def load_submission(audit_file: Path):
|
|
147
|
+
"""The submission in ``audit_file``, or the one an existing audit record re-reviews."""
|
|
148
|
+
from openinterp.cli import _clear_costs
|
|
149
|
+
from openinterp.core.schemas import AuditRecord, NeuronAuditSubmission
|
|
150
|
+
|
|
151
|
+
raw = audit_file.read_text()
|
|
152
|
+
if "submission" in json.loads(raw): # an existing audit record: re-review its submission
|
|
153
|
+
submission = AuditRecord.model_validate_json(raw).submission
|
|
154
|
+
return _clear_costs(submission) # its audit belonged to another experiment
|
|
155
|
+
return NeuronAuditSubmission.model_validate_json(raw)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def sisters_text(identifier, wiki_dir, role: str, exclude=()) -> str:
|
|
159
|
+
"""Return the agent's section on the unit's studied sister neurons.
|
|
160
|
+
|
|
161
|
+
Returns "" if there are none. ``exclude`` leaves out the pole that a sister study tests.
|
|
162
|
+
"""
|
|
163
|
+
from openinterp.cli import _skip_wiki
|
|
164
|
+
from openinterp.sisters import sister_context
|
|
165
|
+
|
|
166
|
+
if _skip_wiki(wiki_dir):
|
|
167
|
+
return ""
|
|
168
|
+
text = sister_context(identifier, wiki_dir, role, exclude=exclude)
|
|
169
|
+
if text:
|
|
170
|
+
click.echo(f"Showing the {role} {text.count(chr(10)) - 1} studied sister neurons", err=True)
|
|
171
|
+
return text
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def aligned_latents_text(submission, wiki_dir) -> str:
|
|
175
|
+
"""The reviewer's section on aligned dictionary latents, or "" if there are none."""
|
|
176
|
+
from openinterp.builder.wiki import load_archive
|
|
177
|
+
from openinterp.cli import _skip_wiki
|
|
178
|
+
from openinterp.crossref import describe, format_for_reviewer, load_rows
|
|
179
|
+
from openinterp.crossref import registry_explanations as explanations
|
|
180
|
+
|
|
181
|
+
rows = load_rows().get(submission.identifier.slug, [])
|
|
182
|
+
if not rows:
|
|
183
|
+
click.echo(f"warning: no aligned latents for {submission.identifier.slug}", err=True)
|
|
184
|
+
return ""
|
|
185
|
+
registry = {} if _skip_wiki(wiki_dir) else explanations(load_archive(wiki_dir))
|
|
186
|
+
return format_for_reviewer(describe(rows, registry))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def write_review_prompt(
|
|
190
|
+
submission, emit_prompt: Path, n_probes: int, full_snippets: bool, latents: str = ""
|
|
191
|
+
):
|
|
192
|
+
"""Write the reviewer's prompt for ``submission`` to ``emit_prompt``."""
|
|
193
|
+
from openinterp.agents.reviewer import ProbeSet, ReviewerAgent
|
|
194
|
+
from openinterp.cli_evidence import answer_path
|
|
195
|
+
from openinterp.core.paths import PROBES_SUFFIX
|
|
196
|
+
|
|
197
|
+
answer = answer_path(emit_prompt, PROBES_SUFFIX)
|
|
198
|
+
task = ReviewerAgent.probe_prompt(submission, n_probes, full_snippets, latents)
|
|
199
|
+
emit_prompt.parent.mkdir(parents=True, exist_ok=True)
|
|
200
|
+
emit_prompt.write_text(ReviewerAgent.render_prompt(ProbeSet, task, answer))
|
|
201
|
+
click.echo(f"Reviewer prompt: {emit_prompt} (answer in {answer})")
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def write_token_check(submission, check_probes: Path):
|
|
205
|
+
"""Write a prompt that shows the reviewer how its probes tokenize."""
|
|
206
|
+
from transformers import AutoTokenizer
|
|
207
|
+
|
|
208
|
+
from openinterp.agents.reviewer import ProbeSet, ReviewerAgent
|
|
209
|
+
|
|
210
|
+
probes = ProbeSet.model_validate_json(check_probes.read_text()).probes
|
|
211
|
+
revision = submission.provenance and submission.provenance.model_revision
|
|
212
|
+
tokenizer = AutoTokenizer.from_pretrained(submission.identifier.model_name, revision=revision)
|
|
213
|
+
task = ReviewerAgent.token_check_prompt(submission, probes, tokenizer)
|
|
214
|
+
stem = check_probes.name.removesuffix(".json").removesuffix(".probes")
|
|
215
|
+
prompt = check_probes.with_name(stem + ".check.prompt.md")
|
|
216
|
+
prompt.write_text(ReviewerAgent.render_prompt(ProbeSet, task, check_probes))
|
|
217
|
+
click.echo(f"Token check: {prompt} (answer by rewriting {check_probes})")
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def check_review_options(wiki_dir, out, probes_file, agent_model, scores_file, corpus) -> None:
|
|
221
|
+
"""Reject review options that do not go together, before any model loads."""
|
|
222
|
+
from openinterp.cli import _skip_wiki
|
|
223
|
+
|
|
224
|
+
if _skip_wiki(wiki_dir) and out is None:
|
|
225
|
+
raise click.UsageError("--out is required when --wiki-dir '' skips the registry.")
|
|
226
|
+
if probes_file is not None and not agent_model:
|
|
227
|
+
raise click.UsageError(
|
|
228
|
+
"--agent-model is required with --probes: name the model that wrote them."
|
|
229
|
+
)
|
|
230
|
+
if scores_file is None and (probes_file is not None or corpus is None):
|
|
231
|
+
raise click.UsageError(
|
|
232
|
+
"--explanation-scores is required (from `open-interpretability score`, "
|
|
233
|
+
"docs/PROTOCOL.md §4); "
|
|
234
|
+
"without --probes, --corpus <the scan's corpus> also works."
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def refuse_unisolated(probes_file: Path) -> None:
|
|
239
|
+
"""Stop if the usage file of ``probes_file`` says that the reviewer read blocked paths.
|
|
240
|
+
|
|
241
|
+
``open-interpretability usage`` writes that list from the reviewer's transcript
|
|
242
|
+
(``openinterp/core/isolation.py``). With no usage file, nothing is checked.
|
|
243
|
+
"""
|
|
244
|
+
from openinterp.core.costs import ExternalUsage, usage_path
|
|
245
|
+
|
|
246
|
+
if not (usage_file := usage_path(probes_file)).exists():
|
|
247
|
+
return
|
|
248
|
+
read = ExternalUsage.model_validate_json(usage_file.read_text()).forbidden_reads
|
|
249
|
+
if read:
|
|
250
|
+
raise click.ClickException(
|
|
251
|
+
f"The reviewer that wrote {probes_file} read blocked paths ({', '.join(read)}). "
|
|
252
|
+
"They can hold probe activations (docs/PROTOCOL.md §2). Ask a new reviewer."
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def warn_self_review(reviewer_model: str, submission) -> None:
|
|
257
|
+
"""Warn when the reviewer model also proposed the hypothesis."""
|
|
258
|
+
from openinterp.core.agent_names import same_agent
|
|
259
|
+
|
|
260
|
+
if same_agent(reviewer_model, submission.submitting_agent_model):
|
|
261
|
+
click.echo(
|
|
262
|
+
f"warning: {reviewer_model} both proposed and reviewed this, so the review is not "
|
|
263
|
+
"independent. It will be recorded and badged as self-reviewed; a different "
|
|
264
|
+
"reviewer model is preferred (docs/PROTOCOL.md §2).",
|
|
265
|
+
err=True,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def warn_unequal_lengths(probes, runner, submission) -> None:
|
|
270
|
+
"""Warn when the probe groups differ in length, as advice and not a gate (issue #3525)."""
|
|
271
|
+
from openinterp.harness.confounds import length_warning
|
|
272
|
+
|
|
273
|
+
if runner.tokenizer is None:
|
|
274
|
+
return
|
|
275
|
+
unequal = length_warning(probes, runner.tokenizer, bool(submission.first_token_floor))
|
|
276
|
+
if unequal:
|
|
277
|
+
click.echo(f"warning: {unequal}", err=True)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def review_probes(
|
|
281
|
+
reviewer, submission, probes_file, n_probes: int, full_snippets: bool, latents: str = ""
|
|
282
|
+
):
|
|
283
|
+
"""The probes in ``probes_file`` (an external reviewer's), or the reviewer LLM's."""
|
|
284
|
+
from openinterp.agents.reviewer import ProbeSet
|
|
285
|
+
from openinterp.core import costs
|
|
286
|
+
from openinterp.core.paths import PROBES_SUFFIX, PROMPT_SUFFIX
|
|
287
|
+
|
|
288
|
+
if probes_file is not None:
|
|
289
|
+
probes = ProbeSet.model_validate_json(probes_file.read_text()).probes
|
|
290
|
+
refuse_unisolated(probes_file)
|
|
291
|
+
# --emit-prompt review.prompt.md is answered in review.probes.json.
|
|
292
|
+
stem = probes_file.name.removesuffix(PROBES_SUFFIX)
|
|
293
|
+
prompt_file = probes_file.with_name(stem + PROMPT_SUFFIX)
|
|
294
|
+
costs.add_external("reviewer", reviewer.model, prompt_file, probes_file)
|
|
295
|
+
return probes
|
|
296
|
+
click.echo(f"Generating {n_probes} adversarial probes with {reviewer.model}…", err=True)
|
|
297
|
+
return reviewer.generate_probes(
|
|
298
|
+
submission, n=n_probes, full_snippets=full_snippets, related=latents
|
|
299
|
+
).probes
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
cli.add_command(verify, name="review")
|
openinterp/cli_scan.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Two commands: a scan's per-token data, and the scan cache.
|
|
2
|
+
|
|
3
|
+
The commands are ``open-interpretability activations`` and ``open-interpretability scan-cache``.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
|
|
15
|
+
from openinterp.cli import cli, with_model_options
|
|
16
|
+
from openinterp.cli_evidence import open_scan_cache, read_corpus, scan_cache_option
|
|
17
|
+
from openinterp.core.schemas import ACTIVATION_THRESHOLD
|
|
18
|
+
from openinterp.corpus import DEFAULT_SAMPLE_SEED, DEFAULT_SAMPLE_TOKENS
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
import torch
|
|
22
|
+
|
|
23
|
+
COLUMNS = ("doc", "window_start", "index", "token_id", "token", "strength", "context")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _tsv(value) -> str:
|
|
27
|
+
text = str(value)
|
|
28
|
+
for raw, escaped in (("\\", "\\\\"), ("\t", "\\t"), ("\n", "\\n"), ("\r", "\\r")):
|
|
29
|
+
text = text.replace(raw, escaped)
|
|
30
|
+
return text
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@cli.command()
|
|
34
|
+
@click.option("--model", "model_name", required=True, help="Hugging Face repo.")
|
|
35
|
+
@click.option("--layer", type=int, required=True)
|
|
36
|
+
@click.option("--neuron", type=int, required=True)
|
|
37
|
+
@click.option("--pole", type=click.Choice(["positive", "negative"]), default="positive")
|
|
38
|
+
@click.option(
|
|
39
|
+
"--corpus",
|
|
40
|
+
type=click.Path(exists=True, path_type=Path),
|
|
41
|
+
help="The audit's corpus (same sample and windows options), so its cached scan is reused.",
|
|
42
|
+
)
|
|
43
|
+
@click.option("--max-tokens", default=DEFAULT_SAMPLE_TOKENS, show_default=True, type=int)
|
|
44
|
+
@click.option("--sample-seed", default=DEFAULT_SAMPLE_SEED, show_default=True, type=int)
|
|
45
|
+
@click.option("--window-tokens", type=click.IntRange(2))
|
|
46
|
+
@click.option("--stride", type=click.IntRange(1))
|
|
47
|
+
@click.option(
|
|
48
|
+
"--top",
|
|
49
|
+
default=50,
|
|
50
|
+
show_default=True,
|
|
51
|
+
type=click.IntRange(0),
|
|
52
|
+
help="The strongest N tokens, strongest first. 0: every matching token in corpus order.",
|
|
53
|
+
)
|
|
54
|
+
@click.option("--token", "tokens", multiple=True, help="Only this token (exact text; repeat).")
|
|
55
|
+
@click.option("--min-strength", type=float, help="Only tokens at least this strong.")
|
|
56
|
+
@click.option("--context", default=12, show_default=True, help="Tokens of left context shown.")
|
|
57
|
+
@click.option("--format", "fmt", type=click.Choice(["tsv", "jsonl"]), default="tsv")
|
|
58
|
+
@click.option("--out", type=click.Path(dir_okay=False, path_type=Path), help="Default: stdout.")
|
|
59
|
+
@scan_cache_option
|
|
60
|
+
@with_model_options
|
|
61
|
+
def activations(
|
|
62
|
+
model_name,
|
|
63
|
+
layer,
|
|
64
|
+
neuron,
|
|
65
|
+
pole,
|
|
66
|
+
corpus,
|
|
67
|
+
max_tokens,
|
|
68
|
+
sample_seed,
|
|
69
|
+
window_tokens,
|
|
70
|
+
stride,
|
|
71
|
+
top,
|
|
72
|
+
tokens,
|
|
73
|
+
min_strength,
|
|
74
|
+
context,
|
|
75
|
+
fmt,
|
|
76
|
+
out,
|
|
77
|
+
use_scan_cache,
|
|
78
|
+
device,
|
|
79
|
+
dtype,
|
|
80
|
+
):
|
|
81
|
+
"""Per-token pole strengths of one neuron over the scan corpus.
|
|
82
|
+
|
|
83
|
+
Reads the layer's cached scan when there is one, else scans and caches the layer.
|
|
84
|
+
An audit of any neuron in the layer, with the same corpus options, stores that scan.
|
|
85
|
+
Each token counts once, as in the audit's histogram. Use this rather than writing
|
|
86
|
+
a scan loop: e.g. every occurrence of a token with --token ' the' --top 0.
|
|
87
|
+
"""
|
|
88
|
+
from openinterp.cli import _load_runner
|
|
89
|
+
from openinterp.harness.scan import cut_pieces
|
|
90
|
+
|
|
91
|
+
if stride is not None and (window_tokens is None or stride > window_tokens):
|
|
92
|
+
raise click.UsageError("--stride needs --window-tokens, and can't exceed it.")
|
|
93
|
+
runner = _load_runner(model_name, device, dtype)
|
|
94
|
+
tokenizer = runner.tokenizer
|
|
95
|
+
texts, *_ = read_corpus(corpus)
|
|
96
|
+
if max_tokens:
|
|
97
|
+
from openinterp.corpus import sample_tokens
|
|
98
|
+
|
|
99
|
+
texts, _ = sample_tokens(tokenizer, texts, max_tokens, sample_seed)
|
|
100
|
+
pieces = list(cut_pieces(tokenizer, texts, window_tokens, stride))
|
|
101
|
+
scan = _neuron_strengths(runner, pieces, layer, neuron, pole, open_scan_cache(use_scan_cache))
|
|
102
|
+
rows = _select(scan, tokenizer, min_strength, tokens, top)
|
|
103
|
+
windowed = window_tokens is not None
|
|
104
|
+
lines = [_format(scan, pieces, row, tokenizer, context, windowed, fmt) for row in rows.tolist()]
|
|
105
|
+
if fmt == "tsv":
|
|
106
|
+
lines.insert(0, "\t".join(COLUMNS))
|
|
107
|
+
text = "\n".join(lines) + "\n"
|
|
108
|
+
if out is None:
|
|
109
|
+
click.echo(text, nl=False)
|
|
110
|
+
else:
|
|
111
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
out.write_text(text, encoding="utf-8")
|
|
113
|
+
click.echo(f"Wrote {len(rows):,} tokens to {out}", err=True)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class _Strengths:
|
|
118
|
+
"""Every scanned token: its id, pole strength, whether it counts, piece and index."""
|
|
119
|
+
|
|
120
|
+
ids: torch.Tensor
|
|
121
|
+
values: torch.Tensor
|
|
122
|
+
counted: torch.Tensor
|
|
123
|
+
piece_of: list[int]
|
|
124
|
+
index: list[int]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _neuron_strengths(runner, pieces, layer: int, neuron: int, pole, cache) -> _Strengths:
|
|
128
|
+
import torch
|
|
129
|
+
|
|
130
|
+
from openinterp.harness.hooks import pole_strength
|
|
131
|
+
from openinterp.harness.scan import scan_chunks
|
|
132
|
+
|
|
133
|
+
ids, strengths, counted, piece_of, index = [], [], [], [], []
|
|
134
|
+
next_piece = 0
|
|
135
|
+
for chunk in scan_chunks(runner, pieces, layer, cache=cache):
|
|
136
|
+
ids.append(chunk.ids)
|
|
137
|
+
strengths.append(pole_strength(chunk.acts[:, neuron].float(), pole))
|
|
138
|
+
counted.append(chunk.counted)
|
|
139
|
+
for n in chunk.lengths: # chunks hold consecutive pieces
|
|
140
|
+
piece_of.extend([next_piece] * n)
|
|
141
|
+
index.extend(range(n))
|
|
142
|
+
next_piece += 1
|
|
143
|
+
if cache is not None and cache.last is not None:
|
|
144
|
+
verb = "Replayed" if cache.replayed else "Cached"
|
|
145
|
+
click.echo(f"{verb} layer {layer}'s scan: {cache.last}", err=True)
|
|
146
|
+
return _Strengths(torch.cat(ids), torch.cat(strengths), torch.cat(counted), piece_of, index)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _select(scan: _Strengths, tokenizer, min_strength, tokens, top: int) -> torch.Tensor:
|
|
150
|
+
"""The rows to print, after reporting the scan's size, peak and active count."""
|
|
151
|
+
import torch
|
|
152
|
+
|
|
153
|
+
ids_t, values = scan.ids, scan.values
|
|
154
|
+
keep = scan.counted.clone()
|
|
155
|
+
peak = float(values[keep].max()) if keep.any() else 0.0
|
|
156
|
+
active = int((values[keep] >= ACTIVATION_THRESHOLD * peak).sum()) if peak > 0 else 0
|
|
157
|
+
click.echo(
|
|
158
|
+
f"{int(keep.sum()):,} tokens; max {peak:.4g}; {active:,} at or above "
|
|
159
|
+
f"{ACTIVATION_THRESHOLD:g} × max (the protocol's active threshold)",
|
|
160
|
+
err=True,
|
|
161
|
+
)
|
|
162
|
+
if min_strength is not None:
|
|
163
|
+
keep &= values >= min_strength
|
|
164
|
+
if tokens:
|
|
165
|
+
seen = ids_t[keep].unique().tolist()
|
|
166
|
+
wanted = [i for i in seen if tokenizer.decode([i]) in set(tokens)]
|
|
167
|
+
keep &= torch.isin(ids_t, torch.tensor(wanted, dtype=ids_t.dtype))
|
|
168
|
+
rows = keep.nonzero().flatten()
|
|
169
|
+
if top:
|
|
170
|
+
rows = rows[values[rows].argsort(descending=True, stable=True)[:top]]
|
|
171
|
+
return rows
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _format(scan: _Strengths, pieces, row: int, tokenizer, context, windowed, fmt) -> str:
|
|
175
|
+
ids_t, index = scan.ids, scan.index
|
|
176
|
+
piece = pieces[scan.piece_of[row]]
|
|
177
|
+
first = row - index[row]
|
|
178
|
+
left = ids_t[max(first, row - context) : row].tolist() if context else []
|
|
179
|
+
record = {
|
|
180
|
+
"doc": piece.doc,
|
|
181
|
+
"window_start": piece.start if windowed else 0,
|
|
182
|
+
"index": index[row],
|
|
183
|
+
"token_id": int(ids_t[row]),
|
|
184
|
+
"token": tokenizer.decode([int(ids_t[row])]),
|
|
185
|
+
"strength": round(float(scan.values[row]), 5),
|
|
186
|
+
"context": tokenizer.decode(left) if left else "",
|
|
187
|
+
}
|
|
188
|
+
if fmt == "jsonl":
|
|
189
|
+
return json.dumps(record, ensure_ascii=False)
|
|
190
|
+
return "\t".join(_tsv(record[c]) for c in COLUMNS)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@cli.command("scan-cache")
|
|
194
|
+
@click.option("--clear", is_flag=True, help="Delete every cached layer scan.")
|
|
195
|
+
def scan_cache(clear):
|
|
196
|
+
"""List (or clear) the cached layer scans that audits and `activations` replay.
|
|
197
|
+
|
|
198
|
+
\b
|
|
199
|
+
Location: $OPENINTERP_SCAN_CACHE (\"off\" disables it), else
|
|
200
|
+
$XDG_CACHE_HOME/openinterp/scans or ~/.cache/openinterp/scans.
|
|
201
|
+
Size cap: $OPENINTERP_SCAN_CACHE_GB (default 40); least recently used go first.
|
|
202
|
+
"""
|
|
203
|
+
from openinterp.harness.layer_cache import LayerCache
|
|
204
|
+
|
|
205
|
+
cache = LayerCache.default()
|
|
206
|
+
if cache is None:
|
|
207
|
+
click.echo("The scan cache is off ($OPENINTERP_SCAN_CACHE).")
|
|
208
|
+
return
|
|
209
|
+
if clear:
|
|
210
|
+
click.echo(f"Deleted {cache.clear()} cached scans from {cache.root}")
|
|
211
|
+
return
|
|
212
|
+
entries = cache.entries()
|
|
213
|
+
total = sum(e.size for e in entries)
|
|
214
|
+
click.echo(
|
|
215
|
+
f"{cache.root}: {len(entries)} scans, {total / 1e9:.1f} of {cache.max_bytes / 1e9:.0f} GB"
|
|
216
|
+
)
|
|
217
|
+
for entry in reversed(entries): # most recently used first
|
|
218
|
+
f = entry.meta["fields"]
|
|
219
|
+
click.echo(
|
|
220
|
+
f" {f['model']} L{f['layer']} {entry.meta['rows']:,} tokens "
|
|
221
|
+
f"{entry.size / 1e9:.2f} GB {f['dtype']} on {f['device']} "
|
|
222
|
+
f"pieces {f['pieces_sha256'][:12]} {entry.path.name}"
|
|
223
|
+
)
|