open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,82 @@
1
+ """Schemas for causal metrics: how much patching a unit moves the model's prediction.
2
+
3
+ Recorded in every verdict, never gated (docs/PROTOCOL.md §4). The harness that
4
+ measures them is ``openinterp.harness.causal``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Literal
10
+
11
+ from pydantic import Field
12
+
13
+ from openinterp.core.strict import StrictModel, UnitFloat
14
+
15
+ CausalTarget = Literal["claimed_tokens", "next_token_kl"]
16
+
17
+
18
+ class PatchEffect(StrictModel):
19
+ """One patch of one minimal pair: the unit's effect and its matched null's."""
20
+
21
+ effect: float = Field(description="The unit's signed effect (logits or KL, per target).")
22
+ percentile: UnitFloat | None = Field(
23
+ default=None, description="Fraction of the matched units' |effect| below the unit's."
24
+ )
25
+ null_median: float = Field(ge=0.0, description="Median |effect| of the matched units.")
26
+ null_max: float = Field(ge=0.0, description="Largest |effect| of the matched units.")
27
+
28
+
29
+ class CausalPairEffect(StrictModel):
30
+ """Both patches of one non-control minimal pair, for the per-pair causal chart."""
31
+
32
+ pair_index: int = Field(ge=0, description="Index into the submission's counterfactuals.")
33
+ sufficiency: PatchEffect
34
+ necessity: PatchEffect
35
+
36
+
37
+ class CausalMetrics(StrictModel):
38
+ """Causal effect of the neuron pole on the model's next-token prediction.
39
+
40
+ Since protocol 0.5, the harness patches each minimal pair in both directions. It
41
+ scores each patch on a target that favours no particular neuron:
42
+
43
+ - ``claimed_tokens``: for a lookahead claim, the mean logit of the upcoming tokens.
44
+ - ``next_token_kl``: for other claims, the next-token KL that the patch causes.
45
+
46
+ The comparison set is the ``null_neurons`` same-layer neurons whose strength changes
47
+ most like the pole's between the pair's prompts. The harness patches them the same
48
+ way and scores them on the same target. ``sufficiency_score`` (clean → corrupted) and
49
+ ``necessity_score`` (corrupted → clean) give the median percentile of the pole's
50
+ |effect| among theirs. So 0.5 means "no more influential than a neuron that changes
51
+ as much". The harness skips control pairs, where the pole barely moves.
52
+
53
+ Records with ``target = None`` predate 0.5. Their scores are the fraction of the
54
+ clean/corrupted gap in the top-1 next-token logit that the patch restores.
55
+ """
56
+
57
+ sufficiency_score: UnitFloat
58
+ necessity_score: UnitFloat
59
+ hydra_mitigated: bool = Field(
60
+ description="True if downstream MLP outputs were frozen to block self-repair (hydra)."
61
+ )
62
+ target: CausalTarget | None = None
63
+ target_tokens: list[str] = Field(
64
+ default_factory=list,
65
+ description="The claimed upcoming tokens scored (``claimed_tokens`` only).",
66
+ )
67
+ sufficiency_effect: float | None = Field(
68
+ default=None,
69
+ description="Median effect of the clean → corrupted patch (logits or KL, per target).",
70
+ )
71
+ necessity_effect: float | None = Field(
72
+ default=None,
73
+ description="Median effect of the corrupted → clean patch (logits or KL, per target).",
74
+ )
75
+ null_neurons: int | None = Field(default=None, ge=0)
76
+ pairs_scored: int | None = Field(
77
+ default=None, ge=0, description="Minimal pairs where the pole moved (controls skipped)."
78
+ )
79
+ pairs: list[CausalPairEffect] = Field(
80
+ default_factory=list,
81
+ description="Per-pair effects behind the medians (empty in older records).",
82
+ )
@@ -0,0 +1,103 @@
1
+ """Schemas for confound profiles.
2
+
3
+ A confound profile says how much trivial explanations (token identity, position,
4
+ per-token labels such as language) account for a unit's activation.
5
+
6
+ Recorded in every scan, never gated (docs/PROTOCOL.md §4). The harness that measures
7
+ them is ``openinterp.harness.confounds`` and, for labels, ``openinterp.harness.annotations``.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from pydantic import Field
13
+
14
+ from openinterp.core.strict import StrictModel, UnitFloat
15
+
16
+ # The label set of a document's language (a corpus's ``lang`` field; ``corpus.py``).
17
+ LANGUAGE = "language"
18
+ # The label set of a blended corpus's document's preset (its ``source`` field).
19
+ SOURCE = "source"
20
+ # The label set of a code document's programming language (a corpus's ``code_language``).
21
+ PROGRAMMING_LANGUAGE = "programming language"
22
+ # The label sets of a code document's syntax tree (`openinterp/syntax.py`).
23
+ SYNTAX_NODE = "syntax node"
24
+ BRACKET_DEPTH = "bracket depth"
25
+ ROUND_DEPTH = "round depth"
26
+ INDENT_DEPTH = "indent depth"
27
+ SYNTAX_CONSTRUCT = "syntax construct"
28
+ # The label sets of what `sensitive.py`, `toxicity.py` and `names.py` find in a document,
29
+ # and the label of every other token. Tokens with no label leave R² and mutual
30
+ # information, so a label set that named only the matches would always measure zero.
31
+ PERSONAL_DATA = "personal data"
32
+ TOXICITY = "toxicity" # a model's guess (`openinterp/toxicity.py`)
33
+ HATE_SPEECH = "hate speech" # a model's guess (`openinterp/toxicity.py`)
34
+ NAMES = "names" # a model's guess (`openinterp/names.py`)
35
+ POSTAL_ADDRESSES = "postal addresses"
36
+ NO_MATCH = "none"
37
+ # Each word's universal part-of-speech tag, from the names model (`openinterp/names.py`).
38
+ # It labels every word, so it is not a match set: its whitespace gaps are NO_MATCH.
39
+ PART_OF_SPEECH = "part of speech"
40
+ MATCH_LABEL_SETS = (PERSONAL_DATA, TOXICITY, HATE_SPEECH, NAMES, POSTAL_ADDRESSES)
41
+ # Recorded, but not in the proposer's prompt. Most units have an active token in some name,
42
+ # so every prompt would carry this noisy label.
43
+ NOT_SHOWN_TO_PROPOSER = (NAMES,)
44
+
45
+
46
+ class TokenStat(StrictModel):
47
+ """One token's mean strength over the scan, for the strongest-tokens table."""
48
+
49
+ token: str
50
+ token_id: int = Field(ge=0)
51
+ mean_strength: float = Field(ge=0.0)
52
+ count: int = Field(gt=0)
53
+
54
+
55
+ class LabelStat(StrictModel):
56
+ """One label's count, mean strength and active tokens over the labelled tokens."""
57
+
58
+ label: str
59
+ count: int = Field(gt=0)
60
+ mean_strength: float = Field(ge=0.0)
61
+ active: int = Field(ge=0, description="Tokens at or above the protocol's activation line.")
62
+
63
+
64
+ class LabelConfound(StrictModel):
65
+ """How much one label set explains of the pole's activation.
66
+
67
+ A label set is, for example, a token's language or its part of speech. The fields
68
+ are ``r2``, as ``token_r2`` is for token identity, and the binned,
69
+ bias-corrected mutual information between strength and label, in bits. Both are
70
+ over labelled tokens only. ``levels`` lists the most frequent labels.
71
+ """
72
+
73
+ r2: UnitFloat
74
+ mutual_information_bits: float = Field(ge=0.0)
75
+ labelled_tokens: int = Field(ge=0)
76
+ unlabelled_tokens: int = Field(ge=0)
77
+ levels: list[LabelStat]
78
+
79
+ @property
80
+ def has_active_match(self) -> bool:
81
+ """Return whether a match label set has active tokens; most units have none."""
82
+ return any(s.label != NO_MATCH and s.active > 0 for s in self.levels)
83
+
84
+
85
+ class ConfoundProfile(StrictModel):
86
+ """How much of the pole's activation simple confounds explain over the scan corpus.
87
+
88
+ ``token_r2`` / ``position_r2`` are the variance explained by a per-token-id / per-
89
+ position mean model. ``trigger_tokens`` (highest mean strength, ``count >= min_count``)
90
+ and ``position_means`` drive the trivial baselines a hypothesis must match or beat.
91
+ """
92
+
93
+ token_r2: UnitFloat
94
+ position_r2: UnitFloat
95
+ frequency_correlation: float = Field(ge=-1.0, le=1.0)
96
+ min_count: int = Field(gt=0)
97
+ trigger_tokens: list[TokenStat]
98
+ position_means: list[float]
99
+ labels: dict[str, LabelConfound] | None = Field(
100
+ default=None,
101
+ description="Confounds from per-token labels (e.g. 'language'), by label set; "
102
+ "None when the scan had none. Reported, not gated.",
103
+ )
@@ -0,0 +1,26 @@
1
+ """The human contributor behind an experiment's proposal or review, beside the agent model.
2
+
3
+ A contributor is a public handle such as a GitHub username, never an email, and is only
4
+ recorded when given: ``--contributor`` or ``$OPENINTERP_CONTRIBUTOR``. Git config is never
5
+ read, so no record names a person who did not choose it. The site credits a record with no
6
+ contributor to the author of the PR that merged it (:mod:`openinterp.builder.users`).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+
13
+ CONTRIBUTOR_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9._-]{0,38}$"
14
+ CONTRIBUTOR_ENV = "OPENINTERP_CONTRIBUTOR"
15
+
16
+
17
+ def contributor_problem(handle: str) -> str | None:
18
+ """Why ``handle`` can't be recorded as a contributor, or None if it can."""
19
+ if "@" in handle:
20
+ return "a contributor is a public handle (e.g. a GitHub username), never an email"
21
+ if not re.fullmatch(CONTRIBUTOR_PATTERN, handle):
22
+ return (
23
+ "a contributor is 1-39 letters, digits, '.', '_' or '-', starting with a letter "
24
+ "or digit"
25
+ )
26
+ return None
@@ -0,0 +1,126 @@
1
+ """Schemas for where a scan's corpus came from.
2
+
3
+ :class:`CorpusSource` is the sidecar of the dataset sample that ``open-interpretability corpus``
4
+ wrote. :class:`CorpusSample` is the token-bounded part of it that an audit read. Both
5
+ are built by ``openinterp.corpus``.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from pydantic import Field
11
+
12
+ from openinterp.core.strict import StrictModel
13
+
14
+
15
+ class CorpusSource(StrictModel):
16
+ """Where a scan corpus came from.
17
+
18
+ ``open-interpretability corpus`` writes it next to the corpus.
19
+
20
+ Enough to rebuild the same documents: the dataset at an exact commit, the files and
21
+ column read, the seed and sizes. ``license`` is the dataset card's licence, so pages
22
+ that show excerpts can attribute them.
23
+ """
24
+
25
+ dataset: str = Field(min_length=1, examples=["HuggingFaceFW/fineweb-edu"])
26
+ revision: str | None = Field(default=None, description="Dataset commit hash.")
27
+ files: str = Field(
28
+ description="Glob of the Parquet files sampled, within the dataset; several, "
29
+ "comma-separated, each gave an equal share of the documents."
30
+ )
31
+ text_column: str = Field(
32
+ default="text",
33
+ description="Column of the text; several, comma-separated, joined into one document "
34
+ "with a blank line between them.",
35
+ )
36
+ documents: int = Field(ge=0)
37
+ seed: int
38
+ max_chars: int = Field(gt=0)
39
+ docs_per_row_group: int | None = Field(default=None, gt=0, description="None: a blend.")
40
+ layout: str | None = Field(
41
+ default=None,
42
+ description="How rows were joined into documents (e.g. 'wikitext'); None: one per row.",
43
+ )
44
+ first: bool = False # the first documents in file order, not a seeded sample (no strip)
45
+ ascii: bool = False # non-ASCII \u-escaped in the JSONL, as in the registry's corpus
46
+ lang_pattern: str | None = Field(
47
+ default=None,
48
+ description="Regular expression whose group, matched in each file glob, gave its "
49
+ "documents' 'lang' field; None: documents carry no language.",
50
+ )
51
+ lang_detector: str | None = Field(
52
+ default=None,
53
+ description="The detector that gave each document's 'lang' field "
54
+ "(openinterp.langid.DETECTOR), when the dataset doesn't say; documents it was "
55
+ "unsure of have none. None: no detected languages.",
56
+ )
57
+ code_language_column: str | None = Field(
58
+ default=None,
59
+ description="The dataset column that gave each document's 'code_language' field; "
60
+ "None: documents carry no programming language.",
61
+ )
62
+ code_detector: str | None = Field(
63
+ default=None,
64
+ description="The detector that gave each document's 'code_language' field "
65
+ "(openinterp.codeid.detector()), when the dataset doesn't say; documents it was "
66
+ "unsure of, and prose, have none. None: no detected programming languages.",
67
+ )
68
+ chat_template: str | None = Field(
69
+ default=None,
70
+ description="The model whose tokenizer's chat template wrote each conversation in "
71
+ "the text column as a document; None: the column's text, as it is.",
72
+ )
73
+ chat_template_revision: str | None = Field(
74
+ default=None, description="Commit of the chat template's model."
75
+ )
76
+ license: str | None = None
77
+ sha256: str = Field(description="SHA-256 of the corpus file this describes.")
78
+ blend: str | None = Field(
79
+ default=None,
80
+ description="The versioned blend of presets this corpus is (openinterp.blends.BLENDS), "
81
+ "e.g. 'mixed-v1'; its documents are its components' in order, each recording its "
82
+ "preset as 'source'. None: one dataset.",
83
+ )
84
+ components: list[CorpusSource] | None = Field(
85
+ default=None,
86
+ description="A blend's samples, one per preset, each as `open-interpretability corpus` "
87
+ "writes that preset alone with the same seed and its share of the documents.",
88
+ )
89
+
90
+ def __str__(self) -> str:
91
+ how = f"{self.documents} docs, seed {self.seed}"
92
+ if self.blend:
93
+ return f"{self.blend} ({len(self.components or [])} presets, {how})"
94
+ at = f"@{self.revision[:8]}" if self.revision else ""
95
+ how = f"first {self.documents} docs" if self.first else how
96
+ if self.chat_template:
97
+ how += f", {self.chat_template}'s chat template"
98
+ return f"{self.dataset}{at} ({self.files}, {how})"
99
+
100
+
101
+ class CorpusSample(StrictModel):
102
+ """The seeded, token-bounded sample of the corpus a scan read (protocol 0.7).
103
+
104
+ The sampler visits documents in ``random.Random(seed)`` order. It takes each document
105
+ whole if it fits in what is left of ``max_tokens``, and always takes the first. Then
106
+ it puts the documents back in corpus order. The model's tokenizer counts the tokens,
107
+ without special tokens.
108
+ Re-drawing with the same corpus, tokenizer, seed and budget gives the same
109
+ documents, which ``sha256`` pins.
110
+ """
111
+
112
+ seed: int
113
+ max_tokens: int = Field(gt=0, description="Token budget the sample was drawn to.")
114
+ documents: int = Field(gt=0)
115
+ corpus_documents: int = Field(gt=0, description="Documents in the corpus it was drawn from.")
116
+ tokens: int = Field(ge=0, description="Tokenizer tokens in the sampled documents.")
117
+ sha256: str = Field(
118
+ description='SHA-256 of the sampled documents as JSONL ({"text": ...} per line), '
119
+ "the bytes `open-interpretability corpus` would write for them."
120
+ )
121
+
122
+ def __str__(self) -> str:
123
+ return (
124
+ f"{self.documents} of {self.corpus_documents} docs, {self.tokens:,} tokens "
125
+ f"(≤ {self.max_tokens:,}, seed {self.seed})"
126
+ )