open-interpretability 0.0.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. open_interpretability-0.0.20.dist-info/METADATA +157 -0
  2. open_interpretability-0.0.20.dist-info/RECORD +256 -0
  3. open_interpretability-0.0.20.dist-info/WHEEL +4 -0
  4. open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
  5. open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
  6. openinterp/__init__.py +6 -0
  7. openinterp/agents/__init__.py +25 -0
  8. openinterp/agents/base.py +229 -0
  9. openinterp/agents/judge.py +149 -0
  10. openinterp/agents/proposer.py +471 -0
  11. openinterp/agents/proposer_profiles.py +79 -0
  12. openinterp/agents/review_measurements.py +173 -0
  13. openinterp/agents/reviewer.py +498 -0
  14. openinterp/agreement.py +63 -0
  15. openinterp/audit_options.py +79 -0
  16. openinterp/baseline.py +254 -0
  17. openinterp/benchmark.py +450 -0
  18. openinterp/blends.py +239 -0
  19. openinterp/builder/__init__.py +6 -0
  20. openinterp/builder/agents.py +260 -0
  21. openinterp/builder/aligned.py +41 -0
  22. openinterp/builder/charts.py +411 -0
  23. openinterp/builder/circuit_pages.py +37 -0
  24. openinterp/builder/circuits.py +264 -0
  25. openinterp/builder/compiler.py +495 -0
  26. openinterp/builder/corpora.py +98 -0
  27. openinterp/builder/coverage.py +46 -0
  28. openinterp/builder/dashboard.py +255 -0
  29. openinterp/builder/disk_memo.py +65 -0
  30. openinterp/builder/experiment.py +105 -0
  31. openinterp/builder/heat.py +121 -0
  32. openinterp/builder/hrefs.py +71 -0
  33. openinterp/builder/learned_baseline.py +204 -0
  34. openinterp/builder/model_facts.py +138 -0
  35. openinterp/builder/neuronpedia_export.py +107 -0
  36. openinterp/builder/parallel.py +100 -0
  37. openinterp/builder/parquet_export.py +458 -0
  38. openinterp/builder/pieces.py +127 -0
  39. openinterp/builder/pii.py +211 -0
  40. openinterp/builder/registry_stats.py +144 -0
  41. openinterp/builder/research.py +109 -0
  42. openinterp/builder/sections.py +28 -0
  43. openinterp/builder/significance.py +103 -0
  44. openinterp/builder/sister_graph.py +180 -0
  45. openinterp/builder/standing.py +185 -0
  46. openinterp/builder/statuses.py +34 -0
  47. openinterp/builder/stopped.py +91 -0
  48. openinterp/builder/stops.py +30 -0
  49. openinterp/builder/superpositions.py +102 -0
  50. openinterp/builder/templates/_agent_role_cells.html +9 -0
  51. openinterp/builder/templates/_causal_chart.html +67 -0
  52. openinterp/builder/templates/_circuit.html +8 -0
  53. openinterp/builder/templates/_dashboard.html +133 -0
  54. openinterp/builder/templates/_experiment.html +500 -0
  55. openinterp/builder/templates/_macros.html +82 -0
  56. openinterp/builder/templates/_patchscope.html +33 -0
  57. openinterp/builder/templates/_prior_art.html +24 -0
  58. openinterp/builder/templates/_specificity.html +21 -0
  59. openinterp/builder/templates/_steering.html +34 -0
  60. openinterp/builder/templates/_superposition.html +62 -0
  61. openinterp/builder/templates/_vocab_chart.html +88 -0
  62. openinterp/builder/templates/_wiring.html +44 -0
  63. openinterp/builder/templates/about.html +95 -0
  64. openinterp/builder/templates/agent.html +69 -0
  65. openinterp/builder/templates/agents.html +80 -0
  66. openinterp/builder/templates/base.html +103 -0
  67. openinterp/builder/templates/circuit.html +64 -0
  68. openinterp/builder/templates/circuits.html +46 -0
  69. openinterp/builder/templates/experiment.html +21 -0
  70. openinterp/builder/templates/hub.html +16 -0
  71. openinterp/builder/templates/index.html +387 -0
  72. openinterp/builder/templates/layer.html +69 -0
  73. openinterp/builder/templates/learned-baseline.html +44 -0
  74. openinterp/builder/templates/model.html +151 -0
  75. openinterp/builder/templates/models.html +47 -0
  76. openinterp/builder/templates/neuron.html +234 -0
  77. openinterp/builder/templates/paper.html +39 -0
  78. openinterp/builder/templates/redirect.html +1 -0
  79. openinterp/builder/templates/research.html +31 -0
  80. openinterp/builder/templates/user.html +54 -0
  81. openinterp/builder/templates/users.html +39 -0
  82. openinterp/builder/templates/wiki/experiment.md +271 -0
  83. openinterp/builder/templates/wiki/index.md +32 -0
  84. openinterp/builder/templates/wiki/neuron.md +30 -0
  85. openinterp/builder/templates/wiki/stop.md +36 -0
  86. openinterp/builder/tree.py +470 -0
  87. openinterp/builder/users.py +142 -0
  88. openinterp/builder/wiki.py +427 -0
  89. openinterp/checks.py +429 -0
  90. openinterp/cli.py +494 -0
  91. openinterp/cli_baseline.py +332 -0
  92. openinterp/cli_benchmark.py +300 -0
  93. openinterp/cli_build.py +128 -0
  94. openinterp/cli_check.py +51 -0
  95. openinterp/cli_concepts.py +76 -0
  96. openinterp/cli_corpus.py +252 -0
  97. openinterp/cli_costs.py +257 -0
  98. openinterp/cli_coverage.py +76 -0
  99. openinterp/cli_crossref.py +86 -0
  100. openinterp/cli_errata.py +142 -0
  101. openinterp/cli_evidence.py +495 -0
  102. openinterp/cli_evidence_store.py +93 -0
  103. openinterp/cli_explain.py +101 -0
  104. openinterp/cli_loop.py +125 -0
  105. openinterp/cli_memory.py +28 -0
  106. openinterp/cli_neuronpedia.py +175 -0
  107. openinterp/cli_pause.py +45 -0
  108. openinterp/cli_pole_effects.py +133 -0
  109. openinterp/cli_probing.py +119 -0
  110. openinterp/cli_registry.py +348 -0
  111. openinterp/cli_reproduce.py +446 -0
  112. openinterp/cli_review.py +302 -0
  113. openinterp/cli_scan.py +223 -0
  114. openinterp/cli_score.py +446 -0
  115. openinterp/cli_sisters.py +86 -0
  116. openinterp/cli_submission.py +217 -0
  117. openinterp/cli_superpositions.py +118 -0
  118. openinterp/cli_wiring.py +54 -0
  119. openinterp/codeid.py +73 -0
  120. openinterp/concepts.py +400 -0
  121. openinterp/core/__init__.py +59 -0
  122. openinterp/core/agent_names.py +76 -0
  123. openinterp/core/attribution.py +101 -0
  124. openinterp/core/cache_dirs.py +15 -0
  125. openinterp/core/causal.py +82 -0
  126. openinterp/core/confounds.py +103 -0
  127. openinterp/core/contributors.py +26 -0
  128. openinterp/core/corpora.py +126 -0
  129. openinterp/core/costs.py +430 -0
  130. openinterp/core/draws.py +109 -0
  131. openinterp/core/errata.py +244 -0
  132. openinterp/core/evidence.py +160 -0
  133. openinterp/core/examples.py +93 -0
  134. openinterp/core/exceptions.py +29 -0
  135. openinterp/core/explanation.py +130 -0
  136. openinterp/core/fence.py +42 -0
  137. openinterp/core/heavy_lock.py +114 -0
  138. openinterp/core/identifiers.py +198 -0
  139. openinterp/core/isolation.py +149 -0
  140. openinterp/core/ledger.py +130 -0
  141. openinterp/core/licenses.py +28 -0
  142. openinterp/core/manifold.py +117 -0
  143. openinterp/core/partners.py +32 -0
  144. openinterp/core/patchscope.py +59 -0
  145. openinterp/core/paths.py +40 -0
  146. openinterp/core/pause.py +96 -0
  147. openinterp/core/pricing.py +178 -0
  148. openinterp/core/prior_art.py +44 -0
  149. openinterp/core/probing.py +146 -0
  150. openinterp/core/profiles.py +221 -0
  151. openinterp/core/project.py +28 -0
  152. openinterp/core/run_environment.py +38 -0
  153. openinterp/core/schemas.py +484 -0
  154. openinterp/core/specificity.py +49 -0
  155. openinterp/core/stats.py +172 -0
  156. openinterp/core/steering.py +49 -0
  157. openinterp/core/strict.py +19 -0
  158. openinterp/core/takedowns.py +136 -0
  159. openinterp/core/transcripts.py +130 -0
  160. openinterp/core/wiring.py +80 -0
  161. openinterp/corpus.py +476 -0
  162. openinterp/corpus_cache.py +145 -0
  163. openinterp/corpus_layouts.py +108 -0
  164. openinterp/corpus_presets.py +168 -0
  165. openinterp/corpus_rebuild.py +93 -0
  166. openinterp/coverage.py +155 -0
  167. openinterp/crossref.py +442 -0
  168. openinterp/data/__init__.py +9 -0
  169. openinterp/data/offensive_words.txt +58 -0
  170. openinterp/data/public_suffix_list.dat +15771 -0
  171. openinterp/data/sample_corpus.txt +40 -0
  172. openinterp/draw_steps.py +359 -0
  173. openinterp/errata_checks.py +129 -0
  174. openinterp/evidence.py +462 -0
  175. openinterp/evidence_branch.py +119 -0
  176. openinterp/evidence_lenses.py +75 -0
  177. openinterp/experiment_loop.py +357 -0
  178. openinterp/explain.py +252 -0
  179. openinterp/half_precision.py +79 -0
  180. openinterp/harness/__init__.py +37 -0
  181. openinterp/harness/ane/__init__.py +9 -0
  182. openinterp/harness/ane/backend.py +324 -0
  183. openinterp/harness/ane/cache.py +276 -0
  184. openinterp/harness/ane/chunks.py +163 -0
  185. openinterp/harness/ane/convert.py +116 -0
  186. openinterp/harness/ane/forward.py +258 -0
  187. openinterp/harness/ane/qualified.py +40 -0
  188. openinterp/harness/annotations.py +329 -0
  189. openinterp/harness/attribution.py +96 -0
  190. openinterp/harness/causal.py +264 -0
  191. openinterp/harness/confounds.py +184 -0
  192. openinterp/harness/covariance.py +301 -0
  193. openinterp/harness/crossref.py +195 -0
  194. openinterp/harness/cuda_devices.py +88 -0
  195. openinterp/harness/dead.py +124 -0
  196. openinterp/harness/devices.py +401 -0
  197. openinterp/harness/dictionaries.py +496 -0
  198. openinterp/harness/dreaming.py +116 -0
  199. openinterp/harness/environment.py +64 -0
  200. openinterp/harness/explanation_scores.py +371 -0
  201. openinterp/harness/features.py +276 -0
  202. openinterp/harness/first_position.py +137 -0
  203. openinterp/harness/heads.py +226 -0
  204. openinterp/harness/hooks.py +499 -0
  205. openinterp/harness/input_attribution.py +367 -0
  206. openinterp/harness/jlens.py +183 -0
  207. openinterp/harness/layer_cache.py +301 -0
  208. openinterp/harness/layer_major.py +274 -0
  209. openinterp/harness/lookahead.py +93 -0
  210. openinterp/harness/manifold.py +200 -0
  211. openinterp/harness/memory.py +471 -0
  212. openinterp/harness/moe.py +134 -0
  213. openinterp/harness/patcher.py +322 -0
  214. openinterp/harness/patchscope.py +177 -0
  215. openinterp/harness/probing.py +383 -0
  216. openinterp/harness/scan.py +462 -0
  217. openinterp/harness/scan_backend.py +201 -0
  218. openinterp/harness/sister_passes.py +463 -0
  219. openinterp/harness/sisters.py +363 -0
  220. openinterp/harness/sites.py +101 -0
  221. openinterp/harness/specificity.py +107 -0
  222. openinterp/harness/steering.py +159 -0
  223. openinterp/harness/superpositions.py +116 -0
  224. openinterp/harness/units.py +203 -0
  225. openinterp/harness/wiring.py +432 -0
  226. openinterp/harness/wiring_activations.py +285 -0
  227. openinterp/langid.py +73 -0
  228. openinterp/names.py +129 -0
  229. openinterp/neuronpedia_baseline.py +88 -0
  230. openinterp/neuronpedia_review.py +148 -0
  231. openinterp/planner.py +466 -0
  232. openinterp/planner_effects.py +89 -0
  233. openinterp/planner_heads.py +92 -0
  234. openinterp/planner_models.py +174 -0
  235. openinterp/planner_sisters.py +122 -0
  236. openinterp/planner_states.py +57 -0
  237. openinterp/planner_superpositions.py +114 -0
  238. openinterp/pole_effects.py +102 -0
  239. openinterp/prior_art.py +143 -0
  240. openinterp/reproduce.py +450 -0
  241. openinterp/revision.py +45 -0
  242. openinterp/runner_cache.py +35 -0
  243. openinterp/score_controls.py +169 -0
  244. openinterp/sensitive.py +243 -0
  245. openinterp/show.py +236 -0
  246. openinterp/signing.py +209 -0
  247. openinterp/sisters.py +476 -0
  248. openinterp/superposition_finders.py +225 -0
  249. openinterp/sync/__init__.py +5 -0
  250. openinterp/sync/r2.py +280 -0
  251. openinterp/syntax.py +399 -0
  252. openinterp/takedown_checks.py +76 -0
  253. openinterp/toxicity.py +158 -0
  254. openinterp/verdict_regression.py +79 -0
  255. openinterp/wiring_backfill.py +170 -0
  256. openinterp/wordnet.py +362 -0
@@ -0,0 +1,88 @@
1
+ """Device planning on a machine with CUDA GPUs: the first free GPU, else the CPU.
2
+
3
+ A GPU is busy when another ``open-interpretability`` process has a lease on it (``cuda0``,
4
+ ``cuda1``, … in the lease folder of ``harness/devices.py``). It is also busy when its
5
+ utilization is more than the planner's threshold. That is the mean of
6
+ :data:`~openinterp.harness.devices.GPU_SAMPLES` readings of ``nvidia-smi``. If every
7
+ GPU is busy, the job runs on the CPU. A machine with one GPU uses ``cuda``, and a
8
+ machine with more uses ``cuda:<index>``.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import subprocess
14
+ import time
15
+
16
+ from openinterp.harness.devices import (
17
+ GPU_SAMPLE_SECONDS,
18
+ GPU_SAMPLES,
19
+ DevicePlan,
20
+ holders,
21
+ )
22
+
23
+ NVIDIA_SMI = ["nvidia-smi", "--query-gpu=index,utilization.gpu", "--format=csv,noheader,nounits"]
24
+
25
+
26
+ def parse_nvidia_smi(output: str) -> dict[int, float]:
27
+ """Each GPU's utilization from ``nvidia-smi`` output, by index. Bad lines are skipped."""
28
+ found = {}
29
+ for line in output.splitlines():
30
+ parts = [p.strip() for p in line.split(",")]
31
+ if len(parts) == 2 and parts[0].isdigit():
32
+ try:
33
+ found[int(parts[0])] = float(parts[1])
34
+ except ValueError:
35
+ continue
36
+ return found
37
+
38
+
39
+ def _read() -> dict[int, float] | None:
40
+ try:
41
+ out = subprocess.run(NVIDIA_SMI, capture_output=True, text=True, timeout=10, check=True)
42
+ except (OSError, subprocess.SubprocessError):
43
+ return None
44
+ return parse_nvidia_smi(out.stdout) or None
45
+
46
+
47
+ def utilization(samples: int = GPU_SAMPLES, seconds: float = GPU_SAMPLE_SECONDS):
48
+ """Each GPU's mean utilization over ``samples`` readings, or None if it can't be read."""
49
+ readings: list[dict[int, float]] = []
50
+ for i in range(samples):
51
+ if i:
52
+ time.sleep(seconds)
53
+ reading = _read()
54
+ if reading is None:
55
+ return None
56
+ readings.append(reading)
57
+ gpus = set().union(*readings)
58
+ return {g: sum(r.get(g, 0.0) for r in readings) / len(readings) for g in gpus}
59
+
60
+
61
+ def choose_cuda(
62
+ count: int, jobs: dict[int, list[int]], percents: dict[int, float] | None, busy: float
63
+ ) -> DevicePlan:
64
+ """The first of ``count`` GPUs that is free, else the CPU."""
65
+ reasons = []
66
+ for gpu in range(count):
67
+ others = jobs.get(gpu, [])
68
+ percent = (percents or {}).get(gpu)
69
+ if others:
70
+ ids = ", ".join(str(pid) for pid in others)
71
+ reasons.append(f"GPU {gpu} is in use by another openinterp job (process {ids})")
72
+ elif percent is not None and percent > busy:
73
+ reasons.append(f"GPU {gpu} is {percent:.0f}% busy, more than {busy:.0f}%")
74
+ else:
75
+ device = "cuda" if count == 1 else f"cuda:{gpu}"
76
+ return DevicePlan(device, f"GPU {gpu} is free", hint="cuda")
77
+ reason = "; ".join(reasons) or "this machine has no CUDA GPU"
78
+ return DevicePlan("cpu", reason, fallback=count > 0, hint="cuda")
79
+
80
+
81
+ def plan_cuda(busy: float) -> DevicePlan:
82
+ """The plan on this machine's CUDA GPUs."""
83
+ import torch
84
+
85
+ count = torch.cuda.device_count()
86
+ jobs = {gpu: holders(f"cuda{gpu}") for gpu in range(count)}
87
+ free = [gpu for gpu in range(count) if not jobs[gpu]]
88
+ return choose_cuda(count, jobs, utilization() if free else None, busy)
@@ -0,0 +1,124 @@
1
+ """Prove from the weights that a neuron pole can't fire on any input.
2
+
3
+ A scan that never sees a pole fire shows only that the corpus doesn't make it fire. This
4
+ module gives a proof. The norm before the MLP puts the MLP input on a ball:
5
+
6
+ - LayerNorm: ``γ⊙ẑ + β``, where ``ẑ`` has mean 0 and ``‖ẑ‖ < √d``.
7
+ - RMSNorm: ``γ⊙x̂``, where ``‖x̂‖ < √d``.
8
+
9
+ So a neuron's pre-activation ``h = w·input + b`` lies in ``(c − R, c + R)``.
10
+
11
+ - LayerNorm: ``c = w·β + b`` and ``R = √d·‖centred(w⊙γ)‖``.
12
+ - RMSNorm: ``c = b`` and ``R = √d·‖w⊙γ‖``.
13
+
14
+ Take an ungated MLP with a sign-preserving activation (GELU, SiLU). Its positive pole
15
+ fires only when ``h > 0``, and its negative pole only when ``h < 0``. So
16
+ ``c + R ≤ 0`` proves the positive pole dead, and ``c − R ≥ 0`` proves the negative pole
17
+ dead. With ReLU, the negative pole is always dead.
18
+
19
+ A gated MLP (SwiGLU) multiplies two reads, so the sign of the product is free on a ball.
20
+ Its pole is proven dead only when a read is exactly zero and the MLP has no bias.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import math
26
+ from dataclasses import dataclass
27
+
28
+ import torch
29
+ from torch import nn
30
+
31
+ from openinterp.core.identifiers import NeuronIdentifier
32
+ from openinterp.harness.hooks import NeuronHookRunner, _try_path
33
+ from openinterp.harness.memory import loaded
34
+ from openinterp.harness.wiring import MLP_READ_PATHS, _matrix
35
+
36
+ # Activations whose output has the sign of their input: f(h) > 0 iff h > 0, f(h) < 0 iff
37
+ # h < 0. ReLU is apart: it is never negative.
38
+ SIGN_PRESERVING = {
39
+ "gelu",
40
+ "gelu_new",
41
+ "gelu_fast",
42
+ "gelu_pytorch_tanh",
43
+ "quick_gelu",
44
+ "silu",
45
+ "swish",
46
+ }
47
+ RELU = "relu"
48
+ # The bound must clear 0 by this share of its scale, so float64 rounding can't prove it.
49
+ MARGIN = 1e-9
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class DeadProof:
54
+ """The result of the weight check on one neuron pole."""
55
+
56
+ proven: bool
57
+ # The largest value of h (positive pole) or −h (negative pole) that any input can
58
+ # give, with the margin added. None when the proof doesn't use a bound.
59
+ bound: float | None
60
+ reason: str
61
+
62
+
63
+ def _activation(runner: NeuronHookRunner) -> str | None:
64
+ config = getattr(runner.model, "config", None)
65
+ name = getattr(config, "activation_function", None) or getattr(config, "hidden_act", None)
66
+ return name if isinstance(name, str) else None
67
+
68
+
69
+ def _is_rms(norm: nn.Module) -> bool:
70
+ """An RMSNorm that scales by ``weight`` (Gemma's scales by ``1 + weight``: not this)."""
71
+ name = type(norm).__name__
72
+ return name.endswith("RMSNorm") and not name.startswith("Gemma")
73
+
74
+
75
+ def _reads(runner: NeuronHookRunner, layer: int) -> dict[str, nn.Module]:
76
+ block = runner.layers[layer]
77
+ for paths in MLP_READ_PATHS:
78
+ modules = {c: _try_path(block, p) for c, p in paths.items()}
79
+ if all(isinstance(m, nn.Module) for m in modules.values()):
80
+ return modules
81
+ return {}
82
+
83
+
84
+ def _range(norm: nn.Module, proj: nn.Module, neuron: int) -> tuple[float, float] | None:
85
+ """``(c, R)``: the pre-activation lies in ``(c − R, c + R)`` for every input."""
86
+ w = _matrix(proj)[neuron].double().cpu()
87
+ with loaded(proj), loaded(norm): # streamed: the weights are real only inside this block
88
+ bias = getattr(proj, "bias", None)
89
+ b = float(bias.detach()[neuron]) if bias is not None else 0.0
90
+ gain = getattr(norm, "weight", None)
91
+ v = w * gain.detach().double().cpu() if gain is not None else w
92
+ shift = getattr(norm, "bias", None)
93
+ shift = shift.detach().double().cpu() if shift is not None else None
94
+ if isinstance(norm, nn.LayerNorm):
95
+ c = b + (float(w @ shift) if shift is not None else 0.0)
96
+ return c, math.sqrt(v.numel()) * float((v - v.mean()).norm())
97
+ if _is_rms(norm):
98
+ return b, math.sqrt(v.numel()) * float(v.norm())
99
+ return None
100
+
101
+
102
+ def prove_dead(runner: NeuronHookRunner, ident: NeuronIdentifier) -> DeadProof:
103
+ """Check whether no input can make ``ident``'s pole fire."""
104
+ act, reads = _activation(runner), _reads(runner, ident.layer)
105
+ norm = runner.mlp_norm(ident.layer)
106
+ if set(reads) == {"in"}:
107
+ if act == RELU and ident.pole == "negative":
108
+ return DeadProof(True, None, "ReLU is never negative")
109
+ if act not in SIGN_PRESERVING and act != RELU:
110
+ return DeadProof(False, None, f"no bound for the activation {act!r}")
111
+ span = _range(norm, reads["in"], ident.neuron_idx)
112
+ if span is None:
113
+ return DeadProof(False, None, f"no bound for the norm {type(norm).__name__}")
114
+ c, r = span
115
+ bound = (c + r if ident.pole == "positive" else r - c) + MARGIN * (abs(c) + r)
116
+ return DeadProof(bound <= 0, bound, f"pre-activation in ({c - r:.4g}, {c + r:.4g})")
117
+ if set(reads) == {"gate", "up"}:
118
+ if any(getattr(m, "bias", None) is not None for m in reads.values()):
119
+ return DeadProof(False, None, "no bound for a gated MLP with biases")
120
+ rows = [_matrix(m)[ident.neuron_idx] for m in reads.values()]
121
+ if any(not torch.any(row) for row in rows):
122
+ return DeadProof(True, 0.0, "a read of the gated MLP is zero")
123
+ return DeadProof(False, None, "a gated MLP's pole is proven dead only by a zero read")
124
+ return DeadProof(False, None, "MLP input projection not found")
@@ -0,0 +1,401 @@
1
+ """Device planner: with ``--device auto``, use a GPU only when it is free.
2
+
3
+ This module plans for Apple's GPU (MPS). ``cuda_devices.py`` plans for CUDA GPUs with
4
+ the same leases and threshold, and chooses the first free GPU.
5
+
6
+ Two things make the GPU busy. One is another ``open-interpretability`` process that uses it. Each
7
+ process that runs a model on MPS keeps a *lease*. That is an empty file named for its
8
+ process id, in ``$OPENINTERP_DEVICE_LEASES`` (default ``~/.cache/openinterp/devices/mps/``).
9
+ The planner ignores and removes the lease of a process that stopped. The other is load
10
+ from other programs. The planner reads the GPU's ``Device Utilization %`` from ``ioreg``.
11
+ It takes the mean of :data:`GPU_SAMPLES` readings, because one reading changes much from
12
+ second to second. Above ``$OPENINTERP_GPU_BUSY_PERCENT`` (default
13
+ :data:`DEFAULT_GPU_BUSY_PERCENT`), the GPU is busy.
14
+
15
+ When the GPU is busy, the order is MPS, then the ANE, then the CPU. The ANE is Apple's
16
+ Neural Engine. If this Mac can run Core ML and no other job has a lease on the ANE, the
17
+ scans' forward passes run there (``harness/ane/``). The rest runs on the CPU. Else everything runs
18
+ on the CPU. The CPU is slower than a free GPU. But two processes on one GPU can slow down
19
+ each other, and one such run gave wrong results (docs/experiments.md, Devices).
20
+ ``--device mps`` uses the GPU even when it is busy. ``--device ane`` runs the scans on
21
+ the ANE, and the rest where the planner puts it.
22
+
23
+ A model that streams (``--memory stream``) is the exception. Its scan on the CPU reads
24
+ each layer once for the whole corpus, and with the CPU free that is faster than the ANE.
25
+ On a 50,000-token Qwen2.5-0.5B scan, the CPU's pass took about 16 s. The ANE's took
26
+ 39 s, and it needed 40 s more to measure the top pieces again. So a streamed model's
27
+ scans use the ANE only if the CPU is busy too. That is a load average of at least
28
+ ``$OPENINTERP_CPU_BUSY_PERCENT`` (default :data:`DEFAULT_CPU_BUSY_PERCENT`) of its cores.
29
+
30
+ A process plans once, and every model it loads after that uses the same device.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import atexit
36
+ import os
37
+ import platform
38
+ import re
39
+ import subprocess
40
+ import threading
41
+ import time
42
+ from contextlib import suppress
43
+ from dataclasses import dataclass, field
44
+ from pathlib import Path
45
+
46
+ from openinterp.core.cache_dirs import cache_root
47
+
48
+ LEASES_ENV = "OPENINTERP_DEVICE_LEASES" # lease directory, or "off"
49
+ # Set by the retry after an MPS failure: every model of the command loads on this device.
50
+ FORCE_ENV = "OPENINTERP_DEVICE_FORCE"
51
+ GPU_BUSY_ENV = "OPENINTERP_GPU_BUSY_PERCENT"
52
+ # Idle reads 1%, one job 93-97% (docs/analyses/device-contention.md).
53
+ DEFAULT_GPU_BUSY_PERCENT = 50.0
54
+ CPU_BUSY_ENV = "OPENINTERP_CPU_BUSY_PERCENT"
55
+ DEFAULT_CPU_BUSY_PERCENT = 50.0 # the 1-minute load average, as a share of the cores
56
+ # The longest the first operation on MPS may take before the process uses the CPU.
57
+ START_ENV = "OPENINTERP_MPS_START_SECONDS"
58
+ DEFAULT_START_SECONDS = 120.0
59
+ GPU_SAMPLES = 5
60
+ GPU_SAMPLE_SECONDS = 0.4
61
+
62
+ _UTILIZATION = re.compile(r'"Device Utilization %"=(\d+)')
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class Load:
67
+ """What the planner saw: other processes on each device, and the GPU's utilization."""
68
+
69
+ jobs: dict[str, list[int]] = field(default_factory=dict) # device -> process ids
70
+ gpu_percent: float | None = None # mean utilization, or None if it was not read
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class DevicePlan:
75
+ """The device to use, and why."""
76
+
77
+ device: str
78
+ reason: str
79
+ fallback: bool = False # True when a busy device was not used
80
+ scan: str | None = None # "ane" when scans run on Apple's Neural Engine
81
+ hint: str = "mps" # the --device value that uses the GPU even when it is busy
82
+
83
+
84
+ def choose(
85
+ load: Load, mps_available: bool, busy_percent: float, ane_ok: bool = False
86
+ ) -> DevicePlan:
87
+ """The device for ``load``: MPS if it is free, else the ANE for scans, else the CPU.
88
+
89
+ With the ANE, the scans' forward passes run there, and the rest runs on the CPU.
90
+ ``ane_ok`` says that this machine can run Core ML on the ANE.
91
+ """
92
+ if not mps_available:
93
+ return DevicePlan("cpu", "this machine has no Apple GPU")
94
+ busy = _busy(load.jobs.get("mps", []), "the GPU", load.gpu_percent, busy_percent)
95
+ if busy is None:
96
+ return DevicePlan("mps", "the GPU is free")
97
+ if ane_ok and not load.jobs.get("ane"):
98
+ return DevicePlan("cpu", busy, True, scan="ane")
99
+ if ane_ok:
100
+ busy += " and " + _busy(load.jobs["ane"], "the ANE", None, busy_percent)
101
+ return DevicePlan("cpu", busy, True)
102
+
103
+
104
+ def _busy(others: list[int], name: str, percent: float | None, limit: float) -> str | None:
105
+ """Why ``name`` is busy, or None if it is free."""
106
+ if others:
107
+ ids = ", ".join(str(pid) for pid in others)
108
+ noun = "job uses" if len(others) == 1 else "jobs use"
109
+ return f"another openinterp {noun} {name} (process {ids})"
110
+ if percent is not None and percent > limit:
111
+ return f"{name} is {percent:.0f}% busy, more than {limit:.0f}%"
112
+ return None
113
+
114
+
115
+ def busy_percent() -> float:
116
+ """``$OPENINTERP_GPU_BUSY_PERCENT``, else :data:`DEFAULT_GPU_BUSY_PERCENT`."""
117
+ return float(os.environ.get(GPU_BUSY_ENV) or DEFAULT_GPU_BUSY_PERCENT)
118
+
119
+
120
+ def read_load() -> Load:
121
+ """The other processes on MPS and the ANE, and the GPU's utilization if no job is on it."""
122
+ jobs = {"mps": holders("mps"), "ane": holders("ane")}
123
+ return Load(jobs, None if jobs["mps"] else gpu_utilization())
124
+
125
+
126
+ def gpu_utilization(samples: int = GPU_SAMPLES, seconds: float = GPU_SAMPLE_SECONDS):
127
+ """The mean of ``samples`` readings of the GPU's utilization, or None if ``ioreg`` fails."""
128
+ readings = []
129
+ for i in range(samples):
130
+ if i:
131
+ time.sleep(seconds)
132
+ value = _read_utilization()
133
+ if value is None:
134
+ return None
135
+ readings.append(value)
136
+ return sum(readings) / len(readings)
137
+
138
+
139
+ def _read_utilization() -> float | None:
140
+ try:
141
+ out = subprocess.run(
142
+ ["ioreg", "-r", "-d", "1", "-c", "IOAccelerator"],
143
+ capture_output=True,
144
+ text=True,
145
+ timeout=5,
146
+ check=True,
147
+ ).stdout
148
+ except (OSError, subprocess.SubprocessError):
149
+ return None
150
+ return parse_utilization(out)
151
+
152
+
153
+ def parse_utilization(ioreg_output: str) -> float | None:
154
+ """The largest ``Device Utilization %`` in ``ioreg`` output, or None if there is none."""
155
+ values = [float(v) for v in _UTILIZATION.findall(ioreg_output)]
156
+ return max(values) if values else None
157
+
158
+
159
+ def lease_root() -> Path | None:
160
+ """``$OPENINTERP_DEVICE_LEASES``, else ``$XDG_CACHE_HOME/openinterp/devices``; None if off."""
161
+ return cache_root(LEASES_ENV, "devices")
162
+
163
+
164
+ def holders(device: str) -> list[int]:
165
+ """Process ids of the other running processes with a lease on ``device``.
166
+
167
+ It removes the leases of processes that have stopped.
168
+ """
169
+ root = lease_root()
170
+ folder = root / device if root else None
171
+ if folder is None or not folder.is_dir():
172
+ return []
173
+ found = []
174
+ for lease in folder.iterdir():
175
+ if not lease.name.isdigit() or int(lease.name) == os.getpid():
176
+ continue
177
+ if _running(int(lease.name)):
178
+ found.append(int(lease.name))
179
+ else:
180
+ with suppress(OSError):
181
+ lease.unlink()
182
+ return sorted(found)
183
+
184
+
185
+ _used: set[str] = set() # devices that this process ran models on
186
+
187
+
188
+ def lease_name(device: str) -> str | None:
189
+ """The lease folder of a GPU device (``mps``, ``cuda0``, ``cuda1``, …), or None."""
190
+ if device == "mps":
191
+ return "mps"
192
+ if device.startswith("cuda"):
193
+ return f"cuda{device.partition(':')[2] or 0}"
194
+ return None
195
+
196
+
197
+ def take_lease(device: str) -> None:
198
+ """Record that this process uses the GPU ``device``, until it stops."""
199
+ name = lease_name(device)
200
+ if name is None:
201
+ return
202
+ _used.add(name)
203
+ root = lease_root()
204
+ if root is None:
205
+ return
206
+ lease = root / name / str(os.getpid())
207
+ if lease.exists():
208
+ return
209
+ with suppress(OSError):
210
+ lease.parent.mkdir(parents=True, exist_ok=True)
211
+ lease.touch()
212
+ atexit.register(_drop, lease)
213
+
214
+
215
+ def _drop(lease: Path) -> None:
216
+ with suppress(OSError):
217
+ lease.unlink()
218
+
219
+
220
+ def _running(pid: int) -> bool:
221
+ try:
222
+ os.kill(pid, 0)
223
+ except ProcessLookupError:
224
+ return False
225
+ except PermissionError: # it runs, as another user
226
+ return True
227
+ return True
228
+
229
+
230
+ _choice: str | None = None # how this process chose its device, for the record
231
+
232
+
233
+ def note_choice(text: str) -> None:
234
+ """Record how this process chose its device (``RunEnvironment.device_choice``)."""
235
+ global _choice
236
+ _choice = text
237
+
238
+
239
+ def choice() -> str | None:
240
+ """How this process chose its device, or None if nothing chose it (e.g. in a test)."""
241
+ return _choice
242
+
243
+
244
+ _planned: DevicePlan | None = None
245
+ _noticed = False
246
+ _forced_ane = False
247
+
248
+
249
+ def force_ane() -> None:
250
+ """Run scans on the ANE whatever the plan says (``--device ane``)."""
251
+ global _forced_ane
252
+ _forced_ane = True
253
+
254
+
255
+ def cpu_load_percent() -> float:
256
+ """The 1-minute load average, as a percentage of the CPU's cores."""
257
+ return 100.0 * os.getloadavg()[0] / (os.cpu_count() or 1)
258
+
259
+
260
+ def streamed_scans_skip_ane() -> str | None:
261
+ """Why a streamed model's scans run on the CPU and not on the planned ANE, or None.
262
+
263
+ The CPU's scan of a streamed model is faster than the ANE's when the CPU is free.
264
+ ``--device ane`` keeps the ANE.
265
+ """
266
+ if _forced_ane:
267
+ return None
268
+ load = cpu_load_percent()
269
+ if load >= float(os.environ.get(CPU_BUSY_ENV) or DEFAULT_CPU_BUSY_PERCENT):
270
+ return None
271
+ return f"the model streams and the CPU is free (load {load:.0f}% of its cores)"
272
+
273
+
274
+ def scans_on_ane() -> bool:
275
+ """True if scans should run on the ANE: ``--device ane``, or the plan's choice."""
276
+ return _forced_ane or (_planned is not None and _planned.scan == "ane")
277
+
278
+
279
+ def notice() -> str | None:
280
+ """At the first call only: why the plan did not use a busy GPU. Else None."""
281
+ global _noticed
282
+ if _noticed or _planned is None or not _planned.fallback:
283
+ return None
284
+ _noticed = True
285
+ ane = " Scans run on Apple's Neural Engine." if _planned.scan == "ane" else ""
286
+ return (
287
+ f"Using the CPU, because {_planned.reason}.{ane} "
288
+ f"Pass --device {_planned.hint} to share the GPU."
289
+ )
290
+
291
+
292
+ def planned() -> DevicePlan:
293
+ """This process's plan, made at the first call."""
294
+ global _planned
295
+ if _planned is None:
296
+ import torch
297
+
298
+ if torch.cuda.is_available():
299
+ from openinterp.harness.cuda_devices import plan_cuda
300
+
301
+ _planned = plan_cuda(busy_percent())
302
+ else:
303
+ mps = torch.backends.mps.is_available()
304
+ _planned = choose(read_load(), mps, busy_percent(), mps and ane_possible())
305
+ return _planned
306
+
307
+
308
+ def ane_possible() -> bool:
309
+ """True if this machine can run Core ML models on the ANE: a Mac with coremltools."""
310
+ from openinterp.harness.ane import convert
311
+
312
+ return platform.system() == "Darwin" and platform.machine() == "arm64" and convert.available()
313
+
314
+
315
+ def forget() -> None:
316
+ """Make the next :func:`planned` plan again (for tests)."""
317
+ global _planned, _noticed, _forced_ane, _start_checked, _choice
318
+ _planned, _noticed, _forced_ane, _start_checked, _choice = None, False, False, False, None
319
+ _used.clear()
320
+
321
+
322
+ def forced() -> str | None:
323
+ """The device that a retry after a GPU failure set, or None."""
324
+ return os.environ.get(FORCE_ENV) or None
325
+
326
+
327
+ def gpu_failed(exc: BaseException) -> str | None:
328
+ """``MPS`` or ``CUDA`` if ``exc`` is a GPU failure that a retry on the CPU can get past.
329
+
330
+ It is one if this process ran a model on that GPU and an exception in the chain
331
+ names it. A command that is a retry already is not retried. Else None.
332
+ """
333
+ if forced():
334
+ return None
335
+ kinds = {"mps"} & _used | ({"cuda"} if any(u.startswith("cuda") for u in _used) else set())
336
+ seen: set[int] = set()
337
+ while exc is not None and id(exc) not in seen:
338
+ seen.add(id(exc))
339
+ text = str(exc).lower() if isinstance(exc, Exception) else ""
340
+ if found := next((k for k in sorted(kinds) if k in text), None):
341
+ return found.upper()
342
+ exc = exc.__cause__ or exc.__context__
343
+ return None
344
+
345
+
346
+ def rerun_on_cpu(exc: BaseException, gpu: str = "MPS") -> None:
347
+ """Run this command again from the start, in a new process, with every model on the CPU.
348
+
349
+ The new process has none of the GPU's state. Files that the first run wrote are
350
+ written again.
351
+ """
352
+ import sys
353
+
354
+ first = str(exc).strip().splitlines()[0][:200] if str(exc).strip() else type(exc).__name__
355
+ print(
356
+ f"The GPU ({gpu}) failed ({first}). Running the command again on the CPU.", file=sys.stderr
357
+ )
358
+ sys.stderr.flush()
359
+ sys.stdout.flush()
360
+ # The same command line that started this process, so nothing new runs.
361
+ os.execve(sys.executable, sys.orig_argv, {**os.environ, FORCE_ENV: "cpu"}) # noqa: S606
362
+
363
+
364
+ _start_checked: str | None | bool = False # False: not checked yet
365
+
366
+
367
+ def mps_stuck() -> str | None:
368
+ """Why this process can't use MPS, or None if a first operation on it ends in time.
369
+
370
+ Several processes that start on MPS at the same moment can each stop for good in
371
+ Metal's compiler. It opens Metal's shared cache of compiled programs on disk
372
+ (docs/analyses/device-contention.md). So a first small operation runs on a
373
+ background thread. The process waits for it at most ``$OPENINTERP_MPS_START_SECONDS``
374
+ (default :data:`DEFAULT_START_SECONDS`). The check runs once in a process.
375
+ """
376
+ global _start_checked
377
+ if _start_checked is False:
378
+ _start_checked = _probe_mps(float(os.environ.get(START_ENV) or DEFAULT_START_SECONDS))
379
+ return _start_checked
380
+
381
+
382
+ def _probe_mps(seconds: float) -> str | None:
383
+ done, failed = threading.Event(), []
384
+
385
+ def probe() -> None:
386
+ try:
387
+ import torch
388
+
389
+ torch.ones(1024, device="mps").mul(2).cpu()
390
+ torch.mps.synchronize()
391
+ except Exception as exc: # a GPU that fails at once can't be used
392
+ failed.append(exc)
393
+ finally:
394
+ done.set()
395
+
396
+ threading.Thread(target=probe, daemon=True, name="mps-start").start()
397
+ if not done.wait(timeout=seconds):
398
+ return f"the GPU (MPS) didn't start in {seconds:g} s (${START_ENV})"
399
+ if failed:
400
+ return f"the GPU (MPS) failed at once ({type(failed[0]).__name__}: {failed[0]})"
401
+ return None