open-interpretability 0.0.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_interpretability-0.0.20.dist-info/METADATA +157 -0
- open_interpretability-0.0.20.dist-info/RECORD +256 -0
- open_interpretability-0.0.20.dist-info/WHEEL +4 -0
- open_interpretability-0.0.20.dist-info/entry_points.txt +2 -0
- open_interpretability-0.0.20.dist-info/licenses/LICENSE +21 -0
- openinterp/__init__.py +6 -0
- openinterp/agents/__init__.py +25 -0
- openinterp/agents/base.py +229 -0
- openinterp/agents/judge.py +149 -0
- openinterp/agents/proposer.py +471 -0
- openinterp/agents/proposer_profiles.py +79 -0
- openinterp/agents/review_measurements.py +173 -0
- openinterp/agents/reviewer.py +498 -0
- openinterp/agreement.py +63 -0
- openinterp/audit_options.py +79 -0
- openinterp/baseline.py +254 -0
- openinterp/benchmark.py +450 -0
- openinterp/blends.py +239 -0
- openinterp/builder/__init__.py +6 -0
- openinterp/builder/agents.py +260 -0
- openinterp/builder/aligned.py +41 -0
- openinterp/builder/charts.py +411 -0
- openinterp/builder/circuit_pages.py +37 -0
- openinterp/builder/circuits.py +264 -0
- openinterp/builder/compiler.py +495 -0
- openinterp/builder/corpora.py +98 -0
- openinterp/builder/coverage.py +46 -0
- openinterp/builder/dashboard.py +255 -0
- openinterp/builder/disk_memo.py +65 -0
- openinterp/builder/experiment.py +105 -0
- openinterp/builder/heat.py +121 -0
- openinterp/builder/hrefs.py +71 -0
- openinterp/builder/learned_baseline.py +204 -0
- openinterp/builder/model_facts.py +138 -0
- openinterp/builder/neuronpedia_export.py +107 -0
- openinterp/builder/parallel.py +100 -0
- openinterp/builder/parquet_export.py +458 -0
- openinterp/builder/pieces.py +127 -0
- openinterp/builder/pii.py +211 -0
- openinterp/builder/registry_stats.py +144 -0
- openinterp/builder/research.py +109 -0
- openinterp/builder/sections.py +28 -0
- openinterp/builder/significance.py +103 -0
- openinterp/builder/sister_graph.py +180 -0
- openinterp/builder/standing.py +185 -0
- openinterp/builder/statuses.py +34 -0
- openinterp/builder/stopped.py +91 -0
- openinterp/builder/stops.py +30 -0
- openinterp/builder/superpositions.py +102 -0
- openinterp/builder/templates/_agent_role_cells.html +9 -0
- openinterp/builder/templates/_causal_chart.html +67 -0
- openinterp/builder/templates/_circuit.html +8 -0
- openinterp/builder/templates/_dashboard.html +133 -0
- openinterp/builder/templates/_experiment.html +500 -0
- openinterp/builder/templates/_macros.html +82 -0
- openinterp/builder/templates/_patchscope.html +33 -0
- openinterp/builder/templates/_prior_art.html +24 -0
- openinterp/builder/templates/_specificity.html +21 -0
- openinterp/builder/templates/_steering.html +34 -0
- openinterp/builder/templates/_superposition.html +62 -0
- openinterp/builder/templates/_vocab_chart.html +88 -0
- openinterp/builder/templates/_wiring.html +44 -0
- openinterp/builder/templates/about.html +95 -0
- openinterp/builder/templates/agent.html +69 -0
- openinterp/builder/templates/agents.html +80 -0
- openinterp/builder/templates/base.html +103 -0
- openinterp/builder/templates/circuit.html +64 -0
- openinterp/builder/templates/circuits.html +46 -0
- openinterp/builder/templates/experiment.html +21 -0
- openinterp/builder/templates/hub.html +16 -0
- openinterp/builder/templates/index.html +387 -0
- openinterp/builder/templates/layer.html +69 -0
- openinterp/builder/templates/learned-baseline.html +44 -0
- openinterp/builder/templates/model.html +151 -0
- openinterp/builder/templates/models.html +47 -0
- openinterp/builder/templates/neuron.html +234 -0
- openinterp/builder/templates/paper.html +39 -0
- openinterp/builder/templates/redirect.html +1 -0
- openinterp/builder/templates/research.html +31 -0
- openinterp/builder/templates/user.html +54 -0
- openinterp/builder/templates/users.html +39 -0
- openinterp/builder/templates/wiki/experiment.md +271 -0
- openinterp/builder/templates/wiki/index.md +32 -0
- openinterp/builder/templates/wiki/neuron.md +30 -0
- openinterp/builder/templates/wiki/stop.md +36 -0
- openinterp/builder/tree.py +470 -0
- openinterp/builder/users.py +142 -0
- openinterp/builder/wiki.py +427 -0
- openinterp/checks.py +429 -0
- openinterp/cli.py +494 -0
- openinterp/cli_baseline.py +332 -0
- openinterp/cli_benchmark.py +300 -0
- openinterp/cli_build.py +128 -0
- openinterp/cli_check.py +51 -0
- openinterp/cli_concepts.py +76 -0
- openinterp/cli_corpus.py +252 -0
- openinterp/cli_costs.py +257 -0
- openinterp/cli_coverage.py +76 -0
- openinterp/cli_crossref.py +86 -0
- openinterp/cli_errata.py +142 -0
- openinterp/cli_evidence.py +495 -0
- openinterp/cli_evidence_store.py +93 -0
- openinterp/cli_explain.py +101 -0
- openinterp/cli_loop.py +125 -0
- openinterp/cli_memory.py +28 -0
- openinterp/cli_neuronpedia.py +175 -0
- openinterp/cli_pause.py +45 -0
- openinterp/cli_pole_effects.py +133 -0
- openinterp/cli_probing.py +119 -0
- openinterp/cli_registry.py +348 -0
- openinterp/cli_reproduce.py +446 -0
- openinterp/cli_review.py +302 -0
- openinterp/cli_scan.py +223 -0
- openinterp/cli_score.py +446 -0
- openinterp/cli_sisters.py +86 -0
- openinterp/cli_submission.py +217 -0
- openinterp/cli_superpositions.py +118 -0
- openinterp/cli_wiring.py +54 -0
- openinterp/codeid.py +73 -0
- openinterp/concepts.py +400 -0
- openinterp/core/__init__.py +59 -0
- openinterp/core/agent_names.py +76 -0
- openinterp/core/attribution.py +101 -0
- openinterp/core/cache_dirs.py +15 -0
- openinterp/core/causal.py +82 -0
- openinterp/core/confounds.py +103 -0
- openinterp/core/contributors.py +26 -0
- openinterp/core/corpora.py +126 -0
- openinterp/core/costs.py +430 -0
- openinterp/core/draws.py +109 -0
- openinterp/core/errata.py +244 -0
- openinterp/core/evidence.py +160 -0
- openinterp/core/examples.py +93 -0
- openinterp/core/exceptions.py +29 -0
- openinterp/core/explanation.py +130 -0
- openinterp/core/fence.py +42 -0
- openinterp/core/heavy_lock.py +114 -0
- openinterp/core/identifiers.py +198 -0
- openinterp/core/isolation.py +149 -0
- openinterp/core/ledger.py +130 -0
- openinterp/core/licenses.py +28 -0
- openinterp/core/manifold.py +117 -0
- openinterp/core/partners.py +32 -0
- openinterp/core/patchscope.py +59 -0
- openinterp/core/paths.py +40 -0
- openinterp/core/pause.py +96 -0
- openinterp/core/pricing.py +178 -0
- openinterp/core/prior_art.py +44 -0
- openinterp/core/probing.py +146 -0
- openinterp/core/profiles.py +221 -0
- openinterp/core/project.py +28 -0
- openinterp/core/run_environment.py +38 -0
- openinterp/core/schemas.py +484 -0
- openinterp/core/specificity.py +49 -0
- openinterp/core/stats.py +172 -0
- openinterp/core/steering.py +49 -0
- openinterp/core/strict.py +19 -0
- openinterp/core/takedowns.py +136 -0
- openinterp/core/transcripts.py +130 -0
- openinterp/core/wiring.py +80 -0
- openinterp/corpus.py +476 -0
- openinterp/corpus_cache.py +145 -0
- openinterp/corpus_layouts.py +108 -0
- openinterp/corpus_presets.py +168 -0
- openinterp/corpus_rebuild.py +93 -0
- openinterp/coverage.py +155 -0
- openinterp/crossref.py +442 -0
- openinterp/data/__init__.py +9 -0
- openinterp/data/offensive_words.txt +58 -0
- openinterp/data/public_suffix_list.dat +15771 -0
- openinterp/data/sample_corpus.txt +40 -0
- openinterp/draw_steps.py +359 -0
- openinterp/errata_checks.py +129 -0
- openinterp/evidence.py +462 -0
- openinterp/evidence_branch.py +119 -0
- openinterp/evidence_lenses.py +75 -0
- openinterp/experiment_loop.py +357 -0
- openinterp/explain.py +252 -0
- openinterp/half_precision.py +79 -0
- openinterp/harness/__init__.py +37 -0
- openinterp/harness/ane/__init__.py +9 -0
- openinterp/harness/ane/backend.py +324 -0
- openinterp/harness/ane/cache.py +276 -0
- openinterp/harness/ane/chunks.py +163 -0
- openinterp/harness/ane/convert.py +116 -0
- openinterp/harness/ane/forward.py +258 -0
- openinterp/harness/ane/qualified.py +40 -0
- openinterp/harness/annotations.py +329 -0
- openinterp/harness/attribution.py +96 -0
- openinterp/harness/causal.py +264 -0
- openinterp/harness/confounds.py +184 -0
- openinterp/harness/covariance.py +301 -0
- openinterp/harness/crossref.py +195 -0
- openinterp/harness/cuda_devices.py +88 -0
- openinterp/harness/dead.py +124 -0
- openinterp/harness/devices.py +401 -0
- openinterp/harness/dictionaries.py +496 -0
- openinterp/harness/dreaming.py +116 -0
- openinterp/harness/environment.py +64 -0
- openinterp/harness/explanation_scores.py +371 -0
- openinterp/harness/features.py +276 -0
- openinterp/harness/first_position.py +137 -0
- openinterp/harness/heads.py +226 -0
- openinterp/harness/hooks.py +499 -0
- openinterp/harness/input_attribution.py +367 -0
- openinterp/harness/jlens.py +183 -0
- openinterp/harness/layer_cache.py +301 -0
- openinterp/harness/layer_major.py +274 -0
- openinterp/harness/lookahead.py +93 -0
- openinterp/harness/manifold.py +200 -0
- openinterp/harness/memory.py +471 -0
- openinterp/harness/moe.py +134 -0
- openinterp/harness/patcher.py +322 -0
- openinterp/harness/patchscope.py +177 -0
- openinterp/harness/probing.py +383 -0
- openinterp/harness/scan.py +462 -0
- openinterp/harness/scan_backend.py +201 -0
- openinterp/harness/sister_passes.py +463 -0
- openinterp/harness/sisters.py +363 -0
- openinterp/harness/sites.py +101 -0
- openinterp/harness/specificity.py +107 -0
- openinterp/harness/steering.py +159 -0
- openinterp/harness/superpositions.py +116 -0
- openinterp/harness/units.py +203 -0
- openinterp/harness/wiring.py +432 -0
- openinterp/harness/wiring_activations.py +285 -0
- openinterp/langid.py +73 -0
- openinterp/names.py +129 -0
- openinterp/neuronpedia_baseline.py +88 -0
- openinterp/neuronpedia_review.py +148 -0
- openinterp/planner.py +466 -0
- openinterp/planner_effects.py +89 -0
- openinterp/planner_heads.py +92 -0
- openinterp/planner_models.py +174 -0
- openinterp/planner_sisters.py +122 -0
- openinterp/planner_states.py +57 -0
- openinterp/planner_superpositions.py +114 -0
- openinterp/pole_effects.py +102 -0
- openinterp/prior_art.py +143 -0
- openinterp/reproduce.py +450 -0
- openinterp/revision.py +45 -0
- openinterp/runner_cache.py +35 -0
- openinterp/score_controls.py +169 -0
- openinterp/sensitive.py +243 -0
- openinterp/show.py +236 -0
- openinterp/signing.py +209 -0
- openinterp/sisters.py +476 -0
- openinterp/superposition_finders.py +225 -0
- openinterp/sync/__init__.py +5 -0
- openinterp/sync/r2.py +280 -0
- openinterp/syntax.py +399 -0
- openinterp/takedown_checks.py +76 -0
- openinterp/toxicity.py +158 -0
- openinterp/verdict_regression.py +79 -0
- openinterp/wiring_backfill.py +170 -0
- openinterp/wordnet.py +362 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Device planning on a machine with CUDA GPUs: the first free GPU, else the CPU.
|
|
2
|
+
|
|
3
|
+
A GPU is busy when another ``open-interpretability`` process has a lease on it (``cuda0``,
|
|
4
|
+
``cuda1``, … in the lease folder of ``harness/devices.py``). It is also busy when its
|
|
5
|
+
utilization is more than the planner's threshold. That is the mean of
|
|
6
|
+
:data:`~openinterp.harness.devices.GPU_SAMPLES` readings of ``nvidia-smi``. If every
|
|
7
|
+
GPU is busy, the job runs on the CPU. A machine with one GPU uses ``cuda``, and a
|
|
8
|
+
machine with more uses ``cuda:<index>``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import subprocess
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
from openinterp.harness.devices import (
|
|
17
|
+
GPU_SAMPLE_SECONDS,
|
|
18
|
+
GPU_SAMPLES,
|
|
19
|
+
DevicePlan,
|
|
20
|
+
holders,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
NVIDIA_SMI = ["nvidia-smi", "--query-gpu=index,utilization.gpu", "--format=csv,noheader,nounits"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def parse_nvidia_smi(output: str) -> dict[int, float]:
|
|
27
|
+
"""Each GPU's utilization from ``nvidia-smi`` output, by index. Bad lines are skipped."""
|
|
28
|
+
found = {}
|
|
29
|
+
for line in output.splitlines():
|
|
30
|
+
parts = [p.strip() for p in line.split(",")]
|
|
31
|
+
if len(parts) == 2 and parts[0].isdigit():
|
|
32
|
+
try:
|
|
33
|
+
found[int(parts[0])] = float(parts[1])
|
|
34
|
+
except ValueError:
|
|
35
|
+
continue
|
|
36
|
+
return found
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _read() -> dict[int, float] | None:
|
|
40
|
+
try:
|
|
41
|
+
out = subprocess.run(NVIDIA_SMI, capture_output=True, text=True, timeout=10, check=True)
|
|
42
|
+
except (OSError, subprocess.SubprocessError):
|
|
43
|
+
return None
|
|
44
|
+
return parse_nvidia_smi(out.stdout) or None
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def utilization(samples: int = GPU_SAMPLES, seconds: float = GPU_SAMPLE_SECONDS):
|
|
48
|
+
"""Each GPU's mean utilization over ``samples`` readings, or None if it can't be read."""
|
|
49
|
+
readings: list[dict[int, float]] = []
|
|
50
|
+
for i in range(samples):
|
|
51
|
+
if i:
|
|
52
|
+
time.sleep(seconds)
|
|
53
|
+
reading = _read()
|
|
54
|
+
if reading is None:
|
|
55
|
+
return None
|
|
56
|
+
readings.append(reading)
|
|
57
|
+
gpus = set().union(*readings)
|
|
58
|
+
return {g: sum(r.get(g, 0.0) for r in readings) / len(readings) for g in gpus}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def choose_cuda(
|
|
62
|
+
count: int, jobs: dict[int, list[int]], percents: dict[int, float] | None, busy: float
|
|
63
|
+
) -> DevicePlan:
|
|
64
|
+
"""The first of ``count`` GPUs that is free, else the CPU."""
|
|
65
|
+
reasons = []
|
|
66
|
+
for gpu in range(count):
|
|
67
|
+
others = jobs.get(gpu, [])
|
|
68
|
+
percent = (percents or {}).get(gpu)
|
|
69
|
+
if others:
|
|
70
|
+
ids = ", ".join(str(pid) for pid in others)
|
|
71
|
+
reasons.append(f"GPU {gpu} is in use by another openinterp job (process {ids})")
|
|
72
|
+
elif percent is not None and percent > busy:
|
|
73
|
+
reasons.append(f"GPU {gpu} is {percent:.0f}% busy, more than {busy:.0f}%")
|
|
74
|
+
else:
|
|
75
|
+
device = "cuda" if count == 1 else f"cuda:{gpu}"
|
|
76
|
+
return DevicePlan(device, f"GPU {gpu} is free", hint="cuda")
|
|
77
|
+
reason = "; ".join(reasons) or "this machine has no CUDA GPU"
|
|
78
|
+
return DevicePlan("cpu", reason, fallback=count > 0, hint="cuda")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def plan_cuda(busy: float) -> DevicePlan:
|
|
82
|
+
"""The plan on this machine's CUDA GPUs."""
|
|
83
|
+
import torch
|
|
84
|
+
|
|
85
|
+
count = torch.cuda.device_count()
|
|
86
|
+
jobs = {gpu: holders(f"cuda{gpu}") for gpu in range(count)}
|
|
87
|
+
free = [gpu for gpu in range(count) if not jobs[gpu]]
|
|
88
|
+
return choose_cuda(count, jobs, utilization() if free else None, busy)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Prove from the weights that a neuron pole can't fire on any input.
|
|
2
|
+
|
|
3
|
+
A scan that never sees a pole fire shows only that the corpus doesn't make it fire. This
|
|
4
|
+
module gives a proof. The norm before the MLP puts the MLP input on a ball:
|
|
5
|
+
|
|
6
|
+
- LayerNorm: ``γ⊙ẑ + β``, where ``ẑ`` has mean 0 and ``‖ẑ‖ < √d``.
|
|
7
|
+
- RMSNorm: ``γ⊙x̂``, where ``‖x̂‖ < √d``.
|
|
8
|
+
|
|
9
|
+
So a neuron's pre-activation ``h = w·input + b`` lies in ``(c − R, c + R)``.
|
|
10
|
+
|
|
11
|
+
- LayerNorm: ``c = w·β + b`` and ``R = √d·‖centred(w⊙γ)‖``.
|
|
12
|
+
- RMSNorm: ``c = b`` and ``R = √d·‖w⊙γ‖``.
|
|
13
|
+
|
|
14
|
+
Take an ungated MLP with a sign-preserving activation (GELU, SiLU). Its positive pole
|
|
15
|
+
fires only when ``h > 0``, and its negative pole only when ``h < 0``. So
|
|
16
|
+
``c + R ≤ 0`` proves the positive pole dead, and ``c − R ≥ 0`` proves the negative pole
|
|
17
|
+
dead. With ReLU, the negative pole is always dead.
|
|
18
|
+
|
|
19
|
+
A gated MLP (SwiGLU) multiplies two reads, so the sign of the product is free on a ball.
|
|
20
|
+
Its pole is proven dead only when a read is exactly zero and the MLP has no bias.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import math
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
|
|
28
|
+
import torch
|
|
29
|
+
from torch import nn
|
|
30
|
+
|
|
31
|
+
from openinterp.core.identifiers import NeuronIdentifier
|
|
32
|
+
from openinterp.harness.hooks import NeuronHookRunner, _try_path
|
|
33
|
+
from openinterp.harness.memory import loaded
|
|
34
|
+
from openinterp.harness.wiring import MLP_READ_PATHS, _matrix
|
|
35
|
+
|
|
36
|
+
# Activations whose output has the sign of their input: f(h) > 0 iff h > 0, f(h) < 0 iff
|
|
37
|
+
# h < 0. ReLU is apart: it is never negative.
|
|
38
|
+
SIGN_PRESERVING = {
|
|
39
|
+
"gelu",
|
|
40
|
+
"gelu_new",
|
|
41
|
+
"gelu_fast",
|
|
42
|
+
"gelu_pytorch_tanh",
|
|
43
|
+
"quick_gelu",
|
|
44
|
+
"silu",
|
|
45
|
+
"swish",
|
|
46
|
+
}
|
|
47
|
+
RELU = "relu"
|
|
48
|
+
# The bound must clear 0 by this share of its scale, so float64 rounding can't prove it.
|
|
49
|
+
MARGIN = 1e-9
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class DeadProof:
|
|
54
|
+
"""The result of the weight check on one neuron pole."""
|
|
55
|
+
|
|
56
|
+
proven: bool
|
|
57
|
+
# The largest value of h (positive pole) or −h (negative pole) that any input can
|
|
58
|
+
# give, with the margin added. None when the proof doesn't use a bound.
|
|
59
|
+
bound: float | None
|
|
60
|
+
reason: str
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _activation(runner: NeuronHookRunner) -> str | None:
|
|
64
|
+
config = getattr(runner.model, "config", None)
|
|
65
|
+
name = getattr(config, "activation_function", None) or getattr(config, "hidden_act", None)
|
|
66
|
+
return name if isinstance(name, str) else None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _is_rms(norm: nn.Module) -> bool:
|
|
70
|
+
"""An RMSNorm that scales by ``weight`` (Gemma's scales by ``1 + weight``: not this)."""
|
|
71
|
+
name = type(norm).__name__
|
|
72
|
+
return name.endswith("RMSNorm") and not name.startswith("Gemma")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _reads(runner: NeuronHookRunner, layer: int) -> dict[str, nn.Module]:
|
|
76
|
+
block = runner.layers[layer]
|
|
77
|
+
for paths in MLP_READ_PATHS:
|
|
78
|
+
modules = {c: _try_path(block, p) for c, p in paths.items()}
|
|
79
|
+
if all(isinstance(m, nn.Module) for m in modules.values()):
|
|
80
|
+
return modules
|
|
81
|
+
return {}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _range(norm: nn.Module, proj: nn.Module, neuron: int) -> tuple[float, float] | None:
|
|
85
|
+
"""``(c, R)``: the pre-activation lies in ``(c − R, c + R)`` for every input."""
|
|
86
|
+
w = _matrix(proj)[neuron].double().cpu()
|
|
87
|
+
with loaded(proj), loaded(norm): # streamed: the weights are real only inside this block
|
|
88
|
+
bias = getattr(proj, "bias", None)
|
|
89
|
+
b = float(bias.detach()[neuron]) if bias is not None else 0.0
|
|
90
|
+
gain = getattr(norm, "weight", None)
|
|
91
|
+
v = w * gain.detach().double().cpu() if gain is not None else w
|
|
92
|
+
shift = getattr(norm, "bias", None)
|
|
93
|
+
shift = shift.detach().double().cpu() if shift is not None else None
|
|
94
|
+
if isinstance(norm, nn.LayerNorm):
|
|
95
|
+
c = b + (float(w @ shift) if shift is not None else 0.0)
|
|
96
|
+
return c, math.sqrt(v.numel()) * float((v - v.mean()).norm())
|
|
97
|
+
if _is_rms(norm):
|
|
98
|
+
return b, math.sqrt(v.numel()) * float(v.norm())
|
|
99
|
+
return None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def prove_dead(runner: NeuronHookRunner, ident: NeuronIdentifier) -> DeadProof:
|
|
103
|
+
"""Check whether no input can make ``ident``'s pole fire."""
|
|
104
|
+
act, reads = _activation(runner), _reads(runner, ident.layer)
|
|
105
|
+
norm = runner.mlp_norm(ident.layer)
|
|
106
|
+
if set(reads) == {"in"}:
|
|
107
|
+
if act == RELU and ident.pole == "negative":
|
|
108
|
+
return DeadProof(True, None, "ReLU is never negative")
|
|
109
|
+
if act not in SIGN_PRESERVING and act != RELU:
|
|
110
|
+
return DeadProof(False, None, f"no bound for the activation {act!r}")
|
|
111
|
+
span = _range(norm, reads["in"], ident.neuron_idx)
|
|
112
|
+
if span is None:
|
|
113
|
+
return DeadProof(False, None, f"no bound for the norm {type(norm).__name__}")
|
|
114
|
+
c, r = span
|
|
115
|
+
bound = (c + r if ident.pole == "positive" else r - c) + MARGIN * (abs(c) + r)
|
|
116
|
+
return DeadProof(bound <= 0, bound, f"pre-activation in ({c - r:.4g}, {c + r:.4g})")
|
|
117
|
+
if set(reads) == {"gate", "up"}:
|
|
118
|
+
if any(getattr(m, "bias", None) is not None for m in reads.values()):
|
|
119
|
+
return DeadProof(False, None, "no bound for a gated MLP with biases")
|
|
120
|
+
rows = [_matrix(m)[ident.neuron_idx] for m in reads.values()]
|
|
121
|
+
if any(not torch.any(row) for row in rows):
|
|
122
|
+
return DeadProof(True, 0.0, "a read of the gated MLP is zero")
|
|
123
|
+
return DeadProof(False, None, "a gated MLP's pole is proven dead only by a zero read")
|
|
124
|
+
return DeadProof(False, None, "MLP input projection not found")
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""Device planner: with ``--device auto``, use a GPU only when it is free.
|
|
2
|
+
|
|
3
|
+
This module plans for Apple's GPU (MPS). ``cuda_devices.py`` plans for CUDA GPUs with
|
|
4
|
+
the same leases and threshold, and chooses the first free GPU.
|
|
5
|
+
|
|
6
|
+
Two things make the GPU busy. One is another ``open-interpretability`` process that uses it. Each
|
|
7
|
+
process that runs a model on MPS keeps a *lease*. That is an empty file named for its
|
|
8
|
+
process id, in ``$OPENINTERP_DEVICE_LEASES`` (default ``~/.cache/openinterp/devices/mps/``).
|
|
9
|
+
The planner ignores and removes the lease of a process that stopped. The other is load
|
|
10
|
+
from other programs. The planner reads the GPU's ``Device Utilization %`` from ``ioreg``.
|
|
11
|
+
It takes the mean of :data:`GPU_SAMPLES` readings, because one reading changes much from
|
|
12
|
+
second to second. Above ``$OPENINTERP_GPU_BUSY_PERCENT`` (default
|
|
13
|
+
:data:`DEFAULT_GPU_BUSY_PERCENT`), the GPU is busy.
|
|
14
|
+
|
|
15
|
+
When the GPU is busy, the order is MPS, then the ANE, then the CPU. The ANE is Apple's
|
|
16
|
+
Neural Engine. If this Mac can run Core ML and no other job has a lease on the ANE, the
|
|
17
|
+
scans' forward passes run there (``harness/ane/``). The rest runs on the CPU. Else everything runs
|
|
18
|
+
on the CPU. The CPU is slower than a free GPU. But two processes on one GPU can slow down
|
|
19
|
+
each other, and one such run gave wrong results (docs/experiments.md, Devices).
|
|
20
|
+
``--device mps`` uses the GPU even when it is busy. ``--device ane`` runs the scans on
|
|
21
|
+
the ANE, and the rest where the planner puts it.
|
|
22
|
+
|
|
23
|
+
A model that streams (``--memory stream``) is the exception. Its scan on the CPU reads
|
|
24
|
+
each layer once for the whole corpus, and with the CPU free that is faster than the ANE.
|
|
25
|
+
On a 50,000-token Qwen2.5-0.5B scan, the CPU's pass took about 16 s. The ANE's took
|
|
26
|
+
39 s, and it needed 40 s more to measure the top pieces again. So a streamed model's
|
|
27
|
+
scans use the ANE only if the CPU is busy too. That is a load average of at least
|
|
28
|
+
``$OPENINTERP_CPU_BUSY_PERCENT`` (default :data:`DEFAULT_CPU_BUSY_PERCENT`) of its cores.
|
|
29
|
+
|
|
30
|
+
A process plans once, and every model it loads after that uses the same device.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import atexit
|
|
36
|
+
import os
|
|
37
|
+
import platform
|
|
38
|
+
import re
|
|
39
|
+
import subprocess
|
|
40
|
+
import threading
|
|
41
|
+
import time
|
|
42
|
+
from contextlib import suppress
|
|
43
|
+
from dataclasses import dataclass, field
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
|
|
46
|
+
from openinterp.core.cache_dirs import cache_root
|
|
47
|
+
|
|
48
|
+
LEASES_ENV = "OPENINTERP_DEVICE_LEASES" # lease directory, or "off"
|
|
49
|
+
# Set by the retry after an MPS failure: every model of the command loads on this device.
|
|
50
|
+
FORCE_ENV = "OPENINTERP_DEVICE_FORCE"
|
|
51
|
+
GPU_BUSY_ENV = "OPENINTERP_GPU_BUSY_PERCENT"
|
|
52
|
+
# Idle reads 1%, one job 93-97% (docs/analyses/device-contention.md).
|
|
53
|
+
DEFAULT_GPU_BUSY_PERCENT = 50.0
|
|
54
|
+
CPU_BUSY_ENV = "OPENINTERP_CPU_BUSY_PERCENT"
|
|
55
|
+
DEFAULT_CPU_BUSY_PERCENT = 50.0 # the 1-minute load average, as a share of the cores
|
|
56
|
+
# The longest the first operation on MPS may take before the process uses the CPU.
|
|
57
|
+
START_ENV = "OPENINTERP_MPS_START_SECONDS"
|
|
58
|
+
DEFAULT_START_SECONDS = 120.0
|
|
59
|
+
GPU_SAMPLES = 5
|
|
60
|
+
GPU_SAMPLE_SECONDS = 0.4
|
|
61
|
+
|
|
62
|
+
_UTILIZATION = re.compile(r'"Device Utilization %"=(\d+)')
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Load:
|
|
67
|
+
"""What the planner saw: other processes on each device, and the GPU's utilization."""
|
|
68
|
+
|
|
69
|
+
jobs: dict[str, list[int]] = field(default_factory=dict) # device -> process ids
|
|
70
|
+
gpu_percent: float | None = None # mean utilization, or None if it was not read
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class DevicePlan:
|
|
75
|
+
"""The device to use, and why."""
|
|
76
|
+
|
|
77
|
+
device: str
|
|
78
|
+
reason: str
|
|
79
|
+
fallback: bool = False # True when a busy device was not used
|
|
80
|
+
scan: str | None = None # "ane" when scans run on Apple's Neural Engine
|
|
81
|
+
hint: str = "mps" # the --device value that uses the GPU even when it is busy
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def choose(
|
|
85
|
+
load: Load, mps_available: bool, busy_percent: float, ane_ok: bool = False
|
|
86
|
+
) -> DevicePlan:
|
|
87
|
+
"""The device for ``load``: MPS if it is free, else the ANE for scans, else the CPU.
|
|
88
|
+
|
|
89
|
+
With the ANE, the scans' forward passes run there, and the rest runs on the CPU.
|
|
90
|
+
``ane_ok`` says that this machine can run Core ML on the ANE.
|
|
91
|
+
"""
|
|
92
|
+
if not mps_available:
|
|
93
|
+
return DevicePlan("cpu", "this machine has no Apple GPU")
|
|
94
|
+
busy = _busy(load.jobs.get("mps", []), "the GPU", load.gpu_percent, busy_percent)
|
|
95
|
+
if busy is None:
|
|
96
|
+
return DevicePlan("mps", "the GPU is free")
|
|
97
|
+
if ane_ok and not load.jobs.get("ane"):
|
|
98
|
+
return DevicePlan("cpu", busy, True, scan="ane")
|
|
99
|
+
if ane_ok:
|
|
100
|
+
busy += " and " + _busy(load.jobs["ane"], "the ANE", None, busy_percent)
|
|
101
|
+
return DevicePlan("cpu", busy, True)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _busy(others: list[int], name: str, percent: float | None, limit: float) -> str | None:
|
|
105
|
+
"""Why ``name`` is busy, or None if it is free."""
|
|
106
|
+
if others:
|
|
107
|
+
ids = ", ".join(str(pid) for pid in others)
|
|
108
|
+
noun = "job uses" if len(others) == 1 else "jobs use"
|
|
109
|
+
return f"another openinterp {noun} {name} (process {ids})"
|
|
110
|
+
if percent is not None and percent > limit:
|
|
111
|
+
return f"{name} is {percent:.0f}% busy, more than {limit:.0f}%"
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def busy_percent() -> float:
|
|
116
|
+
"""``$OPENINTERP_GPU_BUSY_PERCENT``, else :data:`DEFAULT_GPU_BUSY_PERCENT`."""
|
|
117
|
+
return float(os.environ.get(GPU_BUSY_ENV) or DEFAULT_GPU_BUSY_PERCENT)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def read_load() -> Load:
|
|
121
|
+
"""The other processes on MPS and the ANE, and the GPU's utilization if no job is on it."""
|
|
122
|
+
jobs = {"mps": holders("mps"), "ane": holders("ane")}
|
|
123
|
+
return Load(jobs, None if jobs["mps"] else gpu_utilization())
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def gpu_utilization(samples: int = GPU_SAMPLES, seconds: float = GPU_SAMPLE_SECONDS):
|
|
127
|
+
"""The mean of ``samples`` readings of the GPU's utilization, or None if ``ioreg`` fails."""
|
|
128
|
+
readings = []
|
|
129
|
+
for i in range(samples):
|
|
130
|
+
if i:
|
|
131
|
+
time.sleep(seconds)
|
|
132
|
+
value = _read_utilization()
|
|
133
|
+
if value is None:
|
|
134
|
+
return None
|
|
135
|
+
readings.append(value)
|
|
136
|
+
return sum(readings) / len(readings)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _read_utilization() -> float | None:
|
|
140
|
+
try:
|
|
141
|
+
out = subprocess.run(
|
|
142
|
+
["ioreg", "-r", "-d", "1", "-c", "IOAccelerator"],
|
|
143
|
+
capture_output=True,
|
|
144
|
+
text=True,
|
|
145
|
+
timeout=5,
|
|
146
|
+
check=True,
|
|
147
|
+
).stdout
|
|
148
|
+
except (OSError, subprocess.SubprocessError):
|
|
149
|
+
return None
|
|
150
|
+
return parse_utilization(out)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def parse_utilization(ioreg_output: str) -> float | None:
|
|
154
|
+
"""The largest ``Device Utilization %`` in ``ioreg`` output, or None if there is none."""
|
|
155
|
+
values = [float(v) for v in _UTILIZATION.findall(ioreg_output)]
|
|
156
|
+
return max(values) if values else None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def lease_root() -> Path | None:
|
|
160
|
+
"""``$OPENINTERP_DEVICE_LEASES``, else ``$XDG_CACHE_HOME/openinterp/devices``; None if off."""
|
|
161
|
+
return cache_root(LEASES_ENV, "devices")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def holders(device: str) -> list[int]:
|
|
165
|
+
"""Process ids of the other running processes with a lease on ``device``.
|
|
166
|
+
|
|
167
|
+
It removes the leases of processes that have stopped.
|
|
168
|
+
"""
|
|
169
|
+
root = lease_root()
|
|
170
|
+
folder = root / device if root else None
|
|
171
|
+
if folder is None or not folder.is_dir():
|
|
172
|
+
return []
|
|
173
|
+
found = []
|
|
174
|
+
for lease in folder.iterdir():
|
|
175
|
+
if not lease.name.isdigit() or int(lease.name) == os.getpid():
|
|
176
|
+
continue
|
|
177
|
+
if _running(int(lease.name)):
|
|
178
|
+
found.append(int(lease.name))
|
|
179
|
+
else:
|
|
180
|
+
with suppress(OSError):
|
|
181
|
+
lease.unlink()
|
|
182
|
+
return sorted(found)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
_used: set[str] = set() # devices that this process ran models on
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def lease_name(device: str) -> str | None:
|
|
189
|
+
"""The lease folder of a GPU device (``mps``, ``cuda0``, ``cuda1``, …), or None."""
|
|
190
|
+
if device == "mps":
|
|
191
|
+
return "mps"
|
|
192
|
+
if device.startswith("cuda"):
|
|
193
|
+
return f"cuda{device.partition(':')[2] or 0}"
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def take_lease(device: str) -> None:
|
|
198
|
+
"""Record that this process uses the GPU ``device``, until it stops."""
|
|
199
|
+
name = lease_name(device)
|
|
200
|
+
if name is None:
|
|
201
|
+
return
|
|
202
|
+
_used.add(name)
|
|
203
|
+
root = lease_root()
|
|
204
|
+
if root is None:
|
|
205
|
+
return
|
|
206
|
+
lease = root / name / str(os.getpid())
|
|
207
|
+
if lease.exists():
|
|
208
|
+
return
|
|
209
|
+
with suppress(OSError):
|
|
210
|
+
lease.parent.mkdir(parents=True, exist_ok=True)
|
|
211
|
+
lease.touch()
|
|
212
|
+
atexit.register(_drop, lease)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _drop(lease: Path) -> None:
|
|
216
|
+
with suppress(OSError):
|
|
217
|
+
lease.unlink()
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _running(pid: int) -> bool:
|
|
221
|
+
try:
|
|
222
|
+
os.kill(pid, 0)
|
|
223
|
+
except ProcessLookupError:
|
|
224
|
+
return False
|
|
225
|
+
except PermissionError: # it runs, as another user
|
|
226
|
+
return True
|
|
227
|
+
return True
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
_choice: str | None = None # how this process chose its device, for the record
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def note_choice(text: str) -> None:
|
|
234
|
+
"""Record how this process chose its device (``RunEnvironment.device_choice``)."""
|
|
235
|
+
global _choice
|
|
236
|
+
_choice = text
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def choice() -> str | None:
|
|
240
|
+
"""How this process chose its device, or None if nothing chose it (e.g. in a test)."""
|
|
241
|
+
return _choice
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
_planned: DevicePlan | None = None
|
|
245
|
+
_noticed = False
|
|
246
|
+
_forced_ane = False
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def force_ane() -> None:
|
|
250
|
+
"""Run scans on the ANE whatever the plan says (``--device ane``)."""
|
|
251
|
+
global _forced_ane
|
|
252
|
+
_forced_ane = True
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def cpu_load_percent() -> float:
|
|
256
|
+
"""The 1-minute load average, as a percentage of the CPU's cores."""
|
|
257
|
+
return 100.0 * os.getloadavg()[0] / (os.cpu_count() or 1)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def streamed_scans_skip_ane() -> str | None:
|
|
261
|
+
"""Why a streamed model's scans run on the CPU and not on the planned ANE, or None.
|
|
262
|
+
|
|
263
|
+
The CPU's scan of a streamed model is faster than the ANE's when the CPU is free.
|
|
264
|
+
``--device ane`` keeps the ANE.
|
|
265
|
+
"""
|
|
266
|
+
if _forced_ane:
|
|
267
|
+
return None
|
|
268
|
+
load = cpu_load_percent()
|
|
269
|
+
if load >= float(os.environ.get(CPU_BUSY_ENV) or DEFAULT_CPU_BUSY_PERCENT):
|
|
270
|
+
return None
|
|
271
|
+
return f"the model streams and the CPU is free (load {load:.0f}% of its cores)"
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def scans_on_ane() -> bool:
|
|
275
|
+
"""True if scans should run on the ANE: ``--device ane``, or the plan's choice."""
|
|
276
|
+
return _forced_ane or (_planned is not None and _planned.scan == "ane")
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def notice() -> str | None:
|
|
280
|
+
"""At the first call only: why the plan did not use a busy GPU. Else None."""
|
|
281
|
+
global _noticed
|
|
282
|
+
if _noticed or _planned is None or not _planned.fallback:
|
|
283
|
+
return None
|
|
284
|
+
_noticed = True
|
|
285
|
+
ane = " Scans run on Apple's Neural Engine." if _planned.scan == "ane" else ""
|
|
286
|
+
return (
|
|
287
|
+
f"Using the CPU, because {_planned.reason}.{ane} "
|
|
288
|
+
f"Pass --device {_planned.hint} to share the GPU."
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def planned() -> DevicePlan:
|
|
293
|
+
"""This process's plan, made at the first call."""
|
|
294
|
+
global _planned
|
|
295
|
+
if _planned is None:
|
|
296
|
+
import torch
|
|
297
|
+
|
|
298
|
+
if torch.cuda.is_available():
|
|
299
|
+
from openinterp.harness.cuda_devices import plan_cuda
|
|
300
|
+
|
|
301
|
+
_planned = plan_cuda(busy_percent())
|
|
302
|
+
else:
|
|
303
|
+
mps = torch.backends.mps.is_available()
|
|
304
|
+
_planned = choose(read_load(), mps, busy_percent(), mps and ane_possible())
|
|
305
|
+
return _planned
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def ane_possible() -> bool:
|
|
309
|
+
"""True if this machine can run Core ML models on the ANE: a Mac with coremltools."""
|
|
310
|
+
from openinterp.harness.ane import convert
|
|
311
|
+
|
|
312
|
+
return platform.system() == "Darwin" and platform.machine() == "arm64" and convert.available()
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def forget() -> None:
|
|
316
|
+
"""Make the next :func:`planned` plan again (for tests)."""
|
|
317
|
+
global _planned, _noticed, _forced_ane, _start_checked, _choice
|
|
318
|
+
_planned, _noticed, _forced_ane, _start_checked, _choice = None, False, False, False, None
|
|
319
|
+
_used.clear()
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def forced() -> str | None:
|
|
323
|
+
"""The device that a retry after a GPU failure set, or None."""
|
|
324
|
+
return os.environ.get(FORCE_ENV) or None
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def gpu_failed(exc: BaseException) -> str | None:
|
|
328
|
+
"""``MPS`` or ``CUDA`` if ``exc`` is a GPU failure that a retry on the CPU can get past.
|
|
329
|
+
|
|
330
|
+
It is one if this process ran a model on that GPU and an exception in the chain
|
|
331
|
+
names it. A command that is a retry already is not retried. Else None.
|
|
332
|
+
"""
|
|
333
|
+
if forced():
|
|
334
|
+
return None
|
|
335
|
+
kinds = {"mps"} & _used | ({"cuda"} if any(u.startswith("cuda") for u in _used) else set())
|
|
336
|
+
seen: set[int] = set()
|
|
337
|
+
while exc is not None and id(exc) not in seen:
|
|
338
|
+
seen.add(id(exc))
|
|
339
|
+
text = str(exc).lower() if isinstance(exc, Exception) else ""
|
|
340
|
+
if found := next((k for k in sorted(kinds) if k in text), None):
|
|
341
|
+
return found.upper()
|
|
342
|
+
exc = exc.__cause__ or exc.__context__
|
|
343
|
+
return None
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def rerun_on_cpu(exc: BaseException, gpu: str = "MPS") -> None:
|
|
347
|
+
"""Run this command again from the start, in a new process, with every model on the CPU.
|
|
348
|
+
|
|
349
|
+
The new process has none of the GPU's state. Files that the first run wrote are
|
|
350
|
+
written again.
|
|
351
|
+
"""
|
|
352
|
+
import sys
|
|
353
|
+
|
|
354
|
+
first = str(exc).strip().splitlines()[0][:200] if str(exc).strip() else type(exc).__name__
|
|
355
|
+
print(
|
|
356
|
+
f"The GPU ({gpu}) failed ({first}). Running the command again on the CPU.", file=sys.stderr
|
|
357
|
+
)
|
|
358
|
+
sys.stderr.flush()
|
|
359
|
+
sys.stdout.flush()
|
|
360
|
+
# The same command line that started this process, so nothing new runs.
|
|
361
|
+
os.execve(sys.executable, sys.orig_argv, {**os.environ, FORCE_ENV: "cpu"}) # noqa: S606
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
_start_checked: str | None | bool = False # False: not checked yet
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def mps_stuck() -> str | None:
|
|
368
|
+
"""Why this process can't use MPS, or None if a first operation on it ends in time.
|
|
369
|
+
|
|
370
|
+
Several processes that start on MPS at the same moment can each stop for good in
|
|
371
|
+
Metal's compiler. It opens Metal's shared cache of compiled programs on disk
|
|
372
|
+
(docs/analyses/device-contention.md). So a first small operation runs on a
|
|
373
|
+
background thread. The process waits for it at most ``$OPENINTERP_MPS_START_SECONDS``
|
|
374
|
+
(default :data:`DEFAULT_START_SECONDS`). The check runs once in a process.
|
|
375
|
+
"""
|
|
376
|
+
global _start_checked
|
|
377
|
+
if _start_checked is False:
|
|
378
|
+
_start_checked = _probe_mps(float(os.environ.get(START_ENV) or DEFAULT_START_SECONDS))
|
|
379
|
+
return _start_checked
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _probe_mps(seconds: float) -> str | None:
|
|
383
|
+
done, failed = threading.Event(), []
|
|
384
|
+
|
|
385
|
+
def probe() -> None:
|
|
386
|
+
try:
|
|
387
|
+
import torch
|
|
388
|
+
|
|
389
|
+
torch.ones(1024, device="mps").mul(2).cpu()
|
|
390
|
+
torch.mps.synchronize()
|
|
391
|
+
except Exception as exc: # a GPU that fails at once can't be used
|
|
392
|
+
failed.append(exc)
|
|
393
|
+
finally:
|
|
394
|
+
done.set()
|
|
395
|
+
|
|
396
|
+
threading.Thread(target=probe, daemon=True, name="mps-start").start()
|
|
397
|
+
if not done.wait(timeout=seconds):
|
|
398
|
+
return f"the GPU (MPS) didn't start in {seconds:g} s (${START_ENV})"
|
|
399
|
+
if failed:
|
|
400
|
+
return f"the GPU (MPS) failed at once ({type(failed[0]).__name__}: {failed[0]})"
|
|
401
|
+
return None
|