gpu-stack-inspector 0.1.0__py3-none-any.whl

gpu_stack_inspector/rules/compatibility.py

@@ -0,0 +1,256 @@
+"""Checks of the CUDA toolchain against what the driver supports."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from enum import Enum
+
+from gpu_stack_inspector.compatibility.loader import load
+from gpu_stack_inspector.compatibility.versions import major_minor
+from gpu_stack_inspector.models import (
+    Confidence,
+    Evidence,
+    Finding,
+    Severity,
+    Snapshot,
+    Status,
+)
+from gpu_stack_inspector.rules.catalog import RULES
+from gpu_stack_inspector.rules.engine import RuleContext
+from gpu_stack_inspector.rules.torch_build import TorchBuildKind, classify_torch
+
+_MINOR_URL = "https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html"
+
+
+class _Support(Enum):
+    SUPPORTED = "supported"
+    NEWER_MINOR = "newer minor"
+    NEWER_MAJOR = "newer major"
+
+
+def _support(target: tuple[int, int], driver_max: tuple[int, int]) -> _Support:
+    if target <= driver_max:
+        return _Support.SUPPORTED
+    if target[0] == driver_max[0] and target[0] >= load().minor_version_from_major:
+        return _Support.NEWER_MINOR  # minor version compatibility applies
+    return _Support.NEWER_MAJOR
+
+
+def _driver_cause(snapshot: Snapshot) -> str | None:
+    """Which host finding explains a missing driver CUDA version, if any."""
+    nvidia = snapshot.nvidia
+    if nvidia is not None and nvidia.smi_path is None:
+        return "GSI-SYS-001"
+    if nvidia is not None and nvidia.driver_version is None:
+        return "GSI-SYS-002"
+    return None
+
+
+def _not_compared(snapshot: Snapshot, rule_id: str, what: str, subject: str = "host") -> Finding:
+    """No driver CUDA version: not tested because of a host finding, or plainly unknown."""
+    cause = _driver_cause(snapshot)
+    if cause is not None:
+        return Finding(
+            id=rule_id,
+            status=Status.NOT_TESTED,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary=f"{what} was not compared with the driver",
+            reason="The driver's CUDA version is not known.",
+            subject=subject,
+            caused_by=cause,
+        )
+    return Finding(
+        id=rule_id,
+        status=Status.UNKNOWN,
+        severity=Severity.LOW,
+        confidence=Confidence.LOW,
+        summary=f"{what} could not be compared with the driver",
+        reason="The driver answered but did not report the newest CUDA version it supports.",
+        subject=subject,
+        recommended_action="Check the 'CUDA Version' in the header of `nvidia-smi` by hand.",
+        verify_command="nvidia-smi",
+    )
+
+
+def _minimum_driver_sentence(major: int) -> str:
+    row = load().minimum_driver.get(major)
+    return f" CUDA {major}.x needs driver {row[0]} or newer." if row else ""
+
+
+@RULES.rule("GSI-CUDA-001", "nvcc is installed")
+def nvcc_installed(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether the CUDA compiler nvcc is on PATH. Missing is not a problem by itself.
+
+    Prebuilt PyTorch wheels bring their own CUDA runtime and do not need a local CUDA
+    toolkit. nvcc only matters for compiling CUDA code yourself, such as building an
+    extension from source.
+    """
+    toolkit = snapshot.toolkit
+    if toolkit is None:
+        return
+    if toolkit.nvcc_path is not None:
+        yield Finding(
+            id="GSI-CUDA-001",
+            status=Status.PASS,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary=f"nvcc {toolkit.nvcc_version or '(unknown version)'} is installed",
+            reason=f"Found at {toolkit.nvcc_path}.",
+            evidence=(Evidence("nvcc", "version", toolkit.nvcc_version),),
+        )
+        return
+    yield Finding(
+        id="GSI-CUDA-001",
+        status=Status.NOT_APPLICABLE,
+        severity=Severity.INFO,
+        confidence=Confidence.HIGH,
+        summary="No CUDA compiler (nvcc) on PATH",
+        reason=(
+            "Prebuilt PyTorch wheels do not need it; it is only needed to compile CUDA code, "
+            "such as building an extension from source."
+        ),
+        evidence=(Evidence("nvcc", "path", None),),
+    )
+
+
+@RULES.rule("GSI-CUDA-002", "The driver supports nvcc's CUDA version", sources=(_MINOR_URL,))
+def nvcc_supported(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether programs compiled with the local nvcc can run on this driver.
+
+    nvcc from CUDA 13.x produces programs that need a driver supporting CUDA 13. Within one
+    major release family, NVIDIA's minor version compatibility usually lets newer-minor
+    programs run, with a limited feature set; code compiled to PTX does not.
+    """
+    toolkit, nvidia = snapshot.toolkit, snapshot.nvidia
+    nvcc = major_minor(toolkit.nvcc_version) if toolkit else None
+    if nvcc is None:
+        return
+    driver_max = major_minor(nvidia.max_cuda_version) if nvidia else None
+    if driver_max is None:
+        if nvidia is not None:
+            yield _not_compared(snapshot, "GSI-CUDA-002", "nvcc")
+        return
+    version = toolkit.nvcc_version if toolkit else None
+    driver_text = nvidia.max_cuda_version if nvidia else None
+    evidence = (
+        Evidence("nvcc", "version", version),
+        Evidence("nvidia-smi", "max_cuda_version", driver_text),
+    )
+    support = _support(nvcc, driver_max)
+    if support is _Support.SUPPORTED:
+        yield Finding(
+            id="GSI-CUDA-002",
+            status=Status.PASS,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary=f"The driver supports nvcc's CUDA {nvcc[0]}.{nvcc[1]}",
+            reason=f"The driver supports CUDA up to {driver_text}.",
+            evidence=evidence,
+        )
+        return
+    yield Finding(
+        id="GSI-CUDA-002",
+        status=Status.WARN,
+        severity=Severity.MEDIUM if support is _Support.NEWER_MAJOR else Severity.LOW,
+        confidence=Confidence.HIGH,
+        summary=f"Programs built with nvcc {version} may need a newer driver",
+        reason=(
+            f"nvcc builds for CUDA {nvcc[0]}.{nvcc[1]}, but the driver supports CUDA up to "
+            f"{driver_text}."
+            + (
+                _minimum_driver_sentence(nvcc[0])
+                if support is _Support.NEWER_MAJOR
+                else " Minor version compatibility usually lets them run, but code compiled "
+                "to PTX will not."
+            )
+        ),
+        evidence=evidence,
+        recommended_action="Update the NVIDIA driver, or use a CUDA toolkit the driver supports.",
+        verify_command="nvidia-smi",
+    )
+
+
+@RULES.rule(
+    "GSI-CUDA-003",
+    "The driver supports the CUDA version PyTorch was built for",
+    sources=(_MINOR_URL,),
+)
+def torch_cuda_supported(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether the driver can run the CUDA version each CUDA build of PyTorch targets.
+
+    A build for a newer CUDA major release than the driver supports cannot initialise
+    CUDA at all (unless NVIDIA's forward-compatibility package is installed, which is
+    rare outside data centres). A newer minor version of the same release family usually
+    works through minor version compatibility; operations that need the newer driver fail
+    with cudaErrorCallRequiresNewerDriver.
+    """
+    nvidia = snapshot.nvidia
+    driver_max = major_minor(nvidia.max_cuda_version) if nvidia else None
+    driver_text = nvidia.max_cuda_version if nvidia else None
+    for interpreter in snapshot.interpreters:
+        if interpreter.torch is None:
+            continue
+        build = classify_torch(interpreter.torch)
+        target = major_minor(build.cuda_version)
+        if build.kind is not TorchBuildKind.CUDA or target is None:
+            continue
+        if driver_max is None:
+            if nvidia is not None:
+                what = f"CUDA {build.cuda_version}"
+                yield _not_compared(snapshot, "GSI-CUDA-003", what, interpreter.label)
+            continue
+        evidence = (
+            Evidence("torch", "cuda_version", build.cuda_version),
+            Evidence("nvidia-smi", "max_cuda_version", driver_text),
+        )
+        support = _support(target, driver_max)
+        if support is _Support.SUPPORTED:
+            yield Finding(
+                id="GSI-CUDA-003",
+                status=Status.PASS,
+                severity=Severity.INFO,
+                confidence=Confidence.HIGH,
+                summary=f"The driver supports CUDA {build.cuda_version}",
+                reason=(
+                    f"torch targets CUDA {build.cuda_version}; "
+                    f"the driver supports up to {driver_text}."
+                ),
+                subject=interpreter.label,
+                evidence=evidence,
+            )
+            continue
+        newer_major = support is _Support.NEWER_MAJOR
+        yield Finding(
+            id="GSI-CUDA-003",
+            status=Status.FAIL if newer_major else Status.WARN,
+            severity=Severity.HIGH if newer_major else Severity.MEDIUM,
+            confidence=Confidence.HIGH,
+            summary=(
+                f"torch is built for CUDA {build.cuda_version}, newer than the driver supports"
+            ),
+            reason=(
+                f"The driver supports CUDA up to {driver_text}."
+                + (
+                    _minimum_driver_sentence(target[0]) + " CUDA will not initialise."
+                    if newer_major
+                    else " Minor version compatibility usually lets it run, but operations "
+                    "that need a newer driver fail with cudaErrorCallRequiresNewerDriver."
+                )
+            ),
+            subject=interpreter.label,
+            evidence=evidence,
+            recommended_action=(
+                f"Update the NVIDIA driver, or install a torch build for CUDA {driver_text} or "
+                "older from https://pytorch.org/get-started/locally/."
+            ),
+            verify_command="nvidia-smi",
+            limitations=(
+                (
+                    "NVIDIA's forward-compatibility package (cuda-compat) can bridge major "
+                    "versions on supported data-centre GPUs; it is not detected.",
+                )
+                if newer_major
+                else ()
+            ),
+        )

gpu_stack_inspector/rules/differences.py

@@ -0,0 +1,64 @@
+"""Checks across the inspected interpreters: what tells them apart."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from gpu_stack_inspector.models import (
+    Confidence,
+    Evidence,
+    Finding,
+    Severity,
+    Snapshot,
+    Status,
+)
+from gpu_stack_inspector.rules.catalog import RULES
+from gpu_stack_inspector.rules.comparison import torch_build_label
+from gpu_stack_inspector.rules.engine import RuleContext
+
+COMPARISON = "comparison"
+"""Subject of findings about the interpreters as a group."""
+
+
+@RULES.rule("GSI-DIFF-001", "The compared environments have the same PyTorch build")
+def same_torch_build(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether every compared environment has the same kind of PyTorch build.
+
+    When one environment has a CUDA build and another a CPU-only build (or no torch, or a
+    build for another CUDA version), that difference is the most likely reason the GPU
+    works in one and not the other. Only interpreters that could be inspected count, and
+    only when at least two of them are compared.
+    """
+    usable = [i for i in snapshot.interpreters if i.version is not None]
+    if len(usable) < 2:
+        return
+    builds = [(i.label, torch_build_label(i) or "unknown") for i in usable]
+    evidence = tuple(Evidence("comparison", label, build) for label, build in builds)
+    listing = ", ".join(f"{label} {build}" for label, build in builds)
+    if len({build for _, build in builds}) == 1:
+        yield Finding(
+            id="GSI-DIFF-001",
+            status=Status.PASS,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary=f"All compared environments have the same PyTorch build ({builds[0][1]})",
+            reason="The difference, if any, is elsewhere; see the comparison table.",
+            subject=COMPARISON,
+            evidence=evidence,
+        )
+        return
+    yield Finding(
+        id="GSI-DIFF-001",
+        status=Status.WARN,
+        severity=Severity.MEDIUM,
+        confidence=Confidence.HIGH,
+        summary=f"The environments have different PyTorch builds: {listing}",
+        reason=("This is the most likely reason CUDA works in one environment and not in another."),
+        subject=COMPARISON,
+        evidence=evidence,
+        recommended_action=(
+            "Install the same CUDA build of torch in the environment that should use the GPU; "
+            "GSI-TORCH-002 says which build each one has."
+        ),
+        verify_command="gpu-stack-inspector compare --python … --python …",
+    )

gpu_stack_inspector/rules/engine.py

@@ -0,0 +1,101 @@
+"""Rules: pure functions from a snapshot to findings, registered under stable codes."""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable, Iterable, Iterator
+from dataclasses import dataclass
+
+from gpu_stack_inspector.models import FINDING_ID, Confidence, Finding, Severity, Snapshot, Status
+
+
+@dataclass(frozen=True, slots=True)
+class RuleContext:
+    """What the user asked for, which changes how some facts are judged.
+
+    ``cuda_expected``: the user wants these environments to use an NVIDIA GPU (the
+    reason to run this tool), so a missing driver is a problem rather than a fact.
+    ``shell_python``: the interpreter the user's shell runs, when known.
+    """
+
+    cuda_expected: bool = True
+    shell_python: str | None = None
+
+
+RuleCheck = Callable[[Snapshot, RuleContext], Iterable[Finding]]
+
+
+@dataclass(frozen=True, slots=True)
+class Rule:
+    id: str
+    title: str
+    explanation: str
+    check: RuleCheck
+    sources: tuple[str, ...] = ()
+
+
+class Registry:
+    """Rules in registration order, which is the order they are evaluated and listed."""
+
+    def __init__(self) -> None:
+        self._rules: dict[str, Rule] = {}
+
+    def rule(
+        self, rule_id: str, title: str, *, sources: Iterable[str] = ()
+    ) -> Callable[[RuleCheck], RuleCheck]:
+        """Register a check; its docstring becomes the explanation shown by `explain`."""
+
+        def register(check: RuleCheck) -> RuleCheck:
+            if not FINDING_ID.fullmatch(rule_id):
+                raise ValueError(f"rule id {rule_id!r} does not match GSI-<AREA>-<NNN>")
+            if rule_id in self._rules:
+                raise ValueError(f"rule {rule_id} is registered twice")
+            explanation = inspect.cleandoc(check.__doc__ or "")
+            if not explanation:
+                raise ValueError(f"rule {rule_id} has no docstring to explain it")
+            self._rules[rule_id] = Rule(rule_id, title, explanation, check, tuple(sources))
+            return check
+
+        return register
+
+    def __iter__(self) -> Iterator[Rule]:
+        return iter(self._rules.values())
+
+    def get(self, rule_id: str) -> Rule | None:
+        return self._rules.get(rule_id.upper())
+
+
+def evaluate(
+    snapshot: Snapshot, rules: Iterable[Rule], context: RuleContext | None = None
+) -> tuple[Finding, ...]:
+    """Run every rule in order and collect their findings.
+
+    A rule may only report findings under its own code. A rule that raises is a bug in
+    this tool; it becomes an ``unknown`` finding for that check instead of hiding every
+    other result, and never a pass.
+    """
+    context = context or RuleContext()
+    findings: list[Finding] = []
+    for rule in rules:
+        try:
+            produced = list(rule.check(snapshot, context))
+        except Exception as exc:  # report the broken check, keep the others
+            findings.append(_internal_error(rule, exc))
+            continue
+        for finding in produced:
+            if finding.id != rule.id:
+                raise ValueError(f"rule {rule.id} reported a finding as {finding.id}")
+        findings.extend(produced)
+    return tuple(findings)
+
+
+def _internal_error(rule: Rule, exc: Exception) -> Finding:
+    return Finding(
+        id=rule.id,
+        status=Status.UNKNOWN,
+        severity=Severity.LOW,
+        confidence=Confidence.LOW,
+        summary=f"{rule.title}: not checked because of an internal error",
+        reason=f"The check failed with {type(exc).__name__}: {exc}",
+        recommended_action="Please report this as a bug, with the redacted report.",
+    )

gpu_stack_inspector/rules/host.py

@@ -0,0 +1,259 @@
+"""Checks of the machine: driver tools, the driver, the GPUs, variables that hide GPUs."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from gpu_stack_inspector.models import (
+    Confidence,
+    Evidence,
+    Finding,
+    Severity,
+    Snapshot,
+    Status,
+)
+from gpu_stack_inspector.rules.catalog import RULES
+from gpu_stack_inspector.rules.engine import RuleContext
+
+SMI = "nvidia-smi"
+
+
+@RULES.rule("GSI-SYS-001", "NVIDIA driver tools are installed")
+def driver_tools(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether nvidia-smi, which every NVIDIA driver installs, is on PATH.
+
+    Without it the driver and the GPUs cannot be checked. Its absence usually means no
+    NVIDIA driver is installed, but some container images run GPU workloads without it,
+    so this is a warning, not a failure. When CUDA is not expected it does not apply.
+    """
+    nvidia = snapshot.nvidia
+    if nvidia is None:
+        return
+    if nvidia.smi_path is not None:
+        yield Finding(
+            id="GSI-SYS-001",
+            status=Status.PASS,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary="nvidia-smi is installed",
+            reason=f"Found at {nvidia.smi_path}.",
+            evidence=(Evidence(SMI, "path", nvidia.smi_path),),
+        )
+        return
+    yield Finding(
+        id="GSI-SYS-001",
+        status=Status.WARN if context.cuda_expected else Status.NOT_APPLICABLE,
+        severity=Severity.MEDIUM if context.cuda_expected else Severity.INFO,
+        confidence=Confidence.MEDIUM,
+        summary="nvidia-smi was not found",
+        reason=(
+            "Without NVIDIA's driver tools on PATH the driver and the GPUs cannot be "
+            "checked. A system with a working NVIDIA driver normally provides nvidia-smi."
+        ),
+        evidence=(Evidence(SMI, "path", None),),
+        recommended_action=(
+            "Install the NVIDIA driver for your distribution (it provides nvidia-smi), or "
+            "add its directory to PATH. On WSL it is in /usr/lib/wsl/lib."
+        ),
+        verify_command="nvidia-smi",
+        limitations=("Some container images run GPU workloads without nvidia-smi installed.",),
+    )
+
+
+@RULES.rule("GSI-SYS-002", "The NVIDIA driver answers")
+def driver_answers(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether nvidia-smi could talk to the NVIDIA driver and report its version.
+
+    nvidia-smi installed but failing usually means the kernel module is not loaded: after
+    a driver update without a reboot, when Secure Boot blocks an unsigned module, or when
+    the installed module does not match the running kernel. The failure message from
+    nvidia-smi is quoted as evidence.
+    """
+    nvidia = snapshot.nvidia
+    if nvidia is None:
+        return
+    if nvidia.smi_path is None:
+        yield Finding(
+            id="GSI-SYS-002",
+            status=Status.NOT_TESTED,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary="The NVIDIA driver was not checked",
+            reason="nvidia-smi is not installed, so there was nothing to ask.",
+            caused_by="GSI-SYS-001",
+        )
+        return
+    if nvidia.driver_version is None:
+        details = [f.detail for f in nvidia.failures] or ["nvidia-smi reported no driver version"]
+        yield Finding(
+            id="GSI-SYS-002",
+            status=Status.FAIL if context.cuda_expected else Status.WARN,
+            severity=Severity.HIGH,
+            confidence=Confidence.HIGH,
+            summary="nvidia-smi could not get an answer from the NVIDIA driver",
+            reason=details[0],
+            evidence=tuple(Evidence(SMI, "failure", detail) for detail in details),
+            recommended_action=(
+                "Check that the NVIDIA kernel module is loaded (`lsmod | grep nvidia`). After "
+                "a driver update a reboot is usually needed; with Secure Boot an unsigned "
+                "module is refused."
+            ),
+            verify_command="nvidia-smi",
+        )
+        return
+    max_cuda = nvidia.max_cuda_version or "an unknown version"
+    yield Finding(
+        id="GSI-SYS-002",
+        status=Status.PASS,
+        severity=Severity.INFO,
+        confidence=Confidence.HIGH,
+        summary=f"NVIDIA driver {nvidia.driver_version} answers",
+        reason=(
+            f"The driver supports CUDA up to {max_cuda}. That is the newest CUDA it can run, "
+            "not a CUDA toolkit installed on this machine."
+        ),
+        evidence=(
+            Evidence(SMI, "driver_version", nvidia.driver_version),
+            Evidence(SMI, "max_cuda_version", nvidia.max_cuda_version),
+        ),
+        limitations=tuple(f.detail for f in nvidia.failures),
+    )
+
+
+@RULES.rule("GSI-SYS-003", "The driver sees a GPU")
+def gpu_visible(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether the NVIDIA driver lists at least one GPU.
+
+    A driver that answers but lists no GPU means the GPU is switched off or not handed to
+    this system: a hybrid-graphics mode that powers the NVIDIA GPU down, a BIOS setting,
+    or a VM or container the GPU was not passed through to.
+    """
+    nvidia = snapshot.nvidia
+    if nvidia is None:
+        return
+    if nvidia.smi_path is None or nvidia.driver_version is None:
+        cause = "GSI-SYS-001" if nvidia.smi_path is None else "GSI-SYS-002"
+        yield Finding(
+            id="GSI-SYS-003",
+            status=Status.NOT_TESTED,
+            severity=Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary="GPUs were not listed",
+            reason="The driver could not be asked.",
+            caused_by=cause,
+        )
+        return
+    if not nvidia.gpus:
+        yield Finding(
+            id="GSI-SYS-003",
+            status=Status.FAIL if context.cuda_expected else Status.NOT_APPLICABLE,
+            severity=Severity.HIGH if context.cuda_expected else Severity.INFO,
+            confidence=Confidence.HIGH,
+            summary="The NVIDIA driver answers but sees no GPU",
+            reason=f"Driver {nvidia.driver_version} reported no devices.",
+            evidence=(Evidence(SMI, "gpus", "0"),),
+            recommended_action=(
+                "Make sure the NVIDIA GPU is switched on for this system: the hybrid-graphics "
+                "mode (for example prime-select or supergfxctl), the BIOS setting, or the "
+                "GPU passthrough of your VM or container."
+            ),
+            verify_command="nvidia-smi -L",
+        )
+        return
+    names = ", ".join(
+        f"{gpu.name} (compute {gpu.compute_capability or 'unknown'})" for gpu in nvidia.gpus
+    )
+    count = len(nvidia.gpus)
+    yield Finding(
+        id="GSI-SYS-003",
+        status=Status.PASS,
+        severity=Severity.INFO,
+        confidence=Confidence.HIGH,
+        summary=f"{count} GPU{'s' if count != 1 else ''} visible to the driver",
+        reason=names,
+        evidence=tuple(Evidence(SMI, f"gpu{gpu.index}", gpu.name) for gpu in nvidia.gpus),
+    )
+
+
+_HIDE_ALL = frozenset({"", "-1", "none", "nodevfiles"})
+
+
+@RULES.rule(
+    "GSI-ENV-003",
+    "CUDA_VISIBLE_DEVICES does not hide the GPUs",
+    sources=("https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars",),
+)
+def visible_devices(snapshot: Snapshot, context: RuleContext) -> Iterator[Finding]:
+    """Whether CUDA_VISIBLE_DEVICES, when set, leaves the GPUs visible to CUDA programs.
+
+    An empty value (or -1) hides every GPU from CUDA while nvidia-smi still lists it,
+    which looks exactly like a broken install. CUDA also stops at the first index that
+    does not exist: with "1,0" on a one-GPU machine no GPU is visible at all. GPU UUIDs
+    are not checked, because the tool does not collect them.
+    """
+    variable = next(
+        (v for v in snapshot.cuda_environment if v.name == "CUDA_VISIBLE_DEVICES"), None
+    )
+    if variable is None:
+        return
+    value = variable.value.strip()
+    evidence = (Evidence("environment", "CUDA_VISIBLE_DEVICES", variable.value),)
+    shown = repr(variable.value)
+    if value.lower() in _HIDE_ALL:
+        yield Finding(
+            id="GSI-ENV-003",
+            status=Status.WARN,
+            severity=Severity.HIGH,
+            confidence=Confidence.HIGH,
+            summary="CUDA_VISIBLE_DEVICES hides every GPU",
+            reason=(
+                f"It is set to {shown}, so CUDA programs started from this environment see no "
+                "GPU, even though nvidia-smi lists them."
+            ),
+            evidence=evidence,
+            recommended_action=(
+                "Unset CUDA_VISIBLE_DEVICES, or set it to the GPU index to use (e.g. 0)."
+            ),
+            verify_command='echo "$CUDA_VISIBLE_DEVICES"',
+        )
+        return
+    entries = [entry.strip() for entry in value.split(",")]
+    count = len(snapshot.nvidia.gpus) if snapshot.nvidia else 0
+    if count and all(entry.isdigit() for entry in entries):
+        invalid = next((int(e) for e in entries if int(e) >= count), None)
+        if invalid is not None:
+            usable = entries[: entries.index(str(invalid))]
+            yield Finding(
+                id="GSI-ENV-003",
+                status=Status.WARN,
+                severity=Severity.HIGH if not usable else Severity.MEDIUM,
+                confidence=Confidence.HIGH,
+                summary=f"CUDA_VISIBLE_DEVICES names GPU {invalid}, which does not exist",
+                reason=(
+                    f"It is set to {shown}, but the driver lists {count} GPU(s). CUDA stops at "
+                    "the first index that does not exist, so "
+                    + (
+                        f"only GPU(s) {', '.join(usable)} are visible."
+                        if usable
+                        else "no GPU is visible."
+                    )
+                ),
+                evidence=evidence,
+                recommended_action="Use indices from `nvidia-smi -L`, or unset the variable.",
+                verify_command="nvidia-smi -L",
+            )
+            return
+    yield Finding(
+        id="GSI-ENV-003",
+        status=Status.PASS,
+        severity=Severity.INFO,
+        confidence=Confidence.MEDIUM if not all(e.isdigit() for e in entries) else Confidence.HIGH,
+        summary=f"CUDA_VISIBLE_DEVICES limits CUDA to {value}",
+        reason="CUDA programs only see the GPUs listed there.",
+        evidence=evidence,
+        limitations=(
+            ("GPU UUIDs and MIG devices are not checked: the tool does not collect them.",)
+            if not all(e.isdigit() for e in entries)
+            else ()
+        ),
+    )