gpu-stack-inspector 0.1.0__py3-none-any.whl

gpu_stack_inspector/collectors/toolkit.py

@@ -0,0 +1,160 @@
+"""CUDA toolkits on disk and the nvcc on PATH."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import shutil
+from collections.abc import Callable, Mapping, Sequence
+from pathlib import Path
+
+from gpu_stack_inspector.models import CudaToolkit, FailureKind, ProbeFailure, ToolkitFacts
+from gpu_stack_inspector.runner import CommandRunner, probe_failure
+
+TIMEOUT = 8.0
+DEFAULT_ROOTS: tuple[Path, ...] = () if os.name == "nt" else (Path("/usr/local"), Path("/opt"))
+"""Where toolkits are installed: /usr/local/cuda-12.4 (NVIDIA's installer), /opt/cuda (Arch)."""
+
+_MAX_FILE_BYTES = 64 * 1024
+_PROBE = "cuda-toolkits"
+_ENV_DIRS = ("CUDA_HOME", "CUDA_PATH", "CUDA_ROOT")
+"""Variables naming a toolkit; Windows also sets one CUDA_PATH_V<major>_<minor> per version."""
+_TOOLKIT_DIR = re.compile(r"cuda(?:-[0-9]+(?:\.[0-9]+)*)?")
+_VERSION_TXT = re.compile(r"CUDA Version\s+(?P<version>[0-9]+(?:\.[0-9]+)+)")
+_CUDA_H = re.compile(r"^\s*#\s*define\s+CUDA_VERSION\s+(?P<number>[0-9]{4,6})\b", re.MULTILINE)
+_ENV = {"LC_ALL": "C"}
+_NVCC_RELEASE = re.compile(
+    r"release\s+(?P<release>[0-9]+(?:\.[0-9]+)*),\s*V(?P<full>[0-9]+(?:\.[0-9]+)*)"
+)
+
+
+def collect_toolkits(
+    runner: CommandRunner,
+    *,
+    which: Callable[[str], str | None] = shutil.which,
+    roots: Sequence[Path] = DEFAULT_ROOTS,
+    environ: Mapping[str, str] = os.environ,
+    timeout: float = TIMEOUT,
+) -> ToolkitFacts:
+    """Find CUDA toolkit directories and the nvcc on PATH.
+
+    Directories are only listed and small version files read; the only command run is
+    ``nvcc --version``.
+    """
+    toolkits, dir_failures = find_toolkits(roots, environ)
+    nvcc_facts = _collect_nvcc(runner, which, timeout)
+    return ToolkitFacts(
+        nvcc_path=nvcc_facts.nvcc_path,
+        nvcc_version=nvcc_facts.nvcc_version,
+        toolkits=tuple(toolkits),
+        failures=(*dir_failures, *nvcc_facts.failures),
+    )
+
+
+def find_toolkits(
+    roots: Sequence[Path], environ: Mapping[str, str] | None = None
+) -> tuple[list[CudaToolkit], list[ProbeFailure]]:
+    """Toolkit directories under each root and named by CUDA_HOME/CUDA_PATH/….
+
+    Each directory is reported once under its real path; symlinks to it, such as
+    ``/usr/local/cuda``, are listed in ``linked_from``. A variable that names a missing
+    directory is a failure worth reporting: tools that read it will not find CUDA.
+    """
+    candidates: list[Path] = []
+    for root in roots:
+        try:
+            entries = sorted(root.iterdir())
+        except OSError:  # a missing or unreadable root is normal
+            continue
+        candidates.extend(entry for entry in entries if _TOOLKIT_DIR.fullmatch(entry.name))
+
+    failures: list[ProbeFailure] = []
+    env = environ or {}
+    names = [*_ENV_DIRS, *sorted(n for n in env if n.startswith("CUDA_PATH_V"))]
+    for name in names:
+        value = env.get(name, "")
+        if not value:
+            continue
+        if Path(value).is_dir():
+            candidates.append(Path(value))
+        else:
+            failures.append(
+                ProbeFailure(
+                    _PROBE, FailureKind.NOT_FOUND, f"{name} names {value}, which is not a directory"
+                )
+            )
+
+    aliases: dict[Path, list[str]] = {}
+    for candidate in candidates:
+        try:
+            target = candidate.resolve(strict=True)
+        except OSError:  # dangling symlink
+            continue
+        if not target.is_dir():
+            continue
+        links = aliases.setdefault(target, [])
+        if candidate.is_symlink() and str(candidate) not in links:
+            links.append(str(candidate))
+
+    toolkits = []
+    for target, links in aliases.items():
+        version, source = read_toolkit_version(target)
+        toolkits.append(CudaToolkit(str(target), version, source, tuple(links)))
+    return toolkits, failures
+
+
+def read_toolkit_version(directory: Path) -> tuple[str | None, str | None]:
+    """The toolkit's version and the file it came from, or ``(None, None)``."""
+    text = _read_small(directory / "version.json")
+    if text is not None:
+        try:
+            version = json.loads(text)["cuda"]["version"]
+        except (ValueError, KeyError, TypeError):
+            version = None
+        if isinstance(version, str) and version:
+            return version, "version.json"
+    text = _read_small(directory / "version.txt")  # CUDA 10.x and older
+    match = _VERSION_TXT.search(text) if text is not None else None
+    if match:
+        return match["version"], "version.txt"
+    text = _read_small(directory / "include" / "cuda.h")
+    match = _CUDA_H.search(text) if text is not None else None
+    if match:
+        # CUDA_VERSION is 1000 * major + 10 * minor: 12040 is 12.4.
+        number = int(match["number"])
+        return f"{number // 1000}.{number % 1000 // 10}", "include/cuda.h"
+    return None, None
+
+
+def _read_small(path: Path) -> str | None:
+    try:
+        with path.open("rb") as handle:
+            return handle.read(_MAX_FILE_BYTES).decode("utf-8", errors="replace")
+    except OSError:
+        return None
+
+
+def _collect_nvcc(
+    runner: CommandRunner, which: Callable[[str], str | None], timeout: float
+) -> ToolkitFacts:
+    nvcc = which("nvcc")
+    if nvcc is None:
+        return ToolkitFacts()
+    result = runner.run([nvcc, "--version"], timeout=timeout, env_overrides=_ENV)
+    failure = probe_failure("nvcc", result)
+    if failure is not None:
+        return ToolkitFacts(nvcc_path=nvcc, failures=(failure,))
+    version = parse_nvcc_version(result.stdout)
+    failures: tuple[ProbeFailure, ...] = ()
+    if version is None:
+        failures = (
+            ProbeFailure("nvcc", FailureKind.MALFORMED_OUTPUT, "nvcc --version printed no release"),
+        )
+    return ToolkitFacts(nvcc_path=nvcc, nvcc_version=version, failures=failures)
+
+
+def parse_nvcc_version(stdout: str) -> str | None:
+    """``Cuda compilation tools, release 12.4, V12.4.131`` → ``12.4.131``."""
+    match = _NVCC_RELEASE.search(stdout)
+    return match["full"] if match else None

gpu_stack_inspector/compatibility/__init__.py

@@ -0,0 +1 @@
+"""Versioned, source-tagged compatibility data and its loader."""

gpu_stack_inspector/compatibility/data.json

@@ -0,0 +1,24 @@
+{
+  "schema": 1,
+  "retrieved": "2026-11-03",
+  "sources": {
+    "nvidia-cuda-compatibility": {
+      "title": "NVIDIA: CUDA Compatibility",
+      "url": "https://docs.nvidia.com/deploy/cuda-compatibility/latest/"
+    },
+    "nvidia-minor-version-compatibility": {
+      "title": "NVIDIA: CUDA Compatibility, Minor Version Compatibility",
+      "url": "https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html"
+    }
+  },
+  "minor_version_compatibility": {
+    "from_cuda_major": 11,
+    "source": "nvidia-minor-version-compatibility",
+    "note": "Applications built within a CUDA major release family run on a driver of that family's minimum or newer, with a limited feature set; code compiled to PTX does not work on older drivers."
+  },
+  "minimum_driver": [
+    {"cuda_major": 11, "driver": "450", "source": "nvidia-minor-version-compatibility"},
+    {"cuda_major": 12, "driver": "525", "source": "nvidia-minor-version-compatibility"},
+    {"cuda_major": 13, "driver": "580", "source": "nvidia-minor-version-compatibility"}
+  ]
+}

gpu_stack_inspector/compatibility/loader.py

@@ -0,0 +1,75 @@
+"""Loads data.json and refuses data that is not tied to a source."""
+
+from __future__ import annotations
+
+import functools
+import json
+import re
+from dataclasses import dataclass
+from importlib.resources import files
+from typing import Any
+
+
+@dataclass(frozen=True, slots=True)
+class Source:
+    id: str
+    title: str
+    url: str
+
+
+@dataclass(frozen=True, slots=True)
+class CompatibilityData:
+    """CUDA compatibility facts, each tied to the document it was taken from."""
+
+    retrieved: str
+    sources: dict[str, Source]
+    minor_version_from_major: int
+    minor_version_source: Source
+    minor_version_note: str
+    minimum_driver: dict[int, tuple[str, Source]]
+    """CUDA major version → (minimum driver version, source)."""
+
+
+class CompatibilityDataError(ValueError):
+    pass
+
+
+@functools.cache
+def load() -> CompatibilityData:
+    raw = files("gpu_stack_inspector.compatibility").joinpath("data.json").read_text("utf-8")
+    return parse(json.loads(raw))
+
+
+def parse(data: dict[str, Any]) -> CompatibilityData:
+    if data.get("schema") != 1:
+        raise CompatibilityDataError(f"unsupported schema {data.get('schema')!r}")
+    retrieved = data.get("retrieved", "")
+    if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(retrieved)):
+        raise CompatibilityDataError("retrieved must be a YYYY-MM-DD date")
+    sources = {
+        key: Source(key, value["title"], value["url"]) for key, value in data["sources"].items()
+    }
+    for source in sources.values():
+        if not source.url.startswith("https://"):
+            raise CompatibilityDataError(f"source {source.id} has no https URL")
+
+    def source_of(entry: dict[str, Any]) -> Source:
+        key = entry.get("source")
+        if key not in sources:
+            raise CompatibilityDataError(f"unknown source {key!r} in {entry}")
+        return sources[key]
+
+    minor = data["minor_version_compatibility"]
+    minimum: dict[int, tuple[str, Source]] = {}
+    for row in data["minimum_driver"]:
+        if not re.fullmatch(r"\d+(\.\d+)*", str(row["driver"])):
+            raise CompatibilityDataError(f"driver version {row['driver']!r} is not a version")
+        minimum[int(row["cuda_major"])] = (str(row["driver"]), source_of(row))
+    return CompatibilityData(
+        retrieved=retrieved,
+        sources=sources,
+        minor_version_from_major=int(minor["from_cuda_major"]),
+        minor_version_source=source_of(minor),
+        minor_version_note=str(minor["note"]),
+        minimum_driver=minimum,
+    )

gpu_stack_inspector/compatibility/versions.py

@@ -0,0 +1,25 @@
+"""Comparing dotted version numbers as reported by drivers, nvcc and wheels."""
+
+from __future__ import annotations
+
+import re
+
+_LEADING_VERSION = re.compile(r"[0-9]+(?:\.[0-9]+)*")
+
+
+def version_tuple(text: str | None) -> tuple[int, ...] | None:
+    """``12.8`` → ``(12, 8)``, ``595.99.02`` → ``(595, 99, 2)``; no leading number → None."""
+    if not text:
+        return None
+    match = _LEADING_VERSION.match(text.strip())
+    if match is None:
+        return None
+    return tuple(int(part) for part in match.group(0).split("."))
+
+
+def major_minor(text: str | None) -> tuple[int, int] | None:
+    """The CUDA-relevant part of a version: ``12.4.131`` → ``(12, 4)``, ``13`` → ``(13, 0)``."""
+    parts = version_tuple(text)
+    if parts is None:
+        return None
+    return parts[0], parts[1] if len(parts) > 1 else 0

gpu_stack_inspector/exit_codes.py

@@ -0,0 +1,22 @@
+"""Exit codes are a contract: scripts and CI jobs depend on them."""
+
+from __future__ import annotations
+
+from enum import IntEnum
+
+
+class ExitCode(IntEnum):
+    OK = 0
+    """The inspection ran. Findings, even failing ones, do not change this."""
+
+    STRICT_FAILURE = 1
+    """``--strict`` was given and a finding reached the configured fail level."""
+
+    ERROR = 2
+    """Wrong usage, or the tool itself failed."""
+
+    CONTEXT_UNAVAILABLE = 3
+    """Something the command needs is missing, e.g. an interpreter that was named."""
+
+    INTERRUPTED = 130
+    """Stopped with Ctrl+C (128 + SIGINT, as shells report it)."""

gpu_stack_inspector/models.py

@@ -0,0 +1,349 @@
+"""Typed facts and findings shared by collectors, rules and renderers."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from enum import StrEnum
+
+SCHEMA_VERSION = 1
+"""Version of the snapshot/JSON report layout; bumped only for incompatible changes."""
+
+FINDING_ID = re.compile(r"GSI-[A-Z]+-[0-9]{3}")
+"""Finding codes are part of the public contract: ``GSI-<AREA>-<NNN>``, e.g. ``GSI-TORCH-002``."""
+
+
+class Status(StrEnum):
+    """Outcome of one check.
+
+    Only ``PASS`` means that a check ran and succeeded. A check that could not run is
+    ``UNKNOWN``; one that was deliberately skipped (an opt-in probe nobody asked for) is
+    ``NOT_TESTED``; one that does not apply to this machine is ``NOT_APPLICABLE``.
+    """
+
+    PASS = "pass"  # noqa: S105 - a status name, not a password
+    WARN = "warn"
+    FAIL = "fail"
+    UNKNOWN = "unknown"
+    NOT_APPLICABLE = "not_applicable"
+    NOT_TESTED = "not_tested"
+
+
+class Severity(StrEnum):
+    """How much a finding matters to the user's goal, independent of certainty."""
+
+    INFO = "info"
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+
+
+class Confidence(StrEnum):
+    """How well the evidence supports a finding."""
+
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+
+
+@dataclass(frozen=True, slots=True)
+class Evidence:
+    """One observed fact that a finding relies on.
+
+    ``source`` names where the fact came from (``nvidia-smi``, ``python-metadata``…),
+    ``key`` what it is (``driver_version``…). ``value`` is ``None`` when the fact is
+    missing, which is itself evidence.
+    """
+
+    source: str
+    key: str
+    value: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class Finding:
+    """A diagnosis with everything a user needs to judge and verify it.
+
+    ``subject`` says what the finding is about: ``host`` or the label of an inspected
+    environment. ``verify_command`` is a command the user can run to check the result
+    after acting on ``recommended_action``. ``caused_by`` names the finding this one
+    follows from (``GSI-SYS-001`` for "driver not checked because nvidia-smi is missing"),
+    so reports can show one root cause instead of a cascade.
+    """
+
+    id: str
+    status: Status
+    severity: Severity
+    confidence: Confidence
+    summary: str
+    reason: str
+    subject: str = "host"
+    evidence: tuple[Evidence, ...] = ()
+    recommended_action: str | None = None
+    verify_command: str | None = None
+    limitations: tuple[str, ...] = ()
+    caused_by: str | None = None
+
+    def __post_init__(self) -> None:
+        if not FINDING_ID.fullmatch(self.id):
+            raise ValueError(f"finding id {self.id!r} does not match GSI-<AREA>-<NNN>")
+        if self.caused_by is not None and not FINDING_ID.fullmatch(self.caused_by):
+            raise ValueError(f"caused_by {self.caused_by!r} does not match GSI-<AREA>-<NNN>")
+        if not self.summary.strip():
+            raise ValueError(f"finding {self.id} has an empty summary")
+
+
+class FailureKind(StrEnum):
+    """Why a probe produced no usable facts."""
+
+    NOT_FOUND = "not_found"
+    PERMISSION_DENIED = "permission_denied"
+    TIMEOUT = "timeout"
+    NONZERO_EXIT = "nonzero_exit"
+    MALFORMED_OUTPUT = "malformed_output"
+    OS_ERROR = "os_error"
+    PROBE_ERROR = "probe_error"
+    """The probe ran, but reported that part of its own work failed."""
+
+
+@dataclass(frozen=True, slots=True)
+class ProbeFailure:
+    """A probe that could not deliver its facts.
+
+    Collectors return these instead of raising, so one broken tool never hides the
+    results of the others, and rules can report ``unknown`` with the reason.
+    """
+
+    probe: str
+    kind: FailureKind
+    detail: str
+
+
+def utc_timestamp() -> str:
+    """Current time as an ISO 8601 UTC string with second precision."""
+    return datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+@dataclass(frozen=True, slots=True)
+class RedactionCount:
+    """How many values of one kind (``home_path``, ``secret_value``…) were redacted."""
+
+    kind: str
+    count: int
+
+
+@dataclass(frozen=True, slots=True)
+class Snapshot:
+    """Everything one run observed and concluded; the input of every renderer.
+
+    ``probe_failures`` collects the failures of every collector in one place, so a report
+    can list what could not be checked.
+    """
+
+    tool_version: str
+    created_at: str
+    schema_version: int = SCHEMA_VERSION
+    host: HostFacts | None = None
+    nvidia: NvidiaFacts | None = None
+    toolkit: ToolkitFacts | None = None
+    cuda_environment: tuple[EnvironmentVariable, ...] = ()
+    interpreters: tuple[InterpreterFacts, ...] = ()
+    comparison: tuple[FieldComparison, ...] = ()
+    """Field-by-field differences between the interpreters, when they were compared."""
+    probe_failures: tuple[ProbeFailure, ...] = ()
+    findings: tuple[Finding, ...] = ()
+    redactions: tuple[RedactionCount, ...] = ()
+    """What redaction removed before this snapshot was output; empty if nothing was."""
+
+
+@dataclass(frozen=True, slots=True)
+class HostFacts:
+    """The operating system the inspection ran on."""
+
+    system: str
+    """``platform.system()``: ``Linux``, ``Windows``, ``Darwin``…"""
+    machine: str
+    """CPU architecture as the OS reports it: ``x86_64``, ``AMD64``, ``aarch64``…"""
+    kernel: str | None
+    distribution: str | None = None
+    """``PRETTY_NAME`` from os-release on Linux, e.g. ``Ubuntu 26.04.1 LTS``."""
+    wsl: bool = False
+    """Running inside Windows Subsystem for Linux (a fact only; WSL is not validated yet)."""
+
+
+@dataclass(frozen=True, slots=True)
+class GpuDevice:
+    """One GPU as nvidia-smi lists it. Values it reports as N/A are ``None``."""
+
+    index: int
+    name: str
+    memory_total_mib: int | None = None
+    compute_capability: str | None = None
+    """``major.minor``, e.g. ``8.6``; decides which CUDA kernels a wheel can run."""
+
+
+@dataclass(frozen=True, slots=True)
+class NvidiaFacts:
+    """What nvidia-smi reported.
+
+    ``smi_path`` is ``None`` when nvidia-smi is not installed (or not on ``PATH``), which
+    is a fact, not a failure. ``failures`` say why other fields are missing.
+    """
+
+    smi_path: str | None
+    driver_version: str | None = None
+    max_cuda_version: str | None = None
+    """The newest CUDA version the driver supports (nvidia-smi's "CUDA Version").
+
+    It is not the version of any installed CUDA toolkit or of PyTorch's CUDA runtime.
+    """
+    gpus: tuple[GpuDevice, ...] = ()
+    failures: tuple[ProbeFailure, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class CudaToolkit:
+    """A CUDA toolkit directory found on disk.
+
+    ``linked_from`` lists symlinks that point at it, e.g. ``/usr/local/cuda`` for the
+    default toolkit. ``version_source`` names the file the version was read from.
+    """
+
+    path: str
+    version: str | None = None
+    version_source: str | None = None
+    linked_from: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class ToolkitFacts:
+    """Locally installed CUDA toolkits and the ``nvcc`` on ``PATH``.
+
+    Prebuilt PyTorch wheels bring their own CUDA runtime, so a missing toolkit is not a
+    problem by itself; it matters for compiling extensions.
+    """
+
+    nvcc_path: str | None = None
+    nvcc_version: str | None = None
+    toolkits: tuple[CudaToolkit, ...] = ()
+    failures: tuple[ProbeFailure, ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class EnvironmentVariable:
+    """An allowlisted variable that was set. An empty value is kept: for
+    ``CUDA_VISIBLE_DEVICES`` it means "no GPU is visible"."""
+
+    name: str
+    value: str
+
+
+@dataclass(frozen=True, slots=True)
+class Distribution:
+    """An installed distribution relevant to GPU use, as the target interpreter sees it.
+
+    ``location`` is the directory it was found in; the same name in two locations (a
+    user site and the environment, say) is how shadowed installs show up.
+    """
+
+    name: str
+    version: str
+    location: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class TorchFacts:
+    """Static evidence about the torch an interpreter would import. Nothing was imported.
+
+    ``built_cuda``/``built_hip`` come from torch/version.py; they are only meaningful when
+    ``version_py_found`` is true, since ``None`` there means "built without CUDA".
+    ``conda_records`` is ``None`` outside conda environments.
+    """
+
+    version: str
+    location: str | None = None
+    requires_nvidia: tuple[str, ...] = ()
+    cuda_library: bool | None = None
+    hip_library: bool | None = None
+    version_py_found: bool = False
+    built_cuda: str | None = None
+    built_hip: str | None = None
+    conda_records: tuple[str, ...] | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RuntimeStep:
+    """One step of the runtime probe: import torch, cuda available, devices, tensor operation."""
+
+    step: str
+    ok: bool
+    seconds: float | None = None
+    error: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RuntimeFacts:
+    """What happened when torch was imported and CUDA used (only with --runtime).
+
+    ``failure`` is set when the interpreter itself crashed, hung or printed no result; the
+    steps then end with the one it was in.
+    """
+
+    torch_version: str | None = None
+    cuda_version: str | None = None
+    """torch.version.cuda at runtime: the CUDA version the build targets."""
+    cuda_available: bool | None = None
+    devices: tuple[GpuDevice, ...] = ()
+    tensor_ok: bool | None = None
+    steps: tuple[RuntimeStep, ...] = ()
+    failure: ProbeFailure | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class InterpreterFacts:
+    """What one target Python interpreter reported about itself.
+
+    ``path`` is the interpreter as the user named it, made absolute but with symlinks
+    kept: a virtual environment's ``bin/python`` is a symlink, and following it would
+    start the base interpreter outside the environment.
+    """
+
+    label: str
+    path: str
+    executable: str | None = None
+    version: str | None = None
+    implementation: str | None = None
+    prefix: str | None = None
+    base_prefix: str | None = None
+    conda: bool | None = None
+    platform: str | None = None
+    user_site: bool | None = None
+    pythonpath_set: bool | None = None
+    distributions: tuple[Distribution, ...] = ()
+    torch: TorchFacts | None = None
+    runtime: RuntimeFacts | None = None
+    """None unless the runtime probe was run (``probe --runtime``)."""
+    failures: tuple[ProbeFailure, ...] = ()
+
+    @property
+    def virtualenv(self) -> bool | None:
+        """Whether the interpreter runs inside a venv/virtualenv, if that is known."""
+        if self.prefix is None or self.base_prefix is None:
+            return None
+        return self.prefix != self.base_prefix
+
+
+@dataclass(frozen=True, slots=True)
+class FieldComparison:
+    """One field across the compared interpreters, in their order.
+
+    ``None`` means the interpreter has no value for it (not installed, not inspected).
+    """
+
+    field: str
+    values: tuple[str | None, ...]
+
+    @property
+    def differs(self) -> bool:
+        return len(set(self.values)) > 1