gpucheck-cli 0.1.0__py3-none-any.whl

gpucheck/fakes.py

@@ -0,0 +1,140 @@
+"""Scripted fake GPUs for tests and the CLI --fake flag.
+
+Each profile is a generator of TelemetrySample. After the scripted phase the
+generator keeps yielding the final steady state, so it can be sampled forever.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+
+from .adapter import GpuInventory, TelemetryAdapter, TelemetrySample, hash_uuid
+
+MODEL = "NVIDIA H100 80GB HBM3"
+FAKE_UUID = "GPU-00000000-0000-0000-0000-000000000000"
+
+HEALTHY_TFLOPS = 950.0
+DEGRADED_TFLOPS = 600.0
+
+
+def _inventory(**overrides) -> GpuInventory:
+    base = dict(
+        model_name=MODEL,
+        uuid_hash=hash_uuid(FAKE_UUID),
+        driver="550.54.15",
+        cuda="12.4",
+        ecc_enabled=True,
+        power_limit_w=700.0,
+        default_power_limit_w=700.0,
+        mig_enabled=False,
+        fractional=False,
+        retired_pages=0,
+        remapped_rows=0,
+    )
+    base.update(overrides)
+    return GpuInventory(**base)
+
+
+def _sample(t: float, temp: float, **kw) -> TelemetrySample:
+    fields = dict(
+        t_s=t,
+        temp_c=temp,
+        power_w=650.0,
+        sm_clock_mhz=1980.0,
+        fan_pct=None,
+        throttle_reasons=frozenset(),
+        ecc_volatile=0,
+        achieved_tflops=HEALTHY_TFLOPS,
+    )
+    fields.update(kw)
+    return TelemetrySample(**fields)
+
+
+def _wobble(i: int) -> float:
+    return (0.0, 0.5, 1.0, 0.5)[i % 4] - 0.5  # deterministic, in [-0.5, 0.5]
+
+
+IDLE_SAMPLES = 61  # preflight sample + the longest (60 s) baseline
+
+
+def _healthy() -> Iterator[TelemetrySample]:
+    i = 0
+    while True:
+        if i < IDLE_SAMPLES:  # cool, low-power idle through preflight and baseline
+            yield _sample(float(i), 35.0 + _wobble(i), power_w=80.0, sm_clock_mhz=210.0)
+        else:
+            warm = min(i - IDLE_SAMPLES, 20) / 20  # ramp 35 -> 62 C over 20 s, then steady
+            yield _sample(float(i), 35.0 + 27.0 * warm + _wobble(i))
+        i += 1
+
+
+def _hot_env() -> Iterator[TelemetrySample]:
+    i = 0
+    while True:
+        yield _sample(float(i), 55.0 + _wobble(i), power_w=80.0, sm_clock_mhz=210.0)
+        i += 1
+
+
+def _tim_degraded() -> Iterator[TelemetrySample]:
+    i = 0
+    while True:
+        temp = min(87.0, 35.0 + i * 0.52)  # 87 C at sample 100
+        throttled = temp > 83.0
+        if temp <= 80.0:
+            tflops = HEALTHY_TFLOPS
+        else:
+            tflops = HEALTHY_TFLOPS - (HEALTHY_TFLOPS - DEGRADED_TFLOPS) * (temp - 80.0) / 7.0
+        yield _sample(
+            float(i),
+            temp,
+            sm_clock_mhz=1980.0 if not throttled else 1500.0,
+            throttle_reasons=frozenset({"thermal"}) if throttled else frozenset(),
+            achieved_tflops=tflops,
+        )
+        i += 1
+
+
+def _power_capped() -> Iterator[TelemetrySample]:
+    i = 0
+    while True:
+        yield _sample(
+            float(i),
+            55.0 + 8.0 * min(i, 20) / 20 + _wobble(i),
+            power_w=400.0,
+            sm_clock_mhz=1400.0,
+            throttle_reasons=frozenset({"power"}),
+            achieved_tflops=700.0,
+        )
+        i += 1
+
+
+_PROFILES: dict[str, tuple[Callable[[], Iterator[TelemetrySample]], dict]] = {
+    "healthy": (_healthy, {}),
+    "tim_degraded": (_tim_degraded, {}),
+    "hot_env": (_hot_env, {}),
+    "power_capped": (_power_capped, {"power_limit_w": 400.0, "default_power_limit_w": 700.0}),
+    "mig": (_healthy, {"mig_enabled": True}),
+}
+
+
+class FakeAdapter:
+    def __init__(self, inventory: GpuInventory, samples: Iterator[TelemetrySample]) -> None:
+        self._inventory = inventory
+        self._samples = samples
+
+    def inventory(self) -> GpuInventory:
+        return self._inventory
+
+    def sample(self) -> TelemetrySample:
+        return next(self._samples)
+
+    def busy(self) -> bool | None:
+        return False
+
+
+def make_fake(profile: str) -> TelemetryAdapter:
+    try:
+        factory, inv = _PROFILES[profile]
+    except KeyError:
+        raise ValueError(f"unknown fake profile {profile!r}; choose from {sorted(_PROFILES)}") from None
+    return FakeAdapter(_inventory(**inv), factory())

gpucheck/load.py

@@ -0,0 +1,126 @@
+"""Torch bf16 GEMM load generator. torch is imported lazily (optional extra)."""
+
+from __future__ import annotations
+
+import math
+import os
+import time
+
+MIN_N = 2048
+MAX_N = 16384
+MEM_FRACTION = 0.6
+BYTES_PER_ELEM = 2  # bf16
+TICK_S = 0.8
+
+
+class LoadUnavailable(Exception):
+    """torch or a CUDA device is not available."""
+
+
+def pick_gemm_size(free_mem_bytes: int) -> int:
+    """Largest n with 3 bf16 n*n matrices within 60% of free memory, clamped."""
+    budget = int(MEM_FRACTION * max(free_mem_bytes, 0))
+    n = math.isqrt(budget // (3 * BYTES_PER_ELEM))
+    return max(MIN_N, min(MAX_N, n))
+
+
+def apply_workload(base_n: int, offsets: list[int]) -> list[int]:
+    """Resolve prescribed gemm_n offsets against an auto-sized base N.
+
+    The base is first clamped into [MIN_N - min(offsets), MAX_N - max(offsets)]
+    so that base + offset never needs clamping: the server verifies pairwise
+    differences between the sizes, which clamping an individual value would break.
+    """
+    lo = MIN_N - min(offsets)
+    hi = MAX_N - max(offsets)
+    base = max(lo, min(hi, base_n))
+    return [base + o for o in offsets]
+
+
+def tflops(n: int, iters: int, elapsed_s: float) -> float:
+    return 2 * n**3 * iters / elapsed_s / 1e12
+
+
+class TorchGemmLoad:
+    def __init__(self, workload: dict | None = None) -> None:
+        # Make torch's cuda:0 the same physical card NVML index 0 reads; CUDA's
+        # default "fastest first" order can differ from NVML's PCI bus order.
+        # Must be set before torch initialises CUDA, so before the import.
+        os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID")
+        try:
+            import torch
+        except ImportError as e:
+            raise LoadUnavailable("torch is not installed") from e
+        try:
+            if not torch.cuda.is_available():
+                raise LoadUnavailable("CUDA is not available")
+        except RuntimeError as e:
+            raise LoadUnavailable(f"CUDA probe failed: {e}") from e
+        self._torch = torch
+        self._workload = dict(workload) if workload is not None else None
+        self._ns: list[int] = []
+        self._tick_i = 0
+        self.resolved_workload: dict | None = None
+        self._a = self._b = self._c = None
+
+    def start(self) -> None:
+        t = self._torch
+        # Memory sizing creates this process's CUDA context, so it must not
+        # happen in __init__: preflight reads the GPU's process list after the
+        # load is constructed, and a context of our own makes the card look busy.
+        try:
+            free, _total = t.cuda.mem_get_info()
+        except RuntimeError as e:
+            raise LoadUnavailable(f"CUDA probe failed: {e}") from e
+        n = pick_gemm_size(free)
+        self._ns = [n]
+        if self._workload is not None:
+            self._ns = apply_workload(n, list(self._workload["gemm_n_offsets"]))
+            self.resolved_workload = {
+                "gemm_n": list(self._ns),
+                "order_seed": self._workload["order_seed"],
+            }
+        # One flat buffer at the largest size; each tick views the first n*n
+        # elements as a contiguous n x n matrix (no per-size reallocation).
+        # Without a workload this is the plain single-size n x n path.
+        m = max(self._ns)
+        shape = (m * m,) if self.resolved_workload else (m, m)
+        self._a = t.randn(*shape, device="cuda", dtype=t.bfloat16)
+        self._b = t.randn(*shape, device="cuda", dtype=t.bfloat16)
+        self._c = t.empty(*shape, device="cuda", dtype=t.bfloat16)
+        a, b, c = self._views(self._ns[0])
+        t.matmul(a, b, out=c)  # warmup
+        t.cuda.synchronize()
+
+    def _views(self, n: int):
+        if not self.resolved_workload:
+            return self._a, self._b, self._c
+        k = n * n
+        return (
+            self._a[:k].view(n, n),
+            self._b[:k].view(n, n),
+            self._c[:k].view(n, n),
+        )
+
+    def tick(self) -> float | None:
+        t = self._torch
+        n = self._ns[self._tick_i % len(self._ns)]  # cycle prescribed sizes per tick
+        self._tick_i += 1
+        a, b, c = self._views(n)
+        iters = 0
+        t.cuda.synchronize()
+        t0 = time.perf_counter()
+        while time.perf_counter() - t0 < TICK_S:
+            t.matmul(a, b, out=c)
+            # Wait for each kernel: matmul only enqueues, so without this the loop
+            # would queue thousands of kernels in 0.8 s and the tick (and the
+            # watchdog sampling between ticks) would stall for minutes.
+            t.cuda.synchronize()
+            iters += 1
+        t.cuda.synchronize()
+        elapsed = time.perf_counter() - t0
+        return tflops(n, iters, elapsed) if iters else None
+
+    def stop(self) -> None:
+        self._a = self._b = self._c = None
+        self._torch.cuda.empty_cache()

gpucheck/metrics.py

@@ -0,0 +1,116 @@
+"""Derived metrics computed from a RawRun.
+
+Every metric whose inputs are all None is None and its metric-source field
+name is recorded in `telemetry_gaps`; nothing is guessed. Load-phase sample
+timestamps are absolute, so time_to_throttle_s is measured from the first
+load sample.
+"""
+
+from __future__ import annotations
+
+import statistics
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from .adapter import TelemetrySample
+from .run import RawRun
+from .specs import GpuSpec
+
+THERMAL_REASON = "thermal"
+FINAL_FRACTION = 0.2
+
+
+@dataclass(frozen=True)
+class Metrics:
+    idle_temp_c: float | None
+    time_to_throttle_s: float | None
+    steady_temp_c: float | None
+    sustained_tflops: float | None
+    tflops_ratio_vs_spec: float | None
+    sm_clock_sigma: float | None
+    power_sigma_w: float | None
+    ecc_delta: int | None
+    throttle_reasons_seen: list[str]
+    telemetry_gaps: list[str]
+
+
+def _present(values: Iterable[float | int | None]) -> list[float | int]:
+    return [v for v in values if v is not None]
+
+
+def _mean(values: Iterable[float | None]) -> float | None:
+    vals = _present(values)
+    return sum(vals) / len(vals) if vals else None
+
+
+def _pstdev(values: Iterable[float | None]) -> float | None:
+    vals = _present(values)
+    return statistics.pstdev(vals) if len(vals) >= 2 else None
+
+
+def derive(run: RawRun, spec: GpuSpec | None) -> Metrics:
+    gaps: list[str] = []
+
+    def gap(name: str) -> None:
+        if name not in gaps:
+            gaps.append(name)
+
+    baseline = run.baseline
+    load = run.load
+
+    idle_temp = _mean(s.temp_c for s in baseline)
+    if idle_temp is None:
+        gap("temp_c")
+
+    time_to_throttle: float | None = None
+    known_reasons = [s for s in load if s.throttle_reasons is not None]
+    if not known_reasons:
+        gap("throttle_reasons")  # unknown throttle state is a gap, never "no throttling"
+    for s in known_reasons:
+        assert s.throttle_reasons is not None
+        if THERMAL_REASON in s.throttle_reasons:
+            time_to_throttle = s.t_s - load[0].t_s
+            break
+
+    window: list[TelemetrySample] = []
+    if load:
+        window = load[-max(1, round(FINAL_FRACTION * len(load))) :]
+
+    steady_temp = _mean(s.temp_c for s in window)
+    if steady_temp is None:
+        gap("temp_c")
+
+    sustained = _mean(s.achieved_tflops for s in window)
+    if sustained is None:
+        gap("achieved_tflops")
+    ratio = sustained / spec.peak_bf16_tflops if sustained is not None and spec is not None else None
+
+    sm_sigma = _pstdev(s.sm_clock_mhz for s in load)
+    if sm_sigma is None:
+        gap("sm_clock_mhz")
+    power_sigma = _pstdev(s.power_w for s in load)
+    if power_sigma is None:
+        gap("power_w")
+
+    all_ecc = _present(s.ecc_volatile for s in baseline + load)
+    if not all_ecc:
+        gap("ecc_volatile")
+    ecc_delta: int | None = None
+    load_ecc = _present(s.ecc_volatile for s in load)
+    if load_ecc and len(all_ecc) >= 2:
+        ecc_delta = load_ecc[-1] - all_ecc[0]
+
+    reasons_seen = sorted({r for s in known_reasons for r in (s.throttle_reasons or ())})
+
+    return Metrics(
+        idle_temp_c=idle_temp,
+        time_to_throttle_s=time_to_throttle,
+        steady_temp_c=steady_temp,
+        sustained_tflops=sustained,
+        tflops_ratio_vs_spec=ratio,
+        sm_clock_sigma=sm_sigma,
+        power_sigma_w=power_sigma,
+        ecc_delta=ecc_delta,
+        throttle_reasons_seen=reasons_seen,
+        telemetry_gaps=gaps,
+    )

gpucheck/preflight.py

@@ -0,0 +1,95 @@
+"""Preflight policy: decide whether a card may be tested, and how.
+
+Precedence when several rules apply: refuse beats measurements_only beats full.
+`reasons` collects every rule that fired, in evaluation order.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, fields
+from typing import Literal
+
+from .adapter import TelemetryAdapter, TelemetrySample
+from .specs import lookup
+
+Mode = Literal["full", "measurements_only", "refuse"]
+
+# TelemetrySample fields that are None by design at preflight, not missing hardware.
+_EXPECTED_NONE = frozenset({"achieved_tflops"})
+
+# "Active at idle" heuristic: a process list hidden by a container can make busy()
+# report False/None while something else is running. Flag (never refuse) when the
+# idle sample draws over half the power limit with SM clocks near boost.
+ACTIVE_POWER_FRACTION = 0.5
+ACTIVE_CLOCK_FRACTION = 0.9
+ACTIVE_REASON = "GPU appears active at idle"
+
+
+@dataclass(frozen=True)
+class Preflight:
+    mode: Mode
+    reasons: list[str] = field(default_factory=list)
+    missing_telemetry: list[str] = field(default_factory=list)
+
+
+def run_preflight(adapter: TelemetryAdapter) -> Preflight:
+    inv = adapter.inventory()
+    busy = adapter.busy()
+    shared = inv.mig_enabled or inv.fractional
+
+    reasons: list[str] = []
+    refuse = False
+    measurements_only = False
+
+    # Rule 1: a compute process on a full (non-shared) card refuses the run.
+    if busy is True:
+        if shared:
+            reasons.append("busy neighbor on shared GPU (expected)")
+        else:
+            refuse = True
+            reasons.append("GPU busy: compute process running on a full card")
+
+    # Rule 2: shared configurations get measurements only.
+    if inv.mig_enabled:
+        measurements_only = True
+        reasons.append("MIG enabled: measurements only")
+    elif inv.fractional:
+        measurements_only = True
+        reasons.append("fractional sharing enabled: measurements only")
+
+    # Rule 3: busy state cannot be read (process list hidden); proceed in full mode.
+    if busy is None:
+        reasons.append("busy-state unknown")
+
+    try:
+        sample: TelemetrySample | None = adapter.sample()
+    except Exception:  # an unreadable sample is missing telemetry, not a crash
+        sample = None
+    missing = [
+        f.name
+        for f in fields(TelemetrySample)
+        if f.name not in _EXPECTED_NONE and (sample is None or getattr(sample, f.name) is None)
+    ]
+
+    # Rule 4: best-effort "something is already running" check (adds a reason only).
+    if busy is not True and sample is not None and _looks_active(sample, inv):
+        reasons.append(ACTIVE_REASON)
+
+    if refuse:
+        mode: Mode = "refuse"
+    elif measurements_only:
+        mode = "measurements_only"
+    else:
+        mode = "full"
+    return Preflight(mode=mode, reasons=reasons, missing_telemetry=missing)
+
+
+def _looks_active(sample: TelemetrySample, inv) -> bool:
+    if sample.power_w is None or not inv.power_limit_w:
+        return False
+    if sample.power_w <= ACTIVE_POWER_FRACTION * inv.power_limit_w:
+        return False
+    spec = lookup(inv.model_name)
+    if spec is None or sample.sm_clock_mhz is None:
+        return spec is None  # no clock reference: high power alone is the signal
+    return sample.sm_clock_mhz >= ACTIVE_CLOCK_FRACTION * spec.boost_clock_mhz

gpucheck/report.py

@@ -0,0 +1,101 @@
+"""Plain-text terminal report rendered from a result dict (stdlib only)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+_MEASUREMENTS: list[tuple[str, str, str]] = [
+    ("idle_temp_c", "Idle temperature", "C"),
+    ("steady_temp_c", "Steady-state temperature", "C"),
+    ("time_to_throttle_s", "Time to thermal throttle", "s"),
+    ("sustained_tflops", "Sustained throughput", "TFLOPS"),
+    ("tflops_ratio_vs_spec", "Throughput vs. datasheet", "x"),
+    ("sm_clock_sigma", "SM clock variation (sigma)", "MHz"),
+    ("power_sigma_w", "Power variation (sigma)", "W"),
+    ("ecc_delta", "ECC errors during run", ""),
+]
+
+_ABORT_TEXT = {
+    "thermal": "GPU temperature exceeded the safety limit",
+    "ecc": "ECC errors jumped during the run",
+    "telemetry": "telemetry became unreadable",
+    "preflight_refused": "preflight refused the run",
+}
+
+
+def _fmt(value: Any, unit: str) -> str:
+    if value is None:
+        return "n/a"
+    if isinstance(value, float):
+        text = f"{value:.2f}".rstrip("0").rstrip(".")
+    else:
+        text = str(value)
+    return f"{text} {unit}".rstrip()
+
+
+def render_report(result: dict) -> str:
+    inv = result.get("inventory") or {}
+    pre = result.get("preflight") or {}
+    met = result.get("metrics") or {}
+    out: list[str] = ["gpucheck report", "=" * 15, ""]
+
+    if result.get("aborted") and result["aborted"] != "preflight_refused":
+        why = _ABORT_TEXT.get(result["aborted"], result["aborted"])
+        out += [f"!! Run aborted early ({why}); showing what was collected.", ""]
+    if pre.get("mode") == "measurements_only":
+        out += [
+            "NOTE: MIG/fractional GPU configuration - measurements only, no grade is given.",
+            "",
+        ]
+
+    out.append("Inventory")
+    out.append("---------")
+    rows = [
+        ("Model", inv.get("model_name")),
+        ("Driver", inv.get("driver")),
+        ("CUDA", inv.get("cuda")),
+        ("ECC enabled", inv.get("ecc_enabled")),
+        ("Power limit", _fmt(inv.get("power_limit_w"), "W")),
+        ("Default power limit", _fmt(inv.get("default_power_limit_w"), "W")),
+        ("MIG enabled", inv.get("mig_enabled")),
+        ("Fractional sharing", inv.get("fractional")),
+        ("Retired pages", inv.get("retired_pages")),
+        ("Remapped rows", inv.get("remapped_rows")),
+    ]
+    out += [f"  {k + ':':<22}{'n/a' if v is None else v}" for k, v in rows]
+    for reason in pre.get("reasons") or []:
+        out.append(f"  preflight: {reason}")
+    out.append("")
+
+    out.append("Measurements")
+    out.append("------------")
+    out += [f"  {label + ':':<30}{_fmt(met.get(key), unit)}" for key, label, unit in _MEASUREMENTS]
+    seen = met.get("throttle_reasons_seen") or []
+    if seen:
+        seen_text = ", ".join(seen)
+    elif "throttle_reasons" in (met.get("telemetry_gaps") or []):
+        seen_text = "unknown"
+    else:
+        seen_text = "none"
+    out.append(f"  {'Throttle reasons seen:':<30}{seen_text}")
+    out.append("")
+
+    out.append("Findings")
+    out.append("--------")
+    findings = result.get("findings") or []
+    if findings:
+        for f in findings:
+            out.append(f"  [{f['severity'].upper()}] {f['code']} (confidence: {f['confidence']})")
+            out.append(f"      {f['message']}")
+    else:
+        out.append("  none")
+    out.append("")
+
+    out.append("Telemetry gaps")
+    out.append("--------------")
+    gaps = list(met.get("telemetry_gaps") or [])
+    for m in pre.get("missing_telemetry") or []:
+        if m not in gaps:
+            gaps.append(m)
+    out.append("  " + (", ".join(gaps) if gaps else "none"))
+    return "\n".join(out) + "\n"

gpucheck/resultjson.py

@@ -0,0 +1,75 @@
+"""Canonical result.json: build the result dict and serialize it reproducibly.
+
+canonical_dumps is deterministic and idempotent: keys sorted, compact
+separators, every float rounded to 4 decimal places, sets/frozensets emitted
+as sorted lists, NaN/Infinity rejected (ValueError). Dataclass instances are
+also accepted defensively and converted via dataclasses.asdict.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import json
+from typing import Any
+
+from .diagnose import Finding
+from .metrics import Metrics
+from .run import RawRun
+
+SCHEMA_VERSION = "1"
+FLOAT_DP = 4
+
+
+def _normalize(obj: Any) -> Any:
+    if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
+        obj = dataclasses.asdict(obj)
+    if isinstance(obj, dict):
+        return {k: _normalize(v) for k, v in obj.items()}
+    if isinstance(obj, (set, frozenset)):
+        return sorted(_normalize(v) for v in obj)
+    if isinstance(obj, (list, tuple)):
+        return [_normalize(v) for v in obj]
+    if isinstance(obj, float):
+        # round(nan/inf) returns them unchanged; json.dumps(allow_nan=False) rejects.
+        return round(obj, FLOAT_DP) + 0.0  # + 0.0 turns -0.0 into 0.0
+    return obj
+
+
+def build_result(
+    run: RawRun,
+    metrics: Metrics,
+    findings: list[Finding],
+    challenge: dict | None = None,
+    challenge_workload: dict | None = None,
+) -> dict:
+    result = _normalize(
+        {
+            "schema_version": SCHEMA_VERSION,
+            "inventory": run.inventory,
+            "preflight": run.preflight,
+            "config": run.config,
+            "metrics": metrics,
+            "findings": list(findings),
+            "samples": {"baseline": list(run.baseline), "load": list(run.load)},
+            "aborted": run.aborted,
+        }
+    )
+    # Both omitted -> output is byte-identical to v1 (run_ids must not shift).
+    if challenge is not None:
+        result["challenge"] = _normalize(
+            {"challenge_id": challenge["challenge_id"], "nonce": challenge["nonce"]}
+        )
+    if challenge_workload is not None:
+        result["config"]["challenge_workload"] = _normalize(challenge_workload)
+    return result
+
+
+def canonical_dumps(d: dict) -> str:
+    return json.dumps(
+        _normalize(d), sort_keys=True, separators=(",", ":"), allow_nan=False
+    )
+
+
+def run_id(d: dict) -> str:
+    return hashlib.sha256(canonical_dumps(d).encode()).hexdigest()[:16]