gpucheck-cli 0.1.0__py3-none-any.whl

gpucheck/__init__.py

@@ -0,0 +1 @@
+"""gpucheck: health-check NVIDIA GPUs."""

gpucheck/adapter.py

@@ -0,0 +1,192 @@
+"""Telemetry types and the adapter protocol, plus the real NVML adapter.
+
+All NVML access here is read-only: no NVML setter is ever called.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import time
+import warnings
+from dataclasses import dataclass
+from typing import Protocol
+
+
+class AdapterUnavailable(Exception):
+    """Raised when no GPU or NVIDIA driver is available."""
+
+
+@dataclass(frozen=True)
+class GpuInventory:
+    model_name: str
+    uuid_hash: str
+    driver: str
+    cuda: str | None
+    ecc_enabled: bool | None
+    power_limit_w: float | None
+    default_power_limit_w: float | None
+    mig_enabled: bool
+    fractional: bool
+    retired_pages: int | None
+    remapped_rows: int | None  # correctable + uncorrectable
+    row_remap_pending: bool | None = None
+    row_remap_failure: bool | None = None
+
+
+@dataclass(frozen=True)
+class TelemetrySample:
+    t_s: float
+    temp_c: float | None
+    power_w: float | None
+    sm_clock_mhz: float | None
+    fan_pct: float | None
+    throttle_reasons: frozenset[str] | None  # None = NVML could not report reasons (unknown)
+    ecc_volatile: int | None
+    achieved_tflops: float | None
+
+
+class TelemetryAdapter(Protocol):
+    def inventory(self) -> GpuInventory: ...
+
+    def sample(self) -> TelemetrySample: ...
+
+    def busy(self) -> bool | None: ...
+
+
+def hash_uuid(uuid: str) -> str:
+    return hashlib.sha256(("gpucheck.v1." + uuid).encode()).hexdigest()[:16]
+
+
+def _text(value: object) -> str:
+    return value.decode() if isinstance(value, bytes) else str(value)
+
+
+def _try(fn, *args):
+    """Call an NVML query; None if the GPU does not support it."""
+    try:
+        return fn(*args)
+    except Exception:
+        return None
+
+
+# NVML ABI values, used only when pynvml exposes neither naming family.
+_ABI = {
+    "SwPowerCap": 0x4,
+    "SwThermalSlowdown": 0x20,
+    "HwThermalSlowdown": 0x40,
+    "HwPowerBrakeSlowdown": 0x80,
+}
+
+
+def _const(nv, *names: str, default: int) -> int:
+    """First of `names` that pynvml defines, else `default`; never raises."""
+    for name in names:
+        value = getattr(nv, name, None)
+        if isinstance(value, int):
+            return value
+    return default
+
+
+def _reason_bit(nv, suffix: str) -> int:
+    # pynvml renamed "ClocksThrottleReason" to "ClocksEventReason" (NVML 12.x); the
+    # upper-case NVML_CLOCKS_THROTTLE_REASON_* spelling never existed in pynvml.
+    return _const(
+        nv,
+        f"nvmlClocksEventReason{suffix}",
+        f"nvmlClocksThrottleReason{suffix}",
+        default=_ABI[suffix],
+    )
+
+
+class NvmlAdapter(TelemetryAdapter):
+    """Reads GPU 0 through NVML (pynvml)."""
+
+    def __init__(self, index: int = 0) -> None:
+        try:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", FutureWarning)  # pynvml deprecation notice
+                import pynvml
+
+            pynvml.nvmlInit()
+            self._nv = pynvml
+            self._h = pynvml.nvmlDeviceGetHandleByIndex(index)
+        except Exception as exc:  # ImportError, NVMLError, missing library...
+            raise AdapterUnavailable(f"NVML unavailable: {exc}") from exc
+        self._t0 = time.monotonic()
+
+    def _q(self, name: str, *args):
+        """Optional NVML query by name: None if pynvml lacks it or the GPU does not support it."""
+        fn = getattr(self._nv, name, None)
+        return None if fn is None else _try(fn, *args)
+
+    def inventory(self) -> GpuInventory:
+        nv, h = self._nv, self._h
+        uuid = _text(nv.nvmlDeviceGetUUID(h))
+        cuda_raw = self._q("nvmlSystemGetCudaDriverVersion")
+        cuda = f"{cuda_raw // 1000}.{(cuda_raw % 1000) // 10}" if cuda_raw else None
+        ecc = self._q("nvmlDeviceGetEccMode", h)
+        limit = self._q("nvmlDeviceGetPowerManagementLimit", h)
+        default = self._q("nvmlDeviceGetPowerManagementDefaultLimit", h)
+        mig = self._q("nvmlDeviceGetMigMode", h)
+        cause = _const(nv, "NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS", default=0)
+        retired = self._q("nvmlDeviceGetRetiredPages", h, cause)
+        # (correctable rows, uncorrectable rows, remap pending, remap failure occurred)
+        remap = self._q("nvmlDeviceGetRemappedRows", h)
+        remap_ok = isinstance(remap, (tuple, list)) and len(remap) >= 4
+        return GpuInventory(
+            model_name=_text(nv.nvmlDeviceGetName(h)),
+            uuid_hash=hash_uuid(uuid),
+            driver=_text(nv.nvmlSystemGetDriverVersion()),
+            cuda=cuda,
+            ecc_enabled=bool(ecc[0]) if ecc else None,
+            power_limit_w=limit / 1000.0 if limit is not None else None,
+            default_power_limit_w=default / 1000.0 if default is not None else None,
+            mig_enabled=bool(mig and mig[0]),
+            fractional=False,
+            retired_pages=len(retired) if retired is not None else None,
+            remapped_rows=int(remap[0]) + int(remap[1]) if remap_ok else None,
+            row_remap_pending=bool(remap[2]) if remap_ok else None,
+            row_remap_failure=bool(remap[3]) if remap_ok else None,
+        )
+
+    def _throttle_mask(self) -> int | None:
+        for name in ("nvmlDeviceGetCurrentClocksEventReasons", "nvmlDeviceGetCurrentClocksThrottleReasons"):
+            mask = self._q(name, self._h)
+            if mask is not None:
+                return int(mask)
+        return None  # unknown: recorded as a telemetry gap, never as "no throttling"
+
+    def sample(self) -> TelemetrySample:
+        nv, h = self._nv, self._h
+        temp = self._q("nvmlDeviceGetTemperature", h, _const(nv, "NVML_TEMPERATURE_GPU", default=0))
+        power = self._q("nvmlDeviceGetPowerUsage", h)
+        clock = self._q("nvmlDeviceGetClockInfo", h, _const(nv, "NVML_CLOCK_SM", default=1))
+        fan = self._q("nvmlDeviceGetFanSpeed", h)
+        mask = self._throttle_mask()
+        ecc = self._q(
+            "nvmlDeviceGetTotalEccErrors",
+            h,
+            _const(nv, "NVML_MEMORY_ERROR_TYPE_UNCORRECTED", default=1),
+            _const(nv, "NVML_VOLATILE_ECC", default=0),
+        )
+        reasons: frozenset[str] | None = None
+        if mask is not None:
+            names = {
+                "thermal": _reason_bit(nv, "HwThermalSlowdown") | _reason_bit(nv, "SwThermalSlowdown"),
+                "power": _reason_bit(nv, "SwPowerCap") | _reason_bit(nv, "HwPowerBrakeSlowdown"),
+            }
+            reasons = frozenset(name for name, bits in names.items() if mask & bits)
+        return TelemetrySample(
+            t_s=time.monotonic() - self._t0,
+            temp_c=float(temp) if temp is not None else None,
+            power_w=power / 1000.0 if power is not None else None,
+            sm_clock_mhz=float(clock) if clock is not None else None,
+            fan_pct=float(fan) if fan is not None else None,
+            throttle_reasons=reasons,
+            ecc_volatile=int(ecc) if ecc is not None else None,
+            achieved_tflops=None,  # supplied by the load generator, not NVML
+        )
+
+    def busy(self) -> bool | None:
+        procs = self._q("nvmlDeviceGetComputeRunningProcesses", self._h)
+        return None if procs is None else len(procs) > 0

gpucheck/cli.py

@@ -0,0 +1,237 @@
+"""Command-line entry point: run, analyse, write result.json, print report."""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .adapter import AdapterUnavailable, NvmlAdapter
+from .diagnose import diagnose
+from .fakes import make_fake
+from .load import LoadUnavailable, TorchGemmLoad
+from .metrics import derive
+from .report import render_report
+from .resultjson import build_result, canonical_dumps
+from .run import NullLoad, RunConfig, execute_run
+from .specs import lookup
+from .upload import ChallengeUnavailable, UploadOutcome, fetch_challenge, upload
+
+DEFAULT_SERVER = "https://gpucheck-api.fly.dev"
+
+EXIT_OK, EXIT_REFUSED, EXIT_NO_GPU, EXIT_ABORTED = 0, 1, 2, 3
+
+
+def _parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="gpucheck", description="GPU health report card")
+    g = p.add_mutually_exclusive_group()
+    g.add_argument("--quick", action="store_true", help="shorter run (30s baseline, 150s load)")
+    g.add_argument("--long", action="store_true", help="longer run (60s baseline, 1800s load)")
+    p.add_argument("--offline", action="store_true", help="never upload; report locally only")
+    p.add_argument("--certify", action="store_true", help="request a one-time server challenge so the result can earn a certificate (requires a server; excludes --offline/--dry-run/--quick)")
+    p.add_argument("--yes-upload", action="store_true", help="upload without prompting")
+    p.add_argument("--dry-run", action="store_true", help="show what would be uploaded")
+    p.add_argument("--json", metavar="PATH", default="./gpucheck-result.json", help="where to write result.json")
+    p.add_argument("--server", metavar="URL", default=None, help="upload server URL")
+    p.add_argument("--fake", metavar="PROFILE", help=argparse.SUPPRESS)
+    return p
+
+
+def _parse(argv):
+    p = _parser()
+    args = p.parse_args(argv)
+    if args.certify and args.offline:
+        p.error("--certify cannot be combined with --offline")
+    if args.certify and args.dry_run:
+        p.error("--certify cannot be combined with --dry-run")
+    if args.certify and args.quick:
+        p.error("--certify cannot be combined with --quick")
+    return args
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def challenge_expired(challenge: dict, now: datetime | None = None) -> bool:
+    """True when the challenge's expires_at (ISO 8601) is in the past.
+
+    An unparseable or missing expiry is not treated as expired: the server decides."""
+    raw = challenge.get("expires_at")
+    if not isinstance(raw, str):
+        return False
+    try:
+        exp = datetime.fromisoformat(raw.replace("Z", "+00:00"))
+    except ValueError:
+        return False
+    if exp.tzinfo is None:
+        exp = exp.replace(tzinfo=timezone.utc)
+    return (now or _now()) > exp
+
+
+def format_certify(out: UploadOutcome) -> str | None:
+    """The server's certification verdict for a --certify upload, or None if it said nothing."""
+    if out.certifiable is True:
+        return "Certifiable: yes"
+    if out.certifiable is False:
+        return f"Not certifiable: {out.certify_note or 'no reason given'}"
+    return None
+
+
+def _make_adapter(args):
+    return make_fake(args.fake) if args.fake else NvmlAdapter()
+
+
+def _make_load(args, challenge=None):
+    if args.fake:  # fake adapters supply their own achieved_tflops
+        return NullLoad()
+    if challenge is not None:
+        # Certification requires the real load generator; LoadUnavailable propagates.
+        return TorchGemmLoad(challenge["workload"])
+    try:
+        return TorchGemmLoad()
+    except LoadUnavailable:
+        print(
+            "no torch/CUDA — running without load; performance metrics will be gaps",
+            file=sys.stderr,
+        )
+        return NullLoad()
+
+
+def _config(args) -> RunConfig:
+    if args.quick:
+        return RunConfig(baseline_s=30, load_s=150, hz=1.0)
+    if args.long:
+        return RunConfig(baseline_s=60, load_s=1800, hz=1.0)
+    return RunConfig(baseline_s=60, load_s=600, hz=1.0)
+
+
+def _execute(adapter, args, challenge=None):
+    spec = lookup(adapter.inventory().model_name)
+    load = _make_load(args, challenge)
+    kwargs = {}
+    if args.fake:  # simulated hardware: use a virtual clock instead of real waiting
+        t = [0.0]
+
+        def clock() -> float:
+            return t[0]
+
+        def sleep(s: float) -> None:
+            t[0] += s
+
+        kwargs = {"clock": clock, "sleep": sleep}
+    return spec, load, execute_run(adapter, load, _config(args), spec, **kwargs)
+
+
+def format_grade(out: UploadOutcome, model: str | None) -> str | None:
+    """One-line grade summary from the server's response, or None if it said nothing."""
+    g = out.grade
+    if g:
+        letter = g.get("letter", "?")
+        ratio = g.get("ratio")
+        if not isinstance(ratio, (int, float)) or isinstance(ratio, bool):
+            return f"Grade {letter}"
+        n = g.get("n", out.n)
+        extra = f"n={n}" + (", low confidence" if g.get("low_confidence") else "")
+        return f"Grade {letter} — delivers {ratio * 100:.1f}% of a healthy {model or 'card of this model'} ({extra})"
+    if out.reason:
+        return f"Grade: {out.reason}"
+    if out.n is not None:
+        return f"Grade: insufficient reference (n={out.n})"
+    return None
+
+
+def _confirm(server: str) -> bool:
+    try:
+        return input(f"Upload this result to {server}? [y/N] ").strip().lower() == "y"
+    except (EOFError, KeyboardInterrupt):  # closed stdin or Ctrl-C at the prompt means "no"
+        print()
+        return False
+
+
+def _maybe_upload(result: dict, args, challenge: dict | None = None) -> None:
+    if args.dry_run:
+        print(canonical_dumps(result))
+        return
+    server = args.server or DEFAULT_SERVER
+    if not args.yes_upload:
+        if not sys.stdin.isatty():
+            print("Not uploaded. To upload this result, re-run with --yes-upload.")
+            return
+        if not _confirm(server):
+            return
+    out = upload(result, server)
+    if not out.ok:
+        print(f"warning: upload failed ({out.error}); result.json is saved locally")
+        return
+    if out.url:
+        print(f"Result page: {out.url}")
+    line = format_grade(out, (result.get("inventory") or {}).get("model_name"))
+    if line:
+        print(line)
+    if challenge is not None:
+        cline = format_certify(out)
+        if cline:
+            print(cline)
+    if out.delete_token:
+        print(f"Delete token: {out.delete_token}")
+        print("(save this token to delete the page later)")
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parse(argv)
+    try:
+        adapter = _make_adapter(args)
+    except AdapterUnavailable as e:
+        print(f"gpucheck: no NVIDIA GPU or driver available ({e}).", file=sys.stderr)
+        return EXIT_NO_GPU
+    except ValueError as e:  # only _make_adapter resolves --fake, so this is a bad profile name
+        print(f"gpucheck: {e}", file=sys.stderr)
+        return EXIT_NO_GPU
+    challenge = None
+    if args.certify:
+        try:
+            challenge = fetch_challenge(args.server or DEFAULT_SERVER)
+        except ChallengeUnavailable as e:
+            print(f"gpucheck: certification unavailable: {e}", file=sys.stderr)
+            return EXIT_REFUSED
+    try:
+        spec, load, run = _execute(adapter, args, challenge)
+    except LoadUnavailable as e:
+        print(f"gpucheck: certification requires the GPU load generator ({e}).", file=sys.stderr)
+        return EXIT_REFUSED
+    except AdapterUnavailable as e:
+        print(f"gpucheck: no NVIDIA GPU or driver available ({e}).", file=sys.stderr)
+        return EXIT_NO_GPU
+
+    if run.aborted == "preflight_refused":
+        print("gpucheck: preflight refused to run:")
+        for reason in run.preflight.reasons:
+            print(f"  - {reason}")
+        return EXIT_REFUSED
+
+    metrics = derive(run, spec)
+    findings = diagnose(metrics, run.preflight, run.inventory, spec)
+    challenge_workload = None
+    if challenge is not None:
+        challenge_workload = getattr(load, "resolved_workload", None)
+        if challenge_workload is None:
+            # Test/demo-only path (--fake): no real load ran, so synthesize the
+            # echo straight from the prescription. Real runs always echo what
+            # TorchGemmLoad actually resolved.
+            w = challenge["workload"]
+            challenge_workload = {
+                "gemm_n": [8192 + o for o in w["gemm_n_offsets"]],
+                "order_seed": w["order_seed"],
+            }
+    result = build_result(run, metrics, findings, challenge, challenge_workload)
+    Path(args.json).write_text(canonical_dumps(result))  # before any upload consideration
+    print(render_report(result))
+    print(f"Result written to {args.json}")
+    if challenge is not None and challenge_expired(challenge):
+        print("warning: challenge expired before upload; run again to certify — result saved locally")
+    elif not args.offline:
+        _maybe_upload(result, args, challenge)
+    return EXIT_ABORTED if run.aborted else EXIT_OK

gpucheck/diagnose.py

@@ -0,0 +1,203 @@
+"""Diagnosis rule table: turn derived metrics into plain-language findings.
+
+Rules are evaluated in a fixed order (MEMORY_WEAR, TIM_DEGRADATION,
+HOT_ENVIRONMENT, THERMAL_THROTTLING, POWER_CAPPED, UNDERPERFORMING_UNKNOWN,
+INSUFFICIENT_EVIDENCE) and each rule fires at most once. Unknown throttle data
+("throttle_reasons" in telemetry_gaps) is treated as unknown, never as "none". A rule whose required inputs are None does
+not fire. Rules that compare against the datasheet need `spec`; they are
+skipped when it is None.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from .adapter import GpuInventory
+from .metrics import THERMAL_REASON, Metrics
+from .preflight import Preflight
+from .specs import GpuSpec
+
+Severity = Literal["info", "warn", "bad"]
+Confidence = Literal["low", "medium", "high"]
+
+TIM_TIME_TO_THROTTLE_MAX_S = 180.0
+TIM_STEADY_MARGIN_C = 3.0
+TIM_IDLE_MAX_C = 45.0
+TIM_HIGH_CONFIDENCE_IDLE_C = 40.0
+HOT_IDLE_C = 50.0
+POWER_LIMIT_FRACTION = 0.9
+POWER_REASON_HEADROOM_C = 15.0
+UNDERPERFORMING_RATIO = 0.9
+
+
+@dataclass(frozen=True)
+class Finding:
+    code: str
+    severity: Severity
+    confidence: Confidence
+    message: str
+
+
+def _g(value: float) -> str:
+    return f"{value:g}"
+
+
+def _throttled(m: Metrics) -> bool:
+    return m.time_to_throttle_s is not None or THERMAL_REASON in m.throttle_reasons_seen
+
+
+def _memory_wear(m: Metrics, inv: GpuInventory) -> Finding | None:
+    parts: list[str] = []
+    if m.ecc_delta is not None and m.ecc_delta > 0:
+        parts.append(f"ECC error count rose by {m.ecc_delta} during the run")
+    if inv.remapped_rows is not None and inv.remapped_rows > 0:
+        parts.append(f"{inv.remapped_rows} memory row(s) were remapped")
+    if inv.row_remap_failure:
+        parts.append("memory row remapping has failed")
+    if not parts:
+        return None
+    sentence = " and ".join(parts)
+    return Finding(
+        "MEMORY_WEAR",
+        "bad",
+        "high",
+        sentence[0].upper() + sentence[1:] + ", which points to memory wear.",
+    )
+
+
+def _tim_degradation(m: Metrics, spec: GpuSpec | None) -> Finding | None:
+    if spec is None or m.time_to_throttle_s is None or m.steady_temp_c is None or m.idle_temp_c is None:
+        return None
+    if m.idle_temp_c > TIM_IDLE_MAX_C:
+        return None
+    if m.time_to_throttle_s >= TIM_TIME_TO_THROTTLE_MAX_S:
+        return None
+    if m.steady_temp_c < spec.max_temp_c - TIM_STEADY_MARGIN_C:
+        return None
+    confidence: Confidence = "high" if m.idle_temp_c <= TIM_HIGH_CONFIDENCE_IDLE_C else "medium"
+    return Finding(
+        "TIM_DEGRADATION",
+        "bad",
+        confidence,
+        f"Throttled after {_g(m.time_to_throttle_s)} s; steady {_g(m.steady_temp_c)} °C vs "
+        f"{_g(spec.max_temp_c)} °C limit; idle was {_g(m.idle_temp_c)} °C, which suggests the "
+        "thermal paste or cooler contact has degraded.",
+    )
+
+
+def _hot_environment(m: Metrics) -> Finding | None:
+    if m.idle_temp_c is None or m.idle_temp_c <= HOT_IDLE_C:
+        return None
+    return Finding(
+        "HOT_ENVIRONMENT",
+        "warn",
+        "high",
+        f"Idle temperature was {_g(m.idle_temp_c)} °C, above {_g(HOT_IDLE_C)} °C, so room "
+        "temperature or airflow may be the cause of the heat.",
+    )
+
+
+def _power_capped(m: Metrics, inv: GpuInventory, spec: GpuSpec | None) -> Finding | None:
+    limit_arm = (
+        inv.power_limit_w is not None
+        and inv.default_power_limit_w is not None
+        and inv.power_limit_w < POWER_LIMIT_FRACTION * inv.default_power_limit_w
+    )
+    reason_arm = (
+        spec is not None
+        and m.steady_temp_c is not None
+        and any("power" in r for r in m.throttle_reasons_seen)
+        and m.steady_temp_c < spec.max_temp_c - POWER_REASON_HEADROOM_C
+    )
+    if limit_arm:
+        assert inv.power_limit_w is not None and inv.default_power_limit_w is not None
+        return Finding(
+            "POWER_CAPPED",
+            "info",
+            "high",
+            f"Power limit is {_g(inv.power_limit_w)} W, below 90% of the "
+            f"{_g(inv.default_power_limit_w)} W default, so the card is capped below normal power.",
+        )
+    if reason_arm:
+        assert spec is not None and m.steady_temp_c is not None
+        return Finding(
+            "POWER_CAPPED",
+            "info",
+            "high",
+            f"Power capping was seen while steady at {_g(m.steady_temp_c)} °C, well below the "
+            f"{_g(spec.max_temp_c)} °C limit, so the power limit rather than heat held clocks down.",
+        )
+    return None
+
+
+def _thermal_throttling(m: Metrics) -> Finding | None:
+    """Throttling that neither TIM_DEGRADATION nor INSUFFICIENT_EVIDENCE explains."""
+    if not _throttled(m):
+        return None
+    when = f" after {_g(m.time_to_throttle_s)} s of load" if m.time_to_throttle_s is not None else ""
+    return Finding(
+        "THERMAL_THROTTLING",
+        "warn",
+        "medium",
+        f"The card thermal-throttled{when} even though it idled cool, so sustained "
+        "clocks were limited by heat; check airflow and cooler.",
+    )
+
+
+def _underperforming(m: Metrics, spec: GpuSpec | None) -> Finding | None:
+    if spec is None or m.tflops_ratio_vs_spec is None or m.throttle_reasons_seen:
+        return None
+    if m.tflops_ratio_vs_spec >= UNDERPERFORMING_RATIO:
+        return None
+    if "throttle_reasons" in m.telemetry_gaps:
+        context = "and throttle-reason data was unavailable"
+    else:
+        context = "with no throttling seen"
+    return Finding(
+        "UNDERPERFORMING_UNKNOWN",
+        "warn",
+        "low",
+        f"Sustained throughput was {m.tflops_ratio_vs_spec:.1%} of the datasheet peak {context}, "
+        "and the cause is unknown.",
+    )
+
+
+def _insufficient_evidence(m: Metrics, spec: GpuSpec | None) -> Finding | None:
+    if not _throttled(m):
+        return None
+    if m.idle_temp_c is not None and m.idle_temp_c <= TIM_IDLE_MAX_C:
+        return None
+    if m.idle_temp_c is None:
+        detail = "no idle baseline was recorded"
+    else:
+        detail = f"idle was {_g(m.idle_temp_c)} °C, above {_g(TIM_IDLE_MAX_C)} °C"
+    return Finding(
+        "INSUFFICIENT_EVIDENCE",
+        "info",
+        "low",
+        f"Throttling was seen, but {detail}, so a cooler fault cannot be confirmed or ruled out.",
+    )
+
+
+def diagnose(
+    metrics: Metrics,
+    preflight: Preflight,
+    inventory: GpuInventory,
+    spec: GpuSpec | None,
+) -> list[Finding]:
+    """Return the findings for one card, in fixed rule order; empty means healthy."""
+    del preflight  # the v1 rule table does not consult preflight
+    tim = _tim_degradation(metrics, spec)
+    insufficient = _insufficient_evidence(metrics, spec)
+    throttling = _thermal_throttling(metrics) if tim is None and insufficient is None else None
+    rules = (
+        _memory_wear(metrics, inventory),
+        tim,
+        _hot_environment(metrics),
+        throttling,
+        _power_capped(metrics, inventory, spec),
+        _underperforming(metrics, spec),
+        insufficient,
+    )
+    return [f for f in rules if f is not None]