nvq 0.4.0__py3-none-any.whl

nvq/__init__.py

@@ -0,0 +1,186 @@
+"""nvq runs the nvq binary and hands back what it says about NVIDIA cards. The linux x86_64 and aarch64
+wheels carry the binary (bin/nvq, built in manylinux2014: glibc 2.17 and newer), so `pip install nvq` is
+the whole install, and a version of this package runs exactly the binary released with it. The wheel for
+any other platform carries none: NVQ(path=...) names one there.
+
+    n = nvq.NVQ()                       # the package's own binary; NVQ(path=...) runs another
+    cards = n.list()                    # every card, lost ones included; no CUDA context opened
+    probes = n.probe_all()              # does each card run a kernel, and its capability sheet
+    for ev in n.watch(every=1.0): ...   # Xid events, lost cards, samples
+
+Documents are the parsed JSON itself, keys exactly as the contract names them (card["tempC"]). Their
+types (documents.py) are TypedDicts generated from the JSON Schema the binary prints with `nvq schema`
+(`make types`), so a type checker knows every key and a value the card cannot report is a key that is
+absent, never 0. `nvq` on the command line is the binary itself."""
+
+# NVQ.list shadows the builtin inside the class body: annotations there must not be evaluated.
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+from collections.abc import Generator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from . import documents
+from .documents import (
+    Card,
+    EventEccDouble,
+    EventLost,
+    EventSample,
+    EventStart,
+    EventWaitError,
+    EventXid,
+    ListOk,
+    ProbeFailed,
+    ProbeOk,
+)
+
+__all__ = [
+    "BINARY",
+    "NVQ",
+    "SCHEMA",
+    "Card",
+    "Error",
+    "EventEccDouble",
+    "EventLost",
+    "EventSample",
+    "EventStart",
+    "EventWaitError",
+    "EventXid",
+    "ListOk",
+    "ProbeFailed",
+    "ProbeOk",
+    "WatchEvent",
+    "documents",
+]
+
+type WatchEvent = EventStart | EventXid | EventEccDouble | EventLost | EventWaitError | EventSample
+
+SCHEMA = 1
+"""The document version this package reads."""
+
+BINARY = Path(__file__).parent / "bin" / "nvq"
+"""The binary this wheel carries; absent from the wheel for platforms nvq does not run on."""
+
+
+class Error(Exception):
+    """A failure nvq reported as a document: `code` from the schema, nvq's `exit` code. When NVML is
+    unusable, `list` holds the kernel's roster all the same, every card in state "unknown"."""
+
+    def __init__(self, code: str, detail: str, exit: int, list: ListOk | None = None):
+        super().__init__(f"nvq: {code}: {detail} (exit {exit})")
+        self.code = code
+        self.detail = detail
+        self.exit = exit
+        self.list = list
+
+
+@dataclass(frozen=True, kw_only=True)
+class NVQ:
+    """The binary to run: `path`, or when None the one this wheel carries.
+
+    `deadline` (seconds) bounds one list or probe inside nvq itself (--deadline-ms); None keeps nvq's
+    default (10 s list, 60 s probe). Each call's `timeout` bounds the whole process from outside: nvq is
+    killed and subprocess.TimeoutExpired raised."""
+
+    path: Path | str | None = None
+    deadline: float | None = None
+
+    def list(self, timeout: float | None = None) -> ListOk:
+        """Every card the kernel bound. A lost card is a card with state "lost", not an error."""
+        doc, code = self._run(["list"], timeout)
+        if "cards" not in doc:
+            raise _error(doc, code)
+        if "error" in doc:
+            raise Error(doc["error"]["code"], doc["error"]["detail"], code, doc)
+        return doc
+
+    def probe(self, uuid: str, timeout: float | None = None) -> ProbeOk | ProbeFailed:
+        """Runs a kernel on one card, with only that card visible to CUDA. It opens a CUDA context: do
+        not aim it at a card mid-work you care about. A failed step is a ProbeFailed, not an error."""
+        doc, code = self._run(["probe", uuid], timeout)
+        if "ok" not in doc:
+            raise _error(doc, code)
+        return doc
+
+    def probe_all(
+        self, timeout: float | None = None
+    ) -> list[ProbeOk | ProbeFailed | documents.ErrorDocument]:
+        """Probes every card at once, each in its own process: a hung card cannot stall the others. An
+        entry is a child's own document, so a child that failed as a whole is an error document."""
+        doc, code = self._run(["probe", "all"], timeout)
+        if "probes" not in doc:
+            raise _error(doc, code)
+        return doc["probes"]
+
+    def watch(self, every: float | None = None) -> Generator[WatchEvent]:
+        """Events until nvq exits or the iterator is closed (which kills nvq). `every` (seconds) adds a
+        sample per card at that interval. nvq ending on its own raises: watch runs until killed."""
+        args = ["watch", *(["--every-ms", str(round(every * 1000))] if every else [])]
+        with (
+            tempfile.TemporaryFile() as stderr,
+            subprocess.Popen([str(self._binary()), *args], stdout=subprocess.PIPE, stderr=stderr) as p,
+        ):
+            assert p.stdout is not None
+            try:
+                for line in p.stdout:
+                    if not line.strip():
+                        continue
+                    doc = _document(line, args)
+                    if "event" not in doc:
+                        raise _error(doc, p.wait())
+                    yield doc
+                code = p.wait()
+                stderr.seek(0)
+                raise RuntimeError(f"nvq watch ended: exit {code}: {stderr.read().decode().strip()}")
+            finally:
+                p.kill()  # a no-op once nvq exited; leaving the with block reaps it
+
+    def _binary(self) -> Path:
+        return Path(self.path) if self.path is not None else _own_binary()
+
+    def _run(self, args: list[str], timeout: float | None) -> tuple[Any, int]:
+        """The document nvq printed and its exit code. A process that printed none (not found, killed, a
+        usage error) raises here; any other exit is the document's to explain."""
+        deadline = ["--deadline-ms", str(round(self.deadline * 1000))] if self.deadline is not None else []
+        p = subprocess.run(
+            [str(self._binary()), *deadline, *args], capture_output=True, timeout=timeout, check=False
+        )
+        if not p.stdout.strip():
+            stderr = p.stderr.decode(errors="replace").strip()
+            raise RuntimeError(f"nvq {' '.join(args)}: exit {p.returncode}, no document: {stderr}")
+        return _document(p.stdout, args), p.returncode
+
+
+def _own_binary() -> Path:
+    if not BINARY.exists():
+        raise FileNotFoundError(
+            f"nvq: this wheel carries no binary for {sys.platform} (only the linux x86_64 and aarch64 "
+            f"wheels do); pass NVQ(path=...)"
+        )
+    return BINARY
+
+
+def _document(raw: bytes, args: list[str]) -> Any:
+    """A parsed document, typed by the caller: the schema, not this function, vouches for its shape."""
+    doc = json.loads(raw)
+    if not isinstance(doc, dict) or doc.get("nvq") != SCHEMA:
+        raise ValueError(f"nvq {' '.join(args)}: not a version {SCHEMA} document: {raw[:200]!r}")
+    return doc
+
+
+def _error(doc: Any, code: int) -> Error:
+    if not isinstance(doc.get("error"), dict):
+        raise TypeError(f"nvq: neither the expected document nor an error: {json.dumps(doc)[:200]}")
+    return Error(doc["error"]["code"], doc["error"]["detail"], code)
+
+
+def main() -> None:
+    """`nvq` on PATH: becomes the carried binary, arguments and exit code untouched."""
+    binary = _own_binary()
+    os.execv(binary, [str(binary), *sys.argv[1:]])

nvq/documents.py

@@ -0,0 +1,490 @@
+# generated by datamodel-codegen:
+#   filename:  nvq.schema.json
+
+from __future__ import annotations
+from typing import Any, Literal, NotRequired, TypedDict
+
+
+type Nvq = Any
+"""
+NVIDIA cards as JSON. Every document carries "nvq": 1, the version of this schema; additive fields keep it, a removal or a change of meaning bumps it. Readers ignore keys they do not know. A value the card cannot report is absent, never zero, and named under "unsupported".
+"""
+
+
+type SchemaVersion = Literal[1]
+
+
+type Uuid = str
+
+
+type Bus = str
+"""
+PCI location as the kernel prints it
+"""
+
+
+type UnixTime = float
+"""
+seconds since the epoch, millisecond precision
+"""
+
+
+type Code = Literal[
+    "uninitialized",
+    "invalid_argument",
+    "not_supported",
+    "no_permission",
+    "already_initialized",
+    "not_found",
+    "insufficient_size",
+    "insufficient_power",
+    "driver_not_loaded",
+    "timeout",
+    "irq_issue",
+    "library_not_found",
+    "function_not_found",
+    "corrupted_inforom",
+    "gpu_is_lost",
+    "reset_required",
+    "operating_system",
+    "lib_rm_version_mismatch",
+    "in_use",
+    "memory",
+    "no_data",
+    "vgpu_ecc_not_supported",
+    "insufficient_resources",
+    "freq_not_supported",
+    "argument_version_mismatch",
+    "deprecated",
+    "not_ready",
+    "gpu_not_found",
+    "invalid_state",
+    "unknown",
+    "too_many_cards",
+    "deadline",
+]
+"""
+Why something failed: NVML's return codes by name, plus nvq's own.
+"""
+
+
+class ErrorBody(TypedDict):
+    code: Code
+    detail: str
+
+
+type Limit = Literal[
+    "gpu_idle",
+    "applications_clocks",
+    "sw_power_cap",
+    "hw_slowdown",
+    "sync_boost",
+    "sw_thermal",
+    "hw_thermal",
+    "hw_power_brake",
+    "display_clocks",
+]
+"""
+Why the card's clocks are below their maximum right now (NVML clock event reasons).
+"""
+
+
+class MemoryMiB(TypedDict):
+    total: int
+    used: int
+
+
+class PowerW(TypedDict):
+    limit: NotRequired[float]
+    """
+    enforced limit
+    """
+    draw: NotRequired[float]
+
+
+class ClocksMHz(TypedDict):
+    sm: int
+    mem: int
+
+
+class UtilPct(TypedDict):
+    gpu: int
+    mem: int
+
+
+class Pcie(TypedDict):
+    """
+    the link now and at best; a riser shows here (gen 1, width 1)
+    """
+
+    gen: int
+    width: int
+    maxGen: int
+    maxWidth: int
+    replays: NotRequired[int]
+    """
+    link-level retransmissions since the driver loaded; one that keeps rising is a failing riser or slot
+    """
+
+
+class Process(TypedDict):
+    pid: int
+    usedMiB: NotRequired[int]
+    """
+    absent when NVML cannot see it (another container)
+    """
+
+
+type Ms = dict[str, float]
+"""
+milliseconds per step, in the order run
+"""
+
+
+class ProbeFailed(TypedDict):
+    nvq: SchemaVersion
+    uuid: Uuid
+    ok: Literal[False]
+    step: str
+    """
+    load, init, device, uuid, name, memory, driver, attributes, context, alloc, memset, jit, function, launch, sync, copy, verify, free, unload, teardown; in probe all also hang (killed or stuck) and crash (died without a document)
+    """
+    error: str
+    """
+    a CUDA_ERROR_* name, or not_visible, more_than_one_visible, uuid_mismatch, no_compute_capability, wrong_result, library_not_found, function_not_found: <name>, killed_at_deadline, stuck_in_driver, no_output
+    """
+    ms: NotRequired[Ms]
+    signal: NotRequired[int]
+    """
+    crash: the signal that ended the child
+    """
+    exit: NotRequired[int]
+    """
+    crash: the child's exit code
+    """
+    pid: NotRequired[int]
+    """
+    stuck_in_driver: the child that could not be reaped
+    """
+
+
+class Arch(TypedDict):
+    compute: str
+    """
+    "8.6"
+    """
+    sm: str
+    """
+    the compile target, "sm_86"
+    """
+    family: Literal[
+        "kepler", "maxwell", "pascal", "volta", "turing", "ampere", "ada", "hopper", "blackwell", "unknown"
+    ]
+
+
+class Layout(TypedDict):
+    sms: NotRequired[int]
+    """
+    streaming multiprocessors
+    """
+    warpSize: NotRequired[int]
+    maxThreadsPerBlock: NotRequired[int]
+    maxThreadsPerSM: NotRequired[int]
+    maxBlocksPerSM: NotRequired[int]
+    maxWarpsPerSM: NotRequired[int]
+    """
+    maxThreadsPerSM / warpSize
+    """
+    maxBlockDim: NotRequired[list[int]]
+    maxGridDim: NotRequired[list[int]]
+
+
+class Registers(TypedDict):
+    perBlock: NotRequired[int]
+    perSM: NotRequired[int]
+
+
+class SharedBytes(TypedDict):
+    perBlock: NotRequired[int]
+    """
+    without opting in
+    """
+    perBlockOptin: NotRequired[int]
+    """
+    the most a kernel may request with cuFuncSetAttribute
+    """
+    perSM: NotRequired[int]
+
+
+class Memory(TypedDict):
+    totalMiB: int
+    busWidthBits: NotRequired[int]
+    clockMHz: NotRequired[int]
+    peakGBs: NotRequired[float]
+    """
+    clockMHz × 2 (double data rate) × busWidthBits / 8, in GB/s
+    """
+    l2Bytes: NotRequired[int]
+    l2PersistingBytes: NotRequired[int]
+    constantBytes: NotRequired[int]
+
+
+class ProbeOk(TypedDict):
+    nvq: SchemaVersion
+    uuid: Uuid
+    ok: Literal[True]
+    name: str
+    cuda: str
+    """
+    CUDA driver API version, "13.1"
+    """
+    arch: Arch
+    layout: Layout
+    registers: Registers
+    sharedBytes: SharedBytes
+    memory: Memory
+    smClockMHz: NotRequired[int]
+    """
+    the SM's peak clock
+    """
+    copyEngines: NotRequired[int]
+    """
+    copies that can overlap a kernel
+    """
+    concurrentKernels: NotRequired[bool]
+    computeMode: NotRequired[Literal["default", "prohibited", "exclusive_process", "unknown"]]
+    ecc: NotRequired[bool]
+    integrated: NotRequired[bool]
+    multiGpuBoard: NotRequired[bool]
+    unsupported: NotRequired[list[str]]
+    """
+    attributes the driver did not report
+    """
+    ms: Ms
+
+
+class WatchCard(TypedDict):
+    uuid: Uuid
+    bus: Bus
+    state: Literal["ok", "lost", "error"]
+    error: NotRequired[Code]
+    events: NotRequired[list[Literal["xid", "ecc_double"]]]
+    """
+    event kinds registered for this card
+    """
+    eventsError: NotRequired[Code]
+
+
+class EventXid(TypedDict):
+    """
+    An Xid error the driver raised; uuid and bus absent when NVML named a device nvq does not watch.
+    """
+
+    nvq: SchemaVersion
+    event: Literal["xid"]
+    at: UnixTime
+    uuid: NotRequired[Uuid]
+    bus: NotRequired[Bus]
+    xid: int
+    meaning: str
+    """
+    NVIDIA's name for it; empty for an Xid nvq has no name for
+    """
+
+
+class EventEccDouble(TypedDict):
+    """
+    A double-bit ECC error; uuid and bus absent when NVML named a device nvq does not watch.
+    """
+
+    nvq: SchemaVersion
+    event: Literal["ecc_double"]
+    at: UnixTime
+    uuid: NotRequired[Uuid]
+    bus: NotRequired[Bus]
+    data: int
+
+
+class EventLost(TypedDict):
+    """
+    A card NVML stopped reaching, reported once.
+    """
+
+    nvq: SchemaVersion
+    event: Literal["lost"]
+    at: UnixTime
+    uuid: Uuid
+    bus: Bus
+    error: Code
+
+
+class EventWaitError(TypedDict):
+    """
+    Waiting for driver events failed; watch retries after a second.
+    """
+
+    nvq: SchemaVersion
+    event: Literal["wait_error"]
+    at: UnixTime
+    error: Code
+
+
+class EventSample(TypedDict):
+    """
+    One card's readings, every --every-ms; a reading the card cannot give is absent.
+    """
+
+    nvq: SchemaVersion
+    event: Literal["sample"]
+    at: UnixTime
+    uuid: Uuid
+    bus: Bus
+    tempC: NotRequired[int]
+    powerW: NotRequired[float]
+    smMHz: NotRequired[int]
+    utilPct: NotRequired[int]
+    fanPct: NotRequired[int]
+    energyJ: NotRequired[float]
+    """
+    cumulative, as in list
+    """
+    pcieReplays: NotRequired[int]
+    """
+    cumulative, as in list
+    """
+    limits: NotRequired[list[Limit]]
+
+
+class Version(TypedDict):
+    nvq: SchemaVersion
+    version: str
+
+
+class ErrorDocument(TypedDict):
+    """
+    The whole command failed, or (in list) NVML could not be used.
+    """
+
+    nvq: SchemaVersion
+    error: ErrorBody
+
+
+class Card(TypedDict):
+    index: int
+    """
+    position in bus order; not CUDA's or nvidia-smi's index
+    """
+    uuid: Uuid
+    bus: Bus
+    minor: int
+    """
+    /dev/nvidia<minor>; -1 when the kernel did not say
+    """
+    model: str
+    state: Literal["ok", "lost", "error", "unknown"]
+    """
+    ok: NVML reaches it. lost: fell off the bus (gpu_is_lost). error: another NVML failure. unknown: NVML unusable
+    """
+    error: NotRequired[Code]
+    """
+    with state lost or error
+    """
+    compute: NotRequired[str]
+    """
+    CUDA compute capability, "8.6"
+    """
+    memoryMiB: NotRequired[MemoryMiB]
+    powerW: NotRequired[PowerW]
+    tempC: NotRequired[int]
+    fanPct: NotRequired[int]
+    """
+    target fan speed; consumer cards with fan control off report not_supported
+    """
+    clocksMHz: NotRequired[ClocksMHz]
+    utilPct: NotRequired[UtilPct]
+    pstate: NotRequired[int]
+    """
+    performance state, 0 fastest
+    """
+    pcie: NotRequired[Pcie]
+    """
+    the link now and at best; a riser shows here (gen 1, width 1)
+    """
+    energyJ: NotRequired[float]
+    """
+    energy the card used since the driver loaded, joules; the difference of two readings over their interval is average power
+    """
+    persistence: NotRequired[bool]
+    limits: NotRequired[list[Limit]]
+    processes: NotRequired[list[Process]]
+    """
+    compute processes holding the card
+    """
+    unsupported: NotRequired[list[str]]
+    """
+    fields this card does not report
+    """
+    errors: NotRequired[dict[str, Code]]
+    """
+    fields whose query failed, by field
+    """
+
+
+type Probe = ErrorDocument | ProbeOk | ProbeFailed
+
+
+class ProbeAllOk(TypedDict):
+    """
+    One entry per card; a card that failed is an entry that says how, not a failure.
+    """
+
+    nvq: SchemaVersion
+    probes: list[ErrorDocument | ProbeOk | ProbeFailed]
+
+
+class EventStart(TypedDict):
+    """
+    First line: the cards watched and the event kinds registered on each.
+    """
+
+    nvq: SchemaVersion
+    event: Literal["start"]
+    at: UnixTime
+    cards: list[WatchCard]
+
+
+class ListOk(TypedDict):
+    """
+    Every card the kernel bound; lost ones are cards with state lost, not a failure.
+    """
+
+    nvq: SchemaVersion
+    driver: str
+    """
+    kernel module version from /proc, "590.48.01"; empty when unreadable
+    """
+    nvml: NotRequired[str]
+    """
+    NVML library version
+    """
+    cuda: NotRequired[str]
+    """
+    highest CUDA version the driver supports, "13.1"
+    """
+    error: NotRequired[ErrorBody]
+    """
+    NVML could not be used; every card's state is unknown
+    """
+    cards: list[Card]
+
+
+type ProbeAll = ErrorDocument | ProbeAllOk
+
+
+type Event = ErrorDocument | EventStart | EventXid | EventEccDouble | EventLost | EventWaitError | EventSample
+"""
+One line of watch, told apart by "event".
+"""
+
+
+type List = ErrorDocument | ListOk

nvq-0.4.0.dist-info/METADATA

@@ -0,0 +1,25 @@
+Metadata-Version: 2.5
+Name: nvq
+Version: 0.4.0
+Summary: NVIDIA cards as JSON, for programs: list, probe, watch. Carries the nvq binary.
+Project-URL: Source, https://github.com/adriangalilea/nvq
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# nvq
+
+NVIDIA cards as JSON, for programs. This package carries the [nvq](https://github.com/adriangalilea/nvq) binary for linux x86_64 and aarch64 (glibc 2.17 or newer) and runs it: `pip install nvq` is the whole install, and `nvq` lands on PATH too.
+
+```python
+import nvq
+
+n = nvq.NVQ()
+for card in n.list()["cards"]:
+    print(card["model"], card["state"], card.get("tempC"))
+
+for ev in n.watch(every=1.0):
+    if ev["event"] == "xid":
+        print(ev["xid"], ev["meaning"])
+```
+
+Documents are the parsed JSON, keys as the [contract](https://github.com/adriangalilea/nvq/blob/main/schema/nvq.schema.json) names them; a value the card cannot report is an absent key. Their types (`nvq.documents`) are TypedDicts generated from that schema. A command that failed as a whole raises `nvq.Error` with its `code` and `exit`; a lost card or a failed probe step is data, not an exception. Each call takes a `timeout`: past it nvq is killed, so a card hung in the driver never hangs your program.

nvq-0.4.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+nvq/__init__.py,sha256=7I8wXhYMzImZFKZ_bKAD8U1QLWsLS2UeMDr3-s_QS9A,7579
+nvq/documents.py,sha256=rDkiC6EZTzoE83DjklT6FUP9SAU37ZGUROvBR2CP7aE,10661
+nvq-0.4.0.dist-info/METADATA,sha256=GkjO0jzWKsck7M6-JqwtkQSGigV0K_xXdMIZ7un9jbo,1251
+nvq-0.4.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+nvq-0.4.0.dist-info/entry_points.txt,sha256=xREF6LumDfLbWCmIbVcbPvHCvvFNz9tPcdcfNuRRNXQ,33
+nvq-0.4.0.dist-info/RECORD,,

nvq-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

nvq-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nvq = nvq:main