gpustack-runtime 0.2.2.post4__tar.gz → 0.2.2.post6__tar.gz

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpustack-runtime
-Version: 0.2.2.post4
+Version: 0.2.2.post6
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.2.2.post4'
-__version_tuple__ = version_tuple = (0, 2, 2, 'post4')
+__version__ = version = '0.2.2.post6'
+__version_tuple__ = version_tuple = (0, 2, 2, 'post6')
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.2.2.post6/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "79ee112"

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/gpustack_runtime/deployer/__types__.py

@@ -16,6 +16,7 @@ from .. import envs
 from ..detector import (
     ManufacturerEnum,
     detect_devices,
+    expand_mig_devices,
     group_devices_by_manufacturer,
     manufacturer_to_backend,
 )
@@ -1373,9 +1374,11 @@ class Deployer(ABC):
 
         self._materials = {}
 
-        group_devices = group_devices_by_manufacturer(
-            detect_devices(fast=False),
-        )
+        devices = detect_devices(fast=False)
+        if self.allowed_mig_devices:
+            devices = expand_mig_devices(devices)
+
+        group_devices = group_devices_by_manufacturer(devices)
 
         if group_devices:
             for manu, devs in group_devices.items():
@@ -1698,6 +1701,18 @@ class Deployer(ABC):
         """
         return True
 
+    @property
+    def allowed_mig_devices(self) -> bool:
+        """
+        Return whether the deployer addresses the MIG devices of a MIG-partitioned
+        card, instead of the card itself.
+
+        Returns:
+            True if addressed, False otherwise.
+
+        """
+        return True
+
     def close(self):
         if self._pool:
             self._pool.shutdown(cancel_futures=True)

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/gpustack_runtime/deployer/kuberentes.py

@@ -1308,6 +1308,15 @@ class KubernetesDeployer(EndoscopicDeployer):
     def allowed_runtime_uuid_values(self) -> bool:
         return get_resource_injection_policy() != "kdp"
 
+    @property
+    def allowed_mig_devices(self) -> bool:
+        # On Kubernetes a partitioner (e.g. the GPUStack Operator's device
+        # manager) owns MIG: it creates and destroys the instances on demand,
+        # so the card is the item to hold on to, and a pod asks for a slice of
+        # it by resource rather than by addressing an instance that may not
+        # outlive the request.
+        return False
+
     def _prepare_mirrored_deployment(self):
         """
         Prepare for mirrored deployment.

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/gpustack_runtime/detector/__init__.py

@@ -291,6 +291,42 @@ def filter_devices_by_manufacturer(
     return [dev for dev in devices or [] if dev.manufacturer == manufacturer]
 
 
+def expand_mig_devices(devices: Devices | None) -> Devices:
+    """
+    Replace every MIG-partitioned card with its MIG devices.
+
+    Detection reports the physical card and carries its MIG devices in the
+    `mig_devices` appendix, because where a partitioner (e.g. the GPUStack
+    Operator's device manager) owns MIG, the instances come and go and the
+    card is the only stable inventory item. Callers that address MIG devices
+    themselves need them as devices instead: a MIG-enabled card cannot run a
+    workload, only its instances can.
+
+    A MIG-enabled card with no instances is kept as-is: there is nothing to
+    address yet, and dropping it would make the card disappear from the
+    inventory.
+
+    Args:
+        devices:
+            A list of devices to be expanded.
+
+    Returns:
+        A list of devices where MIG-partitioned cards are substituted by their
+        MIG devices.
+
+    """
+    expanded: Devices = []
+    for dev in devices or []:
+        mig_devs = (dev.appendix or {}).get("mig_devices")
+        if not mig_devs:
+            expanded.append(dev)
+            continue
+        expanded.extend(
+            Device(manufacturer=dev.manufacturer, **mig_dev) for mig_dev in mig_devs
+        )
+    return expanded
+
+
 __all__ = [
     "Device",
     "DeviceMemoryStatusEnum",
@@ -302,6 +338,7 @@ __all__ = [
     "backend_to_manufacturer",
     "detect_backend",
     "detect_devices",
+    "expand_mig_devices",
     "filter_devices_by_manufacturer",
     "get_devices_topologies",
     "group_devices_by_manufacturer",

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/gpustack_runtime/detector/__types__.py

@@ -461,6 +461,42 @@ def reduce_devices_distances(
     return result
 
 
+def index_mig_devices(
+    devices: Devices,
+    mig_devices: dict[int, list[dict]],
+    slots: int,
+) -> None:
+    """
+    Number the given cards' MIG devices in place.
+
+    A MIG device carries no driver-side inventory index, so its index is
+    synthetic: every card owns a block of `slots` indexes, and the blocks
+    start above the largest index the physical cards report. The reported
+    index may be the card's minor number
+    (`GPUSTACK_RUNTIME_DETECT_PHYSICAL_INDEX_PRIORITY`), which is not bound by
+    the card count, hence the offset is measured from the reported indexes
+    instead of the count. Sizing a block by the slots a card can host, rather
+    than by the MIG devices it currently has, keeps a card's numbering
+    independent of its neighbours: partitioning one card never renumbers
+    another's MIG devices.
+
+    Args:
+        devices:
+            The detected physical cards, already indexed.
+        mig_devices:
+            The MIG devices to number, keyed by the enumeration index of the
+            card hosting them. Each entry's `index` is the driver slot it was
+            found at, which the block offset is added to.
+        slots:
+            The number of MIG devices a card can host, i.e. the block size.
+
+    """
+    base = max((dev.index for dev in devices), default=-1) + 1
+    for dev_idx, migs in mig_devices.items():
+        for mig in migs:
+            mig["index"] += base + dev_idx * slots
+
+
 class Detector(ABC):
     """
     Base class for all detectors.

{gpustack_runtime-0.2.2.post4 → gpustack_runtime-0.2.2.post6}/gpustack_runtime/detector/nvidia.py

@@ -11,7 +11,14 @@ from functools import lru_cache
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
 from . import DeviceMemoryStatusEnum, Topology, pynvml
-from .__types__ import Detector, Device, Devices, ManufacturerEnum, TopologyDistanceEnum
+from .__types__ import (
+    Detector,
+    Device,
+    Devices,
+    ManufacturerEnum,
+    TopologyDistanceEnum,
+    index_mig_devices,
+)
 from .__utils__ import (
     PCIDevice,
     bitmask_to_str,
@@ -113,6 +120,13 @@ class NVIDIADetector(Detector):
                 sys_runtime_ver_original,
             )
 
+            # MIG devices of every MIG-enabled card, keyed by the card's
+            # enumeration index, and the largest number of MIG devices a card
+            # can host: both are needed to number them once every card is
+            # detected, see index_mig_devices.
+            devs_mig_devices: dict[int, list[dict]] = {}
+            devs_mig_slots = 0
+
             dev_count = pynvml.nvmlDeviceGetCount()
             for dev_idx in range(dev_count):
                 dev = pynvml.nvmlDeviceGetHandleByIndex(dev_idx)
@@ -218,6 +232,27 @@ class NVIDIADetector(Detector):
                     "mig": dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE,
                     "bdf": dev_bdf,
                 }
+                if dev_mig_mode != pynvml.NVML_DEVICE_MIG_DISABLE:
+                    dev_mig_slots = 0
+                    with contextlib.suppress(pynvml.NVMLError):
+                        dev_mig_slots = pynvml.nvmlDeviceGetMaxMigDeviceCount(dev)
+                    devs_mig_slots = max(devs_mig_slots, dev_mig_slots)
+                    dev_mig_devices = _get_mig_devices(
+                        dev,
+                        dev_mig_slots,
+                        dev_cc_t,
+                        sys_driver_ver,
+                        sys_runtime_ver,
+                        sys_runtime_ver_original,
+                        dev_cc,
+                        dev_temp,
+                        dev_power,
+                        dev_power_used,
+                        dev_bdf,
+                        dev_numa,
+                    )
+                    dev_appendix["mig_devices"] = dev_mig_devices
+                    devs_mig_devices[dev_idx] = dev_mig_devices
                 if dev_numa:
                     dev_appendix["numa"] = dev_numa
 
@@ -246,6 +281,8 @@ class NVIDIADetector(Detector):
                         appendix=dev_appendix,
                     ),
                 )
+
+            index_mig_devices(ret, devs_mig_devices, devs_mig_slots)
         except pynvml.NVMLError:
             debug_log_exception(logger, "Failed to fetch devices")
             raise
@@ -558,6 +595,151 @@ def _get_links_state(
     }
 
 
+def _get_mig_devices(
+    dev,
+    dev_mig_slots: int,
+    dev_cc_t,
+    sys_driver_ver,
+    sys_runtime_ver,
+    sys_runtime_ver_original,
+    dev_cc,
+    dev_temp,
+    dev_power,
+    dev_power_used,
+    dev_bdf: str,
+    dev_numa,
+) -> list[dict]:
+    """
+    Enumerate the card's current MIG devices with the same detail a plain
+    device carries (profile name, uuid, compute/memory utilization, memory
+    health, temperature and power), returned as appendix entries of the
+    physical card rather than standalone devices. Empty when MIG is enabled
+    but no GPU instances exist yet.
+
+    Each entry's `index` is the driver slot the MIG device was found at:
+    index_mig_devices turns it into the device index once every card is
+    detected.
+    """
+    ret: list[dict] = []
+    with contextlib.suppress(pynvml.NVMLError):
+        for mdev_idx in range(dev_mig_slots):
+            mdev = None
+            with contextlib.suppress(pynvml.NVMLError):
+                mdev = pynvml.nvmlDeviceGetMigDeviceHandleByIndex(dev, mdev_idx)
+            if not mdev:
+                continue
+
+            mdev_uuid = pynvml.nvmlDeviceGetUUID(mdev)
+
+            mdev_mem = 0
+            mdev_mem_used = 0
+            mdev_mem_status = DeviceMemoryStatusEnum.HEALTHY
+            with contextlib.suppress(pynvml.NVMLError):
+                mdev_mem_info = pynvml.nvmlDeviceGetMemoryInfo(mdev)
+                mdev_mem = byte_to_mebibyte(mdev_mem_info.total)
+                mdev_mem_used = byte_to_mebibyte(mdev_mem_info.used)
+                if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+                    mdev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter(
+                        mdev,
+                        pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED,
+                        pynvml.NVML_AGGREGATE_ECC,
+                        pynvml.NVML_MEMORY_LOCATION_SRAM,
+                    )
+                    if mdev_mem_ecc_errors > 0:
+                        mdev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+
+            mdev_appendix = {
+                "arch_family": _get_arch_family(dev_cc_t),
+                "vgpu": True,
+                "sliced": True,
+                "mig": True,
+                "bdf": dev_bdf,
+            }
+            if dev_numa:
+                mdev_appendix["numa"] = dev_numa
+
+            mdev_gi_id = pynvml.nvmlDeviceGetGpuInstanceId(mdev)
+            mdev_appendix["gpu_instance_id"] = mdev_gi_id
+            mdev_ci_id = pynvml.nvmlDeviceGetComputeInstanceId(mdev)
+            mdev_appendix["compute_instance_id"] = mdev_ci_id
+
+            mdev_cores_util = _get_sm_util_from_gpm_metrics(dev, mdev_gi_id)
+
+            mdev_name = ""
+            mdev_cores = None
+            mdev_gi = pynvml.nvmlDeviceGetGpuInstanceById(dev, mdev_gi_id)
+            mdev_ci = pynvml.nvmlGpuInstanceGetComputeInstanceById(
+                mdev_gi,
+                mdev_ci_id,
+            )
+            mdev_gi_info = pynvml.nvmlGpuInstanceGetInfo(mdev_gi)
+            mdev_ci_info = pynvml.nvmlComputeInstanceGetInfo(mdev_ci)
+            for dev_gi_prf_id in range(pynvml.NVML_GPU_INSTANCE_PROFILE_COUNT):
+                try:
+                    dev_gi_prf = pynvml.nvmlDeviceGetGpuInstanceProfileInfo(
+                        dev,
+                        dev_gi_prf_id,
+                    )
+                    if dev_gi_prf.id != mdev_gi_info.profileId:
+                        continue
+                except pynvml.NVMLError:
+                    continue
+
+                gi_mem = round(math.ceil(dev_gi_prf.memorySizeMB >> 10))
+                gi_prf_name = getattr(dev_gi_prf, "name", None)
+                mdev_name = (
+                    gi_prf_name.removeprefix("MIG ")
+                    if gi_prf_name
+                    else f"{dev_gi_prf.sliceCount}g.{gi_mem}gb"
+                )
+
+                for dev_ci_prf_id in range(
+                    pynvml.NVML_COMPUTE_INSTANCE_PROFILE_COUNT,
+                ):
+                    for dev_cig_prf_id in range(
+                        pynvml.NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT,
+                    ):
+                        try:
+                            mdev_ci_prf = (
+                                pynvml.nvmlGpuInstanceGetComputeInstanceProfileInfo(
+                                    mdev_gi,
+                                    dev_ci_prf_id,
+                                    dev_cig_prf_id,
+                                )
+                            )
+                            if mdev_ci_prf.id != mdev_ci_info.profileId:
+                                continue
+                        except pynvml.NVMLError:
+                            continue
+                        mdev_cores = mdev_ci_prf.multiprocessorCount
+                        break
+
+                break
+
+            ret.append(
+                {
+                    "index": mdev_idx,
+                    "name": mdev_name,
+                    "uuid": mdev_uuid,
+                    "driver_version": sys_driver_ver,
+                    "runtime_version": sys_runtime_ver,
+                    "runtime_version_original": sys_runtime_ver_original,
+                    "compute_capability": dev_cc,
+                    "cores": mdev_cores,
+                    "cores_utilization": mdev_cores_util,
+                    "memory": mdev_mem,
+                    "memory_used": mdev_mem_used,
+                    "memory_utilization": get_utilization(mdev_mem_used, mdev_mem),
+                    "memory_status": mdev_mem_status,
+                    "temperature": dev_temp,
+                    "power": dev_power,
+                    "power_used": dev_power_used,
+                    "appendix": mdev_appendix,
+                },
+            )
+    return ret
+
+
 def _get_arch_family(dev_cc_t: list[int]) -> str:
     """
     Get the architecture family based on the CUDA compute capability.