gpustack-runtime 0.1.41.post3__py3-none-any.whl → 0.1.42.post1__py3-none-any.whl

gpustack_runtime/detector/amd.py

@@ -8,7 +8,14 @@ from pathlib import Path
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
 from . import Topology, pyamdgpu, pyamdsmi, pyhsa, pyrocmcore, pyrocmsmi
-from .__types__ import Detector, Device, Devices, ManufacturerEnum, TopologyDistanceEnum
+from .__types__ import (
+    Detector,
+    Device,
+    DeviceMemoryStatusEnum,
+    Devices,
+    ManufacturerEnum,
+    TopologyDistanceEnum,
+)
 from .__utils__ import (
     PCIDevice,
     byte_to_mebibyte,
@@ -165,20 +172,32 @@ class AMDDetector(Detector):
                     )
                     dev_cores_util = 0
 
-                dev_mem = None
-                dev_mem_used = None
+                dev_mem = 0
+                dev_mem_used = 0
+                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 try:
                     dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
                     dev_mem = dev_gpu_vram_usage.get("vram_total")
                     dev_mem_used = dev_gpu_vram_usage.get("vram_used")
+                    dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
+                        dev,
+                        pyamdsmi.AmdSmiGpuBlock.UMC,
+                    )
+                    if dev_ecc_count.get("uncorrectable_count", 0) > 0:
+                        dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
                 except pyamdsmi.AmdSmiException:
+                    dev_mem = byte_to_mebibyte(  # byte to MiB
+                        pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
+                    )
+                    dev_mem_used = byte_to_mebibyte(  # byte to MiB
+                        pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
+                    )
                     with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                        dev_mem = byte_to_mebibyte(  # byte to MiB
-                            pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
-                        )
-                        dev_mem_used = byte_to_mebibyte(  # byte to MiB
-                            pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
+                        dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(
+                            dev_idx,
                         )
+                        if dev_ecc_count.uncorrectable_err > 0:
+                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
 
                 dev_power = None
                 dev_power_used = None
@@ -201,14 +220,16 @@ class AMDDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_numa = str(pyamdsmi.amdsmi_topo_get_numa_node_number(dev))
+                    with contextlib.suppress(pyamdsmi.AmdSmiException):
+                        dev_numa = str(pyamdsmi.amdsmi_topo_get_numa_node_number(dev))
 
                 dev_appendix = {
                     "arch_family": _get_arch_family(dev_asic_family_id),
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
                 if dev_card_id is not None:
                     dev_appendix["card_id"] = dev_card_id
                 if dev_renderd_id is not None:
@@ -232,6 +253,7 @@ class AMDDetector(Detector):
                         memory=dev_mem,
                         memory_used=dev_mem_used,
                         memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                        memory_status=dev_mem_status,
                         temperature=dev_temp,
                         power=dev_power,
                         power_used=dev_power_used,

gpustack_runtime/detector/ascend.py

@@ -10,6 +10,7 @@ from . import pyacl, pydcmi
 from .__types__ import (
     Detector,
     Device,
+    DeviceMemoryStatusEnum,
     Devices,
     ManufacturerEnum,
     Topology,
@@ -31,10 +32,12 @@ slogger = logger.getChild("internal")
 _TOPOLOGY_DISTANCE_MAPPING: dict[int, int] = {
     pydcmi.DCMI_TOPO_TYPE_SELF: TopologyDistanceEnum.SELF,
     pydcmi.DCMI_TOPO_TYPE_HCCS: TopologyDistanceEnum.LINK,  # Traversing via high-speed interconnect, RoCE, etc.
+    pydcmi.DCMI_TOPO_TYPE_HCCS_SW: TopologyDistanceEnum.LINK,  # Traversing via high-speed interconnect switch.
     pydcmi.DCMI_TOPO_TYPE_PIX: TopologyDistanceEnum.PIX,  # Traversing via a single PCIe bridge.
     pydcmi.DCMI_TOPO_TYPE_PXB: TopologyDistanceEnum.PXB,  # Traversing via multiple PCIe bridges without PCIe Host Bridge.
     pydcmi.DCMI_TOPO_TYPE_PHB: TopologyDistanceEnum.PHB,  # Traversing via a PCIe Host Bridge.
     pydcmi.DCMI_TOPO_TYPE_SYS: TopologyDistanceEnum.SYS,  # Traversing via SMP interconnect across other NUMA nodes.
+    pydcmi.DCMI_TOPO_TYPE_SIO: TopologyDistanceEnum.SYS,  # Traversing via Super I/O or other slower interconnects.
 }
 """
 Mapping of Ascend topology types to distance values.
@@ -108,7 +111,7 @@ class AscendDetector(Detector):
 
             sys_driver_ver = pydcmi.dcmi_get_driver_version()
 
-            sys_runtime_ver_original = pyacl.aclsysGetCANNVersion()
+            sys_runtime_ver_original = pyacl.aclsysGetVersion()
             sys_runtime_ver = get_brief_version(sys_runtime_ver_original)
 
             _, card_list = pydcmi.dcmi_get_card_list()
@@ -128,7 +131,9 @@ class AscendDetector(Detector):
                         dev_is_vgpu = True
                         dev_cores_aicore = dev_virt_info.query_info.computing.aic
                         dev_name = dev_virt_info.query_info.name
-                        dev_mem, dev_mem_used = 0, 0
+                        dev_mem = 0
+                        dev_mem_used = 0
+                        dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                         if hasattr(dev_virt_info.query_info.computing, "memory_size"):
                             dev_mem = dev_virt_info.query_info.computing.memory_size
                         dev_index = dev_virt_info.vdev_id
@@ -143,6 +148,10 @@ class AscendDetector(Detector):
                             dev_card_id,
                             dev_device_id,
                         )
+                        dev_mem_status = _get_device_memory_status(
+                            dev_card_id,
+                            dev_device_id,
+                        )
                         dev_index = pydcmi.dcmi_get_device_logic_id(
                             dev_card_id,
                             dev_device_id,
@@ -191,13 +200,14 @@ class AscendDetector(Detector):
 
                     dev_numa = get_numa_node_by_bdf(dev_bdf)
                     if not dev_numa:
-                        dev_cpu_affinity = (
-                            pydcmi.dcmi_get_affinity_cpu_info_by_device_id(
-                                dev_card_id,
-                                dev_device_id,
+                        with contextlib.suppress(pydcmi.DCMIError):
+                            dev_cpu_affinity = (
+                                pydcmi.dcmi_get_affinity_cpu_info_by_device_id(
+                                    dev_card_id,
+                                    dev_device_id,
+                                )
                             )
-                        )
-                        dev_numa = map_cpu_affinity_to_numa_node(dev_cpu_affinity)
+                            dev_numa = map_cpu_affinity_to_numa_node(dev_cpu_affinity)
 
                     dev_appendix = {
                         "arch_family": (
@@ -206,11 +216,12 @@ class AscendDetector(Detector):
                         ),
                         "vgpu": dev_is_vgpu,
                         "bdf": dev_bdf,
-                        "numa": dev_numa,
                         "card_id": dev_card_id,
                         "device_id": dev_device_id,
                         "device_id_max": device_num_in_card - 1,
                     }
+                    if dev_numa:
+                        dev_appendix["numa"] = dev_numa
 
                     dev_roce_ip, dev_roce_mask, dev_roce_gateway = (
                         _get_device_roce_network_info(
@@ -239,6 +250,7 @@ class AscendDetector(Detector):
                             memory=dev_mem,
                             memory_used=dev_mem_used,
                             memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                            memory_status=dev_mem_status,
                             temperature=dev_temp,
                             power_used=dev_power_used,
                             appendix=dev_appendix,
@@ -332,6 +344,12 @@ def _get_device_memory_info(dev_card_id, dev_device_id) -> tuple[int, int]:
     """
     Get device memory information.
 
+    Args:
+        dev_card_id:
+            The card ID of the device.
+        dev_device_id:
+            The device ID of the device.
+
     Returns:
         A tuple containing total memory and used memory in MiB.
 
@@ -370,6 +388,37 @@ def _get_device_memory_info(dev_card_id, dev_device_id) -> tuple[int, int]:
     return dev_mem, dev_mem_used
 
 
+def _get_device_memory_status(dev_card_id, dev_device_id) -> DeviceMemoryStatusEnum:
+    """
+    Get device memory ECC status.
+
+    Args:
+        dev_card_id:
+            The card ID of the device.
+        dev_device_id:
+            The device ID of the device.
+
+    Returns:
+        DeviceMemoryStatusEnum indicating the ECC status.
+
+    """
+    for dev_mem_type in [pydcmi.DCMI_DEVICE_TYPE_HBM, pydcmi.DCMI_DEVICE_TYPE_DDR]:
+        with contextlib.suppress(pydcmi.DCMIError):
+            dev_ecc_info = pydcmi.dcmi_get_device_ecc_info(
+                dev_card_id,
+                dev_device_id,
+                dev_mem_type,
+            )
+            if dev_ecc_info.enable_flag and (
+                dev_ecc_info.single_bit_error_cnt > 0
+                or dev_ecc_info.double_bit_error_cnt > 0
+            ):
+                return DeviceMemoryStatusEnum.UNHEALTHY
+            return DeviceMemoryStatusEnum.HEALTHY
+
+    return DeviceMemoryStatusEnum.HEALTHY
+
+
 def _get_device_roce_network_info(
     dev_card_id,
     dev_device_id,
@@ -395,7 +444,7 @@ def _get_device_roce_network_info(
             pydcmi.DCMI_PORT_TYPE_ROCE_PORT,
         )
     except pydcmi.DCMIError:
-        debug_log_exception(logger, "Failed to get device roce network info")
+        debug_log_exception(logger, "Failed to get device RoCE network info")
 
     return ip, mask, gateway
 
@@ -456,12 +505,15 @@ _soc_name_version_mapping: dict[str, int] = {
     "Ascend310B3": 242,
     "Ascend310B4": 243,
     "Ascend910_9391": 250,
+    "Ascend910": 250,
     "Ascend910_9392": 251,
     "Ascend910_9381": 252,
     "Ascend910_9382": 253,
     "Ascend910_9372": 254,
     "Ascend910_9362": 255,
     "Ascend910_9579": 260,
+    "Ascend910_95": 260,
+    "Ascend950": 260,
 }
 
 
@@ -477,6 +529,8 @@ def _guess_soc_name_from_dev_name(dev_name: str) -> str | None:
         The guessed SoC name, or None if not found.
 
     """
+    if dev_name.startswith("Ascend"):
+        dev_name = dev_name[6:].strip()
     soc_name = f"Ascend{dev_name}"
     if soc_name in _soc_name_version_mapping:
         return soc_name
@@ -528,11 +582,11 @@ def get_ascend_cann_variant(name: str | None) -> str | None:
     if version < 220:
         return "310p"
     if version < 240:
-        return "910b"
+        return "910b"  # 910b/a2
     if version < 250:
         return "310b"
     if version < 260:
-        return "a3"  # 910c
+        return "a3"  # 910c/a3
     if version < 270:
-        return "a5"  # 910d
+        return "a5"  # 910d/a5
     return None

gpustack_runtime/detector/cambricon.py

@@ -6,6 +6,7 @@ from functools import lru_cache
 
 from .. import envs
 from ..logging import debug_log_exception
+from . import DeviceMemoryStatusEnum
 from .__types__ import Detector, Device, Devices, ManufacturerEnum
 from .__utils__ import (
     PCIDevice,
@@ -100,6 +101,7 @@ class CambriconDetector(Detector):
                 dev_mem_usage_info = dev_info.get("PhysicalMemUsage", {})
                 dev_mem = safe_int(dev_mem_usage_info.get("Total", 0))
                 dev_mem_used = safe_int(dev_mem_usage_info.get("Used", 0))
+                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
 
                 dev_temp_info = dev_info.get("Temperature", {})
                 dev_temp = safe_float(dev_temp_info.get("Chip", 0))
@@ -118,6 +120,7 @@ class CambriconDetector(Detector):
                         memory=dev_mem,
                         memory_used=dev_mem_used,
                         memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                        memory_status=dev_mem_status,
                         temperature=dev_temp,
                         appendix=dev_appendix,
                     ),

gpustack_runtime/detector/hygon.py

@@ -8,7 +8,14 @@ from pathlib import Path
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
 from . import Topology, pyamdgpu, pyhsa, pyrocmcore, pyrocmsmi
-from .__types__ import Detector, Device, Devices, ManufacturerEnum, TopologyDistanceEnum
+from .__types__ import (
+    Detector,
+    Device,
+    DeviceMemoryStatusEnum,
+    Devices,
+    ManufacturerEnum,
+    TopologyDistanceEnum,
+)
 from .__utils__ import (
     PCIDevice,
     byte_to_mebibyte,
@@ -149,6 +156,13 @@ class HygonDetector(Detector):
                 dev_mem_used = byte_to_mebibyte(  # byte to MiB
                     pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
                 )
+                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
+                with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                    dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(
+                        dev_idx,
+                    )
+                    if dev_ecc_count.uncorrectable_err > 0:
+                        dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
 
                 dev_power = pyrocmsmi.rsmi_dev_power_cap_get(dev_idx)
                 dev_power_used = pyrocmsmi.rsmi_dev_power_get(dev_idx)
@@ -157,13 +171,17 @@ class HygonDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_numa = str(pyrocmsmi.rsmi_topo_get_numa_node_number(dev_idx))
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        dev_numa = str(
+                            pyrocmsmi.rsmi_topo_get_numa_node_number(dev_idx),
+                        )
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
                 if dev_card_id is not None:
                     dev_appendix["card_id"] = dev_card_id
                 if dev_renderd_id is not None:
@@ -184,6 +202,7 @@ class HygonDetector(Detector):
                         memory=dev_mem,
                         memory_used=dev_mem_used,
                         memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                        memory_status=dev_mem_status,
                         temperature=dev_temp,
                         power=dev_power,
                         power_used=dev_power_used,

gpustack_runtime/detector/iluvatar.py

@@ -10,6 +10,7 @@ from . import pyixml
 from .__types__ import (
     Detector,
     Device,
+    DeviceMemoryStatusEnum,
     Devices,
     ManufacturerEnum,
     Topology,
@@ -135,6 +136,7 @@ class IluvatarDetector(Detector):
 
                 dev_mem = 0
                 dev_mem_used = 0
+                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 with contextlib.suppress(pyixml.NVMLError):
                     dev_mem_info = pyixml.nvmlDeviceGetMemoryInfo(dev)
                     dev_mem = byte_to_mebibyte(  # byte to MiB
@@ -143,6 +145,9 @@ class IluvatarDetector(Detector):
                     dev_mem_used = byte_to_mebibyte(  # byte to MiB
                         dev_mem_info.used,
                     )
+                    dev_health = pyixml.ixmlDeviceGetHealth(dev)
+                    if dev_health != pyixml.IXML_HEALTH_OK:
+                        dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
 
                 dev_cores_util = None
                 with contextlib.suppress(pyixml.NVMLError):
@@ -185,18 +190,20 @@ class IluvatarDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pyixml.nvmlDeviceGetMemoryAffinity(
-                        dev,
-                        get_numa_nodeset_size(),
-                        pyixml.NVML_AFFINITY_SCOPE_NODE,
-                    )
-                    dev_numa = bitmask_to_str(list(dev_node_affinity))
+                    with contextlib.suppress(pyixml.NVMLError):
+                        dev_node_affinity = pyixml.nvmlDeviceGetMemoryAffinity(
+                            dev,
+                            get_numa_nodeset_size(),
+                            pyixml.NVML_AFFINITY_SCOPE_NODE,
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
 
                 ret.append(
                     Device(
@@ -213,6 +220,7 @@ class IluvatarDetector(Detector):
                         memory=dev_mem,
                         memory_used=dev_mem_used,
                         memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                        memory_status=dev_mem_status,
                         temperature=dev_temp,
                         power=dev_power,
                         power_used=dev_power_used,

gpustack_runtime/detector/metax.py

@@ -1,5 +1,6 @@
 from __future__ import annotations as __future_annotations__
 
+import contextlib
 import logging
 from functools import lru_cache
 from pathlib import Path
@@ -10,6 +11,7 @@ from . import pymxsml
 from .__types__ import (
     Detector,
     Device,
+    DeviceMemoryStatusEnum,
     Devices,
     ManufacturerEnum,
     Topology,
@@ -145,6 +147,11 @@ class MetaXDetector(Detector):
                 dev_mem_used = kibibyte_to_mebibyte(  # KiB to MiB
                     dev_mem_info.vramUse,
                 )
+                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
+                with contextlib.suppress(pymxsml.MXSMLError):
+                    dev_ecc_errors = pymxsml.mxSmlGetTotalEccErrors(dev_idx)
+                    if dev_ecc_errors.dramUE > 0:
+                        dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
 
                 dev_temp = (
                     pymxsml.mxSmlGetTemperatureInfo(
@@ -172,17 +179,19 @@ class MetaXDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pymxsml.mxSmlGetNodeAffinity(
-                        dev_idx,
-                        get_numa_nodeset_size(),
-                    )
-                    dev_numa = bitmask_to_str(list(dev_node_affinity))
+                    with contextlib.suppress(pymxsml.MXSMLError):
+                        dev_node_affinity = pymxsml.mxSmlGetNodeAffinity(
+                            dev_idx,
+                            get_numa_nodeset_size(),
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
                 if dev_card_id is not None:
                     dev_appendix["card_id"] = dev_card_id
                 if dev_renderd_id is not None:
@@ -201,6 +210,7 @@ class MetaXDetector(Detector):
                         memory=dev_mem,
                         memory_used=dev_mem_used,
                         memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                        memory_status=dev_mem_status,
                         temperature=dev_temp,
                         power=dev_power,
                         power_used=dev_power_used,

gpustack_runtime/detector/mthreads.py

@@ -1,5 +1,6 @@
 from __future__ import annotations as __future_annotations__
 
+import contextlib
 import logging
 from functools import lru_cache
 
@@ -7,6 +8,7 @@ import pymtml
 
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
+from . import DeviceMemoryStatusEnum
 from .__types__ import (
     Detector,
     Device,
@@ -140,6 +142,7 @@ class MThreadsDetector(Detector):
 
                 dev_mem = 0
                 dev_mem_used = 0
+                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 with pymtml.mtmlMemoryContext(dev) as devmem:
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         pymtml.mtmlMemoryGetTotal(devmem),
@@ -147,6 +150,14 @@ class MThreadsDetector(Detector):
                     dev_mem_used = byte_to_mebibyte(  # byte to MiB
                         pymtml.mtmlMemoryGetUsed(devmem),
                     )
+                    dev_mem_ecc_errors = pymtml.mtmlMemoryGetEccErrorCounter(
+                        devmem,
+                        pymtml.MTML_MEMORY_ERROR_TYPE_UNCORRECTED,
+                        pymtml.MTML_VOLATILE_ECC,
+                        pymtml.MTML_MEMORY_LOCATION_DRAM,
+                    )
+                    if dev_mem_ecc_errors > 0:
+                        dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
 
                 dev_cores_util = None
                 dev_temp = None
@@ -166,19 +177,21 @@ class MThreadsDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pymtml.mtmlDeviceGetMemoryAffinityWithinNode(
-                        dev,
-                        get_numa_nodeset_size(),
-                    )
-                    dev_numa = bitmask_to_str(
-                        list(dev_node_affinity),
-                    )
+                    with contextlib.suppress(pymtml.MTMLError):
+                        dev_node_affinity = (
+                            pymtml.mtmlDeviceGetMemoryAffinityWithinNode(
+                                dev,
+                                get_numa_nodeset_size(),
+                            )
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
 
                 ret.append(
                     Device(
@@ -192,6 +205,7 @@ class MThreadsDetector(Detector):
                         memory=dev_mem,
                         memory_used=dev_mem_used,
                         memory_utilization=get_utilization(dev_mem_used, dev_mem),
+                        memory_status=dev_mem_status,
                         temperature=dev_temp,
                         power_used=dev_power_used,
                         appendix=dev_appendix,