gpustack-runtime 0.2.4.post3__tar.gz → 0.2.5.post1__tar.gz

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gpustack-runtime
-Version: 0.2.4.post3
+Version: 0.2.5.post1
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues
@@ -16,7 +16,7 @@ Requires-Python: >=3.10
 Requires-Dist: argcomplete>=3.6.3
 Requires-Dist: cachetools>=5.5.2
 Requires-Dist: docker>=7.1.0
-Requires-Dist: gpustack-runner>=0.1.26.post2
+Requires-Dist: gpustack-runner>=0.1.29
 Requires-Dist: kubernetes>=33.1.0
 Requires-Dist: mthreads-ml-py>=2.2.10
 Requires-Dist: nvidia-ml-py>=13.580.65

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.2.4.post3'
-__version_tuple__ = version_tuple = (0, 2, 4, 'post3')
+__version__ = version = '0.2.5.post1'
+__version_tuple__ = version_tuple = (0, 2, 5, 'post1')
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.2.5.post1/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "55bbd00"

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/deployer/cdi/ascend.py

@@ -1,6 +1,9 @@
 from __future__ import annotations as __future_annotations__
 
+import json
 import logging
+from functools import cache
+from pathlib import Path
 
 from ...detector import (
     Devices,
@@ -32,6 +35,28 @@ The CANN variant of the A5 generation, whose UB fabric replaces what the
 earlier generations reach through the shared memory device.
 """
 
+_HCCL_RANKTABLE_PATH = "/etc/hccl_rootinfo.json"
+"""
+The host ranktable ("rootinfo") file. Optional, user-maintained state: it
+belongs to no driver package, and the vendor documents it as generated by
+mindcluster-tools and mounted only when present, see
+https://gitcode.com/Ascend/mind-cluster/blob/master/docs/en/scheduling/references/appendix.md.
+"""
+
+_A5_RANKTABLE_VERSION = "2.0"
+"""
+The ranktable version the A5 generation requires.
+
+The format is bound to the chip generation -- 1.0 for A2, 1.2 for A3, 2.0 for
+A5, see ascend-operator/pkg/ranktable/common/common.go:37-38 and
+ranktable/v2dot0/ranktable.go:27. A5 loads libhccl_v2.so, which rejects an
+older table outright rather than ignoring it.
+
+Only the A5 row of that mapping has a measured failure behind it, so only A5 is
+acted on. An A3 host carrying a 1.2 table is presumed correct rather than
+warned about, because nothing here established otherwise.
+"""
+
 _A5_UB_MOUNT_PATTERNS = [
     "/usr/lib64/libummu*",
     "/usr/lib64/liburma*",
@@ -124,30 +149,51 @@ class AscendGenerator(Generator):
         if not common_device_nodes:
             return None
 
-        common_mounts = []
-        for p in [
-            "/etc/hccl_rootinfo.json",
+        # Device.appendix defaults to None and callers pass their own devices
+        # in, so every read goes through `or {}`.
+        is_a5 = any(
+            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
+            == _A5_CANN_VARIANT
+            for dev in devices
+            if dev
+        )
+
+        mount_paths = [
+            _HCCL_RANKTABLE_PATH,
             "/usr/local/Ascend/driver/topo",
             "/usr/local/Ascend/driver/lib64",
             "/usr/local/Ascend/driver/include",
             "/usr/local/dcmi",
             "/usr/local/bin/npu-smi",
             "/var/queue_schedule",
-        ]:
+        ]
+
+        if is_a5:
+            # A5 loads libhccl_v2.so, which accepts a 2.0 ranktable only. A host
+            # file left over from an A2 fleet is 1.0, and HCCL refuses it with
+            # Config_Error_Ranktable(EI0014) rather than falling back, so every
+            # multi-card init dies. Nothing here writes the file, so the only
+            # lever is not mounting it -- verified by isolation: mounting only
+            # this file into an otherwise-working container reproduces EI0014,
+            # mounting only driver/topo does not.
+            #
+            # This is single-node evidence. A multi-node deployment does need a
+            # ranktable; when that lands it has to be a 2.0 one, not this.
+            #
+            # Only GPUStack's own mount list is covered here. Under the Env
+            # policy ascend-docker-runtime mounts the file itself, which
+            # `warn_incompatible_ranktable` reports and cannot prevent.
+            mount_paths.remove(_HCCL_RANKTABLE_PATH)
+
+        common_mounts = []
+        for p in mount_paths:
             cm = path_to_cdi_mount(
                 path=p,
             )
             if cm:
                 common_mounts.append(cm)
 
-        # Device.appendix defaults to None and callers pass their own devices
-        # in, so every read goes through `or {}`.
-        if any(
-            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
-            == _A5_CANN_VARIANT
-            for dev in devices
-            if dev
-        ):
+        if is_a5:
             for pattern in _A5_UB_MOUNT_PATTERNS:
                 ub_mounts = glob_to_cdi_mounts(pattern=pattern)
                 if not ub_mounts:
@@ -225,3 +271,75 @@ class AscendGenerator(Generator):
                 mounts=common_mounts,
             ),
         )
+
+
+def read_ranktable_version(path: str = _HCCL_RANKTABLE_PATH) -> str | None:
+    """
+    Read the ``version`` of the host ranktable file.
+
+    The path is resolved as this process sees it, exactly as the mount list
+    above resolves the paths it emits. Both therefore agree on whether the file
+    exists.
+
+    Args:
+        path:
+            The ranktable path to read.
+
+    Returns:
+        The declared version, or None if the file is absent, unreadable or not
+        a JSON object carrying a string version.
+
+    """
+    try:
+        content = json.loads(Path(path).read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+
+    if not isinstance(content, dict):
+        return None
+
+    version = content.get("version")
+    return version if isinstance(version, str) else None
+
+
+@cache
+def warn_incompatible_ranktable() -> str | None:
+    """
+    Warn once if this A5 host carries a ranktable A5's HCCL will reject.
+
+    ascend-docker-runtime mounts the host ranktable whenever the file is there
+    -- its `addUBMount` is a bare `os.Stat` with no version check -- so under
+    the default Env policy GPUStack cannot prevent the mount. Reporting is the
+    only available action: the operator has to remove the file or replace it
+    with a 2.0 one.
+
+    Returns:
+        The warning message emitted, or None if there is nothing to warn about.
+
+    """
+    # Ordered so a host without the file pays nothing: the file is absent on
+    # almost every non-Ascend host, and detection is the expensive half.
+    version = read_ranktable_version()
+    if version is None or version == _A5_RANKTABLE_VERSION:
+        return None
+
+    devices = detect_devices(manufacturer=ManufacturerEnum.ASCEND)
+    if not devices or not any(
+        get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
+        == _A5_CANN_VARIANT
+        for dev in devices
+        if dev
+    ):
+        # An older generation's own ranktable is correct for it.
+        return None
+
+    msg = (
+        f"Host ranktable {_HCCL_RANKTABLE_PATH} declares version {version!r}, "
+        f"but this A5 (Ascend 950) host needs {_A5_RANKTABLE_VERSION!r}. "
+        f"Under the default Env policy ascend-docker-runtime mounts it on "
+        f"presence and HCCL refuses it with Config_Error_Ranktable(EI0014), "
+        f"failing every multi-card workload. Remove the file or replace it "
+        f"with a {_A5_RANKTABLE_VERSION} table."
+    )
+    logger.warning(msg)
+    return msg

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/deployer/docker.py

@@ -58,6 +58,9 @@ from .__utils__ import (
     sensitive_env_var,
 )
 from .cdi import dump_config as cdi_dump_config
+from .cdi.ascend import (
+    warn_incompatible_ranktable as warn_incompatible_ascend_ranktable,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator
@@ -1568,6 +1571,9 @@ class DockerDeployer(EndoscopicDeployer):
             raise TypeError(msg)
 
         self._prepare_mirrored_deployment()
+        # ascend-docker-runtime mounts the host ranktable itself, so this can
+        # only report, not prevent. Cached, so it costs one check per process.
+        warn_incompatible_ascend_ranktable()
 
         if isinstance(workload, WorkloadPlan):
             workload = DockerWorkloadPlan(**workload.__dict__)

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/detector/amd.py

@@ -186,28 +186,14 @@ class AMDDetector(Detector):
                             dev_asic_family_id = dev_gpudev_info.family_id
 
                 dev_mem = 0
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 try:
                     dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
                     dev_mem = dev_gpu_vram_usage.get("vram_total")
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
-                            dev,
-                            pyamdsmi.AmdSmiGpuBlock.UMC,
-                        )
-                        if dev_ecc_count.get("uncorrectable_count", 0) > 0:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
                 except pyamdsmi.AmdSmiException:
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
                     )
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                            dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(
-                                dev_idx,
-                            )
-                            if dev_ecc_count.uncorrectable_err > 0:
-                                dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                dev_mem_status = _get_memory_status(dev, dev_idx)
 
                 # The power limit is inventory, so it stays here, while the used
                 # power the same call carries belongs to the usage query.
@@ -331,20 +317,10 @@ class AMDDetector(Detector):
 
                 dev_mem = 0
                 dev_mem_used = 0
-                # Health is reported by both queries, as the operator reports it
-                # from DetectAccelerator and MonitorAccelerator alike.
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 try:
                     dev_gpu_vram_usage = pyamdsmi.amdsmi_get_gpu_vram_usage(dev)
                     dev_mem = dev_gpu_vram_usage.get("vram_total")
                     dev_mem_used = dev_gpu_vram_usage.get("vram_used")
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
-                            dev,
-                            pyamdsmi.AmdSmiGpuBlock.UMC,
-                        )
-                        if dev_ecc_count.get("uncorrectable_count", 0) > 0:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
                 except pyamdsmi.AmdSmiException:
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         pyrocmsmi.rsmi_dev_memory_total_get(dev_idx),
@@ -352,13 +328,9 @@ class AMDDetector(Detector):
                     dev_mem_used = byte_to_mebibyte(  # byte to MiB
                         pyrocmsmi.rsmi_dev_memory_usage_get(dev_idx),
                     )
-                    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                        with contextlib.suppress(pyrocmsmi.ROCMSMIError):
-                            dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(
-                                dev_idx,
-                            )
-                            if dev_ecc_count.uncorrectable_err > 0:
-                                dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                # Health is reported by both queries, as the operator reports it
+                # from DetectAccelerator and MonitorAccelerator alike.
+                dev_mem_status = _get_memory_status(dev, dev_idx)
 
                 # A sentinel reading means the same as a failed call -- AMD SMI
                 # cannot tell the used power -- so both route to ROCm SMI.
@@ -523,6 +495,67 @@ class AMDDetector(Detector):
         return ret
 
 
+def _get_memory_status(dev, dev_idx: int) -> DeviceMemoryStatusEnum:
+    """
+    Get a device's memory health from its uncorrected ECC error count.
+
+    Both queries produce it, mirroring the operator, which reports `Unhealthy`
+    from `DetectAccelerator` and `MonitorAccelerator` alike.
+
+    Args:
+        dev:
+            The AMD SMI device handle.
+        dev_idx:
+            The device index, for the ROCm SMI fallback.
+
+    Returns:
+        The memory status.
+
+    """
+    if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    try:
+        dev_ecc_count = pyamdsmi.amdsmi_get_gpu_ecc_count(
+            dev,
+            pyamdsmi.AmdSmiGpuBlock.UMC,
+        )
+    except pyamdsmi.AmdSmiException as e:
+        # Fail closed: a genuine driver error marks the device unhealthy, while
+        # a card without the counter cannot be judged. The stub binding raises
+        # a codeless error when amdsmi is absent, in which case ROCm SMI is
+        # the fallback, as it is for the memory read.
+        err_code = getattr(e, "err_code", None)
+        if err_code is not None:
+            if err_code != getattr(pyamdsmi, "AMDSMI_STATUS_NOT_SUPPORTED", None):
+                return DeviceMemoryStatusEnum.UNHEALTHY
+            return DeviceMemoryStatusEnum.HEALTHY
+    else:
+        if dev_ecc_count.get("uncorrectable_count", 0) > 0:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    try:
+        dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(dev_idx)
+        if dev_ecc_count.uncorrectable_err > 0:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyrocmsmi.ROCMSMIError as e:
+        # rsmi_status_t comes from the ROCm-installed rsmiBindings, so it is
+        # looked up late: without ROCm the module has no such attribute.
+        rsmi_not_supported = getattr(
+            getattr(pyrocmsmi, "rsmi_status_t", None),
+            "RSMI_STATUS_NOT_SUPPORTED",
+            None,
+        )
+        if e.value not in (
+            pyrocmsmi.ROCMSMI_ERROR_FUNCTION_NOT_FOUND,
+            rsmi_not_supported,
+        ):
+            return DeviceMemoryStatusEnum.UNHEALTHY
+
+    return DeviceMemoryStatusEnum.HEALTHY
+
+
 def _get_reading(dev_info: dict, key: str, default: Any = None) -> Any:
     """
     Read one value out of an AMD SMI answer, treating its sentinel as absent.

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/detector/iluvatar.py

@@ -141,7 +141,6 @@ class IluvatarDetector(Detector):
                     dev_cores = pyixml.nvmlDeviceGetNumGpuCores(dev)
 
                 dev_mem = 0
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 with contextlib.suppress(pyixml.NVMLError):
                     # Prefer the v2 memory structure, falling back to v1 --
                     # mirrors the operator's GetMemoryInfoV, which drops the
@@ -150,11 +149,7 @@ class IluvatarDetector(Detector):
                     dev_mem = byte_to_mebibyte(  # byte to MiB
                         dev_mem_info.total,
                     )
-                if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                    with contextlib.suppress(pyixml.NVMLError):
-                        dev_health = pyixml.ixmlDeviceGetHealth(dev)
-                        if dev_health != pyixml.IXML_HEALTH_OK:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                dev_mem_status = _get_memory_status(dev)
 
                 dev_power = None
                 with contextlib.suppress(pyixml.NVMLError):
@@ -251,7 +246,6 @@ class IluvatarDetector(Detector):
 
                 dev_mem = 0
                 dev_mem_used = 0
-                dev_mem_status = DeviceMemoryStatusEnum.HEALTHY
                 with contextlib.suppress(pyixml.NVMLError):
                     # Same v2-then-v1 fallback as detect_info: the operator's
                     # MonitorAccelerator re-reads the memory info rather than
@@ -263,11 +257,7 @@ class IluvatarDetector(Detector):
                     dev_mem_used = byte_to_mebibyte(  # byte to MiB
                         dev_mem_info.used,
                     )
-                if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-                    with contextlib.suppress(pyixml.NVMLError):
-                        dev_health = pyixml.ixmlDeviceGetHealth(dev)
-                        if dev_health != pyixml.IXML_HEALTH_OK:
-                            dev_mem_status = DeviceMemoryStatusEnum.UNHEALTHY
+                dev_mem_status = _get_memory_status(dev)
 
                 dev_cores_util = None
                 with contextlib.suppress(pyixml.NVMLError):
@@ -405,3 +395,34 @@ def _get_memory_info(dev):
         return pyixml.nvmlDeviceGetMemoryInfo(dev, version=pyixml.nvmlMemory_v2)
     except pyixml.NVMLError:
         return pyixml.nvmlDeviceGetMemoryInfo(dev)
+
+
+def _get_memory_status(dev) -> DeviceMemoryStatusEnum:
+    """
+    Get a device's health from its dedicated health query.
+
+    Both queries produce it, mirroring the operator, which reports `Unhealthy`
+    from `DetectAccelerator` and `MonitorAccelerator` alike.
+
+    Args:
+        dev:
+            The device handle.
+
+    Returns:
+        The memory status.
+
+    """
+    if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    try:
+        dev_health = pyixml.ixmlDeviceGetHealth(dev)
+        if dev_health != pyixml.IXML_HEALTH_OK:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyixml.NVMLError as e:
+        # Fail closed: a query the driver errors on marks the device
+        # unhealthy, while an unsupported query means it cannot be judged.
+        if e.value != pyixml.NVML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+
+    return DeviceMemoryStatusEnum.HEALTHY

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/detector/nvidia.py

@@ -529,7 +529,12 @@ def _get_memory_status(
     memory_location: int,
 ) -> DeviceMemoryStatusEnum:
     """
-    Get a device's memory health from its uncorrected ECC error counter.
+    Get a device's memory health.
+
+    The verdict is the uncorrected ECC error counter plus the driver's
+    recovery state: a GSP failure (Xid 119/154) leaves the ECC counters
+    readable at zero, so the recovery action and reset status fields are
+    probed as well.
 
     Both queries produce it, mirroring the operator, which reports `Unhealthy`
     from `DetectAccelerator` and `MonitorAccelerator` alike.
@@ -549,7 +554,7 @@ def _get_memory_status(
     if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
         return DeviceMemoryStatusEnum.HEALTHY
 
-    with contextlib.suppress(pynvml.NVMLError):
+    try:
         dev_mem_ecc_errors = pynvml.nvmlDeviceGetMemoryErrorCounter(
             dev,
             pynvml.NVML_MEMORY_ERROR_TYPE_UNCORRECTED,
@@ -558,6 +563,29 @@ def _get_memory_status(
         )
         if dev_mem_ecc_errors > 0:
             return DeviceMemoryStatusEnum.UNHEALTHY
+    except pynvml.NVMLError as e:
+        # Fail closed: a query the driver errors on (a wedged GSP answers
+        # NVML_ERROR_UNKNOWN after its RPC timeout) marks the card unhealthy,
+        # while an unsupported counter means the card cannot be judged.
+        if e.value != pynvml.NVML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+
+    try:
+        dev_fields = pynvml.nvmlDeviceGetFieldValues(
+            dev,
+            fieldIds=[
+                pynvml.NVML_FI_DEV_GET_GPU_RECOVERY_ACTION,
+                pynvml.NVML_FI_DEV_RESET_STATUS,
+            ],
+        )
+    except pynvml.NVMLError as e:
+        if e.value != pynvml.NVML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
+        return DeviceMemoryStatusEnum.HEALTHY
+    for dev_field in dev_fields:
+        # A per-field error extracts to None: cannot judge, keep the verdict.
+        if _extract_field_value(dev_field):
+            return DeviceMemoryStatusEnum.UNHEALTHY
 
     return DeviceMemoryStatusEnum.HEALTHY
 

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/detector/thead.py

@@ -508,7 +508,7 @@ def _get_memory_status(
     if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
         return DeviceMemoryStatusEnum.HEALTHY
 
-    with contextlib.suppress(pyhgml.HGMLError):
+    try:
         dev_mem_ecc_errors = pyhgml.hgmlDeviceGetMemoryErrorCounter(
             dev,
             pyhgml.HGML_MEMORY_ERROR_TYPE_UNCORRECTED,
@@ -517,6 +517,11 @@ def _get_memory_status(
         )
         if dev_mem_ecc_errors > 0:
             return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyhgml.HGMLError as e:
+        # Fail closed: a query the driver errors on marks the device
+        # unhealthy, while an unsupported counter means it cannot be judged.
+        if e.value != pyhgml.HGML_ERROR_NOT_SUPPORTED:
+            return DeviceMemoryStatusEnum.UNHEALTHY
 
     return DeviceMemoryStatusEnum.HEALTHY
 

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/gpustack_runtime/envs.py

@@ -46,8 +46,15 @@ if TYPE_CHECKING:
     """
     GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK: bool = True
     """
-    Set true to disable ECC errors check during detection,
+    Set true to disable the health check during detection,
     which is used to determine the health status of the device.
+
+    When enabled (set false), the check reads the uncorrected ECC error
+    counter and, on NVIDIA, the driver's recovery state (GPU reset required,
+    e.g. after a GSP failure); a query the driver errors on reports the device
+    unhealthy. Each query is a driver call per device, and against a wedged
+    device it can block until the driver's RPC timeout (up to 45s with GSP
+    firmware).
     """
     GPUSTACK_RUNTIME_DETECT_BACKEND_MAP_RESOURCE_KEY: dict[str, str] | None = None
     """

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
     # NB(thxCode): Pin to this Podman version for a custom patch. See gpustack_runtime/deployer/__patches__.py.
     "podman==5.6.0",
     "pyyaml",
-    "gpustack-runner>=0.1.26.post2",
+    "gpustack-runner>=0.1.29",
     "nvidia-ml-py>=13.580.65",
     "mthreads-ml-py>=2.2.10",
     "argcomplete>=3.6.3",
@@ -43,13 +43,13 @@ gpustack-runtime = "gpustack_runtime.__main__:main"
 dev=[
     "pyclean==3.4.0",
     "pre-commit==4.5.0",
-    "pytest==8.4.2",
+    "pytest>=9.0.3,<10",
     "hatchling",
     "hatch-vcs",
     "mkdocs>=1.6.1",
     "mkdocstrings-python>=1.18.0",
     "mkdocs-material>=9.6.18",
-    "pymdown-extensions>=10.16.1",
+    "pymdown-extensions>=11.0.1",
     "dockerpty>=0.4.1",
     "amdsmi>=7.0.2",
 ]

{gpustack_runtime-0.2.4.post3 → gpustack_runtime-0.2.5.post1}/specs/2026-08-14-detector-alignment-and-workload-exit-status.md

@@ -128,7 +128,7 @@ covered by F3). Known gaps, each to be closed or documented as a deliberate dive
 | Cambricon | Not a binding at all: a `cnmon info -e -m -u -j` shell-out parsing `cnmon_info.json`, with a `TODO(thxCode)` where the sample output should be. No driver version, no Neuware version, no cores, no power, no BDF, no NUMA, no health/ECC, no PCIe bus id. | New hand-written `pycndev` ctypes binding (following `pydcmi`/`pymxsml`) and a rewritten `cambricon.py` calling `GetDeviceCount`, `GetDeviceHandleByIndex`, `GetUUID`, `GetPCIeInfoV`, `GetCardName`, `GetMemoryInfoV`, `GetCardHealthStateV`, `GetVersionInfo`, `GetUtilizationInfo`, `GetTemperatureInfo`, `GetPowerInfo`, plus the Neuware version from `/usr/local/neuware/version.txt`. `cndev.h` in the operator's `binding/cndev` is the header of record. Four decisions taken while building `T3`, all recorded rather than assumed: **cores and the power limit stay unreported** — `pycndev` binds no core-count entry point and the operator reads neither for this vendor, so closing those two needs a binding addition of its own; **`driver_version` carries `major.minor.build`** where the operator formats only `major.minor`, since it is free precision from the same query and every other vendor here reports the driver's full string; **health comes from `cndevGetCardHealthStateV2().health` alone, with no ECC read**, because that is literally what the operator does (`memoryUnhealthy = healthInfo.Health == 0`); and **memory needs no unit conversion** — the header says MB but the operator, `cnmon` and this repo's Ascend detector all treat the value as MiB, so converting would under-report ~4.8 % against the operator on the same host, which is the very discrepancy Story 1 exists to remove. `C2` confirms the memory figure against `cnmon`. |
 | Cambricon | Per-device failure policy diverges from every sibling **on purpose**: `T3` follows the operator and `continue`s past a card whose required reads fail, where the other eight detectors let the error propagate. | A single faulty card therefore costs one device on Cambricon and **all** devices on the other eight. Graceful degradation is very likely the right behaviour everywhere and the other eight should move toward it, but that is a repo-wide behaviour change deserving its own task — not something to slip in per vendor. |
 | THead | Enumerates MIG-style GPU/compute instances, which the operator does not. | Keep — this is the appendix mechanism F2 preserves. |
-| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. |
+| **All vendors** | `GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK` **defaults to true**, so a default run never reads an ECC counter and every `memory_status` comes back `healthy`. The operator reads the uncorrected-ECC counter unconditionally. | Keep the default (the check costs a driver call per card per pass), but record it as a deliberate divergence: on the same host the runtime can report `healthy` where the operator reports `Unhealthy`, which is a code-path difference of exactly the kind `Story 1` exists to eliminate. `C2` must therefore compare health with the flag switched **off**, or it compares nothing. Found while building `T5`. With the flag off, the NVIDIA check additionally probes the driver's recovery state (`NVML_FI_DEV_GET_GPU_RECOVERY_ACTION` / `NVML_FI_DEV_RESET_STATUS`, i.e. the Xid 154 GPU-reset-required state) and treats an errored query as `unhealthy` rather than swallowing it; against a wedged card such a query can block until the driver's GSP RPC timeout (up to 45s). |
 
 Acceptance criteria:
 - ~~A written parity table lives in the repo (module docstring or `docs/`) listing, per vendor, the