gpustack-runtime 0.2.6__tar.gz → 0.2.7__tar.gz

{gpustack_runtime-0.2.6 → gpustack_runtime-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gpustack-runtime
-Version: 0.2.6
+Version: 0.2.7
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues
@@ -16,7 +16,7 @@ Requires-Python: >=3.10
 Requires-Dist: argcomplete>=3.6.3
 Requires-Dist: cachetools>=5.5.2
 Requires-Dist: docker>=7.1.0
-Requires-Dist: gpustack-runner>=0.1.31
+Requires-Dist: gpustack-runner>=0.1.32.post2
 Requires-Dist: kubernetes>=33.1.0
 Requires-Dist: mthreads-ml-py>=2.2.10
 Requires-Dist: nvidia-ml-py>=13.580.65

{gpustack_runtime-0.2.6 → gpustack_runtime-0.2.7}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.2.6'
-__version_tuple__ = version_tuple = (0, 2, 6)
+__version__ = version = '0.2.7'
+__version_tuple__ = version_tuple = (0, 2, 7)
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.2.7/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "489860c"

{gpustack_runtime-0.2.6 → gpustack_runtime-0.2.7}/gpustack_runtime/detector/__types__.py

@@ -1,6 +1,9 @@
 from __future__ import annotations as __future_annotations__
 
+import copy
 import logging
+import threading
+import time
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from enum import Enum
@@ -9,10 +12,17 @@ from typing import Any
 
 from dataclasses_json import dataclass_json
 
+from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
 
 logger = logging.getLogger(__name__)
 
+_monotonic = time.monotonic
+"""
+The clock the identity cache's time to live is measured on. A module level
+indirection, so tests can inject a clock of their own.
+"""
+
 
 class ManufacturerEnum(str, Enum):
     """
@@ -606,6 +616,285 @@ def _usage_fields_to_merge(usage: Device) -> tuple[str, ...]:
     return tuple(field for field in _DEVICE_USAGE_FIELDS if field != "memory_status")
 
 
+def _clear_health_verdict(devices: Devices) -> None:
+    """
+    Drop the health verdict of the given devices, in place.
+
+    A verdict read when the devices were detected says nothing about them
+    now, and the refresh that could have confirmed it failed. So the devices
+    are served as inventory, with no verdict, rather than with one that went
+    stale.
+
+    Args:
+        devices:
+            The devices to drop the verdict of.
+
+    """
+    for dev in devices:
+        dev.memory_status = DeviceMemoryStatusEnum.UNKNOWN
+        # A MIG device is an entry of the card's appendix, and carries its own
+        # verdict, which the CLI reports as well.
+        for mig_dev in (dev.appendix or {}).get("mig_devices") or []:
+            mig_dev["memory_status"] = DeviceMemoryStatusEnum.UNKNOWN
+
+
+@dataclass
+class _IdentityCacheEntry:
+    """
+    A cached device identity and the moment it expires.
+    """
+
+    devices: Devices
+    """
+    The detected identity, handed out as a copy.
+    """
+
+    expires_at: float
+    """
+    The clock reading after which the identity is stale.
+    """
+
+
+class _RefreshFlight:
+    """
+    One identity refresh of a manufacturer.
+
+    A flight carries its own completion, so a caller that joined it takes
+    that flight's outcome, whatever a later flight does.
+    """
+
+    def __init__(self, stale: Devices | None = None):
+        self.done = threading.Event()
+        self.devices: Devices | None = None
+        self.error: BaseException | None = None
+        self.stale = stale
+        """
+        The last good identity when the flight started: the fallback a failed
+        flight owes its callers.
+        """
+
+    def complete(self, devices: Devices | None, error: BaseException | None) -> None:
+        """
+        Complete the flight with the devices it detected or the error it
+        raised.
+
+        Args:
+            devices:
+                The devices the flight detected, empty when it detected none.
+            error:
+                The exception the flight raised, None when it succeeded.
+
+        """
+        self.devices = devices
+        self.error = error
+        self.done.set()
+
+    def outcome(self) -> Devices | None:
+        """
+        Return the devices the flight detected, or the last good identity
+        when the flight failed.
+
+        The last good identity is served without its health verdict: the
+        verdict was read when the identity was detected, and the refresh that
+        could have confirmed it failed.
+
+        Returns:
+            The devices to serve, as a copy.
+
+        Raises:
+            The exception the flight raised, when no identity is cached to
+            fall back on.
+
+        """
+        if self.error is not None:
+            if self.stale is None:
+                raise self.error
+            # The last good identity is better than nothing, and the next call
+            # retries the detection.
+            devices = copy.deepcopy(self.stale)
+            _clear_health_verdict(devices)
+            return devices
+        return copy.deepcopy(self.devices)
+
+
+class _VendorIdentityState:
+    """
+    The identity cache state of one manufacturer.
+
+    The condition guards the entry and the live flight. A refresh runs
+    outside the lock, so a wedged vendor delays only the callers waiting for
+    that vendor.
+    """
+
+    def __init__(self):
+        self.condition = threading.Condition()
+        self.entry: _IdentityCacheEntry | None = None
+        self.flight: _RefreshFlight | None = None
+        """
+        The refresh running now, if any. A caller that finds one joins it
+        rather than starting a second detection of the same vendor.
+        """
+
+
+class _IdentityCache:
+    """
+    The process wide cache of detected device identities.
+
+    One state per manufacturer, so a slow refresh of one vendor delays only
+    the callers waiting for that vendor.
+    """
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._states: dict[ManufacturerEnum, _VendorIdentityState] = {}
+
+    def state(self, manufacturer: ManufacturerEnum) -> _VendorIdentityState:
+        """
+        Return the cache state of the given manufacturer.
+
+        Args:
+            manufacturer:
+                The manufacturer to take the state of.
+
+        Returns:
+            The state of the manufacturer.
+
+        """
+        with self._lock:
+            state = self._states.get(manufacturer)
+            if state is None:
+                state = _VendorIdentityState()
+                self._states[manufacturer] = state
+            return state
+
+    def reset(self) -> None:
+        """
+        Drop every cached identity.
+        """
+        with self._lock:
+            self._states.clear()
+
+    def identity(self, detector: Detector, ttl: float) -> Devices | None:
+        """
+        Return the identity of the given detector, from the cache when it is
+        still fresh.
+
+        The identity is handed out as a copy: the usage merge writes into the
+        devices it is given, so a shared object would let one caller
+        contaminate the cache.
+
+        A fresh identity is served with the health verdict it was detected
+        with. A stale one is served without any verdict: the refresh that
+        could have confirmed it failed, so the last good identity stands for
+        the inventory only.
+
+        Args:
+            detector:
+                The detector to take the identity of.
+            ttl:
+                The time to live of a cached identity, in seconds.
+                Zero disables the cache.
+
+        Returns:
+            The devices carrying the identity, or None if there is none.
+
+        Raises:
+            The exception the detection raised, when the cache holds no
+            identity to fall back on. An interruption propagates to the caller
+            that asked for the detection, and the waiters of the same refresh
+            get its outcome.
+
+        """
+        if ttl <= 0:
+            # The cache is disabled: the identity is detected on every call.
+            return detector.detect_info()
+
+        state = self.state(detector.manufacturer)
+        with state.condition:
+            entry = state.entry
+            if entry is not None and _monotonic() < entry.expires_at:
+                return copy.deepcopy(entry.devices)
+            flight = state.flight
+            if flight is not None:
+                # A refresh is running: join it, so the callers of one vendor
+                # share one detection. The flight carries its own completion,
+                # so a later flight cannot postpone this caller's outcome.
+                state.condition.wait_for(flight.done.is_set)
+                return flight.outcome()
+            # No refresh is running: start one. The last good identity is the
+            # fallback the flight owes its callers when it fails.
+            flight = _RefreshFlight(
+                stale=entry.devices if entry is not None else None,
+            )
+            state.flight = flight
+
+        # The detection runs outside the lock: a wedged vendor must not hold
+        # the cache for the other vendors.
+        devices: Devices | None = None
+        error: BaseException | None = None
+        try:
+            devices = detector.detect_info()
+        except Exception as e:
+            debug_log_exception(
+                logger,
+                "Failed to detect %s devices identity, keeping the last known one",
+                detector.manufacturer,
+            )
+            error = e
+        except BaseException as e:
+            # An interruption settles the flight as well: a flight left
+            # pending blocks every later caller of the vendor forever. The
+            # interruption keeps propagating to the thread that asked for the
+            # detection, while the waiters get the flight's outcome.
+            debug_log_exception(
+                logger,
+                "Interrupted while detecting %s devices identity",
+                detector.manufacturer,
+            )
+            error = e
+            raise
+        finally:
+            with state.condition:
+                if error is None and devices:
+                    # Only a non empty identity is cached: one enumeration glitch
+                    # must not pin an empty inventory for the time to live.
+                    state.entry = _IdentityCacheEntry(
+                        devices=devices,
+                        expires_at=_monotonic() + ttl,
+                    )
+                flight.complete(devices, error)
+                state.flight = None
+                state.condition.notify_all()
+        return flight.outcome()
+
+
+_IDENTITY_CACHE = _IdentityCache()
+"""
+The process wide cache of detected device identities.
+"""
+
+
+def _identity_cache_ttl() -> float:
+    """
+    Return the time to live of a cached device identity, in seconds.
+
+    Zero disables the cache. The environment variable is read once per
+    process, so it must be set before the first detection.
+
+    Returns:
+        The time to live, in seconds.
+
+    """
+    return float(envs.GPUSTACK_RUNTIME_DETECT_INFO_CACHE_TTL)
+
+
+def _reset_identity_cache() -> None:
+    """
+    Drop every cached device identity.
+    """
+    _IDENTITY_CACHE.reset()
+
+
 class Detector(ABC):
     """
     Base class for all detectors.
@@ -677,6 +966,10 @@ class Detector(ABC):
         """
         Detect devices and return a list of Device objects.
 
+        The identity of the devices is cached process wide, per manufacturer,
+        so a repeated collection pays only the usage query. The usage is read
+        on every call, and so is the health it carries.
+
         Args:
             usage:
                 Whether to fetch the devices' usage as well.
@@ -685,7 +978,7 @@ class Detector(ABC):
             A list of detected Device objects, or None if detection fails.
 
         """
-        devices = self.detect_info()
+        devices = _IDENTITY_CACHE.identity(self, _identity_cache_ttl())
         if usage and devices:
             try:
                 self.detect_usage(devices)

{gpustack_runtime-0.2.6 → gpustack_runtime-0.2.7}/gpustack_runtime/detector/amd.py

@@ -502,6 +502,9 @@ def _get_memory_status(dev, dev_idx: int) -> DeviceMemoryStatusEnum:
     Both queries produce it, mirroring the operator, which reports `Unhealthy`
     from `DetectAccelerator` and `MonitorAccelerator` alike.
 
+    A failed read yields no verdict, as the operator's counter read does: only
+    a readable value may condemn a card.
+
     Args:
         dev:
             The AMD SMI device handle.
@@ -521,14 +524,13 @@ def _get_memory_status(dev, dev_idx: int) -> DeviceMemoryStatusEnum:
             pyamdsmi.AmdSmiGpuBlock.UMC,
         )
     except pyamdsmi.AmdSmiException as e:
-        # Fail closed: a genuine driver error marks the device unhealthy, while
-        # a card without the counter cannot be judged. The stub binding raises
-        # a codeless error when amdsmi is absent, in which case ROCm SMI is
-        # the fallback, as it is for the memory read.
+        # Fail open: a genuine driver error yields no verdict, as the
+        # operator's counter read does. The stub binding raises a codeless
+        # error when amdsmi is absent, in which case ROCm SMI is the
+        # fallback, as it is for the memory read.
         err_code = getattr(e, "err_code", None)
         if err_code is not None:
-            if err_code != getattr(pyamdsmi, "AMDSMI_STATUS_NOT_SUPPORTED", None):
-                return DeviceMemoryStatusEnum.UNHEALTHY
+            debug_log_warning(logger, "Failed to read the UMC ECC error count")
             return DeviceMemoryStatusEnum.HEALTHY
     else:
         if dev_ecc_count.get("uncorrectable_count", 0) > 0:
@@ -539,19 +541,11 @@ def _get_memory_status(dev, dev_idx: int) -> DeviceMemoryStatusEnum:
         dev_ecc_count = pyrocmsmi.rsmi_dev_ecc_count_get(dev_idx)
         if dev_ecc_count.uncorrectable_err > 0:
             return DeviceMemoryStatusEnum.UNHEALTHY
-    except pyrocmsmi.ROCMSMIError as e:
-        # rsmi_status_t comes from the ROCm-installed rsmiBindings, so it is
-        # looked up late: without ROCm the module has no such attribute.
-        rsmi_not_supported = getattr(
-            getattr(pyrocmsmi, "rsmi_status_t", None),
-            "RSMI_STATUS_NOT_SUPPORTED",
-            None,
-        )
-        if e.value not in (
-            pyrocmsmi.ROCMSMI_ERROR_FUNCTION_NOT_FOUND,
-            rsmi_not_supported,
-        ):
-            return DeviceMemoryStatusEnum.UNHEALTHY
+    except pyrocmsmi.ROCMSMIError:
+        # Fail open: a terminal error inside the fallback yields no verdict
+        # either, so the card stays unjudged.
+        debug_log_warning(logger, "Failed to read the ROCm SMI ECC error count")
+        return DeviceMemoryStatusEnum.HEALTHY
 
     return DeviceMemoryStatusEnum.HEALTHY
 

{gpustack_runtime-0.2.6 → gpustack_runtime-0.2.7}/gpustack_runtime/detector/ascend.py

@@ -6,6 +6,7 @@ import os
 import re
 from functools import lru_cache
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
@@ -30,6 +31,9 @@ from .__utils__ import (
     map_numa_node_to_cpu_affinity,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 logger = logging.getLogger(__name__)
 slogger = logger.getChild("internal")
 
@@ -788,33 +792,109 @@ def _get_device_memory_info_v2(dev_id) -> tuple[int, int]:
     return dev_hbm_info.memory_size, dev_hbm_info.memory_usage
 
 
+def _unhealthy(unhealthy: bool) -> DeviceMemoryStatusEnum:
+    """
+    Map a verdict to the device's health status.
+
+    Args:
+        unhealthy:
+            Whether the device is unhealthy.
+
+    Returns:
+        DeviceMemoryStatusEnum.UNHEALTHY if the device is unhealthy,
+        HEALTHY otherwise.
+
+    """
+    if unhealthy:
+        return DeviceMemoryStatusEnum.UNHEALTHY
+    return DeviceMemoryStatusEnum.HEALTHY
+
+
+def _evaluate_device_unhealthy(
+    health: int | None,
+    read_ecc: Callable[[], pydcmi.c_dcmi_ecc_info],
+) -> DeviceMemoryStatusEnum:
+    """
+    Decide whether a device is unhealthy, the way the operator does.
+
+    The DCMI health register is the primary signal: 0 is Normal, 1 is Minor,
+    and 2 or more is Major, Critical or Not Found. The ECC counters are a
+    fallback for a device whose health cannot be read, and only an enabled
+    double-bit error makes it unhealthy. A single-bit error is a corrected
+    error and the device still works, and an unreadable counter says nothing
+    about the device at all.
+
+    Args:
+        health:
+            The device's health status, or None if the query failed.
+        read_ecc:
+            Reads the ECC counters of the device's memory type. It is called
+            only when the health query failed.
+
+    Returns:
+        DeviceMemoryStatusEnum indicating the device's health.
+
+    """
+    if health is not None:
+        return _unhealthy(health >= 2)
+
+    try:
+        ecc_info = read_ecc()
+    except pydcmi.DCMIError:
+        debug_log_exception(logger, "Failed to read the ECC counters of the device")
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    return _unhealthy(ecc_info.enable_flag > 0 and ecc_info.double_bit_error_cnt > 0)
+
+
+def _read_device_health(read_health: Callable[[], int]) -> int | None:
+    """
+    Read the health register of a device.
+
+    Args:
+        read_health:
+            Reads the health status of one device.
+
+    Returns:
+        The device's health status, or None if the query failed.
+
+    """
+    try:
+        return read_health()
+    except pydcmi.DCMIError:
+        debug_log_exception(logger, "Failed to read the health of the device")
+        return None
+
+
 def _get_device_memory_status_v2(dev_id) -> DeviceMemoryStatusEnum:
     """
-    Get device memory ECC status through the V2 API.
+    Get the device's health status through the V2 API.
+
+    The health register decides, as it does in the operator's
+    evaluateDeviceUnhealthy: the device is unhealthy if and only if the
+    status is 2 or more. The ECC counters are read only when the health
+    query fails. V2 declares the HBM call alone, so the counters of that
+    memory type are the only ones a device can have.
 
     Args:
         dev_id:
             The device ID of the device.
 
     Returns:
-        DeviceMemoryStatusEnum indicating the ECC status.
+        DeviceMemoryStatusEnum indicating the device's health.
 
     """
-    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-        for dev_mem_type in [pydcmi.DCMI_DEVICE_TYPE_HBM, pydcmi.DCMI_DEVICE_TYPE_DDR]:
-            with contextlib.suppress(pydcmi.DCMIError):
-                dev_ecc_info = pydcmi.dcmiv2_get_device_ecc_info(
-                    dev_id,
-                    dev_mem_type,
-                )
-                if dev_ecc_info.enable_flag and (
-                    dev_ecc_info.single_bit_error_cnt > 0
-                    or dev_ecc_info.double_bit_error_cnt > 0
-                ):
-                    return DeviceMemoryStatusEnum.UNHEALTHY
-                return DeviceMemoryStatusEnum.HEALTHY
-
-    return DeviceMemoryStatusEnum.HEALTHY
+    if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+        return DeviceMemoryStatusEnum.HEALTHY
+
+    health = _read_device_health(lambda: pydcmi.dcmiv2_get_device_health(dev_id))
+    return _evaluate_device_unhealthy(
+        health,
+        lambda: pydcmi.dcmiv2_get_device_ecc_info(
+            dev_id,
+            pydcmi.DCMI_DEVICE_TYPE_HBM,
+        ),
+    )
 
 
 def _get_device_die(dev_card_id, dev_device_id) -> str:
@@ -964,9 +1044,58 @@ def _get_device_memory_info_without_hbm(dev_card_id, dev_device_id) -> tuple[int
     )
 
 
+def _read_ecc_info_v1(dev_card_id, dev_device_id) -> pydcmi.c_dcmi_ecc_info:
+    """
+    Read the ECC counters of the memory type the device uses.
+
+    The operator picks the memory type from the HBM query: a device that
+    answers it has HBM, and one that does not has DDR. Only that one memory
+    type is read, where the old code looped over both. A device whose HBM
+    query answers with no memory has no HBM either, the same way
+    _get_device_memory_info reads it.
+
+    Args:
+        dev_card_id:
+            The card ID of the device.
+        dev_device_id:
+            The device ID of the device.
+
+    Returns:
+        The ECC counters of the device's memory type.
+
+    Raises:
+        pydcmi.DCMIError: If the driver refuses the ECC query.
+
+    """
+    dev_mem_type = pydcmi.DCMI_DEVICE_TYPE_DDR
+    try:
+        dev_hbm_info = pydcmi.dcmi_get_device_hbm_info(dev_card_id, dev_device_id)
+        if dev_hbm_info.memory_size > 0:
+            dev_mem_type = pydcmi.DCMI_DEVICE_TYPE_HBM
+    except pydcmi.DCMIError:
+        # No HBM means a DDR device, which is what the operator falls back to.
+        debug_log_exception(
+            logger,
+            "Failed to read the HBM info of device %d/%d, reading its DDR counters",
+            dev_card_id,
+            dev_device_id,
+        )
+
+    return pydcmi.dcmi_get_device_ecc_info(
+        dev_card_id,
+        dev_device_id,
+        dev_mem_type,
+    )
+
+
 def _get_device_memory_status(dev_card_id, dev_device_id) -> DeviceMemoryStatusEnum:
     """
-    Get device memory ECC status.
+    Get the device's health status through the V1 API.
+
+    The health register decides, as it does in the operator's
+    evaluateDeviceUnhealthy: the device is unhealthy if and only if the
+    status is 2 or more. The ECC counters are read only when the health
+    query fails, and then only for the memory type the device uses.
 
     Args:
         dev_card_id:
@@ -975,25 +1104,19 @@ def _get_device_memory_status(dev_card_id, dev_device_id) -> DeviceMemoryStatusE
             The device ID of the device.
 
     Returns:
-        DeviceMemoryStatusEnum indicating the ECC status.
+        DeviceMemoryStatusEnum indicating the device's health.
 
     """
-    if not envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
-        for dev_mem_type in [pydcmi.DCMI_DEVICE_TYPE_HBM, pydcmi.DCMI_DEVICE_TYPE_DDR]:
-            with contextlib.suppress(pydcmi.DCMIError):
-                dev_ecc_info = pydcmi.dcmi_get_device_ecc_info(
-                    dev_card_id,
-                    dev_device_id,
-                    dev_mem_type,
-                )
-                if dev_ecc_info.enable_flag and (
-                    dev_ecc_info.single_bit_error_cnt > 0
-                    or dev_ecc_info.double_bit_error_cnt > 0
-                ):
-                    return DeviceMemoryStatusEnum.UNHEALTHY
-                return DeviceMemoryStatusEnum.HEALTHY
+    if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
+        return DeviceMemoryStatusEnum.HEALTHY
 
-    return DeviceMemoryStatusEnum.HEALTHY
+    health = _read_device_health(
+        lambda: pydcmi.dcmi_get_device_health(dev_card_id, dev_device_id),
+    )
+    return _evaluate_device_unhealthy(
+        health,
+        lambda: _read_ecc_info_v1(dev_card_id, dev_device_id),
+    )
 
 
 def _get_device_roce_network_info(

{gpustack_runtime-0.2.6 → gpustack_runtime-0.2.7}/gpustack_runtime/detector/cambricon.py

@@ -313,6 +313,11 @@ def _get_memory_status(device: int) -> DeviceMemoryStatusEnum:
     (`memoryUnhealthy = healthInfo.Health == 0`): CNDev reports no per-memory
     health, and the ECC counters cndevGetECCInfo exposes take no part in it.
 
+    A failed V2 health read falls back to the V1 one, for any reason and not
+    only for a missing V2 symbol, as the operator's evaluateCardUnhealthy does:
+    a driver refusing the V2 API still yields a verdict instead of a blank one.
+    Two unreadable reads yield no verdict, which is healthy.
+
     Args:
         device:
             The device handle.
@@ -324,10 +329,21 @@ def _get_memory_status(device: int) -> DeviceMemoryStatusEnum:
     if envs.GPUSTACK_RUNTIME_DETECT_NO_HEALTH_CHECK:
         return DeviceMemoryStatusEnum.HEALTHY
 
-    with contextlib.suppress(pycndev.CNDevError):
+    try:
         dev_health_state = pycndev.cndevGetCardHealthStateV2(device)
-        if dev_health_state.health == 0:
-            return DeviceMemoryStatusEnum.UNHEALTHY
+    except pycndev.CNDevError:
+        debug_log_warning(
+            logger,
+            "Failed to read the V2 card health state, falling back to V1",
+        )
+        try:
+            dev_health_state = pycndev.cndevGetCardHealthState(device)
+        except pycndev.CNDevError:
+            debug_log_exception(logger, "Failed to read the card health state")
+            return DeviceMemoryStatusEnum.HEALTHY
+
+    if dev_health_state.health == 0:
+        return DeviceMemoryStatusEnum.UNHEALTHY
 
     return DeviceMemoryStatusEnum.HEALTHY