gpustack-runtime 0.1.42__tar.gz → 0.1.42.post1__tar.gz

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpustack-runtime
-Version: 0.1.42
+Version: 0.1.42.post1
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.1.42'
-__version_tuple__ = version_tuple = (0, 1, 42)
+__version__ = version = '0.1.42.post1'
+__version_tuple__ = version_tuple = (0, 1, 42, 'post1')
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.1.42.post1/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "ff610ac"

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/cmds/detector.py

@@ -123,7 +123,7 @@ class GetDevicesTopologySubCommand(SubCommand):
                     "",
                     "Legend (from nearest to farthest):",
                     "  X    = Self",
-                    "  LINK = Connection traversing with High-Speed Link (e.g., AMD XGMI, Ascend HCCS, NVIDIA NVLink)",
+                    "  LINK = Connection traversing with High-Speed Link (e.g., AMD XGMI, Ascend HCCS, MThreads MTLink, NVIDIA NVLink, T-Head ICNLink)",
                     "  PIX  = Connection traversing at most a single PCIe bridge",
                     "  PXB  = Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge)",
                     "  PHB  = Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU)",

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/__init__.py

@@ -82,7 +82,7 @@ def dump_config(
     if cdi_path.exists():
         actual = cdi_path.read_text(encoding="utf-8")
         if actual == expected:
-            return expected, str(cdi_path)
+            return expected, None
 
     cdi_path.write_text(expected, encoding="utf-8")
     return expected, str(cdi_path)

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/__types__.py

@@ -527,7 +527,7 @@ class Config(dict):
         self,
         kind: str,
         devices: list[ConfigDevice],
-        container_edits: list[ConfigContainerEdits] | None = None,
+        container_edits: ConfigContainerEdits | None = None,
         cdi_version: str = _DEFAULT_CDI_VERSION,
         annotations: dict[str, str] | None = None,
     ):
@@ -581,7 +581,7 @@ class Config(dict):
         return self["kind"]
 
     @property
-    def container_edits(self) -> list[ConfigContainerEdits] | None:
+    def container_edits(self) -> ConfigContainerEdits | None:
         """
         Return the list of container edits in the CDI configuration.
 

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/amd.py

@@ -135,12 +135,10 @@ class AMDGenerator(Generator):
         return Config(
             kind=kind,
             devices=cdi_devices,
-            container_edits=[
-                ConfigContainerEdits(
-                    env=[
-                        f"{runtime_env}=void",
-                    ],
-                    device_nodes=common_device_nodes,
-                ),
-            ],
+            container_edits=ConfigContainerEdits(
+                env=[
+                    f"{runtime_env}=void",
+                ],
+                device_nodes=common_device_nodes,
+            ),
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/ascend.py

@@ -152,13 +152,11 @@ class AscendGenerator(Generator):
         return Config(
             kind=kind,
             devices=cdi_devices,
-            container_edits=[
-                ConfigContainerEdits(
-                    env=[
-                        f"{runtime_env}=void",
-                    ],
-                    device_nodes=common_device_nodes,
-                    mounts=common_mounts,
-                ),
-            ],
+            container_edits=ConfigContainerEdits(
+                env=[
+                    f"{runtime_env}=void",
+                ],
+                device_nodes=common_device_nodes,
+                mounts=common_mounts,
+            ),
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/hygon.py

@@ -136,12 +136,10 @@ class HygonGenerator(Generator):
         return Config(
             kind=kind,
             devices=cdi_devices,
-            container_edits=[
-                ConfigContainerEdits(
-                    env=[
-                        f"{runtime_env}=void",
-                    ],
-                    device_nodes=common_device_nodes,
-                ),
-            ],
+            container_edits=ConfigContainerEdits(
+                env=[
+                    f"{runtime_env}=void",
+                ],
+                device_nodes=common_device_nodes,
+            ),
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/iluvatar.py

@@ -125,12 +125,10 @@ class IluvatarGenerator(Generator):
         return Config(
             kind=kind,
             devices=cdi_devices,
-            container_edits=[
-                ConfigContainerEdits(
-                    env=[
-                        f"{runtime_env}=void",
-                    ],
-                    device_nodes=common_device_nodes,
-                ),
-            ],
+            container_edits=ConfigContainerEdits(
+                env=[
+                    f"{runtime_env}=void",
+                ],
+                device_nodes=common_device_nodes,
+            ),
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/metax.py

@@ -137,12 +137,10 @@ class MetaXGenerator(Generator):
         return Config(
             kind=kind,
             devices=cdi_devices,
-            container_edits=[
-                ConfigContainerEdits(
-                    env=[
-                        f"{runtime_env}=void",
-                    ],
-                    device_nodes=common_device_nodes,
-                ),
-            ],
+            container_edits=ConfigContainerEdits(
+                env=[
+                    f"{runtime_env}=void",
+                ],
+                device_nodes=common_device_nodes,
+            ),
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/cdi/thead.py

@@ -126,12 +126,10 @@ class THeadGenerator(Generator):
         return Config(
             kind=kind,
             devices=cdi_devices,
-            container_edits=[
-                ConfigContainerEdits(
-                    env=[
-                        f"{runtime_env}=void",
-                    ],
-                    device_nodes=common_device_nodes,
-                ),
-            ],
+            container_edits=ConfigContainerEdits(
+                env=[
+                    f"{runtime_env}=void",
+                ],
+                device_nodes=common_device_nodes,
+            ),
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/docker.py

@@ -985,11 +985,37 @@ class DockerDeployer(EndoscopicDeployer):
                     # Generate CDI config if not yet.
                     if cdi and envs.GPUSTACK_RUNTIME_DOCKER_CDI_SPECS_GENERATE:
                         for ren in runtime_envs:
-                            r_m = self.get_manufacturer(ren)
-                            cdi_dump_config(
-                                manufacturer=r_m,
+                            manu = self.get_manufacturer(ren)
+                            cdi_config, cdi_config_path = cdi_dump_config(
+                                manufacturer=manu,
                                 output=envs.GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY,
                             )
+                            if cdi_config and cdi_config_path:
+                                if logger.isEnabledFor(logging.DEBUG):
+                                    logger.debug(
+                                        "Generated CDI configuration for '%s' at '%s':\n%s",
+                                        manu,
+                                        cdi_config_path,
+                                        cdi_config,
+                                    )
+                                else:
+                                    logger.info(
+                                        "Generated CDI configuration for '%s' at '%s'",
+                                        manu,
+                                        cdi_config_path,
+                                    )
+                            elif cdi_config:
+                                logger.info(
+                                    "Reuse generated CDI configuration for '%s'",
+                                    manu,
+                                )
+                            else:
+                                logger.warning(
+                                    "Delegated CDI configuration by other tools for '%s', "
+                                    "e.g. for NVIDIA devices, please follow NVIDIA Container Toolkit Manual CDI Specification Generation to generate the CDI configuration, "
+                                    "see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/cdi-support.html#manual-cdi-specification-generation",
+                                    manu,
+                                )
 
                     # Request devices.
                     if r_v == "all":

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/k8s/deviceplugin/__init__.py

@@ -78,28 +78,33 @@ async def serve_async(
             allocation_policy == "cdi"
             and envs.GPUSTACK_RUNTIME_KUBERNETES_KDP_CDI_SPECS_GENERATE
         ):
-            generated_content, generated_path = cdi_dump_config(
+            cdi_config, cdi_config_path = cdi_dump_config(
                 manufacturer=manu,
                 output=cdi_generation_output,
             )
-            if generated_content:
+            if cdi_config and cdi_config_path:
                 if logger.isEnabledFor(logging.DEBUG):
                     logger.debug(
                         "Generated CDI configuration for '%s' at '%s':\n%s",
                         manu,
-                        generated_path,
-                        generated_content,
+                        cdi_config_path,
+                        cdi_config,
                     )
                 else:
                     logger.info(
                         "Generated CDI configuration for '%s' at '%s'",
                         manu,
-                        generated_path,
+                        cdi_config_path,
                     )
+            elif cdi_config:
+                logger.info(
+                    "Reuse generated CDI configuration for '%s'",
+                    manu,
+                )
             else:
                 logger.warning(
-                    "Delegated CDI configuration by other tools for manufacturer '%s', "
-                    "e.g. NVIDIA Container Toolkit Manual CDI Specification Generation, "
+                    "Delegated CDI configuration by other tools for '%s', "
+                    "e.g. for NVIDIA devices, please follow NVIDIA Container Toolkit Manual CDI Specification Generation to generate the CDI configuration, "
                     "see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/cdi-support.html#manual-cdi-specification-generation",
                     manu,
                 )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/k8s/deviceplugin/plugin.py

@@ -12,6 +12,7 @@ from grpc_interceptor import AsyncServerInterceptor
 from grpc_interceptor.exceptions import GrpcException
 
 from ....detector import Device, DeviceMemoryStatusEnum, str_range_to_list
+from ....detector.__utils__ import get_numa_node_size
 from ...cdi import (
     generate_config,
     manufacturer_to_cdi_kind,
@@ -346,8 +347,11 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
                 NUMANode(
                     ID=node_id,
                 )
-                for node_id in str_range_to_list(
-                    self._device.appendix.get("numa", "0"),
+                for node_id in (
+                    str_range_to_list(
+                        self._device.appendix.get("numa", ""),
+                    )
+                    or list(range(get_numa_node_size()))
                 )
             ],
         )

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/deployer/podman.py

@@ -982,11 +982,37 @@ class PodmanDeployer(EndoscopicDeployer):
                     # Generate CDI config if not yet.
                     if envs.GPUSTACK_RUNTIME_PODMAN_CDI_SPECS_GENERATE:
                         for ren in runtime_envs:
-                            r_m = self.get_manufacturer(ren)
-                            cdi_dump_config(
-                                manufacturer=r_m,
+                            manu = self.get_manufacturer(ren)
+                            cdi_config, cdi_config_path = cdi_dump_config(
+                                manufacturer=manu,
                                 output=envs.GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY,
                             )
+                            if cdi_config and cdi_config_path:
+                                if logger.isEnabledFor(logging.DEBUG):
+                                    logger.debug(
+                                        "Generated CDI configuration for '%s' at '%s':\n%s",
+                                        manu,
+                                        cdi_config_path,
+                                        cdi_config,
+                                    )
+                                else:
+                                    logger.info(
+                                        "Generated CDI configuration for '%s' at '%s'",
+                                        manu,
+                                        cdi_config_path,
+                                    )
+                            elif cdi_config:
+                                logger.info(
+                                    "Reuse generated CDI configuration for '%s'",
+                                    manu,
+                                )
+                            else:
+                                logger.warning(
+                                    "Delegated CDI configuration by other tools for '%s', "
+                                    "e.g. for NVIDIA devices, please follow NVIDIA Container Toolkit Manual CDI Specification Generation to generate the CDI configuration, "
+                                    "see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/cdi-support.html#manual-cdi-specification-generation",
+                                    manu,
+                                )
 
                     # Configure device access environment variable.
                     if r_v == "all":

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/__utils__.py

@@ -916,6 +916,9 @@ def str_range_to_list(str_range: str) -> list[int]:
         A list of indices.
 
     """
+    if not str_range:
+        return []
+
     str_range_parts = str_range.split(",")
 
     indices: set[int] = set()

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/amd.py

@@ -220,14 +220,16 @@ class AMDDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_numa = str(pyamdsmi.amdsmi_topo_get_numa_node_number(dev))
+                    with contextlib.suppress(pyamdsmi.AmdSmiException):
+                        dev_numa = str(pyamdsmi.amdsmi_topo_get_numa_node_number(dev))
 
                 dev_appendix = {
                     "arch_family": _get_arch_family(dev_asic_family_id),
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
                 if dev_card_id is not None:
                     dev_appendix["card_id"] = dev_card_id
                 if dev_renderd_id is not None:

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/ascend.py

@@ -32,10 +32,12 @@ slogger = logger.getChild("internal")
 _TOPOLOGY_DISTANCE_MAPPING: dict[int, int] = {
     pydcmi.DCMI_TOPO_TYPE_SELF: TopologyDistanceEnum.SELF,
     pydcmi.DCMI_TOPO_TYPE_HCCS: TopologyDistanceEnum.LINK,  # Traversing via high-speed interconnect, RoCE, etc.
+    pydcmi.DCMI_TOPO_TYPE_HCCS_SW: TopologyDistanceEnum.LINK,  # Traversing via high-speed interconnect switch.
     pydcmi.DCMI_TOPO_TYPE_PIX: TopologyDistanceEnum.PIX,  # Traversing via a single PCIe bridge.
     pydcmi.DCMI_TOPO_TYPE_PXB: TopologyDistanceEnum.PXB,  # Traversing via multiple PCIe bridges without PCIe Host Bridge.
     pydcmi.DCMI_TOPO_TYPE_PHB: TopologyDistanceEnum.PHB,  # Traversing via a PCIe Host Bridge.
     pydcmi.DCMI_TOPO_TYPE_SYS: TopologyDistanceEnum.SYS,  # Traversing via SMP interconnect across other NUMA nodes.
+    pydcmi.DCMI_TOPO_TYPE_SIO: TopologyDistanceEnum.SYS,  # Traversing via Super I/O or other slower interconnects.
 }
 """
 Mapping of Ascend topology types to distance values.
@@ -109,7 +111,7 @@ class AscendDetector(Detector):
 
             sys_driver_ver = pydcmi.dcmi_get_driver_version()
 
-            sys_runtime_ver_original = pyacl.aclsysGetCANNVersion()
+            sys_runtime_ver_original = pyacl.aclsysGetVersion()
             sys_runtime_ver = get_brief_version(sys_runtime_ver_original)
 
             _, card_list = pydcmi.dcmi_get_card_list()
@@ -198,13 +200,14 @@ class AscendDetector(Detector):
 
                     dev_numa = get_numa_node_by_bdf(dev_bdf)
                     if not dev_numa:
-                        dev_cpu_affinity = (
-                            pydcmi.dcmi_get_affinity_cpu_info_by_device_id(
-                                dev_card_id,
-                                dev_device_id,
+                        with contextlib.suppress(pydcmi.DCMIError):
+                            dev_cpu_affinity = (
+                                pydcmi.dcmi_get_affinity_cpu_info_by_device_id(
+                                    dev_card_id,
+                                    dev_device_id,
+                                )
                             )
-                        )
-                        dev_numa = map_cpu_affinity_to_numa_node(dev_cpu_affinity)
+                            dev_numa = map_cpu_affinity_to_numa_node(dev_cpu_affinity)
 
                     dev_appendix = {
                         "arch_family": (
@@ -213,11 +216,12 @@ class AscendDetector(Detector):
                         ),
                         "vgpu": dev_is_vgpu,
                         "bdf": dev_bdf,
-                        "numa": dev_numa,
                         "card_id": dev_card_id,
                         "device_id": dev_device_id,
                         "device_id_max": device_num_in_card - 1,
                     }
+                    if dev_numa:
+                        dev_appendix["numa"] = dev_numa
 
                     dev_roce_ip, dev_roce_mask, dev_roce_gateway = (
                         _get_device_roce_network_info(
@@ -440,7 +444,7 @@ def _get_device_roce_network_info(
             pydcmi.DCMI_PORT_TYPE_ROCE_PORT,
         )
     except pydcmi.DCMIError:
-        debug_log_exception(logger, "Failed to get device roce network info")
+        debug_log_exception(logger, "Failed to get device RoCE network info")
 
     return ip, mask, gateway
 
@@ -501,12 +505,15 @@ _soc_name_version_mapping: dict[str, int] = {
     "Ascend310B3": 242,
     "Ascend310B4": 243,
     "Ascend910_9391": 250,
+    "Ascend910": 250,
     "Ascend910_9392": 251,
     "Ascend910_9381": 252,
     "Ascend910_9382": 253,
     "Ascend910_9372": 254,
     "Ascend910_9362": 255,
     "Ascend910_9579": 260,
+    "Ascend910_95": 260,
+    "Ascend950": 260,
 }
 
 
@@ -522,6 +529,8 @@ def _guess_soc_name_from_dev_name(dev_name: str) -> str | None:
         The guessed SoC name, or None if not found.
 
     """
+    if dev_name.startswith("Ascend"):
+        dev_name = dev_name[6:].strip()
     soc_name = f"Ascend{dev_name}"
     if soc_name in _soc_name_version_mapping:
         return soc_name

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/hygon.py

@@ -171,13 +171,17 @@ class HygonDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_numa = str(pyrocmsmi.rsmi_topo_get_numa_node_number(dev_idx))
+                    with contextlib.suppress(pyrocmsmi.ROCMSMIError):
+                        dev_numa = str(
+                            pyrocmsmi.rsmi_topo_get_numa_node_number(dev_idx),
+                        )
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
                 if dev_card_id is not None:
                     dev_appendix["card_id"] = dev_card_id
                 if dev_renderd_id is not None:

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/iluvatar.py

@@ -190,18 +190,20 @@ class IluvatarDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pyixml.nvmlDeviceGetMemoryAffinity(
-                        dev,
-                        get_numa_nodeset_size(),
-                        pyixml.NVML_AFFINITY_SCOPE_NODE,
-                    )
-                    dev_numa = bitmask_to_str(list(dev_node_affinity))
+                    with contextlib.suppress(pyixml.NVMLError):
+                        dev_node_affinity = pyixml.nvmlDeviceGetMemoryAffinity(
+                            dev,
+                            get_numa_nodeset_size(),
+                            pyixml.NVML_AFFINITY_SCOPE_NODE,
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
 
                 ret.append(
                     Device(

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/metax.py

@@ -179,17 +179,19 @@ class MetaXDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pymxsml.mxSmlGetNodeAffinity(
-                        dev_idx,
-                        get_numa_nodeset_size(),
-                    )
-                    dev_numa = bitmask_to_str(list(dev_node_affinity))
+                    with contextlib.suppress(pymxsml.MXSMLError):
+                        dev_node_affinity = pymxsml.mxSmlGetNodeAffinity(
+                            dev_idx,
+                            get_numa_nodeset_size(),
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
                 if dev_card_id is not None:
                     dev_appendix["card_id"] = dev_card_id
                 if dev_renderd_id is not None:

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/mthreads.py

@@ -1,5 +1,6 @@
 from __future__ import annotations as __future_annotations__
 
+import contextlib
 import logging
 from functools import lru_cache
 
@@ -176,19 +177,21 @@ class MThreadsDetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pymtml.mtmlDeviceGetMemoryAffinityWithinNode(
-                        dev,
-                        get_numa_nodeset_size(),
-                    )
-                    dev_numa = bitmask_to_str(
-                        list(dev_node_affinity),
-                    )
+                    with contextlib.suppress(pymtml.MTMLError):
+                        dev_node_affinity = (
+                            pymtml.mtmlDeviceGetMemoryAffinityWithinNode(
+                                dev,
+                                get_numa_nodeset_size(),
+                            )
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_appendix = {
                     "vgpu": dev_is_vgpu,
                     "bdf": dev_bdf,
-                    "numa": dev_numa,
                 }
+                if dev_numa:
+                    dev_appendix["numa"] = dev_numa
 
                 ret.append(
                     Device(

{gpustack_runtime-0.1.42 → gpustack_runtime-0.1.42.post1}/gpustack_runtime/detector/nvidia.py

@@ -134,12 +134,13 @@ class NVIDIADetector(Detector):
 
                 dev_numa = get_numa_node_by_bdf(dev_bdf)
                 if not dev_numa:
-                    dev_node_affinity = pynvml.nvmlDeviceGetMemoryAffinity(
-                        dev,
-                        get_numa_nodeset_size(),
-                        pynvml.NVML_AFFINITY_SCOPE_NODE,
-                    )
-                    dev_numa = bitmask_to_str(list(dev_node_affinity))
+                    with contextlib.suppress(pynvml.NVMLError):
+                        dev_node_affinity = pynvml.nvmlDeviceGetMemoryAffinity(
+                            dev,
+                            get_numa_nodeset_size(),
+                            pynvml.NVML_AFFINITY_SCOPE_NODE,
+                        )
+                        dev_numa = bitmask_to_str(list(dev_node_affinity))
 
                 dev_temp = None
                 with contextlib.suppress(pynvml.NVMLError):
@@ -224,8 +225,9 @@ class NVIDIADetector(Detector):
                         "arch_family": _get_arch_family(dev_cc_t),
                         "vgpu": dev_is_vgpu,
                         "bdf": dev_bdf,
-                        "numa": dev_numa,
                     }
+                    if dev_numa:
+                        dev_appendix["numa"] = dev_numa
 
                     if dev_fabric_info := _get_fabric_info(dev):
                         dev_appendix.update(dev_fabric_info)
@@ -297,8 +299,9 @@ class NVIDIADetector(Detector):
                         "arch_family": _get_arch_family(dev_cc_t),
                         "vgpu": True,
                         "bdf": dev_bdf,
-                        "numa": dev_numa,
                     }
+                    if dev_numa:
+                        mdev_appendix["numa"] = dev_numa
 
                     mdev_gi_id = pynvml.nvmlDeviceGetGpuInstanceId(mdev)
                     mdev_appendix["gpu_instance_id"] = mdev_gi_id