gpustack-runtime 0.1.41.post3__py3-none-any.whl → 0.1.42.post1__py3-none-any.whl

gpustack_runtime/deployer/kuberentes.py

@@ -3,11 +3,9 @@ from __future__ import annotations as __future_annotations__
 import contextlib
 import json
 import logging
-import operator
 import os
 from dataclasses import dataclass, field
 from enum import Enum
-from functools import reduce
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -43,7 +41,7 @@ from .__utils__ import (
     sensitive_env_var,
     validate_rfc1123_domain_name,
 )
-from .k8s.deviceplugin import cdi_kind_to_kdp_resource, get_resource_injection_policy
+from .k8s.deviceplugin import get_resource_injection_policy
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator
@@ -88,17 +86,6 @@ class KubernetesWorkloadPlan(WorkloadPlan):
             Domain suffix for the cluster. Default is "cluster.local".
         service_type (KubernetesWorkloadServiceTypeEnum):
             Service type for the workload. Default is CLUSTER_IP.
-        resource_key_runtime_env_mapping: (dict[str, str]):
-            Mapping from resource names to environment variable names for device allocation,
-            which is used to tell the Container Runtime which GPUs to mount into the container.
-            For example, {"nvidia.com/gpu": "NVIDIA_VISIBLE_DEVICES"},
-            which sets the "NVIDIA_VISIBLE_DEVICES" environment variable to the allocated GPU device IDs.
-            With privileged mode, the container can access all GPUs even if specified.
-        resource_key_backend_env_mapping: (dict[str, list[str]]):
-            Mapping from resource names to environment variable names for device runtime,
-            which is used to tell the Device Runtime (e.g., ROCm, CUDA, OneAPI) which GPUs to use inside the container.
-            For example, {"nvidia.com/gpu": ["CUDA_VISIBLE_DEVICES"]},
-            which sets the "CUDA_VISIBLE_DEVICES" environment variable to the allocated GPU device IDs.
         namespace (str | None):
             Namespace of the workload.
         name (str):
@@ -993,113 +980,103 @@ class KubernetesDeployer(EndoscopicDeployer):
             # Parameterize resources
             if c.resources:
                 kdp = get_resource_injection_policy() == "kdp"
+                fmt = "kdp" if kdp else "plain"
 
                 resources: dict[str, str] = {}
-                r_k_runtime_env = workload.resource_key_runtime_env_mapping or {}
-                r_k_backend_env = workload.resource_key_backend_env_mapping or {}
-                _, vd_env, vd_cdis, vd_values = self.get_visible_devices_materials()
                 for r_k, r_v in c.resources.items():
                     if r_k in ("cpu", "memory"):
                         resources[r_k] = str(r_v)
+                        continue
+
+                    if (
+                        r_k
+                        in envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES
+                    ):
+                        # Set env if resource key is mapped.
+                        runtime_envs = [
+                            envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES[
+                                r_k
+                            ],
+                        ]
+                    elif r_k == envs.GPUSTACK_RUNTIME_DEPLOY_AUTOMAP_RESOURCE_KEY:
+                        # Set env if auto-mapping key is matched.
+                        runtime_envs = self.get_runtime_envs()
                     else:
-                        if r_k in r_k_runtime_env:
-                            # Set env if resource key is mapped.
-                            runtime_env = [r_k_runtime_env[r_k]]
-                        elif r_k == envs.GPUSTACK_RUNTIME_DEPLOY_AUTOMAP_RESOURCE_KEY:
-                            # Set env if auto-mapping key is matched.
-                            runtime_env = list(vd_env.keys())
-                        else:
-                            resources[r_k] = str(r_v)
-                            continue
-
-                        if r_k in r_k_backend_env:
-                            # Set env if resource key is mapped.
-                            backend_env = r_k_backend_env[r_k]
-                        else:
-                            # Otherwise, use the default backend env names.
-                            backend_env = reduce(operator.add, list(vd_env.values()))
-
-                        privileged = (
+                        resources[r_k] = str(r_v)
+                        continue
+
+                    privileged = (
+                        container.security_context
+                        and container.security_context.privileged
+                    )
+                    resource_values = [x.strip() for x in r_v.split(",")]
+
+                    # Request devices.
+                    if r_v == "all":
+                        # Configure privileged.
+                        container.security_context = (
                             container.security_context
-                            and container.security_context.privileged
+                            or kubernetes.client.V1SecurityContext()
                         )
-
-                        # Configure device access environment variable.
-                        if r_v == "all" and backend_env:
-                            # Configure privileged if requested all devices.
-                            container.security_context = (
-                                container.security_context
-                                or kubernetes.client.V1SecurityContext()
+                        container.security_context.privileged = True
+                        # Request all devices.
+                        for ren in runtime_envs:
+                            r_vs = self.get_runtime_visible_devices(ren, fmt)
+                            # Request device via KDP.
+                            if kdp:
+                                resources.update(
+                                    dict.fromkeys(r_vs, "1"),
+                                )
+                                continue
+                            # Request device via visible devices env.
+                            container.env.append(
+                                kubernetes.client.V1EnvVar(
+                                    name=ren,
+                                    value=",".join(r_vs),
+                                ),
                             )
-                            container.security_context.privileged = True
-                            # Then, set container backend visible devices env to all devices,
-                            # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
-                            # and mount corresponding libs if needed.
-                            for re in runtime_env:
-                                # Request device via KDP.
-                                if kdp:
-                                    for v in vd_values.get(re) or []:
-                                        kdp_resource = cdi_kind_to_kdp_resource(
-                                            cdi_kind=vd_cdis[re],
-                                            device_index=v,
-                                        )
-                                        resources[kdp_resource] = "1"
-                                    continue
-                                # Request device via visible devices env.
-                                rv = ",".join(vd_values.get(re) or ["all"])
-                                container.env.append(
-                                    kubernetes.client.V1EnvVar(
-                                        name=re,
-                                        value=rv,
-                                    ),
+                    else:
+                        # Request specific devices.
+                        for ren in runtime_envs:
+                            # Request all devices if privileged,
+                            # otherwise, normalize requested devices.
+                            if privileged:
+                                r_vs = self.get_runtime_visible_devices(ren, fmt)
+                            else:
+                                r_vs = self.map_runtime_visible_devices(
+                                    ren,
+                                    resource_values,
+                                    fmt,
                                 )
-                        else:
-                            # Set env to the allocated device IDs if no privileged,
-                            # otherwise, set container backend visible devices env to all devices,
-                            # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
-                            # and mount corresponding libs if needed.
-                            for re in runtime_env:
-                                # Request device via KDP.
-                                if kdp:
-                                    if not privileged:
-                                        for v in str(r_v).split(","):
-                                            kdp_resource = cdi_kind_to_kdp_resource(
-                                                cdi_kind=vd_cdis[re],
-                                                device_index=int(v.strip()),
-                                            )
-                                            resources[kdp_resource] = "1"
-                                    else:
-                                        for v in vd_values.get(re) or []:
-                                            kdp_resource = cdi_kind_to_kdp_resource(
-                                                cdi_kind=vd_cdis[re],
-                                                device_index=v,
-                                            )
-                                            resources[kdp_resource] = "1"
-                                    continue
-                                # Request device via visible devices env.
-                                if not privileged:
-                                    rv = str(r_v)
-                                else:
-                                    rv = ",".join(vd_values.get(re) or ["all"])
-                                container.env.append(
-                                    kubernetes.client.V1EnvVar(
-                                        name=re,
-                                        value=rv,
-                                    ),
+                            # Request device via KDP.
+                            if kdp:
+                                resources.update(
+                                    dict.fromkeys(r_vs, "1"),
                                 )
+                                continue
+                            # Request device via visible devices env.
+                            container.env.append(
+                                kubernetes.client.V1EnvVar(
+                                    name=ren,
+                                    value=",".join(r_vs),
+                                ),
+                            )
 
-                        # Configure runtime device access environment variables.
-                        if r_v != "all" and privileged:
-                            for be in backend_env:
-                                container.env.append(
-                                    kubernetes.client.V1EnvVar(
-                                        name=be,
-                                        value=self.align_backend_visible_devices_env_values(
-                                            be,
-                                            str(r_v),
-                                        ),
-                                    ),
+                    # Configure runtime device access environment variables.
+                    if r_v != "all" and privileged:
+                        b_vs = self.map_backend_visible_devices(
+                            runtime_envs,
+                            resource_values,
+                        )
+                        container.env.extend(
+                            [
+                                kubernetes.client.V1EnvVar(
+                                    name=be,
+                                    value=be_v,
                                 )
+                                for be, be_v in b_vs.items()
+                            ],
+                        )
 
                 container.resources = kubernetes.client.V1ResourceRequirements(
                     limits=(resources if resources else None),
@@ -1229,6 +1206,10 @@ class KubernetesDeployer(EndoscopicDeployer):
         self._client = self._get_client()
         self._node_name = envs.GPUSTACK_RUNTIME_KUBERNETES_NODE_NAME
 
+    @property
+    def allowed_uuid_values(self) -> bool:
+        return get_resource_injection_policy() != "kdp"
+
     def _prepare_mirrored_deployment(self):
         """
         Prepare for mirrored deployment.

gpustack_runtime/deployer/podman.py

@@ -4,13 +4,11 @@ import contextlib
 import io
 import json
 import logging
-import operator
 import os
 import socket
 import sys
 import tarfile
 from dataclasses import dataclass, field
-from functools import reduce
 from math import ceil
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
@@ -84,17 +82,6 @@ class PodmanWorkloadPlan(WorkloadPlan):
             Image used for the pause container.
         unhealthy_restart_image (str):
             Image used for unhealthy restart container.
-        resource_key_runtime_env_mapping: (dict[str, str]):
-            Mapping from resource names to environment variable names for device allocation,
-            which is used to tell the Container Runtime which GPUs to mount into the container.
-            For example, {"nvidia.com/gpu": "NVIDIA_VISIBLE_DEVICES"},
-            which sets the "NVIDIA_VISIBLE_DEVICES" environment variable to the allocated GPU device IDs.
-            With privileged mode, the container can access all GPUs even if specified.
-        resource_key_backend_env_mapping: (dict[str, list[str]]):
-            Mapping from resource names to environment variable names for device runtime,
-            which is used to tell the Device Runtime (e.g., ROCm, CUDA, OneAPI) which GPUs to use inside the container.
-            For example, {"nvidia.com/gpu": ["CUDA_VISIBLE_DEVICES"]},
-            which sets the "CUDA_VISIBLE_DEVICES" environment variable to the allocated GPU device IDs.
         namespace (str | None):
             Namespace of the workload.
         name (str):
@@ -952,120 +939,126 @@ class PodmanDeployer(EndoscopicDeployer):
 
             # Parameterize resources.
             if c.resources:
-                r_k_runtime_env = workload.resource_key_runtime_env_mapping or {}
-                r_k_backend_env = workload.resource_key_backend_env_mapping or {}
-                vd_manus, vd_env, vd_cdis, vd_values = (
-                    self.get_visible_devices_materials()
-                )
-                for r_k, r_v in c.resources.items():
-                    match r_k:
-                        case "cpu":
-                            if isinstance(r_v, int | float):
-                                create_options["cpu_shares"] = ceil(r_v * 1024)
-                            elif isinstance(r_v, str) and r_v.isdigit():
-                                create_options["cpu_shares"] = ceil(float(r_v) * 1024)
-                        case "memory":
-                            if isinstance(r_v, int):
-                                create_options["mem_limit"] = r_v
-                                create_options["mem_reservation"] = r_v
-                                create_options["memswap_limit"] = r_v
-                            elif isinstance(r_v, str):
-                                v = r_v.lower().removesuffix("i")
-                                create_options["mem_limit"] = v
-                                create_options["mem_reservation"] = v
-                                create_options["memswap_limit"] = v
-                        case _:
-                            if r_k in r_k_runtime_env:
-                                # Set env if resource key is mapped.
-                                runtime_env = [r_k_runtime_env[r_k]]
-                            elif (
-                                r_k == envs.GPUSTACK_RUNTIME_DEPLOY_AUTOMAP_RESOURCE_KEY
-                            ):
-                                # Set env if auto-mapping key is matched.
-                                runtime_env = list(vd_env.keys())
-                            else:
-                                continue
+                fmt = "cdi"
 
-                            if r_k in r_k_backend_env:
-                                # Set env if resource key is mapped.
-                                backend_env = r_k_backend_env[r_k]
-                            else:
-                                # Otherwise, use the default backend env names.
-                                backend_env = reduce(
-                                    operator.add,
-                                    list(vd_env.values()),
-                                )
+                for r_k, r_v in c.resources.items():
+                    if r_k == "cpu":
+                        if isinstance(r_v, int | float):
+                            create_options["cpu_shares"] = ceil(r_v * 1024)
+                        elif isinstance(r_v, str) and r_v.isdigit():
+                            create_options["cpu_shares"] = ceil(float(r_v) * 1024)
+                        continue
+                    if r_k == "memory":
+                        if isinstance(r_v, int):
+                            create_options["mem_limit"] = r_v
+                            create_options["mem_reservation"] = r_v
+                            create_options["memswap_limit"] = r_v
+                        elif isinstance(r_v, str):
+                            v = r_v.lower().removesuffix("i")
+                            create_options["mem_limit"] = v
+                            create_options["mem_reservation"] = v
+                            create_options["memswap_limit"] = v
+                        continue
 
-                            privileged = create_options.get("privileged", False)
+                    if (
+                        r_k
+                        in envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES
+                    ):
+                        # Set env if resource key is mapped.
+                        runtime_envs = [
+                            envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES[
+                                r_k
+                            ],
+                        ]
+                    elif r_k == envs.GPUSTACK_RUNTIME_DEPLOY_AUTOMAP_RESOURCE_KEY:
+                        # Set env if auto-mapping key is matched.
+                        runtime_envs = self.get_runtime_envs()
+                    else:
+                        continue
 
-                            # Generate CDI config if not yet.
-                            if envs.GPUSTACK_RUNTIME_PODMAN_CDI_SPECS_GENERATE:
-                                for re in runtime_env:
-                                    cdi_dump_config(
-                                        manufacturer=vd_manus[re],
-                                        output=envs.GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY,
+                    privileged = create_options.get("privileged", False)
+                    resource_values = [x.strip() for x in r_v.split(",")]
+
+                    # Generate CDI config if not yet.
+                    if envs.GPUSTACK_RUNTIME_PODMAN_CDI_SPECS_GENERATE:
+                        for ren in runtime_envs:
+                            manu = self.get_manufacturer(ren)
+                            cdi_config, cdi_config_path = cdi_dump_config(
+                                manufacturer=manu,
+                                output=envs.GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY,
+                            )
+                            if cdi_config and cdi_config_path:
+                                if logger.isEnabledFor(logging.DEBUG):
+                                    logger.debug(
+                                        "Generated CDI configuration for '%s' at '%s':\n%s",
+                                        manu,
+                                        cdi_config_path,
+                                        cdi_config,
                                     )
-
-                            # Configure device access environment variable.
-                            if r_v == "all" and backend_env:
-                                # Configure privileged if requested all devices.
-                                create_options["privileged"] = True
-                                # Then, set container backend visible devices env to all devices,
-                                # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
-                                # and mount corresponding libs if needed.
-                                for re in runtime_env:
-                                    # Request device via CDI.
-                                    rv = [
-                                        f"{vd_cdis[re]}={v}"
-                                        for v in (vd_values.get(re) or ["all"])
-                                    ]
-                                    if "devices" not in create_options:
-                                        create_options["devices"] = []
-                                    create_options["devices"].extend(rv)
-                            else:
-                                # Set env to the allocated device IDs if no privileged,
-                                # otherwise, set container backend visible devices env to all devices,
-                                # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
-                                # and mount corresponding libs if needed.
-                                for re in runtime_env:
-                                    # Request device via CDI.
-                                    if not privileged:
-                                        rv = [
-                                            f"{vd_cdis[re]}={v.strip()}"
-                                            for v in r_v.split(",")
-                                        ]
-                                    else:
-                                        rv = [
-                                            f"{vd_cdis[re]}={v}"
-                                            for v in (vd_values.get(re) or ["all"])
-                                        ]
-                                    if "devices" not in create_options:
-                                        create_options["devices"] = []
-                                    create_options["devices"].extend(rv)
-
-                            # Configure runtime device access environment variables.
-                            if r_v != "all" and privileged:
-                                for be in backend_env:
-                                    create_options["environment"][be] = (
-                                        self.align_backend_visible_devices_env_values(
-                                            be,
-                                            str(r_v),
-                                        )
+                                else:
+                                    logger.info(
+                                        "Generated CDI configuration for '%s' at '%s'",
+                                        manu,
+                                        cdi_config_path,
                                     )
+                            elif cdi_config:
+                                logger.info(
+                                    "Reuse generated CDI configuration for '%s'",
+                                    manu,
+                                )
+                            else:
+                                logger.warning(
+                                    "Delegated CDI configuration by other tools for '%s', "
+                                    "e.g. for NVIDIA devices, please follow NVIDIA Container Toolkit Manual CDI Specification Generation to generate the CDI configuration, "
+                                    "see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/cdi-support.html#manual-cdi-specification-generation",
+                                    manu,
+                                )
 
-                            # Configure affinity if applicable.
-                            if (
-                                envs.GPUSTACK_RUNTIME_DEPLOY_CPU_AFFINITY
-                                or envs.GPUSTACK_RUNTIME_DEPLOY_NUMA_AFFINITY
-                            ):
-                                cpus, numas = self.get_visible_devices_affinities(
-                                    runtime_env,
-                                    r_v,
+                    # Configure device access environment variable.
+                    if r_v == "all":
+                        # Configure privileged.
+                        create_options["privileged"] = True
+                        # Request all devices.
+                        for ren in runtime_envs:
+                            r_vs = self.get_runtime_visible_devices(ren, fmt)
+                            # Request device via CDI.
+                            if "devices" not in create_options:
+                                create_options["devices"] = []
+                            create_options["devices"].extend(r_vs)
+                    else:
+                        # Request specific devices.
+                        for ren in runtime_envs:
+                            # Request all devices if privileged,
+                            # otherwise, normalize requested devices.
+                            if privileged:
+                                r_vs = self.get_runtime_visible_devices(ren, fmt)
+                            else:
+                                r_vs = self.map_runtime_visible_devices(
+                                    ren,
+                                    resource_values,
+                                    fmt,
                                 )
-                                if cpus:
-                                    create_options["cpuset_cpus"] = cpus
-                                if numas and envs.GPUSTACK_RUNTIME_DEPLOY_NUMA_AFFINITY:
-                                    create_options["cpuset_mems"] = numas
+                            # Request device via CDI.
+                            if "devices" not in create_options:
+                                create_options["devices"] = []
+                            create_options["devices"].extend(r_vs)
+
+                    # If not requesting all devices but privileged,
+                    # must configure visible devices.
+                    if r_v != "all" and privileged:
+                        b_vs = self.map_backend_visible_devices(
+                            runtime_envs,
+                            resource_values,
+                        )
+                        create_options["environment"].update(b_vs)
+
+                    # Configure affinity if applicable.
+                    create_options.update(
+                        self.map_visible_devices_affinities(
+                            runtime_envs,
+                            resource_values,
+                        ),
+                    )
 
             # Parameterize mounts.
             self._append_container_mounts(

gpustack_runtime/detector/__init__.py

@@ -7,6 +7,7 @@ from ..logging import debug_log_exception
 from .__types__ import (
     Detector,
     Device,
+    DeviceMemoryStatusEnum,
     Devices,
     ManufacturerEnum,
     Topology,
@@ -292,6 +293,7 @@ def filter_devices_by_manufacturer(
 
 __all__ = [
     "Device",
+    "DeviceMemoryStatusEnum",
     "Devices",
     "ManufacturerEnum",
     "Topology",

gpustack_runtime/detector/__types__.py

@@ -122,6 +122,28 @@ def backend_to_manufacturer(backend: str) -> ManufacturerEnum:
     return ManufacturerEnum.UNKNOWN
 
 
+class DeviceMemoryStatusEnum(str, Enum):
+    """
+    Enum for Device Memory Status.
+    """
+
+    HEALTHY = "healthy"
+    """
+    Device is healthy.
+    """
+    UNHEALTHY = "unhealthy"
+    """
+    Device is unhealthy.
+    """
+    UNKNOWN = "unknown"
+    """
+    Device status is unknown.
+    """
+
+    def __str__(self):
+        return self.value
+
+
 @dataclass_json
 @dataclass
 class Device:
@@ -185,6 +207,10 @@ class Device:
     """
     Memory utilization of the device in percentage.
     """
+    memory_status: DeviceMemoryStatusEnum = DeviceMemoryStatusEnum.UNKNOWN
+    """
+    Status of the device.
+    """
     temperature: int | float | None = None
     """
     Temperature of the device in Celsius.

gpustack_runtime/detector/__utils__.py

@@ -916,6 +916,9 @@ def str_range_to_list(str_range: str) -> list[int]:
         A list of indices.
 
     """
+    if not str_range:
+        return []
+
     str_range_parts = str_range.split(",")
 
     indices: set[int] = set()