gpustack-runtime 0.1.41__tar.gz → 0.1.41.post1__tar.gz

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpustack-runtime
-Version: 0.1.41
+Version: 0.1.41.post1
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.1.41'
-__version_tuple__ = version_tuple = (0, 1, 41)
+__version__ = version = '0.1.41.post1'
+__version_tuple__ = version_tuple = (0, 1, 41, 'post1')
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.1.41.post1/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "8671a00"

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/deployer/cdi/__init__.py

@@ -151,6 +151,7 @@ def available_backends() -> list[str]:
 
 
 __all__ = [
+    "Config",
     "available_backends",
     "available_manufacturers",
     "dump_config",

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/deployer/cdi/__utils__.py

@@ -68,6 +68,8 @@ def linux_device_from_path(path: Path | str | None) -> LinuxDevice | None:
         return None
 
     path_stat = path.lstat()
+    if not path_stat:
+        return None
 
     dev_mode = stat.S_IFMT(path_stat.st_mode)
     match dev_mode:

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/deployer/k8s/deviceplugin/__init__.py

@@ -4,12 +4,16 @@ import asyncio
 import contextlib
 import logging
 import signal
+import stat
 import threading
-from typing import TYPE_CHECKING
+from functools import lru_cache
+from typing import TYPE_CHECKING, Literal
 
 from .... import envs
 from ....deployer.cdi import dump_config as cdi_dump_config
 from ....detector import ManufacturerEnum, detect_devices, supported_manufacturers
+from ...cdi import Config, manufacturer_to_cdi_kind
+from ..types.kubelet.deviceplugin.v1beta1 import KubeletSocket
 from .__types__ import GroupedError, PluginServer
 from .plugin import SharableDevicePlugin, cdi_kind_to_kdp_resource
 
@@ -61,9 +65,19 @@ async def serve_async(
         if not devices:
             continue
 
+        allocation_policy = _get_device_allocation_policy(manu)
+        logger.info(
+            "Using device allocation policy '%s' for manufacturer '%s'",
+            allocation_policy,
+            manu,
+        )
+
         # Also works if the manufacturer does not have a CDI generator,
         # which means we are relying on other tools to generate CDI specs.
-        if envs.GPUSTACK_RUNTIME_KUBERNETES_KDP_CDI_SPECS_GENERATE:
+        if (
+            allocation_policy == "cdi"
+            and envs.GPUSTACK_RUNTIME_KUBERNETES_KDP_CDI_SPECS_GENERATE
+        ):
             generated_content, generated_path = cdi_dump_config(
                 manufacturer=manu,
                 output=cdi_generation_output,
@@ -95,6 +109,7 @@ async def serve_async(
                 SharableDevicePlugin(
                     device=dev,
                     id_by="index" if manu == ManufacturerEnum.ASCEND else "uuid",
+                    allocation_policy=allocation_policy,
                 ),
             )
 
@@ -104,6 +119,9 @@ async def serve_async(
         logger.info("Stop event triggered, shutting down...")
         return
 
+    if not kubelet_endpoint:
+        kubelet_endpoint = KubeletSocket
+
     # Create tasks to start all servers.
     serve_tasks = [
         asyncio.create_task(
@@ -233,8 +251,75 @@ def serve(
             pass
 
 
+def is_kubelet_socket_accessible(
+    kubelet_endpoint: Path | None = None,
+) -> bool:
+    """
+    Check if the kubelet socket is accessible.
+
+    Args:
+        kubelet_endpoint:
+            The path to the kubelet endpoint.
+
+    Returns:
+        True if the socket is accessible, False otherwise.
+
+    """
+    if not kubelet_endpoint:
+        kubelet_endpoint = KubeletSocket
+
+    if kubelet_endpoint.exists():
+        path_stat = kubelet_endpoint.lstat()
+        if path_stat and stat.S_ISSOCK(path_stat.st_mode):
+            return True
+    return False
+
+
+@lru_cache
+def _get_device_allocation_policy(
+    manufacturer: ManufacturerEnum,
+) -> Literal["env", "cdi", "opaque"]:
+    """
+    Get the device allocation policy (in lowercase) for the device plugin.
+
+    Args:
+        manufacturer:
+            The manufacturer of the device.
+
+    Returns:
+        The device allocation policy.
+
+    """
+    policy = envs.GPUSTACK_RUNTIME_KUBERNETES_KDP_DEVICE_ALLOCATION_POLICY.lower()
+    if policy != "auto":
+        return policy
+
+    cdi_kind = manufacturer_to_cdi_kind(manufacturer)
+
+    cdi_dir = envs.GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY
+    for suffix in ["*.yaml", "*.yml", "*.json"]:
+        for file in cdi_dir.glob(suffix):
+            with contextlib.suppress(Exception):
+                config = Config.from_file(file)
+                if config and config.kind == cdi_kind:
+                    return "cdi"
+
+    if manufacturer in [
+        ManufacturerEnum.AMD,
+        # ManufacturerEnum.ASCEND, # Prioritize using Env policy for Ascend.
+        ManufacturerEnum.HYGON,
+        ManufacturerEnum.ILUVATAR,
+        ManufacturerEnum.METAX,
+        ManufacturerEnum.THEAD,
+    ]:
+        return "opaque"
+
+    return "env"
+
+
 __all__ = [
     "cdi_kind_to_kdp_resource",
+    "is_kubelet_socket_accessible",
     "serve",
     "serve_async",
 ]

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/deployer/k8s/deviceplugin/__types__.py

@@ -107,7 +107,7 @@ class PluginServer(ABC):
     async def serve(
         self,
         stop_event: asyncio.Event,
-        kubelet_endpoint: Path | None = None,
+        kubelet_endpoint: Path,
         start_timeout: int = 5,
         register_timeout: int = 5,
     ):

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/deployer/k8s/deviceplugin/plugin.py

@@ -31,7 +31,6 @@ from ..types.kubelet.deviceplugin.v1beta1 import (
     DeviceSpec,
     Empty,
     Healthy,
-    KubeletSocket,
     ListAndWatchResponse,
     Mount,
     NUMANode,
@@ -110,6 +109,10 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
     """
     Controls how the device IDs of the Kubernetes Device Plugin are generated.
     """
+    _allocation_policy: Literal["env", "cdi", "opaque"]
+    """
+    Controls the device allocation policy.
+    """
     _max_allocations: int
     """
     Controls the maximum shards per underlying device.
@@ -131,6 +134,7 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
         self,
         device: Device,
         id_by: Literal["uuid", "index"] = "uuid",
+        allocation_policy: Literal["env", "cdi", "opaque"] = "cdi",
         max_allocations: int | None = None,
     ):
         """
@@ -142,6 +146,8 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
             id_by:
                 Controls how the device IDs of the Kubernetes Device Plugin are generated.
                 Either "uuid" or "index". Default is "uuid".
+            allocation_policy:
+                Controls the device allocation policy.
             max_allocations:
                 Controls the maximum allocations per underlying device.
                 If None, uses the environment variable `GPUSTACK_RUNTIME_KUBERNETES_KDP_PER_DEVICE_MAX_ALLOCATIONS`.
@@ -149,6 +155,7 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
         """
         self._device = device
         self._id_by = id_by
+        self._allocation_policy = allocation_policy
         self._max_allocations = max_allocations
         if not self._max_allocations:
             self._max_allocations = (
@@ -229,7 +236,7 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
     async def serve(
         self,
         stop_event: asyncio.Event,
-        kubelet_endpoint: Path | None = None,
+        kubelet_endpoint: Path,
         start_timeout: int = 5,
         register_timeout: int = 5,
     ):
@@ -247,9 +254,6 @@ class SharableDevicePlugin(PluginServer, DevicePluginServicer):
                 The timeout in seconds for registering the device plugin.
 
         """
-        if not kubelet_endpoint:
-            kubelet_endpoint = KubeletSocket
-
         resource_name = self._kdp_resource
         endpoint = kubelet_endpoint.parent / f"{resource_name.replace('/', '.')}.sock"
 

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/deployer/kuberentes.py

@@ -7,9 +7,9 @@ import operator
 import os
 from dataclasses import dataclass, field
 from enum import Enum
-from functools import reduce
+from functools import lru_cache, reduce
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 import kubernetes
 import kubernetes.stream.ws_client
@@ -43,7 +43,7 @@ from .__utils__ import (
     sensitive_env_var,
     validate_rfc1123_domain_name,
 )
-from .k8s.deviceplugin import cdi_kind_to_kdp_resource
+from .k8s.deviceplugin import cdi_kind_to_kdp_resource, is_kubelet_socket_accessible
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Generator
@@ -381,6 +381,22 @@ class KubernetesDeployer(EndoscopicDeployer):
 
         return wrapper
 
+    @staticmethod
+    @lru_cache
+    def _get_resource_injection_policy() -> Literal["env", "kdp"]:
+        """
+        Get the resource injection policy (in lowercase) for the deployer.
+
+        Returns:
+            The resource injection policy.
+
+        """
+        policy = envs.GPUSTACK_RUNTIME_KUBERNETES_RESOURCE_INJECTION_POLICY.lower()
+        if policy != "auto":
+            return policy
+
+        return "kdp" if is_kubelet_socket_accessible() else "env"
+
     def _create_ephemeral_configmaps(
         self,
         workload: KubernetesWorkloadPlan,
@@ -992,10 +1008,7 @@ class KubernetesDeployer(EndoscopicDeployer):
 
             # Parameterize resources
             if c.resources:
-                kdp = (
-                    envs.GPUSTACK_RUNTIME_KUBERNETES_RESOURCE_INJECTION_POLICY.lower()
-                    == "kdp"
-                )
+                kdp = self._get_resource_injection_policy() == "kdp"
 
                 resources: dict[str, str] = {}
                 r_k_runtime_env = workload.resource_key_runtime_env_mapping or {}

{gpustack_runtime-0.1.41 → gpustack_runtime-0.1.41.post1}/gpustack_runtime/envs.py

@@ -280,7 +280,8 @@ if TYPE_CHECKING:
     """
     GPUSTACK_RUNTIME_KUBERNETES_RESOURCE_INJECTION_POLICY: str | None = None
     """
-    Resource injection policy for the Kubernetes deployer (e.g., Env, KDP).
+    Resource injection policy for the Kubernetes deployer (e.g., Auto, Env, KDP).
+    `Auto`: Automatically choose the resource injection policy based on the environment.
     `Env`: Injects resources using standard environment variable, depends on underlying Container Toolkit, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES`.
     `KDP`: Injects resources using Kubernetes Device Plugin, based on `GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_CDI`.
     """
@@ -292,8 +293,9 @@ if TYPE_CHECKING:
     GPUSTACK_RUNTIME_KUBERNETES_KDP_DEVICE_ALLOCATION_POLICY: str | None = None
     """
     Device allocation policy for the Kubernetes Device Plugin (e.g., CDI, Env, Opaque).
-    `CDI`: Allocates devices using generated CDI specifications, making it easy to debug and troubleshoot; requires `GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY` to exist.
+    `Auto`: Automatically choose the device allocation policy based on the environment.
     `Env`: Allocates devices using runtime-visible environment variables; requires Container Toolkit support.
+    `CDI`: Allocates devices using generated CDI specifications, making it easy to debug and troubleshoot; requires `GPUSTACK_RUNTIME_DEPLOY_CDI_SPECS_DIRECTORY` to exist.
     `Opaque`: Uses internal logic for allocation, which is convenient for deployment but difficult to troubleshoot.
     """
     GPUSTACK_RUNTIME_KUBERNETES_KDP_CDI_SPECS_GENERATE: bool = True
@@ -685,8 +687,8 @@ variables: dict[str, Callable[[], Any]] = {
         getenv(
             "GPUSTACK_RUNTIME_KUBERNETES_RESOURCE_INJECTION_POLICY",
         ),
-        options=["Env", "KDP"],
-        default="Env",
+        options=["Auto", "Env", "KDP"],
+        default="Auto",
     ),
     "GPUSTACK_RUNTIME_KUBERNETES_KDP_PER_DEVICE_MAX_ALLOCATIONS": lambda: to_int(
         getenv(
@@ -698,12 +700,12 @@ variables: dict[str, Callable[[], Any]] = {
         getenv(
             "GPUSTACK_RUNTIME_KUBERNETES_KDP_DEVICE_ALLOCATION_POLICY",
         ),
-        options=["CDI", "Env", "Opaque"],
-        default="CDI",
+        options=["Auto", "Env", "CDI", "Opaque"],
+        default="Auto",
     ),
     "GPUSTACK_RUNTIME_KUBERNETES_KDP_CDI_SPECS_GENERATE": lambda: ternary(
         lambda: (
-            getenv("GPUSTACK_RUNTIME_KUBERNETES_RESOURCE_INJECTION_POLICY", "Env")
+            getenv("GPUSTACK_RUNTIME_KUBERNETES_RESOURCE_INJECTION_POLICY", "Auto")
             == "Env"
         ),
         lambda: False,
@@ -711,9 +713,9 @@ variables: dict[str, Callable[[], Any]] = {
             lambda: (
                 getenv(
                     "GPUSTACK_RUNTIME_KUBERNETES_KDP_DEVICE_ALLOCATION_POLICY",
-                    "Opaque",
+                    "Auto",
                 )
-                == "Opaque"
+                not in ["Auto", "CDI"]
             ),
             lambda: False,
             lambda: to_bool(

gpustack_runtime-0.1.41/gpustack_runtime/_version_appendix.py

@@ -1 +0,0 @@
-git_commit = "f706cfd"