gpustack-runtime 0.2.3.post1__tar.gz → 0.2.4__tar.gz

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: gpustack-runtime
-Version: 0.2.3.post1
+Version: 0.2.4
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.2.3.post1'
-__version_tuple__ = version_tuple = (0, 2, 3, 'post1')
+__version__ = version = '0.2.4'
+__version_tuple__ = version_tuple = (0, 2, 4)
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.2.4/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "59ea8c8"

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/cmds/deployer.py

@@ -362,6 +362,7 @@ class DeleteWorkloadSubCommand(SubCommand):
 
     namespace: str
     name: str
+    grace_period_seconds: int | None
 
     @staticmethod
     def register(parser: _SubParsersAction):
@@ -376,6 +377,13 @@ class DeleteWorkloadSubCommand(SubCommand):
             help="Namespace of the workload",
         )
 
+        delete_parser.add_argument(
+            "--grace-period-seconds",
+            type=int,
+            help="Duration in seconds the workload needs to terminate gracefully, "
+            "which overrides the one declared by the workload plan",
+        )
+
         delete_parser.add_argument(
             "name",
             type=str,
@@ -387,6 +395,7 @@ class DeleteWorkloadSubCommand(SubCommand):
     def __init__(self, args: Namespace):
         self.namespace = args.namespace
         self.name = args.name
+        self.grace_period_seconds = args.grace_period_seconds
 
         if not self.name:
             msg = "The name argument is required."
@@ -396,6 +405,7 @@ class DeleteWorkloadSubCommand(SubCommand):
         st = delete_workload(
             name=self.name,
             namespace=self.namespace,
+            grace_period_seconds=self.grace_period_seconds,
         )
         if st:
             print(f"Deleted workload '{self.name}'.")
@@ -428,11 +438,21 @@ class DeleteWorkloadsSubCommand(SubCommand):
             help="Filter workloads by labels (key=value pairs separated by commas)",
         )
 
+        delete_parser.add_argument(
+            "--grace-period-seconds",
+            type=int,
+            help="Duration in seconds the workloads need to terminate gracefully, "
+            "which overrides the one declared by the workload plans. "
+            "The workloads are deleted one after another, "
+            "so the whole deletion may take this duration per workload",
+        )
+
         delete_parser.set_defaults(func=DeleteWorkloadsSubCommand)
 
     def __init__(self, args: Namespace):
         self.namespace = args.namespace
         self.labels = args.labels
+        self.grace_period_seconds = args.grace_period_seconds
 
     def run(self):
         sts: list[WorkloadStatus] = list_workloads(
@@ -443,6 +463,7 @@ class DeleteWorkloadsSubCommand(SubCommand):
             delete_workload(
                 name=st.name,
                 namespace=st.namespace,
+                grace_period_seconds=self.grace_period_seconds,
             )
             print(f"Deleted workload '{st.name}'.")
         if not sts:

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/deployer/__init__.py

@@ -150,6 +150,7 @@ def get_workload(
 def delete_workload(
     name: WorkloadName,
     namespace: WorkloadNamespace | None = None,
+    grace_period_seconds: int | None = None,
 ) -> WorkloadStatus | None:
     """
     Delete the given workload.
@@ -159,11 +160,16 @@ def delete_workload(
             The name of the workload to delete.
         namespace:
             The namespace of the workload.
+        grace_period_seconds:
+            Duration in seconds the workload needs to terminate gracefully,
+            which overrides the one declared by the workload plan.
 
     Return:
         The status if found, None otherwise.
 
     Raises:
+        ValueError:
+            If the grace period is negative.
         UnsupportedError:
             If no deployer supports the given workload.
         OperationError:
@@ -174,7 +180,11 @@ def delete_workload(
         if not dep.is_supported():
             continue
 
-        return dep.delete(name=name, namespace=namespace)
+        return dep.delete(
+            name=name,
+            namespace=namespace,
+            grace_period_seconds=grace_period_seconds,
+        )
 
     raise UnsupportedError(_NO_AVAILABLE_DEPLOYER_MSG)
 

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/deployer/__types__.py

@@ -832,6 +832,12 @@ WorkloadName = str
 Name for a workload.
 """
 
+DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS = 30
+"""
+Default duration in seconds a workload needs to terminate gracefully,
+which aligns with the Kubernetes API server default.
+"""
+
 
 @dataclass_json
 @dataclass
@@ -862,6 +868,8 @@ class WorkloadPlan(WorkloadSecurity):
             The group ID to own the filesystem of the workload.
         sysctls (dict[str, str] | None):
             Sysctls to set for the workload.
+        termination_grace_period_seconds (int):
+            Duration in seconds the containers of the workload need to terminate gracefully.
         containers (list[Container] | None):
             Containers in the workload.
             It must contain at least one "RUN" profile container.
@@ -909,6 +917,12 @@ class WorkloadPlan(WorkloadSecurity):
     Containers in the workload.
     It must contain at least one "RUN" profile container.
     """
+    termination_grace_period_seconds: int = DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS
+    """
+    Duration in seconds the containers of the workload need to terminate gracefully.
+    Containers are signaled to stop, and killed once the duration elapses.
+    Zero means killing immediately.
+    """
 
     ##
     ## The below are internal use only fields.
@@ -957,6 +971,15 @@ class WorkloadPlan(WorkloadSecurity):
             msg = 'Workload must contain at least one "RUN" profile container.'
             raise ValueError(msg)
 
+        # Default and validate termination grace period.
+        if self.termination_grace_period_seconds is None:
+            self.termination_grace_period_seconds = (
+                DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS
+            )
+        elif self.termination_grace_period_seconds < 0:
+            msg = "Workload termination grace period must not be negative."
+            raise ValueError(msg)
+
         # Validate workload labels, including label names and values.
         for ln, lv in self.labels.items():
             for s in ln.split("/"):
@@ -2109,6 +2132,8 @@ class Deployer(ABC):
         self,
         name: WorkloadName,
         namespace: WorkloadNamespace | None = None,
+        *,
+        grace_period_seconds: int | None = None,
         async_mode: bool | None = None,
     ) -> WorkloadStatus | None:
         """
@@ -2119,6 +2144,9 @@ class Deployer(ABC):
                 The name of the workload.
             namespace:
                 The namespace of the workload.
+            grace_period_seconds:
+                Duration in seconds the workload needs to terminate gracefully,
+                which overrides the one declared by the workload plan.
             async_mode:
                 Whether to execute in a separate thread.
 
@@ -2128,16 +2156,23 @@ class Deployer(ABC):
         Raises:
             UnsupportedError:
                 If the deployer is not supported in the current environment.
+            ValueError:
+                If the grace period is negative.
             OperationError:
                 If the workload fails to delete.
 
         """
+        if grace_period_seconds is not None and grace_period_seconds < 0:
+            msg = "Workload termination grace period must not be negative."
+            raise ValueError(msg)
+
         if async_mode:
             try:
                 future = self.pool.submit(
                     self._delete,
                     name,
                     namespace,
+                    grace_period_seconds,
                 )
                 return future.result()
             except OperationError:
@@ -2146,13 +2181,14 @@ class Deployer(ABC):
                 msg = "Asynchronous workload delete failed."
                 raise OperationError(msg) from e
         else:
-            return self._delete(name, namespace)
+            return self._delete(name, namespace, grace_period_seconds)
 
     @abstractmethod
     def _delete(
         self,
         name: WorkloadName,
         namespace: WorkloadNamespace | None = None,
+        grace_period_seconds: int | None = None,
     ) -> WorkloadStatus | None:
         """
         Delete a workload.
@@ -2162,6 +2198,9 @@ class Deployer(ABC):
                 The name of the workload.
             namespace:
                 The namespace of the workload.
+            grace_period_seconds:
+                Duration in seconds the workload needs to terminate gracefully,
+                which overrides the one declared by the workload plan.
 
         Return:
             The status if found, None otherwise.

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/deployer/cdi/__utils__.py

@@ -143,6 +143,56 @@ def device_to_cdi_device_node(
     )
 
 
+def path_to_cdi_device_nodes(
+    path: str,
+    permission: str = "rw",
+    no_user: bool = False,
+) -> list[ConfigDeviceNode]:
+    """
+    Convert a device path, or a directory holding device paths, to ConfigDeviceNodes.
+
+    A generation may expose a bus as a directory of device nodes rather than as
+    a single node -- Ascend's UB does, which is why the operator enumerates it,
+    see addUBDevicesFromDir in
+    https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/cdi/devnode.go.
+    Both shapes are accepted so that a path which is a plain device node on one
+    driver and a directory on another needs no caller-side branch.
+
+    Args:
+        path:
+            Path to the device, or to a directory of devices, on the host.
+        permission:
+            Permissions for the devices.
+        no_user:
+            Whether to omit user and group information.
+
+    Returns:
+        The ConfigDeviceNode objects, empty if the path holds no device.
+
+    """
+    p = Path(path)
+    if not p.is_dir():
+        cdn = device_to_cdi_device_node(
+            path=path,
+            permission=permission,
+            no_user=no_user,
+        )
+        return [cdn] if cdn else []
+
+    return [
+        cdn
+        for entry in sorted(p.iterdir())
+        if not entry.is_dir()
+        and (
+            cdn := device_to_cdi_device_node(
+                path=str(entry),
+                permission=permission,
+                no_user=no_user,
+            )
+        )
+    ]
+
+
 def path_to_cdi_mount(
     path: str,
     container_path: str | None = None,
@@ -181,3 +231,34 @@ def path_to_cdi_mount(
         container_path=container_path,
         options=options,
     )
+
+
+def glob_to_cdi_mounts(
+    pattern: str,
+    options: list[str] | None = None,
+) -> list[ConfigMount]:
+    """
+    Convert every path matching a glob pattern to ConfigMounts.
+
+    A user-space library is versioned in its file name, so the set of files to
+    mount cannot be spelled out ahead of time -- the operator's mount profile
+    lists them as patterns for the same reason.
+
+    Args:
+        pattern:
+            Path on the host whose last segment may carry a wildcard; the
+            directory part is taken literally, so a wildcard there matches
+            nothing. A pattern without a wildcard resolves to that path alone.
+        options:
+            Mount options.
+
+    Returns:
+        The ConfigMount objects, empty if nothing matches.
+
+    """
+    p = Path(pattern)
+    return [
+        cm
+        for path in sorted(p.parent.glob(p.name))
+        if (cm := path_to_cdi_mount(path=str(path), options=options))
+    ]

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/deployer/cdi/ascend.py

@@ -1,11 +1,14 @@
 from __future__ import annotations as __future_annotations__
 
+import logging
+
 from ...detector import (
     Devices,
     ManufacturerEnum,
     detect_devices,
     filter_devices_by_manufacturer,
 )
+from ...detector.ascend import get_ascend_cann_variant
 from .__types__ import (
     Config,
     ConfigContainerEdits,
@@ -14,7 +17,44 @@ from .__types__ import (
     manufacturer_to_cdi_kind,
     manufacturer_to_runtime_env,
 )
-from .__utils__ import device_to_cdi_device_node, path_to_cdi_mount
+from .__utils__ import (
+    device_to_cdi_device_node,
+    glob_to_cdi_mounts,
+    path_to_cdi_device_nodes,
+    path_to_cdi_mount,
+)
+
+logger = logging.getLogger(__name__)
+
+_A5_CANN_VARIANT = "950"
+"""
+The CANN variant of the A5 generation, whose UB fabric replaces what the
+earlier generations reach through the shared memory device.
+"""
+
+_A5_UB_MOUNT_PATTERNS = [
+    "/usr/lib64/libummu*",
+    "/usr/lib64/liburma*",
+    "/usr/lib64/urma",
+    "/usr/lib64/libnl*",
+    "/usr/bin/urma_admin",
+    "/usr/bin/urma_perftest",
+    "/usr/bin/urma_ping",
+]
+"""
+The UB user-space libraries an A5 container needs, mirroring the operator's
+mount profile for the Ascend950 generation, see
+https://gitcode.com/Ascend/mind-cluster/blob/master/component/ascend-common/cdi/mount/profile.go.
+
+These are mounted for the A5 generation only. Some of them -- libnl above all
+-- are ordinary system libraries present on any host, so mounting them
+unconditionally would shadow what an earlier generation's container ships with
+its own image.
+
+The /usr/lib64 prefix is the operator's, and holds on the openEuler and CentOS
+hosts an A5 ships on; a Debian-derived host uses /usr/lib/<triplet>, where none
+of these match -- hence the log line when a pattern finds nothing.
+"""
 
 
 class AscendGenerator(Generator):
@@ -73,16 +113,14 @@ class AscendGenerator(Generator):
                 break
         for p in [
             "/dev/dvpp_cmdlist",
+            # UB exposes a directory of device nodes rather than a single one,
+            # so every entry below it has to be injected.
             "/dev/uburma",
             "/dev/ummu",
             "/dev/devmm_svm",
             "/dev/hisi_hdc",
         ]:
-            cdn = device_to_cdi_device_node(
-                path=p,
-            )
-            if cdn:
-                common_device_nodes.append(cdn)
+            common_device_nodes.extend(path_to_cdi_device_nodes(path=p))
         if not common_device_nodes:
             return None
 
@@ -102,6 +140,25 @@ class AscendGenerator(Generator):
             if cm:
                 common_mounts.append(cm)
 
+        # Device.appendix defaults to None and callers pass their own devices
+        # in, so every read goes through `or {}`.
+        if any(
+            get_ascend_cann_variant((dev.appendix or {}).get("arch_family"))
+            == _A5_CANN_VARIANT
+            for dev in devices
+            if dev
+        ):
+            for pattern in _A5_UB_MOUNT_PATTERNS:
+                ub_mounts = glob_to_cdi_mounts(pattern=pattern)
+                if not ub_mounts:
+                    # Otherwise this surfaces later as a container that cannot
+                    # reach the UB fabric.
+                    logger.debug(
+                        "No UB library matched %s, an A5 container will lack it",
+                        pattern,
+                    )
+                common_mounts.extend(ub_mounts)
+
         cdi_devices: list[ConfigDevice] = []
 
         all_device_nodes = []
@@ -119,7 +176,7 @@ class AscendGenerator(Generator):
             # addressed by the index, which would resolve to another NPU's
             # node. The detector already drops such a device; this guards the
             # devices a caller passes in.
-            cdn_number = dev.appendix.get("physical_id")
+            cdn_number = (dev.appendix or {}).get("physical_id")
             if cdn_number is None:
                 continue
             cdn_path = f"/dev/davinci{cdn_number}"

{gpustack_runtime-0.2.3.post1 → gpustack_runtime-0.2.4}/gpustack_runtime/deployer/docker.py

@@ -8,6 +8,7 @@ import os
 import socket
 import sys
 import tarfile
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from math import ceil
 from pathlib import Path
@@ -27,6 +28,7 @@ from tqdm import tqdm
 from .. import envs
 from ..logging import debug_log_exception, debug_log_warning
 from .__types__ import (
+    DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS,
     Container,
     ContainerCheck,
     ContainerEnv,
@@ -68,6 +70,9 @@ _LABEL_COMPONENT = f"{envs.GPUSTACK_RUNTIME_DEPLOY_LABEL_PREFIX}/component"
 _LABEL_COMPONENT_NAME = f"{_LABEL_COMPONENT}-name"
 _LABEL_COMPONENT_INDEX = f"{_LABEL_COMPONENT}-index"
 _LABEL_COMPONENT_HEAL_PREFIX = f"{_LABEL_COMPONENT}-heal"
+_LABEL_TERMINATION_GRACE_PERIOD_SECONDS = (
+    f"{envs.GPUSTACK_RUNTIME_DEPLOY_LABEL_PREFIX}/termination-grace-period-seconds"
+)
 
 
 @dataclass_json
@@ -104,6 +109,8 @@ class DockerWorkloadPlan(WorkloadPlan):
             The group ID to own the filesystem of the workload.
         sysctls (dict[str, str] | None):
             Sysctls to set for the workload.
+        termination_grace_period_seconds (int):
+            Duration in seconds the containers of the workload need to terminate gracefully.
         containers (list[tuple[int, Container]] | None):
             List of containers in the workload.
             It must contain at least one "RUN" profile container.
@@ -138,6 +145,12 @@ class DockerWorkloadPlan(WorkloadPlan):
         # Default and validate in the base class.
         super().validate_and_default()
 
+        # Carry the termination grace period on the containers,
+        # as the Docker models layer cannot express a create time stop timeout.
+        self.labels[_LABEL_TERMINATION_GRACE_PERIOD_SECONDS] = str(
+            self.termination_grace_period_seconds,
+        )
+
         # Adjust images.
         self.pause_image = adjust_image_with_envs(self.pause_image)
         self.unhealthy_restart_image = adjust_image_with_envs(
@@ -1650,6 +1663,7 @@ class DockerDeployer(EndoscopicDeployer):
         self,
         name: WorkloadName,
         namespace: WorkloadNamespace | None = None,
+        grace_period_seconds: int | None = None,
     ) -> WorkloadStatus | None:
         """
         Delete a Docker workload.
@@ -1659,6 +1673,9 @@ class DockerDeployer(EndoscopicDeployer):
                 The name of the workload.
             namespace:
                 The namespace of the workload.
+            grace_period_seconds:
+                Duration in seconds the workload needs to terminate gracefully,
+                which overrides the one declared by the workload plan.
 
         Return:
             The status if found, None otherwise.
@@ -1678,19 +1695,40 @@ class DockerDeployer(EndoscopicDeployer):
         # Remove all containers with the workload label.
         try:
             d_containers = getattr(workload, "_d_containers", [])
-            # Remove non-pause containers first.
+            # Remove the unhealthy restart container first,
+            # otherwise it restarts the containers draining below.
             for c in d_containers:
-                if "-pause" not in c.name:
+                if c.labels.get(_LABEL_COMPONENT) == "unhealthy-restart":
                     c.remove(
                         force=True,
                     )
-            # Then remove pause containers.
+            # Then drain the non-pause containers concurrently,
+            # so each of them gets the whole grace period,
+            # and remove them.
+            d_drain_containers = [
+                c
+                for c in d_containers
+                if c.labels.get(_LABEL_COMPONENT) not in ("pause", "unhealthy-restart")
+            ]
+            _stop_containers(
+                d_drain_containers,
+                _termination_grace_period_seconds(d_containers, grace_period_seconds),
+            )
+            for c in d_drain_containers:
+                c.remove(
+                    force=True,
+                )
+            # Finally remove the pause containers,
+            # which hold the namespaces shared by the above containers.
             for c in d_containers:
-                if "-pause" in c.name:
+                if c.labels.get(_LABEL_COMPONENT) == "pause":
                     c.remove(
                         force=True,
                     )
-        except docker.errors.APIError as e:
+        # NB(thxCode): Deleting a workload always fails with an OperationError,
+        # as neither the Docker SDK errors nor the transport ones underneath them
+        # share a base class narrow enough to catch on its own.
+        except Exception as e:
             msg = f"Failed to delete containers for workload {name}{_detail_api_call_error(e)}"
             raise OperationError(msg) from e
 
@@ -1711,7 +1749,7 @@ class DockerDeployer(EndoscopicDeployer):
                 v.remove(
                     force=True,
                 )
-        except docker.errors.APIError as e:
+        except Exception as e:
             msg = f"Failed to delete volumes for workload {name}{_detail_api_call_error(e)}"
             raise OperationError(msg) from e
 
@@ -2168,6 +2206,71 @@ def _has_restart_policy(
     )
 
 
+def _stop_containers(
+    containers: list[docker.models.containers.Container],
+    grace_period_seconds: int,
+):
+    """
+    Stop the given containers concurrently,
+    so each of them gets the whole grace period,
+    which mirrors how Kubernetes terminates the containers of a Pod.
+
+    A container failing to stop is left to the forceful removal following this,
+    as the removal is the authoritative teardown.
+
+    Args:
+        containers:
+            List of Docker containers to stop.
+        grace_period_seconds:
+            Duration in seconds the containers need to terminate gracefully.
+
+    """
+    if not containers:
+        return
+
+    def stop(container: docker.models.containers.Container):
+        try:
+            container.stop(
+                timeout=grace_period_seconds,
+            )
+        except Exception:
+            debug_log_exception(logger, f"Failed to stop container {container.name}")
+
+    with ThreadPoolExecutor(max_workers=min(len(containers), 16)) as pool:
+        list(pool.map(stop, containers))
+
+
+def _termination_grace_period_seconds(
+    containers: list[docker.models.containers.Container],
+    override: int | None = None,
+) -> int:
+    """
+    Resolve how long the given containers may take to terminate gracefully,
+    which prefers the given override, then the one declared by the workload plan,
+    and finally the default.
+
+    Args:
+        containers:
+            List of Docker containers in the workload.
+        override:
+            Duration in seconds overriding the one declared by the workload plan.
+
+    Returns:
+        The duration in seconds.
+
+    """
+    if override is not None:
+        return override
+
+    for c in containers:
+        declared = c.labels.get(_LABEL_TERMINATION_GRACE_PERIOD_SECONDS)
+        if declared:
+            with contextlib.suppress(ValueError):
+                return max(0, int(declared))
+
+    return DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS
+
+
 class DockerWorkloadExecStream(WorkloadExecStream):
     """
     A WorkloadExecStream implementation for Docker exec socket streams.
@@ -2202,14 +2305,14 @@ class DockerWorkloadExecStream(WorkloadExecStream):
         self._sock.close()
 
 
-def _detail_api_call_error(err: docker.errors.APIError) -> str:
+def _detail_api_call_error(err: Exception) -> str:
     """
     Explain a Docker API error in a concise way,
     if the envs.GPUSTACK_RUNTIME_DEPLOY_API_CALL_ERROR_DETAIL is enabled.
 
     Args:
         err:
-            The Docker API error.
+            The Docker API error, or the transport error underneath it.
 
     Returns:
         A concise explanation of the error.
@@ -2218,6 +2321,9 @@ def _detail_api_call_error(err: docker.errors.APIError) -> str:
     if not envs.GPUSTACK_RUNTIME_DEPLOY_API_CALL_ERROR_DETAIL:
         return ""
 
+    if not isinstance(err, docker.errors.APIError):
+        return f": {err}"
+
     msg = f": Docker {'Client' if err.is_client_error() else 'Server'} Error"
     if err.explanation:
         msg += f": {err.explanation}"