gpustack-runtime 0.2.0.post5__tar.gz → 0.2.2__tar.gz

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/.gitignore

@@ -3,6 +3,7 @@
 .vscode/
 .DS_Store
 *.swp
+.claude/
 
 # Python-generated files
 _version.py

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gpustack-runtime
-Version: 0.2.0.post5
+Version: 0.2.2
 Summary: GPUStack Runtime is library for detecting GPU resources and launching GPU workloads.
 Project-URL: Homepage, https://github.com/gpustack/runtime
 Project-URL: Bug Tracker, https://github.com/gpustack/gpustack/issues

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/_version.py

@@ -27,8 +27,8 @@ version_tuple: VERSION_TUPLE
 __commit_id__: COMMIT_ID
 commit_id: COMMIT_ID
 
-__version__ = version = '0.2.0.post5'
-__version_tuple__ = version_tuple = (0, 2, 0, 'post5')
+__version__ = version = '0.2.2'
+__version_tuple__ = version_tuple = (0, 2, 2)
 try:
     from ._version_appendix import git_commit
     __commit_id__ = commit_id = git_commit

gpustack_runtime-0.2.2/gpustack_runtime/_version_appendix.py

@@ -0,0 +1 @@
+git_commit = "3ea35d7"

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/deployer/__types__.py

@@ -1135,6 +1135,10 @@ class WorkloadStatus:
             Namespace for the workload.
         labels (dict[str, str] | None):
             Labels for the workload.
+        annotations (dict[str, str] | None):
+            Annotations for the workload.
+        state_message (str):
+            Message accompanying the current state.
         executable (list[WorkloadStatusOperation]):
             The operation for the executable containers of the workload.
         loggable (list[WorkloadStatusOperation]):
@@ -1161,6 +1165,15 @@ class WorkloadStatus:
     """
     Labels for the workload.
     """
+    annotations: dict[str, str] | None = field(default_factory=dict)
+    """
+    Annotations for the workload.
+    """
+    state_message: str = ""
+    """
+    Message accompanying the current state, e.g. the Kubernetes Pod's
+    status message on failure (such as a device-plugin admission rejection).
+    """
     executable: list[WorkloadStatusOperation] | None = field(default_factory=list)
     """
     The operation for the executable containers of the workload.

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/deployer/kuberentes.py

@@ -227,12 +227,18 @@ class KubernetesWorkloadStatus(WorkloadStatus):
             for k, v in (k_pod.metadata.labels or {}).items()
             if not k.startswith("runtime.gpustack.ai/")
         }
+        annotations = {
+            k: v
+            for k, v in (k_pod.metadata.annotations or {}).items()
+            if not k.startswith("runtime.gpustack.ai/")
+        }
 
         super().__init__(
             name=name,
             created_at=created_at,
             namespace=k_pod.metadata.namespace,
             labels=labels,
+            annotations=annotations,
             **kwargs,
         )
 
@@ -259,6 +265,11 @@ class KubernetesWorkloadStatus(WorkloadStatus):
             self.loggable.append(op)
 
         self.state = self.parse_state(k_pod)
+        if k_pod.status.message:
+            # Surface the Pod's status message, e.g. a device-plugin
+            # admission rejection ("UnexpectedAdmissionError: Allocate
+            # failed ...") on Failed Pods.
+            self.state_message = k_pod.status.message
 
 
 _NAME = "kubernetes"
@@ -267,6 +278,29 @@ Name of the Kubernetes deployer.
 """
 
 
+def _pin_pod_for_kueue(
+    pod: kubernetes.client.V1Pod,
+    node_name: str | None,
+) -> None:
+    """
+    Adjust node pinning for Kueue-gated Pods in place.
+
+    Kueue-gated Pods (queue label present) cannot set spec.nodeName:
+    the API forbids nodeName until all schedulingGates have been
+    cleared. Pin via nodeSelector instead; kube-scheduler places
+    the Pod on the same node once Kueue admits it.
+    """
+    if not node_name:
+        return
+    if "kueue.x-k8s.io/queue-name" not in (pod.metadata.labels or {}):
+        return
+    pod.spec.node_name = None
+    pod.spec.node_selector = {
+        **(pod.spec.node_selector or {}),
+        "kubernetes.io/hostname": node_name,
+    }
+
+
 class KubernetesDeployer(EndoscopicDeployer):
     """
     Deployer implementation for Kubernetes.
@@ -1191,6 +1225,8 @@ class KubernetesDeployer(EndoscopicDeployer):
 
         core_api = kubernetes.client.CoreV1Api(self._client)
         try:
+            _pin_pod_for_kueue(pod, self._node_name)
+
             pod = self._mutate_create_pod(pod)
             if envs.GPUSTACK_RUNTIME_DEPLOY_PRINT_CONVERSION:
                 clogger.info(

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/__utils__.py

@@ -16,6 +16,38 @@ from typing import Any
 from gpustack_runtime import envs
 
 
+def clone_exception(exc: BaseException) -> BaseException:
+    """
+    Build a fresh copy of an exception, without its traceback or chaining state.
+
+    The detector library wrappers cache the init failure and re-raise it on every
+    subsequent call to fail fast. Re-raising the *same* object is a memory leak:
+    CPython appends a new frame to its ``__traceback__`` on each raise, and those
+    frames retain the caller's locals indefinitely (gpustack/gpustack#5342).
+
+    Cloning via ``BaseException.__new__`` preserves the original type and
+    attributes regardless of the class's constructor signature -- some binding
+    error types take an error code and leave ``args`` empty (e.g. amdsmi's
+    ``AmdSmiLibraryException``), so ``type(exc)(*exc.args)`` cannot reconstruct them.
+
+    Args:
+        exc:
+            The exception to clone.
+
+    Returns:
+        A fresh exception of the same type, carrying the same args and attributes
+        but no traceback or ``__cause__``/``__context__`` chain.
+
+    """
+    clone = BaseException.__new__(type(exc))
+    clone.args = exc.args
+    # Copy instance state (custom attributes, and __notes__ if any live here too).
+    # Guard against exotic exception types that don't expose __dict__.
+    if hasattr(exc, "__dict__"):
+        clone.__dict__.update(exc.__dict__)
+    return clone
+
+
 @dataclass
 class PCIDevice:
     address: str

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/ascend.py

@@ -658,5 +658,5 @@ def get_ascend_cann_variant(name: str | None) -> str | None:
     if version < 260:
         return "a3"  # 910c/a3
     if version < 270:
-        return "a5"  # 910d/a5
+        return "950"  # 950/a5
     return None

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pyamdsmi/__init__.py

@@ -32,7 +32,12 @@ try:
 
         if _libInitialized:
             if _libInitializedException is not None:
-                raise _libInitializedException
+                # Re-raise a fresh copy: re-raising the same cached exception object
+                # appends a traceback frame on every call, and those frames retain the
+                # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+                from ..__utils__ import clone_exception
+
+                raise clone_exception(_libInitializedException) from None
             return
 
         try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pydcmi/__init__.py

@@ -780,7 +780,12 @@ def dcmi_init():
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pyhgml/__init__.py

@@ -2380,7 +2380,12 @@ def hgmlInitWithFlags(flags):
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pyixml/__init__.py

@@ -2131,7 +2131,12 @@ def nvmlInitWithFlags(flags):
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pymtml/__init__.py

@@ -15,7 +15,12 @@ def mtmlLibraryInit():
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pymxsml/__init__.py

@@ -778,7 +778,12 @@ def mxSmlInit():
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:
@@ -802,7 +807,12 @@ def mxSmlInitWithFlags(flags):
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pynvml/__init__.py

@@ -16,7 +16,12 @@ def nvmlInitWithFlags(flags):
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

{gpustack_runtime-0.2.0.post5 → gpustack_runtime-0.2.2}/gpustack_runtime/detector/pyrocmsmi/__init__.py

@@ -180,7 +180,12 @@ def rsmi_init(flags=0):
 
     if _libInitialized:
         if _libInitializedException is not None:
-            raise _libInitializedException
+            # Re-raise a fresh copy: re-raising the same cached exception object
+            # appends a traceback frame on every call, and those frames retain the
+            # caller's locals, leaking memory over time. See gpustack/gpustack#5342.
+            from ..__utils__ import clone_exception
+
+            raise clone_exception(_libInitializedException) from None
         return
 
     try:

gpustack_runtime-0.2.2/tests/gpustack_runtime/deployer/test_workload_status.py

@@ -0,0 +1,117 @@
+from datetime import datetime, timezone
+
+import kubernetes.client
+
+from gpustack_runtime.deployer.__types__ import WorkloadStatus
+from gpustack_runtime.deployer.kuberentes import (
+    KubernetesWorkloadStatus,
+    _pin_pod_for_kueue,
+)
+
+
+def _pod(annotations=None) -> kubernetes.client.V1Pod:
+    return kubernetes.client.V1Pod(
+        metadata=kubernetes.client.V1ObjectMeta(
+            name="gpustack-test",
+            namespace="default",
+            creation_timestamp=datetime(2026, 7, 29, 8, 0, 0, tzinfo=timezone.utc),
+            labels={"app": "test"},
+            annotations=annotations,
+        ),
+        spec=kubernetes.client.V1PodSpec(
+            containers=[kubernetes.client.V1Container(name="run-0")],
+        ),
+        status=kubernetes.client.V1PodStatus(phase="Pending"),
+    )
+
+
+def test_kubernetes_workload_status_annotations():
+    status = KubernetesWorkloadStatus(
+        name="test",
+        k_pod=_pod(
+            annotations={
+                "device.gpustack.ai/accelerator.allocated": '{"run-0": {}}',
+                "runtime.gpustack.ai/component-run-0-name": "internal",
+            },
+        ),
+    )
+    # Third-party annotations (e.g. device-plugin allocation) are surfaced,
+    # runtime-internal annotations are filtered out.
+    assert status.annotations == {
+        "device.gpustack.ai/accelerator.allocated": '{"run-0": {}}',
+    }
+
+
+def test_kubernetes_workload_status_annotations_absent():
+    status = KubernetesWorkloadStatus(name="test", k_pod=_pod(annotations=None))
+    assert status.annotations == {}
+
+
+def test_workload_status_annotations_default_empty():
+    # Docker/Podman deployers don't populate annotations; the base default is empty.
+    status = WorkloadStatus(name="test", created_at="2026-07-29T08:00:00.000000Z")
+    assert status.annotations == {}
+
+
+def test_workload_status_annotations_json_roundtrip():
+    status = KubernetesWorkloadStatus(
+        name="test",
+        k_pod=_pod(annotations={"device.gpustack.ai/accelerator.allocated": "{}"}),
+    )
+    restored = WorkloadStatus.from_json(status.to_json())
+    assert restored.annotations == status.annotations
+
+
+def _pinning_pod(labels=None) -> kubernetes.client.V1Pod:
+    return kubernetes.client.V1Pod(
+        metadata=kubernetes.client.V1ObjectMeta(name="p", labels=labels or {}),
+        spec=kubernetes.client.V1PodSpec(
+            node_name="node-a",
+            containers=[kubernetes.client.V1Container(name="run-0")],
+        ),
+    )
+
+
+def test_pin_pod_for_kueue_switches_to_node_selector():
+    pod = _pinning_pod(labels={"kueue.x-k8s.io/queue-name": "q"})
+    _pin_pod_for_kueue(pod, "node-a")
+    assert pod.spec.node_name is None
+    assert pod.spec.node_selector == {"kubernetes.io/hostname": "node-a"}
+
+
+def test_pin_pod_for_kueue_merges_existing_node_selector():
+    pod = _pinning_pod(labels={"kueue.x-k8s.io/queue-name": "q"})
+    pod.spec.node_selector = {"disk": "ssd"}
+    _pin_pod_for_kueue(pod, "node-a")
+    assert pod.spec.node_selector == {
+        "disk": "ssd",
+        "kubernetes.io/hostname": "node-a",
+    }
+
+
+def test_pin_pod_for_kueue_noop_without_queue_label():
+    pod = _pinning_pod()
+    _pin_pod_for_kueue(pod, "node-a")
+    assert pod.spec.node_name == "node-a"
+    assert pod.spec.node_selector is None
+
+
+def test_pin_pod_for_kueue_noop_without_node_name():
+    pod = _pinning_pod(labels={"kueue.x-k8s.io/queue-name": "q"})
+    _pin_pod_for_kueue(pod, None)
+    assert pod.spec.node_name == "node-a"
+
+
+def test_kubernetes_workload_status_state_message():
+    pod = _pod()
+    pod.status = kubernetes.client.V1PodStatus(
+        phase="Failed",
+        message="Allocate failed due to no enough GPU devices",
+    )
+    status = KubernetesWorkloadStatus(name="test", k_pod=pod)
+    assert status.state_message == "Allocate failed due to no enough GPU devices"
+
+
+def test_kubernetes_workload_status_state_message_default():
+    status = KubernetesWorkloadStatus(name="test", k_pod=_pod())
+    assert status.state_message == ""

gpustack_runtime-0.2.2/tests/gpustack_runtime/detector/test_detector_utils.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import sys
+from typing import ClassVar
+
+import pytest
+
+from gpustack_runtime.detector.__utils__ import clone_exception
+
+
+# --------------------------------------------------------------------------- #
+# Exception classes that mimic the real binding error types, to exercise the   #
+# constructor shapes clone_exception has to survive.                           #
+# --------------------------------------------------------------------------- #
+class RocmLikeError(Exception):
+    """Custom __init__ that does NOT call super().__init__ (like ROCMSMIError)."""
+
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return f"rocm {self.value}"
+
+
+class DcmiLikeError(Exception):
+    """Custom __new__(cls, value) that requires a positional arg (like DCMIError)."""
+
+    _mapping: ClassVar[dict] = {}
+
+    def __new__(cls, value):
+        target = DcmiLikeError._mapping.get(value, cls)
+        obj = Exception.__new__(target)
+        obj.value = value
+        return obj
+
+    def __str__(self):
+        return f"dcmi {self.value}"
+
+
+class AmdSmiLikeError(Exception):
+    """super().__init__() with no args -> args == (); state lives in __dict__.
+
+    This is the case that `type(exc)(*exc.args)` CANNOT reconstruct.
+    """
+
+    def __init__(self, err_code):
+        super().__init__()
+        self.err_code = err_code
+        self.err_info = f"info-{err_code}"
+
+    def __str__(self):
+        return f"Error code: {self.err_code} | {self.err_info}"
+
+
+def _tb_len(exc: BaseException) -> int:
+    n = 0
+    tb = exc.__traceback__
+    while tb:
+        n += 1
+        tb = tb.tb_next
+    return n
+
+
+ALL_CASES = [
+    RocmLikeError(-99997),
+    DcmiLikeError(-8007),
+    AmdSmiLikeError(43),
+    ValueError("boom"),
+    RuntimeError("multi", "arg", 3),
+]
+
+
+# --------------------------------------------------------------------------- #
+# Correctness: the clone must be usable in place of the original.              #
+# --------------------------------------------------------------------------- #
+def test_clone_preserves_type():
+    for exc in ALL_CASES:
+        clone = clone_exception(exc)
+        assert type(clone) is type(exc)
+
+
+def test_clone_is_a_distinct_object():
+    for exc in ALL_CASES:
+        clone = clone_exception(exc)
+        assert clone is not exc
+
+
+def test_clone_preserves_args_and_attributes_and_str():
+    for exc in ALL_CASES:
+        clone = clone_exception(exc)
+        assert clone.args == exc.args
+        assert clone.__dict__ == exc.__dict__
+        # str() relies on the preserved state; it must render identically.
+        assert str(clone) == str(exc)
+
+
+def test_clone_still_matches_except_clause():
+    # A caller doing `except RocmLikeError` must still catch the clone.
+    with pytest.raises(RocmLikeError) as info:
+        raise clone_exception(RocmLikeError(-1))
+    assert info.value.value == -1
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 11),
+    reason="Exception.add_note() was added in Python 3.11",
+)
+def test_clone_preserves_notes():
+    # __notes__ (add_note, 3.11+) lives in __dict__, so it is carried over by
+    # the __dict__ copy without any special handling.
+    exc = ValueError("boom")
+    exc.add_note("first note")
+    exc.add_note("second note")
+    clone = clone_exception(exc)
+    assert clone.__notes__ == ["first note", "second note"]
+
+
+def test_clone_handles_amdsmi_style_empty_args():
+    # The whole reason we don't use `type(exc)(*exc.args)`: it blows up here.
+    exc = AmdSmiLikeError(7)
+    assert exc.args == ()
+    clone = clone_exception(exc)
+    assert clone.err_code == 7
+    assert clone.err_info == "info-7"
+
+
+# --------------------------------------------------------------------------- #
+# The point of the helper: cloning + re-raising must NOT grow the cached       #
+# exception's traceback (this is the gpustack/gpustack#5342 regression).       #
+# --------------------------------------------------------------------------- #
+def test_reraising_clones_does_not_grow_cached_traceback():
+    # Seed a traceback on the "cached" exception, like a first failed init.
+    with pytest.raises(RocmLikeError) as info:
+        raise RocmLikeError(-99997)
+    cached = info.value
+
+    baseline = _tb_len(cached)
+
+    raised = []
+    for _ in range(500):
+        with pytest.raises(RocmLikeError) as info:
+            raise clone_exception(cached) from None
+        raised.append(info.value)
+
+    # The cached object's traceback is untouched by the re-raises.
+    assert _tb_len(cached) == baseline
+    # Every raise produced a fresh object, never the cached one.
+    assert all(r is not cached for r in raised)
+
+
+def test_contrast_reraising_same_object_would_grow_traceback():
+    # Demonstrates the bug the helper fixes: re-raising the SAME object grows
+    # its traceback unbounded. (Sanity check that our test measures the right
+    # thing; not testing clone_exception here.)
+    with pytest.raises(RocmLikeError) as info:
+        raise RocmLikeError(-99997)
+    cached = info.value
+
+    before = _tb_len(cached)
+    for _ in range(50):
+        with pytest.raises(RocmLikeError):
+            raise cached
+    assert _tb_len(cached) > before
+
+
+def test_clone_severs_cause_and_context_when_raised_from_none():
+    with pytest.raises(ValueError, match="x") as info:
+        raise clone_exception(ValueError("x")) from None
+    assert info.value.__cause__ is None
+    assert info.value.__suppress_context__ is True

gpustack_runtime-0.2.0.post5/gpustack_runtime/_version_appendix.py

@@ -1 +0,0 @@
-git_commit = "94ebfdd"