gpustack-runtime 0.1.39__py3-none-any.whl → 0.1.39.post2__py3-none-any.whl

gpustack_runtime/deployer/kuberentes.py

@@ -5,7 +5,6 @@ import json
 import logging
 import operator
 import os
-import socket
 from dataclasses import dataclass, field
 from enum import Enum
 from functools import lru_cache, reduce
@@ -24,7 +23,7 @@ from .__types__ import (
     Container,
     ContainerPort,
     ContainerProfileEnum,
-    Deployer,
+    EndoscopicDeployer,
     UnsupportedError,
     WorkloadExecStream,
     WorkloadName,
@@ -40,6 +39,7 @@ from .__utils__ import (
     fnv1a_32_hex,
     fnv1a_64_hex,
     safe_yaml,
+    sensitive_env_var,
     validate_rfc1123_domain_name,
 )
 
@@ -277,7 +277,7 @@ Name of the Kubernetes deployer.
 """
 
 
-class KubernetesDeployer(Deployer):
+class KubernetesDeployer(EndoscopicDeployer):
     """
     Deployer implementation for Kubernetes.
     """
@@ -1195,9 +1195,9 @@ class KubernetesDeployer(Deployer):
         self._client = self._get_client()
         self._node_name = envs.GPUSTACK_RUNTIME_KUBERNETES_NODE_NAME
 
-    def _prepare_create(self):
+    def _prepare_mirrored_deployment(self):
         """
-        Prepare for creation.
+        Prepare for mirrored deployment.
 
         """
         # Get the first node name of the cluster if not configured.
@@ -1207,61 +1207,39 @@ class KubernetesDeployer(Deployer):
 
         # Create image pull secrets if default registry credentials are set.
         if not self._image_pull_secret and (
-            envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_USERNAME
-            and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_PASSWORD
+            envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_CONTAINER_REGISTRY_USERNAME
+            and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_CONTAINER_REGISTRY_PASSWORD
         ):
             registry = (
-                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY or "index.docker.io"
+                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_CONTAINER_REGISTRY
+                or "index.docker.io"
             )
             self._image_pull_secret = self._apply_image_pull_secret(
                 registry=f"https://{registry}/v1/",
-                username=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_USERNAME,
-                password=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_PASSWORD,
+                username=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_CONTAINER_REGISTRY_USERNAME,
+                password=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_CONTAINER_REGISTRY_PASSWORD,
             )
 
         # Prepare mirrored deployment if enabled.
         if self._mutate_create_pod:
             return
         self._mutate_create_pod = lambda o: o
-        if not envs.GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT:
-            logger.debug("Mirrored deployment disabled")
-            return
 
         # Retrieve self-pod info.
-        core_api = kubernetes.client.CoreV1Api(self._client)
-        ## - Get Pod name, default to hostname if not set.
-        self_pod_name = envs.GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME
-        if not self_pod_name:
-            self_pod_name = socket.gethostname()
-            logger.warning(
-                "Mirrored deployment enabled, but no Pod name set, using hostname(%s) instead",
-                self_pod_name,
-            )
-        ## - Get Pod namespace, default to "default" if not found.
-        try:
-            self_pod_namespace_f = Path(
-                "/var/run/secrets/kubernetes.io/serviceaccount/namespace",
-            )
-            self_pod_namespace = self_pod_namespace_f.read_text(
-                encoding="utf-8",
-            ).strip()
-        except (FileNotFoundError, OSError):
-            self_pod_namespace = "default"
-            logger.warning(
-                "Mirrored deployment enabled, but no Pod namespace found, using 'default' instead",
-            )
         try:
-            self_pod = core_api.read_namespaced_pod(
-                name=self_pod_name,
-                namespace=self_pod_namespace,
-            )
+            self_pod = self._find_self_pod()
+            if not self_pod:
+                return
         except kubernetes.client.exceptions.ApiException:
             logger.exception(
-                "Mirrored deployment enabled, but failed to get self Pod %s/%s, skipping",
-                self_pod_namespace,
-                self_pod_name,
+                "Mirrored deployment enabled, but failed to get self Pod, skipping",
             )
             return
+
+        self_pod_name = self_pod.metadata.name
+        self_pod_namespace = self_pod.metadata.namespace
+
+        # Retrieve self-container
         ## - Get the first Container, or the Container named "default" if exists.
         self_container = next(
             (c for c in self_pod.spec.containers if c.name == "default"),
@@ -1273,11 +1251,14 @@ class KubernetesDeployer(Deployer):
                 "Mirrored deployment enabled, but no Container named 'default' found, using the first Container instead",
             )
         logger.info(
-            "Mirrored deployment enabled, using self Container %s of self Pod %s for options mirroring",
+            "Mirrored deployment enabled, using self Container %s of self Pod %s/%s for options mirroring",
             self_container.name,
-            f"{self_pod_namespace}/{self_pod_name}",
+            self_pod_namespace,
+            self_pod_name,
         )
 
+        core_api = kubernetes.client.CoreV1Api(self._client)
+
         # Preprocess mirrored deployment options.
         in_same_namespace = (
             self_pod_namespace == envs.GPUSTACK_RUNTIME_KUBERNETES_NAMESPACE
@@ -1460,6 +1441,51 @@ class KubernetesDeployer(Deployer):
 
         self._mutate_create_pod = mutate_create_pod
 
+    def _find_self_pod(self) -> kubernetes.client.V1Pod | None:
+        """
+        Find the self Pod in the cluster.
+
+        Returns:
+            The self Pod object if found, None otherwise.
+
+        Raises:
+            If failed to find itself.
+
+        """
+        if not envs.GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT:
+            logger.debug("Mirrored deployment disabled")
+            return None
+
+        # Get Pod name or hostname.
+        self_pod_name = envs.GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME
+        if not self_pod_name:
+            msg = "Please use env `GPUSTACK_RUNTIME_DEPLOY_MIRRORED_NAME` to specify the exact Pod name"
+            raise kubernetes.client.exceptions.ApiException(
+                status=404,
+                reason=msg,
+            )
+
+        # Get Pod namespace, default to "default" if not found.
+        try:
+            self_pod_namespace_f = Path(
+                "/var/run/secrets/kubernetes.io/serviceaccount/namespace",
+            )
+            self_pod_namespace = self_pod_namespace_f.read_text(
+                encoding="utf-8",
+            ).strip()
+        except (FileNotFoundError, OSError):
+            self_pod_namespace = "default"
+            logger.warning(
+                "Mirrored deployment enabled, but no Pod namespace found, using 'default' instead",
+            )
+
+        core_api = kubernetes.client.CoreV1Api(self._client)
+
+        return core_api.read_namespaced_pod(
+            name=self_pod_name,
+            namespace=self_pod_namespace,
+        )
+
     @_supported
     def _create(self, workload: WorkloadPlan):
         """
@@ -1484,7 +1510,7 @@ class KubernetesDeployer(Deployer):
             msg = f"Invalid workload plan type: {type(workload)}"
             raise TypeError(msg)
 
-        self._prepare_create()
+        self._prepare_mirrored_deployment()
 
         if isinstance(workload, WorkloadPlan):
             workload = KubernetesWorkloadPlan(**workload.__dict__)
@@ -1930,6 +1956,223 @@ class KubernetesDeployer(Deployer):
                 return result
             return KubernetesWorkloadExecStream(result)
 
+    @_supported
+    def _inspect(
+        self,
+        name: WorkloadName,
+        namespace: WorkloadNamespace | None = None,
+    ) -> str | None:
+        """
+        Inspect a Kubernetes workload.
+
+        Args:
+            name:
+                The name of the workload.
+            namespace:
+                The namespace of the workload.
+
+        Returns:
+            The inspection result as a YAML string if found, None otherwise.
+
+        Raises:
+            UnsupportedError:
+                If Kubernetes is not supported in the current environment.
+            OperationError:
+                If the Kubernetes workload fails to inspect.
+
+        """
+        workload = self._get(name=name, namespace=namespace)
+        if not workload:
+            return None
+
+        k_pod = getattr(workload, "_k_pod", None)
+        if not k_pod:
+            return None
+
+        # Remove managed fields to reduce output size
+        k_pod.metadata.managed_fields = None
+        # Mask sensitive environment variables
+        for c in k_pod.spec.containers:
+            for env in c.env or []:
+                if sensitive_env_var(env.name):
+                    env.value = "******"
+
+        return safe_yaml(k_pod, indent=2, sort_keys=False)
+
+    def _find_self_pod_for_endoscopy(self) -> kubernetes.client.V1Pod:
+        """
+        Find the self Pod for endoscopy.
+        Only works in mirrored deployment mode.
+
+        Returns:
+            The self Pod object.
+
+        Raises:
+            UnsupportedError:
+                If endoscopy is not supported in the current environment.
+
+        """
+        try:
+            self_pod = self._find_self_pod()
+        except kubernetes.client.exceptions.ApiException as e:
+            msg = "Endoscopy is not supported in the current environment: Mirrored deployment enabled, but failed to get self Pod"
+            raise UnsupportedError(msg) from e
+        except Exception as e:
+            msg = "Endoscopy is not supported in the current environment: Failed to get self Pod"
+            raise UnsupportedError(msg) from e
+
+        if not self_pod:
+            msg = "Endoscopy is not supported in the current environment: Mirrored deployment disabled"
+            raise UnsupportedError(msg)
+        return self_pod
+
+    def _endoscopic_logs(
+        self,
+        timestamps: bool = False,
+        tail: int | None = None,
+        since: int | None = None,
+        follow: bool = False,
+    ) -> Generator[bytes | str, None, None] | bytes | str:
+        """
+        Get the logs of the deployer itself.
+        Only works in mirrored deployment mode.
+
+        Args:
+            timestamps:
+                Show timestamps in the logs.
+            tail:
+                Number of lines to show from the end of the logs.
+            since:
+                Show logs since the given epoch in seconds.
+            follow:
+                Whether to follow the logs.
+
+        Returns:
+            The logs as a byte string or a generator yielding byte strings if follow is True.
+
+        Raises:
+            UnsupportedError:
+                If endoscopy is not supported in the current environment.
+            OperationError:
+                If the deployer fails to get logs.
+
+        """
+        self_pod = self._find_self_pod_for_endoscopy()
+
+        logs_options = {
+            "timestamps": timestamps,
+            "tail_lines": tail if tail >= 0 else None,
+            "since_seconds": since,
+            "follow": follow,
+            "_preload_content": not follow,
+        }
+
+        self_pod_name = self_pod.metadata.name
+        self_pod_namespace = self_pod.metadata.namespace
+
+        core_api = kubernetes.client.CoreV1Api(self._client)
+
+        try:
+            output = core_api.read_namespaced_pod_log(
+                namespace=self_pod_namespace,
+                name=self_pod_name,
+                **logs_options,
+            )
+        except kubernetes.client.exceptions.ApiException as e:
+            msg = f"Failed to fetch logs for self Pod {self_pod_namespace}/{self_pod_name}{_detail_api_call_error(e)}"
+            raise OperationError(msg) from e
+        else:
+            return output
+
+    def _endoscopic_exec(
+        self,
+        detach: bool = True,
+        command: list[str] | None = None,
+        args: list[str] | None = None,
+    ) -> WorkloadExecStream | bytes | str:
+        """
+        Execute a command in the deployer itself.
+        Only works in mirrored deployment mode.
+
+        Args:
+            detach:
+                Whether to detach from the command.
+            command:
+                The command to execute.
+                If not specified, use /bin/sh and implicitly attach.
+            args:
+                The arguments to pass to the command.
+
+        Returns:
+            If detach is False, return a WorkloadExecStream.
+            otherwise, return the output of the command as a byte string or string.
+
+        Raises:
+            UnsupportedError:
+                If endoscopy is not supported in the current environment.
+            OperationError:
+                If the deployer fails to execute the command.
+
+        """
+        self_pod = self._find_self_pod_for_endoscopy()
+
+        attach = not detach or not command
+        exec_options = {
+            "stdout": True,
+            "stderr": True,
+            "stdin": attach,
+            "tty": attach,
+            "command": [*command, *(args or [])] if command else ["/bin/sh"],
+            "_preload_content": not attach,
+        }
+
+        self_pod_name = self_pod.metadata.name
+        self_pod_namespace = self_pod.metadata.namespace
+
+        core_api = kubernetes.client.CoreV1Api(self._client)
+
+        try:
+            result = kubernetes.stream.stream(
+                core_api.connect_get_namespaced_pod_exec,
+                namespace=self_pod_namespace,
+                name=self_pod_name,
+                **exec_options,
+            )
+        except kubernetes.client.exceptions.ApiException as e:
+            msg = f"Failed to exec command in self Pod {self_pod_namespace}/{self_pod_name}{_detail_api_call_error(e)}"
+            raise OperationError(msg) from e
+        else:
+            if not attach:
+                return result
+            return KubernetesWorkloadExecStream(result)
+
+    def _endoscopic_inspect(self) -> str:
+        """
+        Inspect the deployer itself.
+        Only works in mirrored deployment mode.
+
+        Returns:
+            The inspection result.
+
+        Raises:
+            UnsupportedError:
+                If endoscopy is not supported in the current environment.
+            OperationError:
+                If the deployer fails to execute the command.
+
+        """
+        self_pod = self._find_self_pod_for_endoscopy()
+
+        # Remove managed fields to reduce output size
+        self_pod.metadata.managed_fields = None
+        # Mask sensitive environment variables
+        for c in self_pod.spec.containers:
+            for env in c.env or []:
+                if sensitive_env_var(env.name):
+                    env.value = "******"
+
+        return safe_yaml(self_pod, indent=2, sort_keys=False)
+
 
 def equal_config_maps(
     a: kubernetes.client.V1ConfigMap,