gpustack-runtime 0.1.38.post4__py3-none-any.whl → 0.1.39__py3-none-any.whl

gpustack_runtime/deployer/__types__.py

@@ -1020,11 +1020,11 @@ class WorkloadPlan(WorkloadSecurity):
                 c.execution.command_script = None
             # Add default registry if needed.
             if (
-                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY
-                and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY
+                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY
+                and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY
                 not in ["docker.io", "index.docker.io"]
             ):
-                image_registry = envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_NAMESPACE
+                image_registry = envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_NAMESPACE
                 image_split = c.image.split("/")
                 if len(image_split) == 1:
                     c.image = f"{image_registry}/library/{c.image}"
@@ -1269,6 +1269,17 @@ class Deployer(ABC):
         "AMD_VISIBLE_DEVICES": ["HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"]
     }.
     """
+    _visible_devices_cdis: dict[str, str] | None = None
+    """
+    Recorded visible devices envs to CDI mapping,
+    the key is the runtime visible devices env name,
+    the value is the corresponding CDI key.
+    For example:
+    {
+        "NVIDIA_VISIBLE_DEVICES": "nvidia.com/gpu",
+        "AMD_VISIBLE_DEVICES": "amd.com/gpu"
+    }.
+    """
     _visible_devices_values: dict[str, list[str]] | None = None
     """
     Recorded visible devices values,
@@ -1349,6 +1360,7 @@ class Deployer(ABC):
             return
 
         self._visible_devices_env = {}
+        self._visible_devices_cdis = {}
         self._visible_devices_values = {}
         self._visible_devices_topologies = {}
         self._backend_visible_devices_values_alignment = {}
@@ -1364,33 +1376,33 @@ class Deployer(ABC):
                 ren = envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_RUNTIME_VISIBLE_DEVICES.get(
                     rk,
                 )
-                ben = envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_BACKEND_VISIBLE_DEVICES.get(
+                ben_list = envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_BACKEND_VISIBLE_DEVICES.get(
+                    rk,
+                )
+                cdi = envs.GPUSTACK_RUNTIME_DEPLOY_RESOURCE_KEY_MAP_CONTAINER_DEVICE_INTERFACES.get(
                     rk,
                 )
-                if ren and ben:
+                if ren and ben_list:
+                    valued_uuid = (
+                        ren
+                        in envs.GPUSTACK_RUNTIME_DEPLOY_RUNTIME_VISIBLE_DEVICES_VALUE_UUID
+                    )
                     dev_uuids: list[str] = []
                     dev_indexes: list[str] = []
-                    for dev in devs:
+                    dev_indexes_alignment: dict[str, str] = {}
+                    for dev_i, dev in enumerate(devs):
                         dev_uuids.append(dev.uuid)
                         dev_indexes.append(str(dev.index))
-                    dev_indexes_alignment: dict[str, str] = {
-                        dev_indexes[i]: str(i) for i in range(len(devs))
-                    }
-                    self._visible_devices_env[ren] = ben
+                        dev_indexes_alignment[str(dev.index)] = str(dev_i)
+                    # Map runtime visible devices env <-> backend visible devices env list.
+                    self._visible_devices_env[ren] = ben_list
+                    # Map runtime visible devices env <-> CDI key.
+                    self._visible_devices_cdis[ren] = cdi
+                    # Map runtime visible devices env <-> device indexes or uuids.
                     self._visible_devices_values[ren] = (
-                        dev_uuids
-                        if ren
-                        in envs.GPUSTACK_RUNTIME_DEPLOY_RUNTIME_VISIBLE_DEVICES_VALUE_UUID
-                        else dev_indexes
+                        dev_uuids if valued_uuid else dev_indexes
                     )
-                    for ben_item in ben:
-                        if (
-                            ben_item
-                            in envs.GPUSTACK_RUNTIME_DEPLOY_BACKEND_VISIBLE_DEVICES_VALUE_ALIGNMENT
-                        ):
-                            self._backend_visible_devices_values_alignment[ben_item] = (
-                                dev_indexes_alignment
-                            )
+                    # Map runtime visible devices env <-> topology.
                     if (
                         envs.GPUSTACK_RUNTIME_DEPLOY_CPU_AFFINITY
                         or envs.GPUSTACK_RUNTIME_DEPLOY_NUMA_AFFINITY
@@ -1398,6 +1410,17 @@ class Deployer(ABC):
                         topos = get_devices_topologies(devices=devs)
                         if topos:
                             self._visible_devices_topologies[ren] = topos[0]
+                    # Map backend visible devices env <-> devices alignment.
+                    if not valued_uuid:
+                        for ben in ben_list:
+                            valued_alignment = (
+                                ben
+                                in envs.GPUSTACK_RUNTIME_DEPLOY_BACKEND_VISIBLE_DEVICES_VALUE_ALIGNMENT
+                            )
+                            if valued_alignment:
+                                self._backend_visible_devices_values_alignment[ben] = (
+                                    dev_indexes_alignment
+                                )
 
             if self._visible_devices_env:
                 return
@@ -1406,17 +1429,21 @@ class Deployer(ABC):
         self._visible_devices_env["UNKNOWN_RUNTIME_VISIBLE_DEVICES"] = []
         self._visible_devices_values["UNKNOWN_RUNTIME_VISIBLE_DEVICES"] = ["all"]
 
-    def get_visible_devices_env_values(
+    def get_visible_devices_values(
         self,
-    ) -> (dict[str, list[str]], dict[str, list[str]]):
+    ) -> (dict[str, list[str]], dict[str, str], dict[str, list[str]]):
         """
-        Return the visible devices environment variables and values mappings.
+        Return the visible devices environment variables, cdis and values mappings.
         For example:
         (
             {
                 "NVIDIA_VISIBLE_DEVICES": ["CUDA_VISIBLE_DEVICES"],
                 "AMD_VISIBLE_DEVICES": ["HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"]
-            }.
+            },
+            {
+                "NVIDIA_VISIBLE_DEVICES": "nvidia.com/gpu",
+                "AMD_VISIBLE_DEVICES": "amd.com/gpu"
+            },
             {
                 "NVIDIA_VISIBLE_DEVICES": ["0"],
                 "AMD_VISIBLE_DEVICES": ["0", "1"]
@@ -1428,11 +1455,17 @@ class Deployer(ABC):
             - The first dictionary maps runtime visible devices environment variable names
               to lists of backend visible devices environment variable names.
             - The second dictionary maps runtime visible devices environment variable names
+              to corresponding CDI keys.
+            - The last dictionary maps runtime visible devices environment variable names
               to lists of device indexes or UUIDs.
 
         """
         self._prepare()
-        return self._visible_devices_env, self._visible_devices_values
+        return (
+            self._visible_devices_env,
+            self._visible_devices_cdis,
+            self._visible_devices_values,
+        )
 
     def get_visible_devices_affinities(
         self,

gpustack_runtime/deployer/docker.py

@@ -139,7 +139,7 @@ class DockerWorkloadPlan(WorkloadPlan):
         super().validate_and_default()
 
         # Adjust default image namespace if needed.
-        if namespace := envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_NAMESPACE:
+        if namespace := envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_NAMESPACE:
             self.pause_image = replace_image_with(
                 image=self.pause_image,
                 namespace=namespace,
@@ -175,7 +175,7 @@ class DockerWorkloadStatus(WorkloadStatus):
 
     @staticmethod
     def parse_state(
-        d_containers: list[docker.models.containers],
+        d_containers: list[docker.models.containers.Container],
     ) -> WorkloadStatusStateEnum:
         """
         Parse the state of the workload based on the status of its containers.
@@ -221,7 +221,7 @@ class DockerWorkloadStatus(WorkloadStatus):
                 d_run_state = WorkloadStatusStateEnum.PENDING
             else:
                 health = cr.attrs["State"].get("Health", {})
-                if health and health.get("Status", "healthy") != "healthy":
+                if health and health.get("Status", "healthy") not in ["healthy", ""]:
                     return WorkloadStatusStateEnum.UNHEALTHY
 
         d_init_state = None
@@ -252,7 +252,7 @@ class DockerWorkloadStatus(WorkloadStatus):
     def __init__(
         self,
         name: WorkloadName,
-        d_containers: list[docker.models.containers],
+        d_containers: list[docker.models.containers.Container],
         **kwargs,
     ):
         created_at = d_containers[0].attrs["Created"]
@@ -330,6 +330,12 @@ class DockerDeployer(Deployer):
         if client:
             try:
                 supported = client.ping()
+                if envs.GPUSTACK_RUNTIME_LOG_EXCEPTION:
+                    version_info = client.version()
+                    logger.debug(
+                        "Connected to Docker API server: %s",
+                        version_info,
+                    )
             except docker.errors.APIError:
                 debug_log_exception(logger, "Failed to connect to Docker API server")
 
@@ -352,12 +358,13 @@ class DockerDeployer(Deployer):
                 contextlib.redirect_stdout(dev_null),
                 contextlib.redirect_stderr(dev_null),
             ):
-                if Path("/var/run/docker.sock").exists():
-                    client = docker.DockerClient(base_url="unix://var/run/docker.sock")
-                else:
-                    client = docker.from_env()
-        except docker.errors.DockerException:
-            debug_log_exception(logger, "Failed to get Docker client")
+                os_env = os.environ.copy()
+                if envs.GPUSTACK_RUNTIME_DOCKER_HOST:
+                    os_env["DOCKER_HOST"] = envs.GPUSTACK_RUNTIME_DOCKER_HOST
+                client = docker.from_env(environment=os_env)
+        except docker.errors.DockerException as e:
+            if "FileNotFoundError" not in str(e):
+                debug_log_exception(logger, "Failed to get Docker client")
 
         return client
 
@@ -463,12 +470,12 @@ class DockerDeployer(Deployer):
             tag = tag or "latest"
             auth_config = None
             if (
-                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_USERNAME
-                and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_PASSWORD
+                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_USERNAME
+                and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_PASSWORD
             ):
                 auth_config = {
-                    "username": envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_USERNAME,
-                    "password": envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_PASSWORD,
+                    "username": envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_USERNAME,
+                    "password": envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_PASSWORD,
                 }
 
             logs = self._client.api.pull(
@@ -488,8 +495,10 @@ class DockerDeployer(Deployer):
             msg = f"Failed to pull image {image}, invalid response"
             raise OperationError(msg) from e
         except docker.errors.APIError as e:
-            msg = f"Failed to pull image {image}{_detail_api_call_error(e)}"
-            raise OperationError(msg) from e
+            if "no unqualified-search registries are defined" not in str(e):
+                msg = f"Failed to pull image {image}{_detail_api_call_error(e)}"
+                raise OperationError(msg) from e
+            return self._pull_image(f"docker.io/{image}")
 
     def _get_image(
         self,
@@ -658,6 +667,14 @@ class DockerDeployer(Deployer):
             # TODO(thxCode): check if the container matches the spec
             return d_container
 
+        host_socket_path = None
+        if envs.GPUSTACK_RUNTIME_DOCKER_HOST.startswith("http+unix://"):
+            host_socket_path = envs.GPUSTACK_RUNTIME_DOCKER_HOST[len("http+unix://") :]
+        elif envs.GPUSTACK_RUNTIME_DOCKER_HOST.startswith("unix://"):
+            host_socket_path = envs.GPUSTACK_RUNTIME_DOCKER_HOST[len("unix://") :]
+        if host_socket_path and not host_socket_path.startswith("/"):
+            host_socket_path = f"/{host_socket_path}"
+
         create_options: dict[str, Any] = {
             "name": container_name,
             "restart_policy": {"Name": "always"},
@@ -669,10 +686,17 @@ class DockerDeployer(Deployer):
             "environment": [
                 f"AUTOHEAL_CONTAINER_LABEL={_LABEL_COMPONENT_HEAL_PREFIX}-{workload.name}",
             ],
-            "volumes": [
-                "/var/run/docker.sock:/var/run/docker.sock",
-            ],
         }
+        if host_socket_path:
+            create_options["volumes"] = (
+                [
+                    f"{host_socket_path}:/var/run/docker.sock",
+                ],
+            )
+        elif envs.GPUSTACK_RUNTIME_DOCKER_HOST:
+            create_options["environment"].append(
+                f"DOCKER_SOCK={envs.GPUSTACK_RUNTIME_DOCKER_HOST}",
+            )
 
         if envs.GPUSTACK_RUNTIME_DEPLOY_PRINT_CONVERSION:
             clogger.info(
@@ -760,7 +784,7 @@ class DockerDeployer(Deployer):
                 else:
                     continue
 
-                if m.mode == ContainerMountModeEnum.ROX:
+                if m.mode != ContainerMountModeEnum.RWX:
                     binding["ReadOnly"] = True
 
                 mount_binding.append(binding)
@@ -841,6 +865,7 @@ class DockerDeployer(Deployer):
         workload: DockerWorkloadPlan,
         ephemeral_filename_mapping: dict[tuple[int, str] : str],
         ephemeral_volume_name_mapping: dict[str, str],
+        pause_container: docker.models.containers.Container,
     ) -> (
         list[docker.models.containers.Container],
         list[docker.models.containers.Container],
@@ -860,7 +885,7 @@ class DockerDeployer(Deployer):
         d_init_containers: list[docker.models.containers.Container] = []
         d_run_containers: list[docker.models.containers.Container] = []
 
-        pause_container_namespace = f"container:{workload.name}-pause"
+        pause_container_namespace = f"container:{pause_container.id}"
         for ci, c in enumerate(workload.containers):
             container_name = f"{workload.name}-{c.profile.lower()}-{ci}"
             try:
@@ -942,9 +967,14 @@ class DockerDeployer(Deployer):
 
             # Parameterize resources.
             if c.resources:
+                cdi = (
+                    envs.GPUSTACK_RUNTIME_DOCKER_RESOURCE_INJECTION_POLICY.lower()
+                    == "cdi"
+                )
+
                 r_k_runtime_env = workload.resource_key_runtime_env_mapping or {}
                 r_k_backend_env = workload.resource_key_backend_env_mapping or {}
-                vd_env, vd_values = self.get_visible_devices_env_values()
+                vd_env, vd_cdis, vd_values = self.get_visible_devices_values()
                 for r_k, r_v in c.resources.items():
                     match r_k:
                         case "cpu":
@@ -994,24 +1024,59 @@ class DockerDeployer(Deployer):
                                 # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
                                 # and mount corresponding libs if needed.
                                 for re in runtime_env:
-                                    # Set to "all" if no specific devices detected,
-                                    # maybe the container backend can handle it.
-                                    create_options["environment"][re] = (
-                                        ",".join(vd_values.get(re, [])) or "all"
-                                    )
+                                    # Request device via CDI.
+                                    if cdi:
+                                        rv = [
+                                            f"{vd_cdis[re]}={v}"
+                                            for v in (vd_values.get(re) or ["all"])
+                                        ]
+                                        if "device_requests" not in create_options:
+                                            create_options["device_requests"] = []
+                                        create_options["device_requests"].append(
+                                            docker.types.DeviceRequest(
+                                                driver="cdi",
+                                                count=0,
+                                                device_ids=rv,
+                                            ),
+                                        )
+                                        continue
+                                    # Request device via visible devices env.
+                                    rv = ",".join(vd_values.get(re) or ["all"])
+                                    create_options["environment"][re] = rv
                             else:
                                 # Set env to the allocated device IDs if no privileged,
                                 # otherwise, set container backend visible devices env to all devices,
                                 # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
                                 # and mount corresponding libs if needed.
                                 for re in runtime_env:
-                                    # Set to "all" if no specific devices detected,
-                                    # maybe the container backend can handle it.
-                                    create_options["environment"][re] = (
-                                        str(r_v)
-                                        if not privileged
-                                        else (",".join(vd_values.get(re, [])) or "all")
-                                    )
+                                    # Request device via CDI.
+                                    if cdi:
+                                        if not privileged:
+                                            rv = [
+                                                f"{vd_cdis[re]}={v.strip()}"
+                                                for v in r_v.split(",")
+                                            ]
+                                        else:
+                                            rv = [
+                                                f"{vd_cdis[re]}={v}"
+                                                for v in (vd_values.get(re) or ["all"])
+                                            ]
+                                        if "device_requests" not in create_options:
+                                            create_options["device_requests"] = []
+                                        create_options["device_requests"].append(
+                                            docker.types.DeviceRequest(
+                                                driver="cdi",
+                                                count=0,
+                                                device_ids=rv,
+                                            ),
+                                        )
+                                        continue
+                                    # Request device via visible devices env.
+                                    if not privileged:
+                                        rv = str(r_v)
+                                    else:
+                                        rv = ",".join(vd_values.get(re) or ["all"])
+                                    create_options["environment"][re] = rv
 
                             # Configure runtime device access environment variables.
                             if r_v != "all" and privileged:
@@ -1204,7 +1269,7 @@ class DockerDeployer(Deployer):
             # Always filter out Docker Socket mount.
             m
             for m in (self_container.attrs["Mounts"] or [])
-            if m.get("Destination") != "/var/run/docker.sock"
+            if not m.get("Destination").endswith("/docker.sock")
         ]
         if igs := envs.GPUSTACK_RUNTIME_DEPLOY_MIRRORED_DEPLOYMENT_IGNORE_VOLUMES:
             mirrored_mounts = [
@@ -1491,6 +1556,7 @@ class DockerDeployer(Deployer):
             workload,
             ephemeral_filename_mapping,
             ephemeral_volume_name_mapping,
+            pause_container,
         )
 
         # Create unhealthy restart container if needed.
@@ -1593,10 +1659,18 @@ class DockerDeployer(Deployer):
         # Remove all containers with the workload label.
         try:
             d_containers = getattr(workload, "_d_containers", [])
+            # Remove non-pause containers first.
             for c in d_containers:
-                c.remove(
-                    force=True,
-                )
+                if "-pause" not in c.name:
+                    c.remove(
+                        force=True,
+                    )
+            # Then remove pause containers.
+            for c in d_containers:
+                if "-pause" in c.name:
+                    c.remove(
+                        force=True,
+                    )
         except docker.errors.APIError as e:
             msg = f"Failed to delete containers for workload {name}{_detail_api_call_error(e)}"
             raise OperationError(msg) from e
@@ -1860,7 +1934,7 @@ class DockerDeployer(Deployer):
         }
 
         try:
-            result = container.exec_run(
+            _, output = container.exec_run(
                 detach=False,
                 **exec_options,
             )
@@ -1869,8 +1943,8 @@ class DockerDeployer(Deployer):
             raise OperationError(msg) from e
         else:
             if not attach:
-                return result.output
-            return DockerWorkloadExecStream(result.output)
+                return output
+            return DockerWorkloadExecStream(output)
 
 
 def _has_restart_policy(

gpustack_runtime/deployer/kuberentes.py

@@ -321,14 +321,18 @@ class KubernetesDeployer(Deployer):
                 version_api = kubernetes.client.VersionApi(client)
                 version_info = version_api.get_code()
                 supported = version_info is not None
-            except (
-                urllib3.exceptions.MaxRetryError,
-                kubernetes.client.exceptions.ApiException,
-            ):
+                if envs.GPUSTACK_RUNTIME_LOG_EXCEPTION:
+                    logger.debug(
+                        "Connected to Kubernetes API server: %s",
+                        version_info,
+                    )
+            except kubernetes.client.exceptions.ApiException:
                 debug_log_exception(
                     logger,
                     "Failed to connect to Kubernetes API server",
                 )
+            except urllib3.exceptions.MaxRetryError:
+                pass
 
         return supported
 
@@ -985,7 +989,7 @@ class KubernetesDeployer(Deployer):
                 resources: dict[str, str] = {}
                 r_k_runtime_env = workload.resource_key_runtime_env_mapping or {}
                 r_k_backend_env = workload.resource_key_backend_env_mapping or {}
-                vd_env, vd_values = self.get_visible_devices_env_values()
+                vd_env, _, vd_values = self.get_visible_devices_values()
                 for r_k, r_v in c.resources.items():
                     if r_k in ("cpu", "memory"):
                         resources[r_k] = str(r_v)
@@ -1024,12 +1028,12 @@ class KubernetesDeployer(Deployer):
                             # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
                             # and mount corresponding libs if needed.
                             for re in runtime_env:
-                                # Set to "all" if no specific devices detected,
-                                # maybe the container backend can handle it.
+                                # Request device via visible devices env.
+                                rv = ",".join(vd_values.get(re) or ["all"])
                                 container.env.append(
                                     kubernetes.client.V1EnvVar(
                                         name=re,
-                                        value=",".join(vd_values.get(re, [])) or "all",
+                                        value=rv,
                                     ),
                                 )
                         else:
@@ -1038,18 +1042,15 @@ class KubernetesDeployer(Deployer):
                             # so that the container backend (e.g., NVIDIA Container Toolkit) can handle it,
                             # and mount corresponding libs if needed.
                             for re in runtime_env:
-                                # Set to "all" if no specific devices detected,
-                                # maybe the container backend can handle it.
+                                # Request device via visible devices env.
+                                if not privileged:
+                                    rv = str(r_v)
+                                else:
+                                    rv = ",".join(vd_values.get(re) or ["all"])
                                 container.env.append(
                                     kubernetes.client.V1EnvVar(
                                         name=re,
-                                        value=(
-                                            str(r_v)
-                                            if not privileged
-                                            else (
-                                                ",".join(vd_values.get(re, [])) or "all"
-                                            )
-                                        ),
+                                        value=rv,
                                     ),
                                 )
 
@@ -1206,16 +1207,16 @@ class KubernetesDeployer(Deployer):
 
         # Create image pull secrets if default registry credentials are set.
         if not self._image_pull_secret and (
-            envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_USERNAME
-            and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_PASSWORD
+            envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_USERNAME
+            and envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_PASSWORD
         ):
             registry = (
-                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY or "index.docker.io"
+                envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY or "index.docker.io"
             )
             self._image_pull_secret = self._apply_image_pull_secret(
                 registry=f"https://{registry}/v1/",
-                username=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_USERNAME,
-                password=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_REGISTRY_PASSWORD,
+                username=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_USERNAME,
+                password=envs.GPUSTACK_RUNTIME_DEPLOY_DEFAULT_IMAGE_REGISTRY_PASSWORD,
             )
 
         # Prepare mirrored deployment if enabled.