skypilot-nightly 1.0.0.dev2024053101__py3-none-any.whl → 1.0.0.dev2025022801__py3-none-any.whl

sky/provision/docker_utils.py

@@ -12,9 +12,6 @@ from sky.utils import subprocess_utils
 
 logger = sky_logging.init_logger(__name__)
 
-DOCKER_PERMISSION_DENIED_STR = ('permission denied while trying to connect to '
-                                'the Docker daemon socket')
-
 # Configure environment variables. A docker image can have environment variables
 # set in the Dockerfile with `ENV``. We need to export these variables to the
 # shell environment, so that our ssh session can access them.
@@ -23,9 +20,16 @@ SETUP_ENV_VARS_CMD = (
     '{ if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; } && '
     'printenv | while IFS=\'=\' read -r key value; do echo "export $key=\\\"$value\\\""; done > '  # pylint: disable=line-too-long
     '~/container_env_var.sh && '
-    '$(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh'
+    '$(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh;'
 )
 
+# Docker daemon may not be ready when the machine is firstly started. The error
+# message starts with the following string. We should wait for a while and retry
+# the command.
+DOCKER_PERMISSION_DENIED_STR = ('permission denied while trying to connect to '
+                                'the Docker daemon socket')
+_DOCKER_SOCKET_WAIT_TIMEOUT_SECONDS = 30
+
 
 @dataclasses.dataclass
 class DockerLoginConfig:
@@ -34,6 +38,13 @@ class DockerLoginConfig:
     password: str
     server: str
 
+    def format_image(self, image: str) -> str:
+        """Format the image name with the server prefix."""
+        server_prefix = f'{self.server}/'
+        if not image.startswith(server_prefix):
+            return f'{server_prefix}{image}'
+        return image
+
     @classmethod
     def from_env_vars(cls, d: Dict[str, str]) -> 'DockerLoginConfig':
         return cls(
@@ -106,8 +117,8 @@ def docker_start_cmds(
         '--cap-add=SYS_ADMIN',
         '--device=/dev/fuse',
         '--security-opt=apparmor:unconfined',
+        '--entrypoint=/bin/bash',
         image,
-        'bash',
     ]
     return ' '.join(docker_run)
 
@@ -139,7 +150,9 @@ class DockerInitializer:
     def _run(self,
              cmd,
              run_env='host',
-             wait_for_docker_daemon: bool = False) -> str:
+             wait_for_docker_daemon: bool = False,
+             separate_stderr: bool = False,
+             log_err_when_fail: bool = True) -> str:
 
         if run_env == 'docker':
             cmd = self._docker_expand_user(cmd, any_char=True)
@@ -152,29 +165,38 @@ class DockerInitializer:
                    f' {shlex.quote(cmd)} ')
 
         logger.debug(f'+ {cmd}')
-        cnt = 0
-        retry = 3
+        start = time.time()
         while True:
-            rc, stdout, stderr = self.runner.run(cmd,
-                                                 require_outputs=True,
-                                                 stream_logs=False,
-                                                 log_path=self.log_path)
-            if (not wait_for_docker_daemon or
-                    DOCKER_PERMISSION_DENIED_STR not in stdout + stderr):
-                break
-
-            cnt += 1
-            if cnt > retry:
-                break
-            logger.info(
-                'Failed to run docker command, retrying in 10 seconds... '
-                f'({cnt}/{retry})')
-            time.sleep(10)
+            rc, stdout, stderr = self.runner.run(
+                cmd,
+                require_outputs=True,
+                stream_logs=False,
+                separate_stderr=separate_stderr,
+                log_path=self.log_path)
+            if (DOCKER_PERMISSION_DENIED_STR in stdout + stderr and
+                    wait_for_docker_daemon):
+                if time.time() - start > _DOCKER_SOCKET_WAIT_TIMEOUT_SECONDS:
+                    if rc == 0:
+                        # Set returncode to 1 if failed to connect to docker
+                        # daemon after timeout.
+                        rc = 1
+                    break
+                # Close the cached connection to make the permission update of
+                # ssh user take effect, e.g. usermod -aG docker $USER, called
+                # by cloud-init of Azure.
+                self.runner.close_cached_connection()
+                logger.info('Failed to connect to docker daemon. It might be '
+                            'initializing, retrying in 5 seconds...')
+                time.sleep(5)
+                continue
+            break
         subprocess_utils.handle_returncode(
             rc,
             cmd,
-            error_msg='Failed to run docker setup commands',
-            stderr=stdout + stderr)
+            error_msg='Failed to run docker setup commands.',
+            stderr=stdout + stderr,
+            # Print out the error message if the command failed.
+            stream_logs=log_err_when_fail)
         return stdout.strip()
 
     def initialize(self) -> str:
@@ -205,9 +227,7 @@ class DockerInitializer:
                 wait_for_docker_daemon=True)
             # We automatically add the server prefix to the image name if
             # the user did not add it.
-            server_prefix = f'{docker_login_config.server}/'
-            if not specific_image.startswith(server_prefix):
-                specific_image = f'{server_prefix}{specific_image}'
+            specific_image = docker_login_config.format_image(specific_image)
 
         if self.docker_config.get('pull_before_run', True):
             assert specific_image, ('Image must be included in config if ' +
@@ -238,12 +258,13 @@ class DockerInitializer:
             # issue with nvidia container toolkit:
             # https://github.com/NVIDIA/nvidia-container-toolkit/issues/48
             self._run(
-                '[ -f /etc/docker/daemon.json ] || '
+                '{ which jq || sudo apt update && sudo apt install -y jq; } && '
+                '{ [ -f /etc/docker/daemon.json ] || '
                 'echo "{}" | sudo tee /etc/docker/daemon.json;'
                 'sudo jq \'.["exec-opts"] = ["native.cgroupdriver=cgroupfs"]\' '
                 '/etc/docker/daemon.json > /tmp/daemon.json;'
                 'sudo mv /tmp/daemon.json /etc/docker/daemon.json;'
-                'sudo systemctl restart docker')
+                'sudo systemctl restart docker; } || true')
             user_docker_run_options = self.docker_config.get('run_options', [])
             start_command = docker_start_cmds(
                 specific_image,
@@ -320,12 +341,22 @@ class DockerInitializer:
 
     def _check_docker_installed(self):
         no_exist = 'NoExist'
-        cleaned_output = self._run(
-            f'command -v {self.docker_cmd} || echo {no_exist!r}')
-        if no_exist in cleaned_output or 'docker' not in cleaned_output:
-            logger.error(
-                f'{self.docker_cmd.capitalize()} not installed. Please use an '
-                f'image with {self.docker_cmd.capitalize()} installed.')
+        # SkyPilot: Add the current user to the docker group first (if needed),
+        # before checking if docker is installed to avoid permission issues.
+        docker_cmd = ('id -nG $USER | grep -qw docker || '
+                      'sudo usermod -aG docker $USER > /dev/null 2>&1;'
+                      f'command -v {self.docker_cmd} || echo {no_exist!r}')
+        cleaned_output = self._run(docker_cmd)
+        timeout = 60 * 10  # 10 minute timeout
+        start = time.time()
+        while no_exist in cleaned_output or 'docker' not in cleaned_output:
+            if time.time() - start > timeout:
+                logger.error(
+                    f'{self.docker_cmd.capitalize()} not installed. Please use '
+                    f'an image with {self.docker_cmd.capitalize()} installed.')
+                return
+            time.sleep(5)
+            cleaned_output = self._run(docker_cmd)
 
     def _check_container_status(self):
         if self.initialized:
@@ -340,9 +371,14 @@ class DockerInitializer:
         user_pos = string.find('~')
         if user_pos > -1:
             if self.home_dir is None:
-                self.home_dir = (self._run(
-                    f'{self.docker_cmd} exec {self.container_name} '
-                    'printenv HOME',))
+                cmd = (f'{self.docker_cmd} exec {self.container_name} '
+                       'printenv HOME')
+                self.home_dir = self._run(cmd, separate_stderr=True)
+                # Check for unexpected newline in home directory, which can be
+                # a common issue when the output is mixed with stderr.
+                assert '\n' not in self.home_dir, (
+                    'Unexpected newline in home directory '
+                    f'({{self.home_dir}}) retrieved with {cmd}')
 
             if any_char:
                 return string.replace('~/', self.home_dir + '/')
@@ -360,8 +396,8 @@ class DockerInitializer:
                                     'info -f "{{.Runtimes}}"'))
         if 'nvidia-container-runtime' in runtime_output:
             try:
-                self._run('nvidia-smi')
-                return run_options + ['--runtime=nvidia']
+                self._run('nvidia-smi', log_err_when_fail=False)
+                return run_options + ['--runtime=nvidia', '--gpus all']
             except Exception as e:  # pylint: disable=broad-except
                 logger.debug(
                     'Nvidia Container Runtime is present in the docker image'
@@ -404,8 +440,8 @@ class DockerInitializer:
     def _check_container_exited(self) -> bool:
         if self.initialized:
             return True
-        output = (self._run(check_docker_running_cmd(self.container_name,
-                                                     self.docker_cmd),
-                            wait_for_docker_daemon=True))
-        return 'false' in output.lower(
-        ) and 'no such object' not in output.lower()
+        output = self._run(check_docker_running_cmd(self.container_name,
+                                                    self.docker_cmd),
+                           wait_for_docker_daemon=True)
+        return ('false' in output.lower() and
+                'no such object' not in output.lower())

sky/provision/fluidstack/fluidstack_utils.py

@@ -1,21 +1,22 @@
 """FluidStack API client."""
 
-import functools
 import json
 import os
-from typing import Any, Dict, List, Optional
+import time
+from typing import Any, Dict, List
 import uuid
 
 import requests
 
+from sky.utils import annotations
+
 
 def get_key_suffix():
     return str(uuid.uuid4()).replace('-', '')[:8]
 
 
-ENDPOINT = 'https://api.fluidstack.io/v1/'
+ENDPOINT = 'https://platform.fluidstack.io/'
 FLUIDSTACK_API_KEY_PATH = '~/.fluidstack/api_key'
-FLUIDSTACK_API_TOKEN_PATH = '~/.fluidstack/api_token'
 
 
 def read_contents(path: str) -> str:
@@ -30,7 +31,7 @@ class FluidstackAPIError(Exception):
         super().__init__(message)
 
 
-def raise_fluidstack_error(response: requests.Response) -> None:
+def raise_fluidstack_error(response: 'requests.Response') -> None:
     """Raise FluidstackAPIError if appropriate."""
     status_code = response.status_code
     if response.ok:
@@ -46,109 +47,76 @@ def raise_fluidstack_error(response: requests.Response) -> None:
     raise FluidstackAPIError(f'{message}', status_code)
 
 
-@functools.lru_cache()
-def with_nvidia_drivers(region: str):
-    if region in ['norway_4_eu', 'generic_1_canada']:
-        return False
-    client = FluidstackClient()
-    plans = client.get_plans()
-    for plan in plans:
-        if region in [r['id'] for r in plan['regions']]:
-            if 'Ubuntu 20.04 LTS (Nvidia)' in plan['os_options']:
-                return True
-    return False
-
-
 class FluidstackClient:
     """FluidStack API Client"""
 
     def __init__(self):
         self.api_key = read_contents(
-            os.path.expanduser(FLUIDSTACK_API_KEY_PATH))
-        self.api_token = read_contents(
-            os.path.expanduser(FLUIDSTACK_API_TOKEN_PATH))
+            os.path.expanduser(FLUIDSTACK_API_KEY_PATH)).strip()
 
     def get_plans(self):
-        response = requests.get(ENDPOINT + 'plans')
+        response = requests.get(ENDPOINT + 'list_available_configurations',
+                                headers={'api-key': self.api_key})
         raise_fluidstack_error(response)
         plans = response.json()
-        plans = [
-            plan for plan in plans
-            if plan['minimum_commitment'] == 'hourly' and plan['type'] in
-            ['preconfigured', 'custom'] and plan['gpu_type'] != 'NO GPU'
-        ]
         return plans
 
-    def list_instances(
-            self,
-            tag_filters: Optional[Dict[str,
-                                       str]] = None) -> List[Dict[str, Any]]:
+    def list_instances(self) -> List[Dict[str, Any]]:
         response = requests.get(
-            ENDPOINT + 'servers',
-            auth=(self.api_key, self.api_token),
+            ENDPOINT + 'instances',
+            headers={'api-key': self.api_key},
         )
         raise_fluidstack_error(response)
         instances = response.json()
-        filtered_instances = []
-
-        for instance in instances:
-            if isinstance(instance['tags'], str):
-                instance['tags'] = json.loads(instance['tags'])
-            if not instance['tags']:
-                instance['tags'] = {}
-            if tag_filters:
-                for key in tag_filters:
-                    if instance['tags'].get(key, None) != tag_filters[key]:
-                        break
-                else:
-                    filtered_instances.append(instance)
-            else:
-                filtered_instances.append(instance)
-
-        return filtered_instances
+        return instances
 
     def create_instance(
         self,
         instance_type: str = '',
-        hostname: str = '',
+        name: str = '',
         region: str = '',
         ssh_pub_key: str = '',
         count: int = 1,
     ) -> List[str]:
         """Launch new instances."""
 
-        config: Dict[str, Any] = {}
         plans = self.get_plans()
         regions = self.list_regions()
+        gpu_type, gpu_count = instance_type.split('::')
+        gpu_count = int(gpu_count)
+
         plans = [
-            plan for plan in plans if plan['plan_id'] == instance_type and
-            region in [r['id'] for r in plan['regions']]
+            plan for plan in plans if plan['gpu_type'] == gpu_type and
+            gpu_count in plan['gpu_counts'] and region in plan['regions']
         ]
         if not plans:
             raise FluidstackAPIError(
                 f'Plan {instance_type} out of stock in region {region}')
 
         ssh_key = self.get_or_add_ssh_key(ssh_pub_key)
-        os_id = 'Ubuntu 20.04 LTS'
-        body = dict(plan=None if config else instance_type,
-                    region=regions[region],
-                    os=os_id,
-                    hostname=hostname,
-                    ssh_keys=[ssh_key['id']],
-                    multiplicity=count,
-                    config=config)
-
-        response = requests.post(ENDPOINT + 'server',
-                                 auth=(self.api_key, self.api_token),
-                                 json=body)
-        raise_fluidstack_error(response)
-        instance_ids = response.json().get('multiple')
-        assert all(id is not None for id in instance_ids), instance_ids
+        default_operating_system = 'ubuntu_22_04_lts_nvidia'
+        instance_ids = []
+        for _ in range(count):
+            body = dict(gpu_type=gpu_type,
+                        gpu_count=gpu_count,
+                        region=regions[region],
+                        operating_system_label=default_operating_system,
+                        name=name,
+                        ssh_key=ssh_key['name'])
+
+            response = requests.post(ENDPOINT + 'instances',
+                                     headers={'api-key': self.api_key},
+                                     json=body)
+            raise_fluidstack_error(response)
+            instance_id = response.json().get('id')
+            instance_ids.append(instance_id)
+            time.sleep(1)
+
         return instance_ids
 
     def list_ssh_keys(self):
-        response = requests.get(ENDPOINT + 'ssh',
-                                auth=(self.api_key, self.api_token))
+        response = requests.get(ENDPOINT + 'ssh_keys',
+                                headers={'api-key': self.api_key})
         raise_fluidstack_error(response)
         return response.json()
 
@@ -156,86 +124,50 @@ class FluidstackClient:
         """Add ssh key if not already added."""
         ssh_keys = self.list_ssh_keys()
         for key in ssh_keys:
-            if key['public_key'].strip() == ssh_pub_key.strip():
-                return {
-                    'id': key['id'],
-                    'name': key['name'],
-                    'ssh_key': ssh_pub_key
-                }
+            if key['public_key'].strip().split()[:2] == ssh_pub_key.strip(
+            ).split()[:2]:
+                return {'name': key['name'], 'ssh_key': ssh_pub_key}
         ssh_key_name = 'skypilot-' + get_key_suffix()
         response = requests.post(
-            ENDPOINT + 'ssh',
-            auth=(self.api_key, self.api_token),
+            ENDPOINT + 'ssh_keys',
+            headers={'api-key': self.api_key},
             json=dict(name=ssh_key_name, public_key=ssh_pub_key),
         )
         raise_fluidstack_error(response)
-        key_id = response.json()['id']
-        return {'id': key_id, 'name': ssh_key_name, 'ssh_key': ssh_pub_key}
+        return {'name': ssh_key_name, 'ssh_key': ssh_pub_key}
 
-    @functools.lru_cache()
+    @annotations.lru_cache(scope='global')
     def list_regions(self):
-        response = requests.get(ENDPOINT + 'plans')
-        raise_fluidstack_error(response)
-        plans = response.json()
-        plans = [
-            plan for plan in plans
-            if plan['minimum_commitment'] == 'hourly' and plan['type'] in
-            ['preconfigured', 'custom'] and plan['gpu_type'] != 'NO GPU'
-        ]
+        plans = self.get_plans()
 
         def get_regions(plans: List) -> dict:
             """Return a list of regions where the plan is available."""
             regions = {}
             for plan in plans:
                 for region in plan.get('regions', []):
-                    regions[region['id']] = region['id']
+                    regions[region] = region
             return regions
 
         regions = get_regions(plans)
         return regions
 
     def delete(self, instance_id: str):
-        response = requests.delete(ENDPOINT + 'server/' + instance_id,
-                                   auth=(self.api_key, self.api_token))
+        response = requests.delete(ENDPOINT + 'instances/' + instance_id,
+                                   headers={'api-key': self.api_key})
         raise_fluidstack_error(response)
         return response.json()
 
     def stop(self, instance_id: str):
-        response = requests.put(ENDPOINT + 'server/' + instance_id + '/stop',
-                                auth=(self.api_key, self.api_token))
-        raise_fluidstack_error(response)
-        return response.json()
-
-    def restart(self, instance_id: str):
-        response = requests.post(ENDPOINT + 'server/' + instance_id + '/reboot',
-                                 auth=(self.api_key, self.api_token))
-        raise_fluidstack_error(response)
-        return response.json()
-
-    def info(self, instance_id: str):
-        response = requests.get(ENDPOINT + f'server/{instance_id}',
-                                auth=(self.api_key, self.api_token))
-        raise_fluidstack_error(response)
-        return response.json()
-
-    def status(self, instance_id: str):
-        response = self.info(instance_id)
-        return response['status']
-
-    def add_tags(self, instance_id: str, tags: Dict[str, str]) -> str:
-        response = requests.patch(
-            ENDPOINT + f'server/{instance_id}/tag',
-            auth=(self.api_key, self.api_token),
-            json=dict(tags=json.dumps(tags)),
-        )
+        response = requests.put(ENDPOINT + 'instances/' + instance_id + '/stop',
+                                headers={'api-key': self.api_key})
         raise_fluidstack_error(response)
         return response.json()
 
-    def rename(self, instance_id: str, hostname: str) -> str:
-        response = requests.patch(
-            ENDPOINT + f'server/{instance_id}/rename',
-            auth=(self.api_key, self.api_token),
-            json=dict(name=hostname),
+    def rename(self, instance_id: str, name: str) -> str:
+        response = requests.put(
+            ENDPOINT + f'instances/{instance_id}/rename',
+            headers={'api-key': self.api_key},
+            json=dict(new_instance_name=name),
         )
         raise_fluidstack_error(response)
         return response.json()

sky/provision/fluidstack/instance.py

@@ -1,15 +1,16 @@
 """FluidStack instance provisioning."""
+import os
 import time
 from typing import Any, Dict, List, Optional
 
 from sky import authentication as auth
 from sky import exceptions
 from sky import sky_logging
-from sky import status_lib
 from sky.provision import common
 from sky.provision.fluidstack import fluidstack_utils as utils
 from sky.utils import command_runner
 from sky.utils import common_utils
+from sky.utils import status_lib
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
 
@@ -25,10 +26,11 @@ logger = sky_logging.init_logger(__name__)
 
 def get_internal_ip(node_info: Dict[str, Any]) -> None:
     node_info['internal_ip'] = node_info['ip_address']
+    private_key_path, _ = auth.get_or_generate_keys()
     runner = command_runner.SSHCommandRunner(
-        node_info['ip_address'],
-        ssh_user=node_info['capabilities']['default_user_name'],
-        ssh_private_key=auth.PRIVATE_SSH_KEY_PATH)
+        (node_info['ip_address'], 22),
+        ssh_user='ubuntu',
+        ssh_private_key=os.path.expanduser(private_key_path))
     result = runner.run(_GET_INTERNAL_IP_CMD,
                         require_outputs=True,
                         stream_logs=False)
@@ -61,7 +63,7 @@ def _filter_instances(
         if (include_instances is not None and
                 instance['id'] not in include_instances):
             continue
-        if instance.get('hostname') in possible_names:
+        if instance.get('name') in possible_names:
             filtered_instances[instance['id']] = instance
     return filtered_instances
 
@@ -69,7 +71,7 @@ def _filter_instances(
 def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     head_instance_id = None
     for inst_id, inst in instances.items():
-        if inst['hostname'].endswith('-head'):
+        if inst['name'].endswith('-head'):
             head_instance_id = inst_id
             break
     return head_instance_id
@@ -79,18 +81,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
 
-    pending_status = [
-        'create',
-        'requesting',
-        'provisioning',
-        'customizing',
-        'starting',
-        'stopping',
-        'start',
-        'stop',
-        'reboot',
-        'rebooting',
-    ]
+    pending_status = ['pending', 'provisioning']
     while True:
         instances = _filter_instances(cluster_name_on_cloud, pending_status)
         if len(instances) > config.count:
@@ -127,7 +118,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                         f'{instance_name}')
             rename(instance_id, instance_name)
         if (instance_id != head_instance_id and
-                instance['hostname'].endswith('-head')):
+                instance['name'].endswith('-head')):
             # Multiple head instances exist.
             # This is a rare case when the instance name was manually modified
             # on the cloud or some unexpected behavior happened.
@@ -167,7 +158,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
         node_type = 'head' if head_instance_id is None else 'worker'
         try:
             instance_ids = utils.FluidstackClient().create_instance(
-                hostname=f'{cluster_name_on_cloud}-{node_type}',
+                name=f'{cluster_name_on_cloud}-{node_type}',
                 instance_type=config.node_config['InstanceType'],
                 ssh_pub_key=config.node_config['AuthorizedKey'],
                 region=region)
@@ -184,9 +175,6 @@ def run_instances(region: str, cluster_name_on_cloud: str,
         instances = _filter_instances(cluster_name_on_cloud,
                                       pending_status + ['running'])
         if len(instances) < config.count:
-            # Some of pending instances have been convert to a state that will
-            # not convert to `running` status. This can be due to resource
-            # availability issue.
             all_instances = _filter_instances(
                 cluster_name_on_cloud,
                 status_filters=None,
@@ -253,15 +241,11 @@ def terminate_instances(
     instances = _filter_instances(cluster_name_on_cloud, None)
     for inst_id, inst in instances.items():
         logger.debug(f'Terminating instance {inst_id}: {inst}')
-        if worker_only and inst['hostname'].endswith('-head'):
+        if worker_only and inst['name'].endswith('-head'):
             continue
         try:
             utils.FluidstackClient().delete(inst_id)
         except Exception as e:  # pylint: disable=broad-except
-            if (isinstance(e, utils.FluidstackAPIError) and
-                    'Machine is already terminated' in str(e)):
-                logger.debug(f'Instance {inst_id} is already terminated.')
-                continue
             with ux_utils.print_exception_no_traceback():
                 raise RuntimeError(
                     f'Failed to terminate instance {inst_id}: '
@@ -291,7 +275,7 @@ def get_cluster_info(
                 tags={},
             )
         ]
-        if instance_info['hostname'].endswith('-head'):
+        if instance_info['name'].endswith('-head'):
             head_instance_id = instance_id
 
     return common.ClusterInfo(instances=instances,
@@ -311,22 +295,10 @@ def query_instances(
     instances = _filter_instances(cluster_name_on_cloud, None)
     instances = _filter_instances(cluster_name_on_cloud, None)
     status_map = {
-        'provisioning': status_lib.ClusterStatus.INIT,
-        'requesting': status_lib.ClusterStatus.INIT,
-        'create': status_lib.ClusterStatus.INIT,
-        'customizing': status_lib.ClusterStatus.INIT,
-        'stopping': status_lib.ClusterStatus.STOPPED,
-        'stop': status_lib.ClusterStatus.STOPPED,
-        'start': status_lib.ClusterStatus.INIT,
-        'reboot': status_lib.ClusterStatus.STOPPED,
-        'rebooting': status_lib.ClusterStatus.STOPPED,
+        'pending': status_lib.ClusterStatus.INIT,
         'stopped': status_lib.ClusterStatus.STOPPED,
-        'starting': status_lib.ClusterStatus.INIT,
         'running': status_lib.ClusterStatus.UP,
-        'failed to create': status_lib.ClusterStatus.INIT,
-        'timeout error': status_lib.ClusterStatus.INIT,
-        'out of stock': status_lib.ClusterStatus.INIT,
-        'terminating': None,
+        'unhealthy': status_lib.ClusterStatus.INIT,
         'terminated': None,
     }
     statuses: Dict[str, Optional[status_lib.ClusterStatus]] = {}