skypilot-nightly 1.0.0.dev20250720__py3-none-any.whl → 1.0.0.dev20250724__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '80235386457c47b3c178c1e1842e9b07ed75dc06'
+_SKYPILOT_COMMIT_SHA = '5b5dfe080437e0b2e58b1e7742c5864a1ebe9432'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250720'
+__version__ = '1.0.0.dev20250724'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/admin_policy.py

@@ -121,11 +121,17 @@ class MutatedUserRequest:
                 dict(self.skypilot_config),)).model_dump_json()
 
     @classmethod
-    def decode(cls, mutated_user_request_body: str) -> 'MutatedUserRequest':
+    def decode(cls, mutated_user_request_body: str,
+               original_request: UserRequest) -> 'MutatedUserRequest':
         mutated_user_request_body = _MutatedUserRequestBody.model_validate_json(
             mutated_user_request_body)
-        return cls(task=sky.Task.from_yaml_config(
-            common_utils.read_yaml_all_str(mutated_user_request_body.task)[0]),
+        task = sky.Task.from_yaml_config(
+            common_utils.read_yaml_all_str(mutated_user_request_body.task)[0])
+        # Some internal Task fields are not serialized. We need to manually
+        # restore them from the original request.
+        task.managed_job_dag = original_request.task.managed_job_dag
+        task.service_name = original_request.task.service_name
+        return cls(task=task,
                    skypilot_config=config_utils.Config.from_dict(
                        common_utils.read_yaml_all_str(
                            mutated_user_request_body.skypilot_config)[0],))
@@ -243,7 +249,8 @@ class RestfulAdminPolicy(PolicyTemplate):
                     f'{self.policy_url}: {e}') from None
 
         try:
-            mutated_user_request = MutatedUserRequest.decode(response.json())
+            mutated_user_request = MutatedUserRequest.decode(
+                response.json(), user_request)
         except Exception as e:  # pylint: disable=broad-except
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.RestfulPolicyError(

sky/backends/backend_utils.py

@@ -17,7 +17,6 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
 import uuid
 
 import colorama
-import filelock
 from packaging import version
 from typing_extensions import Literal
 
@@ -45,6 +44,7 @@ from sky.utils import common_utils
 from sky.utils import context_utils
 from sky.utils import controller_utils
 from sky.utils import env_options
+from sky.utils import locks
 from sky.utils import registry
 from sky.utils import resources_utils
 from sky.utils import rich_utils
@@ -110,17 +110,12 @@ _TEST_IP_LIST = ['https://8.8.8.8', 'https://1.1.1.1']
 # Note: This value cannot be too small, otherwise OOM issue may occur.
 DEFAULT_TASK_CPU_DEMAND = 0.5
 
-# Filelocks for the cluster status change.
-CLUSTER_STATUS_LOCK_PATH = os.path.expanduser('~/.sky/.{}.lock')
 CLUSTER_STATUS_LOCK_TIMEOUT_SECONDS = 20
 
 # Time that must elapse since the last status check before we should re-check if
 # the cluster has been terminated or autostopped.
 _CLUSTER_STATUS_CACHE_DURATION_SECONDS = 2
 
-# Filelocks for updating cluster's file_mounts.
-CLUSTER_FILE_MOUNTS_LOCK_PATH = os.path.expanduser(
-    '~/.sky/.{}_file_mounts.lock')
 CLUSTER_FILE_MOUNTS_LOCK_TIMEOUT_SECONDS = 10
 
 # Remote dir that holds our runtime files.
@@ -2005,9 +2000,20 @@ def _update_cluster_status(cluster_name: str) -> Optional[Dict[str, Any]]:
 
             total_nodes = handle.launched_nodes * handle.num_ips_per_node
 
+            cloud_name = repr(handle.launched_resources.cloud).lower()
             for i in range(5):
-                ready_head, ready_workers, output, stderr = (
-                    get_node_counts_from_ray_status(head_runner))
+                try:
+                    ready_head, ready_workers, output, stderr = (
+                        get_node_counts_from_ray_status(head_runner))
+                except RuntimeError as e:
+                    logger.debug(f'Refreshing status ({cluster_name!r}) attempt'
+                                 f' {i}: {common_utils.format_exception(e)}')
+                    if cloud_name != 'kubernetes':
+                        raise e
+                    # We retry for kubernetes because coreweave can have a
+                    # transient network issue.
+                    time.sleep(1)
+                    continue
                 if ready_head + ready_workers == total_nodes:
                     return True
                 logger.debug(f'Refreshing status ({cluster_name!r}) attempt '
@@ -2294,8 +2300,7 @@ def refresh_cluster_record(
 
         # The loop logic allows us to notice if the status was updated in the
         # global_user_state by another process and stop trying to get the lock.
-        # The core loop logic is adapted from FileLock's implementation.
-        lock = filelock.FileLock(CLUSTER_STATUS_LOCK_PATH.format(cluster_name))
+        lock = locks.get_lock(cluster_status_lock_id(cluster_name))
         start_time = time.perf_counter()
 
         # Loop until we have an up-to-date status or until we acquire the lock.
@@ -2319,7 +2324,8 @@ def refresh_cluster_record(
                         return record
                     # Update and return the cluster status.
                     return _update_cluster_status(cluster_name)
-            except filelock.Timeout:
+
+            except locks.LockTimeout:
                 # lock.acquire() will throw a Timeout exception if the lock is not
                 # available and we have blocking=False.
                 pass
@@ -3197,3 +3203,13 @@ def get_endpoints(cluster: str,
         return {
             port_num: urls[0].url() for port_num, urls in port_details.items()
         }
+
+
+def cluster_status_lock_id(cluster_name: str) -> str:
+    """Get the lock ID for cluster status operations."""
+    return f'{cluster_name}_status'
+
+
+def cluster_file_mounts_lock_id(cluster_name: str) -> str:
+    """Get the lock ID for cluster file mounts operations."""
+    return f'{cluster_name}_file_mounts'

sky/backends/cloud_vm_ray_backend.py

@@ -20,7 +20,6 @@ from typing import (Any, Callable, Dict, Iterable, List, Optional, Set, Tuple,
                     Union)
 
 import colorama
-import filelock
 import yaml
 
 import sky
@@ -64,6 +63,7 @@ from sky.utils import common_utils
 from sky.utils import context_utils
 from sky.utils import controller_utils
 from sky.utils import env_options
+from sky.utils import locks
 from sky.utils import log_utils
 from sky.utils import message_utils
 from sky.utils import registry
@@ -2916,9 +2916,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         # Check if the cluster is owned by the current user. Raise
         # exceptions.ClusterOwnerIdentityMismatchError
         backend_utils.check_owner_identity(cluster_name)
-        lock_path = os.path.expanduser(
-            backend_utils.CLUSTER_STATUS_LOCK_PATH.format(cluster_name))
-        with timeline.FileLockEvent(lock_path):
+        lock_id = backend_utils.cluster_status_lock_id(cluster_name)
+        with timeline.DistributedLockEvent(lock_id):
             # Try to launch the exiting cluster first. If no existing cluster,
             # this function will create a to_provision_config with required
             # resources.
@@ -3065,7 +3064,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
 
                 self._update_after_cluster_provisioned(
                     handle, to_provision_config.prev_handle, task,
-                    prev_cluster_status, lock_path, config_hash)
+                    prev_cluster_status, lock_id, config_hash)
                 return handle, False
 
             cluster_config_file = config_dict['ray']
@@ -3137,7 +3136,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
 
             self._update_after_cluster_provisioned(
                 handle, to_provision_config.prev_handle, task,
-                prev_cluster_status, lock_path, config_hash)
+                prev_cluster_status, lock_id, config_hash)
             return handle, False
 
     def _open_ports(self, handle: CloudVmRayResourceHandle) -> None:
@@ -3155,7 +3154,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             prev_handle: Optional[CloudVmRayResourceHandle],
             task: task_lib.Task,
             prev_cluster_status: Optional[status_lib.ClusterStatus],
-            lock_path: str, config_hash: str) -> None:
+            lock_id: str, config_hash: str) -> None:
         usage_lib.messages.usage.update_cluster_resources(
             handle.launched_nodes, handle.launched_resources)
         usage_lib.messages.usage.update_final_cluster_status(
@@ -3237,7 +3236,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 handle.cached_external_ssh_ports, handle.docker_user,
                 handle.ssh_user)
 
-            common_utils.remove_file_if_exists(lock_path)
+            locks.get_lock(lock_id).force_unlock()
 
     def _sync_workdir(self, handle: CloudVmRayResourceHandle,
                       workdir: Union[Path, Dict[str, Any]],
@@ -3819,8 +3818,8 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 is_identity_mismatch_and_purge = True
             else:
                 raise
-        lock_path = os.path.expanduser(
-            backend_utils.CLUSTER_STATUS_LOCK_PATH.format(cluster_name))
+        lock_id = backend_utils.cluster_status_lock_id(cluster_name)
+        lock = locks.get_lock(lock_id)
         # Retry in case new cluster operation comes in and holds the lock
         # right after the lock is removed.
         n_attempts = 2
@@ -3828,7 +3827,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             n_attempts -= 1
             # In case other running cluster operations are still holding the
             # lock.
-            common_utils.remove_file_if_exists(lock_path)
+            lock.force_unlock()
             # We have to kill the cluster requests, because `down` and `stop`
             # should be higher priority than the cluster requests, and we should
             # release the lock from other requests.
@@ -3847,9 +3846,7 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                     f'cluster {handle.cluster_name}: '
                     f'{common_utils.format_exception(e, use_bracket=True)}')
             try:
-                with filelock.FileLock(
-                        lock_path,
-                        backend_utils.CLUSTER_STATUS_LOCK_TIMEOUT_SECONDS):
+                with lock:
                     self.teardown_no_lock(
                         handle,
                         terminate,
@@ -3862,14 +3859,14 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                         refresh_cluster_status=(
                             not is_identity_mismatch_and_purge))
                 if terminate:
-                    common_utils.remove_file_if_exists(lock_path)
+                    lock.force_unlock()
                 break
-            except filelock.Timeout as e:
+            except locks.LockTimeout as e:
                 logger.debug(f'Failed to acquire lock for {cluster_name}, '
                              f'retrying...')
                 if n_attempts <= 0:
                     raise RuntimeError(
-                        f'Cluster {cluster_name!r} is locked by {lock_path}. '
+                        f'Cluster {cluster_name!r} is locked by {lock_id}. '
                         'Check to see if it is still being launched') from e
 
     # --- CloudVMRayBackend Specific APIs ---
@@ -5245,18 +5242,17 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                 # reconstruct them during cluster restart.
                 continue
             storage_mounts_metadata[dst] = storage_obj.handle
-        lock_path = (
-            backend_utils.CLUSTER_FILE_MOUNTS_LOCK_PATH.format(cluster_name))
+        lock_id = backend_utils.cluster_file_mounts_lock_id(cluster_name)
         lock_timeout = backend_utils.CLUSTER_FILE_MOUNTS_LOCK_TIMEOUT_SECONDS
         try:
-            with filelock.FileLock(lock_path, lock_timeout):
+            with locks.get_lock(lock_id, lock_timeout):
                 global_user_state.set_cluster_storage_mounts_metadata(
                     cluster_name, storage_mounts_metadata)
-        except filelock.Timeout as e:
+        except locks.LockTimeout as e:
             raise RuntimeError(
                 f'Failed to store metadata for cluster {cluster_name!r} due to '
                 'a timeout when trying to access local database. Please '
-                f'try again or manually remove the lock at {lock_path}. '
+                f'try again or manually remove the lock at {lock_id}. '
                 f'{common_utils.format_exception(e)}') from None
 
     def get_storage_mounts_metadata(
@@ -5267,19 +5263,18 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
         After retrieving storage_mounts_metadata, it converts back the
         StorageMetadata to Storage object and restores 'storage_mounts.'
         """
-        lock_path = (
-            backend_utils.CLUSTER_FILE_MOUNTS_LOCK_PATH.format(cluster_name))
+        lock_id = backend_utils.cluster_file_mounts_lock_id(cluster_name)
         lock_timeout = backend_utils.CLUSTER_FILE_MOUNTS_LOCK_TIMEOUT_SECONDS
         try:
-            with filelock.FileLock(lock_path, lock_timeout):
+            with locks.get_lock(lock_id, lock_timeout):
                 storage_mounts_metadata = (
                     global_user_state.get_cluster_storage_mounts_metadata(
                         cluster_name))
-        except filelock.Timeout as e:
+        except locks.LockTimeout as e:
             raise RuntimeError(
                 f'Failed to retrieve metadata for cluster {cluster_name!r} '
                 'due to a timeout when trying to access local database. '
-                f'Please try again or manually remove the lock at {lock_path}.'
+                f'Please try again or manually remove the lock at {lock_id}.'
                 f' {common_utils.format_exception(e)}') from None
 
         if storage_mounts_metadata is None:

sky/client/cli/command.py

@@ -2143,7 +2143,9 @@ def logs(
     if sync_down:
         with rich_utils.client_status(
                 ux_utils.spinner_message('Downloading logs')):
-            log_local_path_dict = sdk.download_logs(cluster, job_ids)
+            log_local_path_dict = sdk.download_logs(
+                cluster,
+                list(job_ids) if job_ids else None)
         style = colorama.Style
         fore = colorama.Fore
         for job, log_local_path in log_local_path_dict.items():
@@ -2195,8 +2197,7 @@ def logs(
                 f'{colorama.Style.RESET_ALL}')
 
     # Stream logs from the server.
-    returncode = sdk.tail_logs(cluster, job_id, follow, tail=tail)
-    sys.exit(returncode)
+    sys.exit(sdk.tail_logs(cluster, job_id, follow, tail=tail))
 
 
 @cli.command()
@@ -3023,17 +3024,18 @@ def _down_or_stop_clusters(
                         click.echo(common_utils.format_exception(e))
                     else:
                         raise
-                confirm_str = 'delete'
-                input_prefix = ('Since --purge is set, errors will be ignored '
-                                'and controller will be removed from '
-                                'local state.\n') if purge else ''
-                user_input = click.prompt(
-                    f'{input_prefix}'
-                    f'To proceed, please type {colorama.Style.BRIGHT}'
-                    f'{confirm_str!r}{colorama.Style.RESET_ALL}',
-                    type=str)
-                if user_input != confirm_str:
-                    raise click.Abort()
+                if not purge:
+                    confirm_str = 'delete'
+                    user_input = click.prompt(
+                        f'To proceed, please type {colorama.Style.BRIGHT}'
+                        f'{confirm_str!r}{colorama.Style.RESET_ALL}',
+                        type=str)
+                    if user_input != confirm_str:
+                        raise click.Abort()
+                else:
+                    click.echo('Since --purge is set, errors will be ignored '
+                               'and controller will be removed from '
+                               'local state.\nSkipping confirmation.')
                 no_confirm = True
         names += controllers
 
@@ -3243,7 +3245,7 @@ def show_gpus(
         infra: Optional[str],
         cloud: Optional[str],
         region: Optional[str],
-        all_regions: Optional[bool]):
+        all_regions: bool):
     """Show supported GPU/TPU/accelerators and their prices.
 
     The names and counts shown can be set in the ``accelerators`` field in task
@@ -5397,7 +5399,7 @@ def api():
               required=False,
               help='Enable basic authentication in the SkyPilot API server.')
 @usage_lib.entrypoint
-def api_start(deploy: bool, host: Optional[str], foreground: bool,
+def api_start(deploy: bool, host: str, foreground: bool,
               enable_basic_auth: bool):
     """Starts the SkyPilot API server locally."""
     sdk.api_start(deploy=deploy,
@@ -5508,19 +5510,27 @@ def api_status(request_ids: Optional[List[str]], all_status: bool,
         columns.append('Cluster')
     columns.extend(['Created', 'Status'])
     table = log_utils.create_table(columns)
-    for request in request_list:
-        r_id = request.request_id
-        if not verbose:
-            r_id = common_utils.truncate_long_string(r_id, 36)
-        req_status = requests.RequestStatus(request.status)
-        row = [r_id, request.user_name, request.name]
+    if len(request_list) > 0:
+        for request in request_list:
+            r_id = request.request_id
+            if not verbose:
+                r_id = common_utils.truncate_long_string(r_id, 36)
+            req_status = requests.RequestStatus(request.status)
+            row = [r_id, request.user_name, request.name]
+            if verbose:
+                row.append(request.cluster_name)
+            row.extend([
+                log_utils.readable_time_duration(request.created_at),
+                req_status.colored_str()
+            ])
+            table.add_row(row)
+    else:
+        # add dummy data for when api server is down.
+        dummy_row = ['-'] * 5
         if verbose:
-            row.append(request.cluster_name)
-        row.extend([
-            log_utils.readable_time_duration(request.created_at),
-            req_status.colored_str()
-        ])
-        table.add_row(row)
+            dummy_row.append('-')
+        table.add_row(dummy_row)
+        click.echo()
     click.echo(table)
 
 
@@ -5568,6 +5578,12 @@ def api_login(endpoint: Optional[str], relogin: bool,
     sdk.api_login(endpoint, relogin, service_account_token)
 
 
+@api.command('logout', cls=_DocumentedCodeCommand)
+def api_logout():
+    """Logs out of the api server"""
+    sdk.api_logout()
+
+
 @api.command('info', cls=_DocumentedCodeCommand)
 @flags.config_option(expose_value=False)
 @usage_lib.entrypoint

sky/client/sdk.py

@@ -1854,6 +1854,18 @@ def api_cancel(request_ids: Optional[Union[str, List[str]]] = None,
     return server_common.get_request_id(response)
 
 
+def _local_api_server_running(kill: bool = False) -> bool:
+    """Checks if the local api server is running."""
+    for process in psutil.process_iter(attrs=['pid', 'cmdline']):
+        cmdline = process.info['cmdline']
+        if cmdline and server_common.API_SERVER_CMD in ' '.join(cmdline):
+            if kill:
+                subprocess_utils.kill_children_processes(
+                    parent_pids=[process.pid], force=True)
+            return True
+    return False
+
+
 @usage_lib.entrypoint
 @annotations.client_api
 def api_status(
@@ -1872,6 +1884,10 @@ def api_status(
     Returns:
         A list of request payloads.
     """
+    if server_common.is_api_server_local() and not _local_api_server_running():
+        logger.info('SkyPilot API server is not running.')
+        return []
+
     body = payloads.RequestStatusBody(request_ids=request_ids,
                                       all_status=all_status)
     response = server_common.make_authenticated_request(
@@ -1992,13 +2008,7 @@ def api_stop() -> None:
                 f'Cannot kill the API server at {server_url} because it is not '
                 f'the default SkyPilot API server started locally.')
 
-    found = False
-    for process in psutil.process_iter(attrs=['pid', 'cmdline']):
-        cmdline = process.info['cmdline']
-        if cmdline and server_common.API_SERVER_CMD in ' '.join(cmdline):
-            subprocess_utils.kill_children_processes(parent_pids=[process.pid],
-                                                     force=True)
-            found = True
+    found = _local_api_server_running(kill=True)
 
     # Remove the database for requests.
     server_common.clear_local_api_server_database()
@@ -2067,6 +2077,22 @@ def _save_config_updates(endpoint: Optional[str] = None,
         skypilot_config.reload_config()
 
 
+def _clear_api_server_config() -> None:
+    """Clear endpoint and service account token from config file."""
+    config_path = pathlib.Path(
+        skypilot_config.get_user_config_path()).expanduser()
+    with filelock.FileLock(config_path.with_suffix('.lock')):
+        if not config_path.exists():
+            return
+
+        config = skypilot_config.get_user_config()
+        config = dict(config)
+        del config['api_server']
+
+        common_utils.dump_yaml(str(config_path), config, blank=True)
+        skypilot_config.reload_config()
+
+
 def _validate_endpoint(endpoint: Optional[str]) -> str:
     """Validate and normalize the endpoint URL."""
     if endpoint is None:
@@ -2323,3 +2349,22 @@ def api_login(endpoint: Optional[str] = None,
         endpoint)
     _show_logged_in_message(endpoint, dashboard_url, final_api_server_info.user,
                             server_status)
+
+
+@usage_lib.entrypoint
+@annotations.client_api
+def api_logout() -> None:
+    """Logout of the API server.
+
+    Clears all cookies and settings stored in ~/.sky/config.yaml"""
+    if server_common.is_api_server_local():
+        with ux_utils.print_exception_no_traceback():
+            raise RuntimeError('Local api server cannot be logged out. '
+                               'Use `sky api stop` instead.')
+
+    # no need to clear cookies if it doesn't exist.
+    server_common.set_api_cookie_jar(cookiejar.MozillaCookieJar(),
+                                     create_if_not_exists=False)
+    _clear_api_server_config()
+    logger.info(f'{colorama.Fore.GREEN}Logged out of SkyPilot API server.'
+                f'{colorama.Style.RESET_ALL}')