skypilot-nightly 1.0.0.dev2024053101__py3-none-any.whl → 1.0.0.dev2025022801__py3-none-any.whl

sky/data/storage_utils.py

@@ -1,15 +1,20 @@
 """Utility functions for the storage module."""
+import glob
 import os
+import pathlib
 import shlex
 import subprocess
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional, TextIO, Union
+import warnings
+import zipfile
 
 import colorama
 
 from sky import exceptions
 from sky import sky_logging
+from sky.skylet import constants
+from sky.utils import common_utils
 from sky.utils import log_utils
-from sky.utils.cli_utils import status_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -19,6 +24,8 @@ _FILE_EXCLUSION_FROM_GITIGNORE_FAILURE_MSG = (
     'to the cloud storage for {path!r}'
     'due to the following error: {error_msg!r}')
 
+_LAST_USE_TRUNC_LENGTH = 25
+
 
 def format_storage_table(storages: List[Dict[str, Any]],
                          show_all: bool = False) -> str:
@@ -43,8 +50,8 @@ def format_storage_table(storages: List[Dict[str, Any]],
         if show_all:
             command = row['last_use']
         else:
-            command = status_utils.truncate_long_string(
-                row['last_use'], status_utils.COMMAND_TRUNC_LENGTH)
+            command = common_utils.truncate_long_string(row['last_use'],
+                                                        _LAST_USE_TRUNC_LENGTH)
         storage_table.add_row([
             # NAME
             row['name'],
@@ -63,6 +70,42 @@ def format_storage_table(storages: List[Dict[str, Any]],
         return 'No existing storage.'
 
 
+def get_excluded_files_from_skyignore(src_dir_path: str) -> List[str]:
+    """List files and patterns ignored by the .skyignore file
+    in the given source directory.
+    """
+    excluded_list: List[str] = []
+    expand_src_dir_path = os.path.expanduser(src_dir_path)
+    skyignore_path = os.path.join(expand_src_dir_path,
+                                  constants.SKY_IGNORE_FILE)
+
+    try:
+        with open(skyignore_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith('#'):
+                    # Make parsing consistent with rsync.
+                    # Rsync uses '/' as current directory.
+                    if line.startswith('/'):
+                        line = '.' + line
+                    else:
+                        line = '**/' + line
+                    # Find all files matching the pattern.
+                    matching_files = glob.glob(os.path.join(
+                        expand_src_dir_path, line),
+                                               recursive=True)
+                    # Process filenames to comply with cloud rsync format.
+                    for i in range(len(matching_files)):
+                        matching_files[i] = os.path.relpath(
+                            matching_files[i], expand_src_dir_path)
+                    excluded_list.extend(matching_files)
+    except IOError as e:
+        logger.warning(f'Error reading {skyignore_path}: '
+                       f'{common_utils.format_exception(e, use_bracket=True)}')
+
+    return excluded_list
+
+
 def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
     """ Lists files and patterns ignored by git in the source directory
 
@@ -78,7 +121,8 @@ def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
     expand_src_dir_path = os.path.expanduser(src_dir_path)
 
     git_exclude_path = os.path.join(expand_src_dir_path, '.git/info/exclude')
-    gitignore_path = os.path.join(expand_src_dir_path, '.gitignore')
+    gitignore_path = os.path.join(expand_src_dir_path,
+                                  constants.GIT_IGNORE_FILE)
 
     git_exclude_exists = os.path.isfile(git_exclude_path)
     gitignore_exists = os.path.isfile(gitignore_path)
@@ -162,3 +206,80 @@ def get_excluded_files_from_gitignore(src_dir_path: str) -> List[str]:
                     to_be_excluded += '*'
                 excluded_list.append(to_be_excluded)
     return excluded_list
+
+
+def get_excluded_files(src_dir_path: str) -> List[str]:
+    # TODO: this could return a huge list of files,
+    # should think of ways to optimize.
+    """ List files and directories to be excluded."""
+    expand_src_dir_path = os.path.expanduser(src_dir_path)
+    skyignore_path = os.path.join(expand_src_dir_path,
+                                  constants.SKY_IGNORE_FILE)
+    if os.path.exists(skyignore_path):
+        logger.debug(f'  {colorama.Style.DIM}'
+                     f'Excluded files to sync to cluster based on '
+                     f'{constants.SKY_IGNORE_FILE}.'
+                     f'{colorama.Style.RESET_ALL}')
+        return get_excluded_files_from_skyignore(src_dir_path)
+    logger.debug(f'  {colorama.Style.DIM}'
+                 f'Excluded files to sync to cluster based on '
+                 f'{constants.GIT_IGNORE_FILE}.'
+                 f'{colorama.Style.RESET_ALL}')
+    return get_excluded_files_from_gitignore(src_dir_path)
+
+
+def zip_files_and_folders(items: List[str],
+                          output_file: Union[str, pathlib.Path],
+                          log_file: Optional[TextIO] = None):
+
+    def _store_symlink(zipf, path: str, is_dir: bool):
+        # Get the target of the symlink
+        target = os.readlink(path)
+        # Use relative path as absolute path will not be able to resolve on
+        # remote API server.
+        if os.path.isabs(target):
+            target = os.path.relpath(target, os.path.dirname(path))
+        # Create a ZipInfo instance
+        zi = zipfile.ZipInfo(path + '/') if is_dir else zipfile.ZipInfo(path)
+        # Set external attributes to mark as symlink
+        zi.external_attr = 0xA1ED0000
+        # Write symlink target as content
+        zipf.writestr(zi, target)
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings('ignore',
+                                category=UserWarning,
+                                message='Duplicate name:')
+        with zipfile.ZipFile(output_file, 'w') as zipf:
+            for item in items:
+                item = os.path.expanduser(item)
+                if not os.path.isfile(item) and not os.path.isdir(item):
+                    raise ValueError(f'{item} does not exist.')
+                excluded_files = set(
+                    [os.path.join(item, f) for f in get_excluded_files(item)])
+                if os.path.isfile(item) and item not in excluded_files:
+                    zipf.write(item)
+                elif os.path.isdir(item):
+                    for root, dirs, files in os.walk(item, followlinks=False):
+                        # Store directory entries (important for empty
+                        # directories)
+                        for dir_name in dirs:
+                            dir_path = os.path.join(root, dir_name)
+                            if dir_path in excluded_files:
+                                continue
+                            # If it's a symlink, store it as a symlink
+                            if os.path.islink(dir_path):
+                                _store_symlink(zipf, dir_path, is_dir=True)
+                            else:
+                                zipf.write(dir_path)
+
+                        for file in files:
+                            file_path = os.path.join(root, file)
+                            if file_path in excluded_files:
+                                continue
+                            if os.path.islink(file_path):
+                                _store_symlink(zipf, file_path, is_dir=False)
+                            else:
+                                zipf.write(file_path)
+                if log_file is not None:
+                    log_file.write(f'Zipped {item}\n')

sky/exceptions.py

@@ -1,11 +1,16 @@
 """Exceptions."""
+import builtins
 import enum
+import traceback
+import types
 import typing
-from typing import List, Optional
+from typing import Any, Dict, List, Optional, Sequence
+
+from sky.utils import env_options
 
 if typing.TYPE_CHECKING:
-    from sky import status_lib
     from sky.backends import backend
+    from sky.utils import status_lib
 
 # Return code for keyboard interruption and SIGTSTP
 KEYBOARD_INTERRUPT_CODE = 130
@@ -19,6 +24,107 @@ INSUFFICIENT_PRIVILEGES_CODE = 52
 GIT_FATAL_EXIT_CODE = 128
 
 
+def is_safe_exception(exc: Exception) -> bool:
+    """Returns True if the exception is safe to send to clients.
+
+    Safe exceptions are:
+    1. Built-in exceptions
+    2. SkyPilot's own exceptions
+    """
+    module = type(exc).__module__
+
+    # Builtin exceptions (e.g., ValueError, RuntimeError)
+    if module == 'builtins':
+        return True
+
+    # SkyPilot exceptions
+    if module.startswith('sky.'):
+        return True
+
+    return False
+
+
+def wrap_exception(exc: Exception) -> Exception:
+    """Wraps non-safe exceptions into SkyPilot exceptions
+
+    This is used to wrap exceptions that are not safe to deserialize at clients.
+
+    Examples include exceptions from cloud providers whose packages are not
+    available at clients.
+    """
+    if is_safe_exception(exc):
+        return exc
+
+    return CloudError(message=str(exc),
+                      cloud_provider=type(exc).__module__.split('.')[0],
+                      error_type=type(exc).__name__)
+
+
+def serialize_exception(e: Exception) -> Dict[str, Any]:
+    """Serialize the exception.
+
+    This function also wraps any unsafe exceptions (e.g., cloud exceptions)
+    into SkyPilot's CloudError before serialization to ensure clients can
+    deserialize them without needing cloud provider packages installed.
+    """
+    # Wrap unsafe exceptions before serialization
+    e = wrap_exception(e)
+
+    stacktrace = getattr(e, 'stacktrace', None)
+    attributes = e.__dict__.copy()
+    if 'stacktrace' in attributes:
+        del attributes['stacktrace']
+    for attr_k in list(attributes.keys()):
+        attr_v = attributes[attr_k]
+        if isinstance(attr_v, types.TracebackType):
+            attributes[attr_k] = traceback.format_tb(attr_v)
+
+    data = {
+        'type': e.__class__.__name__,
+        'message': str(e),
+        'args': e.args,
+        'attributes': attributes,
+        'stacktrace': stacktrace,
+    }
+    if isinstance(e, SkyPilotExcludeArgsBaseException):
+        data['args'] = tuple()
+    return data
+
+
+def deserialize_exception(serialized: Dict[str, Any]) -> Exception:
+    """Deserialize the exception."""
+    exception_type = serialized['type']
+    if hasattr(builtins, exception_type):
+        exception_class = getattr(builtins, exception_type)
+    else:
+        exception_class = globals().get(exception_type, None)
+    if exception_class is None:
+        # Unknown exception type.
+        return Exception(f'{exception_type}: {serialized["message"]}')
+    e = exception_class(*serialized['args'], **serialized['attributes'])
+    if serialized['stacktrace'] is not None:
+        setattr(e, 'stacktrace', serialized['stacktrace'])
+    return e
+
+
+class CloudError(Exception):
+    """Wraps cloud-specific errors into a SkyPilot exception."""
+
+    def __init__(self, message: str, cloud_provider: str, error_type: str):
+        super().__init__(message)
+        self.cloud_provider = cloud_provider
+        self.error_type = error_type
+
+    def __str__(self):
+        return (f'{self.cloud_provider} error ({self.error_type}): '
+                f'{super().__str__()}')
+
+
+class InvalidSkyPilotConfigError(ValueError):
+    """Raised when the SkyPilot config is invalid."""
+    pass
+
+
 class ResourcesUnavailableError(Exception):
     """Raised when resources are unavailable.
 
@@ -61,12 +167,12 @@ class ProvisionPrechecksError(Exception):
     the error will be raised.
 
     Args:
-        reasons: (List[Exception]) The reasons why the prechecks failed.
+        reasons: (Sequence[Exception]) The reasons why the prechecks failed.
     """
 
-    def __init__(self, reasons: List[Exception]) -> None:
+    def __init__(self, reasons: Sequence[Exception]) -> None:
         super().__init__()
-        self.reasons = list(reasons)
+        self.reasons = reasons
 
 
 class ManagedJobReachedMaxRetriesError(Exception):
@@ -79,12 +185,34 @@ class ManagedJobReachedMaxRetriesError(Exception):
     pass
 
 
+class ManagedJobStatusError(Exception):
+    """Raised when a managed job task status update is invalid.
+
+    For instance, a RUNNING job cannot become SUBMITTED.
+    """
+    pass
+
+
 class ResourcesMismatchError(Exception):
     """Raised when resources are mismatched."""
     pass
 
 
-class CommandError(Exception):
+class SkyPilotExcludeArgsBaseException(Exception):
+    """Base class for exceptions that don't need args while serialization.
+
+    Due to our serialization/deserialization logic, when an exception does
+    not take `args` as an argument in __init__, `args` should not be included
+    in the serialized exception.
+
+    This is useful when an exception needs to construct the error message based
+    on the arguments passed in instead of directly having the error message as
+    the first argument in __init__. Refer to `CommandError` for an example.
+    """
+    pass
+
+
+class CommandError(SkyPilotExcludeArgsBaseException):
     """Raised when a command fails.
 
     Args:
@@ -100,9 +228,14 @@ class CommandError(Exception):
         self.command = command
         self.error_msg = error_msg
         self.detailed_reason = detailed_reason
+
         if not command:
             message = error_msg
         else:
+            if (len(command) > 100 and
+                    not env_options.Options.SHOW_DEBUG_INFO.get()):
+                # Chunck the command to avoid overflow.
+                command = command[:100] + '...'
             message = (f'Command {command} failed with return code '
                        f'{returncode}.\n{error_msg}')
         super().__init__(message)
@@ -113,7 +246,7 @@ class ClusterNotUpError(Exception):
 
     def __init__(self,
                  message: str,
-                 cluster_status: Optional['status_lib.ClusterStatus'],
+                 cluster_status: Optional['status_lib.ClusterStatus'] = None,
                  handle: Optional['backend.ResourceHandle'] = None) -> None:
         super().__init__(message)
         self.cluster_status = cluster_status
@@ -125,6 +258,13 @@ class ClusterSetUpError(Exception):
     pass
 
 
+class ClusterDoesNotExist(ValueError):
+    """Raise when trying to operate on a cluster that does not exist."""
+    # This extends ValueError for compatibility reasons - we used to throw
+    # ValueError instead of this.
+    pass
+
+
 class NotSupportedError(Exception):
     """Raised when a feature is not supported."""
     pass
@@ -190,6 +330,12 @@ class StorageExternalDeletionError(StorageBucketGetError):
     pass
 
 
+class NonExistentStorageAccountError(StorageExternalDeletionError):
+    # Error raise when storage account provided through config.yaml or read
+    # from store handle(local db) does not exist.
+    pass
+
+
 class FetchClusterInfoError(Exception):
     """Raised when fetching the cluster info fails."""
 
@@ -237,7 +383,7 @@ class NoCloudAccessError(Exception):
     pass
 
 
-class AWSAzFetchingError(Exception):
+class AWSAzFetchingError(SkyPilotExcludeArgsBaseException):
     """Raised when fetching the AWS availability zone fails."""
 
     class Reason(enum.Enum):
@@ -276,3 +422,28 @@ class ServeUserTerminatedError(Exception):
 
 class PortDoesNotExistError(Exception):
     """Raised when the port does not exist."""
+
+
+class UserRequestRejectedByPolicy(Exception):
+    """Raised when a user request is rejected by an admin policy."""
+    pass
+
+
+class NoClusterLaunchedError(Exception):
+    """No cluster launched, so cleanup can be skipped during failover."""
+    pass
+
+
+class RequestCancelled(Exception):
+    """Raised when a request is cancelled."""
+    pass
+
+
+class ApiServerConnectionError(RuntimeError):
+    """Raised when the API server cannot be connected."""
+
+    def __init__(self, server_url: str):
+        super().__init__(
+            f'Could not connect to SkyPilot API server at {server_url}. '
+            f'Please ensure that the server is running. '
+            f'Try: curl {server_url}/api/health')