skypilot-nightly 1.0.0.dev20250627__py3-none-any.whl → 1.0.0.dev20250630__py3-none-any.whl

sky/users/token_service.py

@@ -0,0 +1,196 @@
+"""JWT-based service account token management for SkyPilot."""
+
+import contextlib
+import datetime
+import hashlib
+import os
+import secrets
+from typing import Any, Dict, Generator, Optional
+
+import filelock
+import jwt
+
+from sky import global_user_state
+from sky import sky_logging
+
+logger = sky_logging.init_logger(__name__)
+
+# JWT Configuration
+JWT_ALGORITHM = 'HS256'
+JWT_ISSUER = 'sky'  # Shortened for compact tokens
+JWT_SECRET_DB_KEY = 'jwt_secret'
+
+# File lock for JWT secret initialization
+JWT_SECRET_LOCK_PATH = os.path.expanduser('~/.sky/.jwt_secret_init.lock')
+JWT_SECRET_LOCK_TIMEOUT_SECONDS = 20
+
+
+@contextlib.contextmanager
+def _jwt_secret_lock() -> Generator[None, None, None]:
+    """Context manager for JWT secret initialization lock."""
+    try:
+        with filelock.FileLock(JWT_SECRET_LOCK_PATH,
+                               JWT_SECRET_LOCK_TIMEOUT_SECONDS):
+            yield
+    except filelock.Timeout as e:
+        raise RuntimeError(f'Failed to initialize JWT secret due to a timeout '
+                           f'when trying to acquire the lock at '
+                           f'{JWT_SECRET_LOCK_PATH}. '
+                           'Please try again or manually remove the lock '
+                           f'file if you believe it is stale.') from e
+
+
+class TokenService:
+    """Service for managing JWT-based service account tokens."""
+
+    def __init__(self):
+        self.secret_key = self._get_or_generate_secret()
+
+    def _get_or_generate_secret(self) -> str:
+        """Get JWT secret from database or generate a new one."""
+        with _jwt_secret_lock():
+            # Try to get from database (persistent across deployments)
+            try:
+                db_secret = global_user_state.get_system_config(
+                    JWT_SECRET_DB_KEY)
+                if db_secret:
+                    logger.debug('Retrieved existing JWT secret from database')
+                    return db_secret
+            except Exception as e:  # pylint: disable=broad-except
+                logger.debug(f'Failed to get JWT secret from database: {e}')
+
+            # Generate a new secret and store in database
+            new_secret = secrets.token_urlsafe(64)
+            try:
+                global_user_state.set_system_config(JWT_SECRET_DB_KEY,
+                                                    new_secret)
+                logger.info(
+                    'Generated new JWT secret and stored in database. '
+                    'This secret will persist across API server restarts.')
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(
+                    f'Failed to store new JWT secret in database: {e}. '
+                    f'Using in-memory secret (tokens will not persist '
+                    f'across restarts).')
+
+            return new_secret
+
+    def create_token(self,
+                     creator_user_id: str,
+                     service_account_user_id: str,
+                     token_name: str,
+                     expires_in_days: Optional[int] = None) -> Dict[str, Any]:
+        """Create a new JWT service account token.
+
+        Args:
+            creator_user_id: The creator's user hash
+            service_account_user_id: The service account's own user ID
+            token_name: Descriptive name for the token
+            expires_in_days: Optional expiration in days
+
+        Returns:
+            Dict containing token info including the JWT token
+        """
+        now = datetime.datetime.now(datetime.timezone.utc)
+        token_id = secrets.token_urlsafe(12)  # Shorter ID for JWT
+
+        # Build minimal JWT payload with single-character field names for
+        # compactness
+        payload = {
+            'i': JWT_ISSUER,  # Issuer (use constant)
+            't': int(now.timestamp()),  # Issued at (shortened from 'iat')
+            # Service account user ID (shortened from 'sub')
+            'u': service_account_user_id,
+            'k': token_id,  # Token ID (shortened from 'token_id')
+            'y': 'sa',  # Type: service account (shortened from 'type')
+        }
+
+        # Add expiration if specified
+        expires_at = None
+        if expires_in_days:
+            exp_time = now + datetime.timedelta(days=expires_in_days)
+            payload['e'] = int(
+                exp_time.timestamp())  # Expiration (shortened from 'exp')
+            expires_at = int(exp_time.timestamp())
+
+        # Generate JWT
+        jwt_token = jwt.encode(payload,
+                               self.secret_key,
+                               algorithm=JWT_ALGORITHM)
+
+        # Create token with SkyPilot prefix
+        full_token = f'sky_{jwt_token}'
+
+        # Generate hash for database storage (we still hash the full token)
+        token_hash = hashlib.sha256(full_token.encode()).hexdigest()
+
+        return {
+            'token_id': token_id,
+            'token': full_token,
+            'token_hash': token_hash,
+            'creator_user_id': creator_user_id,
+            'service_account_user_id': service_account_user_id,
+            'token_name': token_name,
+            'created_at': int(now.timestamp()),
+            'expires_at': expires_at,
+        }
+
+    def verify_token(self, token: str) -> Optional[Dict[str, Any]]:
+        """Verify and decode a JWT token.
+
+        Args:
+            token: The full token (with sky_ prefix)
+
+        Returns:
+            Decoded token payload or None if invalid
+        """
+        if not token.startswith('sky_'):
+            return None
+
+        # Remove the sky_ prefix
+        jwt_token = token[4:]
+
+        try:
+            # Decode and verify JWT (without issuer verification)
+            payload = jwt.decode(jwt_token,
+                                 self.secret_key,
+                                 algorithms=[JWT_ALGORITHM])
+
+            # Manually verify issuer using our shortened field name
+            token_issuer = payload.get('i')
+            if token_issuer != JWT_ISSUER:
+                logger.warning(f'Invalid token issuer: {token_issuer}')
+                return None
+
+            # Verify token type
+            token_type = payload.get('y')
+            if token_type != 'sa':
+                logger.warning(f'Invalid token type: {token_type}')
+                return None
+
+            # Convert shortened field names back to standard names for
+            # compatibility
+            normalized_payload = {
+                'iss': payload.get('i'),  # issuer
+                'iat': payload.get('t'),  # issued at
+                'sub': payload.get('u'),  # subject (service account user ID)
+                'token_id': payload.get('k'),  # token ID
+                'type': 'service_account',  # expand shortened type
+            }
+
+            # Add expiration if present
+            if 'e' in payload:
+                normalized_payload['exp'] = payload['e']
+
+            return normalized_payload
+
+        except jwt.ExpiredSignatureError:
+            logger.warning('Token has expired')
+            return None
+        except jwt.InvalidTokenError as e:
+            logger.warning(f'Invalid token: {e}')
+            return None
+
+
+# Singleton instance
+token_service = TokenService()

sky/utils/common_utils.py

@@ -71,11 +71,10 @@ def get_usage_run_id() -> str:
 def is_valid_user_hash(user_hash: Optional[str]) -> bool:
     if user_hash is None:
         return False
-    try:
-        int(user_hash, 16)
-    except (TypeError, ValueError):
-        return False
-    return len(user_hash) == USER_HASH_LENGTH
+    # Must start with a letter, followed by alphanumeric characters and hyphens
+    # This covers both old hex format (e.g., "abc123") and new service account
+    # format (e.g., "sa-abc123-token-xyz")
+    return bool(re.match(r'^[a-zA-Z0-9][a-zA-Z0-9-]*$', user_hash))
 
 
 def generate_user_hash() -> str:

sky/utils/config_utils.py

@@ -226,3 +226,44 @@ def merge_k8s_configs(
                 base_config[key].extend(value)
         else:
             base_config[key] = value
+
+
+def get_cloud_config_value_from_dict(
+        dict_config: Dict[str, Any],
+        cloud: str,
+        keys: Tuple[str, ...],
+        region: Optional[str] = None,
+        default_value: Optional[Any] = None,
+        override_configs: Optional[Dict[str, Any]] = None) -> Any:
+    """Returns the nested key value by reading from config
+    Order to get the property_name value:
+    1. if region is specified,
+       try to get the value from <cloud>/<region_key>/<region>/keys
+    2. if no region or no override,
+       try to get it at the cloud level <cloud>/keys
+    3. if not found at cloud level,
+       return either default_value if specified or None
+    """
+    input_config = Config(dict_config)
+    region_key = None
+    if cloud == 'kubernetes':
+        region_key = 'context_configs'
+
+    per_context_config = None
+    if region is not None and region_key is not None:
+        per_context_config = input_config.get_nested(
+            keys=(cloud, region_key, region) + keys,
+            default_value=None,
+            override_configs=override_configs)
+    # if no override found for specified region
+    general_config = input_config.get_nested(keys=(cloud,) + keys,
+                                             default_value=default_value,
+                                             override_configs=override_configs)
+
+    if (cloud == 'kubernetes' and isinstance(general_config, dict) and
+            isinstance(per_context_config, dict)):
+        merge_k8s_configs(general_config, per_context_config)
+        return general_config
+    else:
+        return (general_config
+                if per_context_config is None else per_context_config)

sky/utils/controller_utils.py

@@ -733,7 +733,11 @@ def _setup_proxy_command_on_controller(
     config = config_utils.Config.from_dict(user_config)
     proxy_command_key = (str(controller_launched_cloud).lower(),
                          'ssh_proxy_command')
-    ssh_proxy_command = config.get_nested(proxy_command_key, None)
+    ssh_proxy_command = skypilot_config.get_effective_region_config(
+        cloud=str(controller_launched_cloud).lower(),
+        region=None,
+        keys=('ssh_proxy_command',),
+        default_value=None)
     if isinstance(ssh_proxy_command, str):
         config.set_nested(proxy_command_key, None)
     elif isinstance(ssh_proxy_command, dict):

sky/utils/log_utils.py

@@ -573,6 +573,74 @@ def readable_time_duration(start: Optional[float],
     return diff
 
 
+def human_duration(start: int, end: Optional[int] = None) -> str:
+    """Calculates the time elapsed between two timestamps and returns
+       it as a human-readable string, similar to Kubernetes' duration format.
+
+    Args:
+        start: The start time as a Unix timestamp (seconds since epoch).
+        end: The end time as a Unix timestamp (seconds since epoch).
+            If None, current time is used.
+
+    Returns:
+        A string representing the duration, e.g., "2d3h", "15m", "30s".
+        Returns "0s" for zero, negative durations, or if the timestamp
+        is invalid.
+    """
+    if not start or start <= 0:
+        return '0s'
+
+    if end is None:
+        end = int(time.time())
+    duration_seconds = end - start
+
+    units = {
+        'y': 365 * 24 * 60 * 60,
+        'd': 60 * 60 * 24,
+        'h': 60 * 60,
+        'm': 60,
+        's': 1,
+    }
+
+    if duration_seconds <= 0:
+        return '0s'
+    elif duration_seconds < 60 * 2:
+        return f'{duration_seconds}s'
+
+    minutes = int(duration_seconds / units['m'])
+    if minutes < 10:
+        s = int(duration_seconds / units['s']) % 60
+        if s == 0:
+            return f'{minutes}m'
+        return f'{minutes}m{s}s'
+    elif minutes < 60 * 3:
+        return f'{minutes}m'
+
+    hours = int(duration_seconds / units['h'])
+    days = int(hours / 24)
+    years = int(hours / 24 / 365)
+    if hours < 8:
+        m = int(duration_seconds / units['m']) % 60
+        if m == 0:
+            return f'{hours}h'
+        return f'{hours}h{m}m'
+    elif hours < 48:
+        return f'{hours}h'
+    elif hours < 24 * 8:
+        h = hours % 24
+        if h == 0:
+            return f'{days}d'
+        return f'{days}d{h}h'
+    elif hours < 24 * 365 * 2:
+        return f'{days}d'
+    elif hours < 24 * 365 * 8:
+        dy = int(hours / 24) % 365
+        if dy == 0:
+            return f'{years}y'
+        return f'{years}y{dy}d'
+    return f'{years}y'
+
+
 def follow_logs(
     file: TextIO,
     *,

sky/utils/resource_checker.py

@@ -0,0 +1,153 @@
+"""Resource checking utilities for finding active clusters and managed jobs."""
+
+import concurrent.futures
+from typing import Any, Callable, Dict, List, Tuple
+
+from sky import exceptions
+from sky import global_user_state
+from sky import sky_logging
+from sky.skylet import constants
+
+logger = sky_logging.init_logger(__name__)
+
+
+def check_no_active_resources_for_users(
+        user_operations: List[Tuple[str, str]]) -> None:
+    """Check if users have active clusters or managed jobs.
+
+    Args:
+        user_operations: List of tuples (user_id, operation) where
+            operation is 'update' or 'delete'.
+
+    Raises:
+        ValueError: If any user has active clusters or managed jobs.
+            The error message will include all users with issues.
+    """
+    if not user_operations:
+        return
+
+    def filter_by_user(user_id: str):
+        return lambda resource: resource.get('user_hash') == user_id
+
+    _check_active_resources(user_operations, filter_by_user, 'user')
+
+
+def check_no_active_resources_for_workspaces(
+        workspace_operations: List[Tuple[str, str]]) -> None:
+    """Check if workspaces have active clusters or managed jobs.
+
+    Args:
+        workspace_operations: List of tuples (workspace_name, operation) where
+            operation is 'update' or 'delete'.
+
+    Raises:
+        ValueError: If any workspace has active clusters or managed jobs.
+            The error message will include all workspaces with issues.
+    """
+    if not workspace_operations:
+        return
+
+    def filter_by_workspace(workspace_name: str):
+        return lambda resource: (resource.get(
+            'workspace', constants.SKYPILOT_DEFAULT_WORKSPACE) == workspace_name
+                                )
+
+    _check_active_resources(workspace_operations, filter_by_workspace,
+                            'workspace')
+
+
+def _check_active_resources(resource_operations: List[Tuple[str, str]],
+                            filter_factory: Callable[[str],
+                                                     Callable[[Dict[str, Any]],
+                                                              bool]],
+                            resource_type: str) -> None:
+    """Check if resource entities have active clusters or managed jobs.
+
+    Args:
+        resource_operations: List of tuples (resource_name, operation) where
+            operation is 'update' or 'delete'.
+        filter_factory: Function that takes a resource_name and returns a filter
+            function for clusters/jobs.
+        resource_type: Type of resource being checked ('user' or 'workspace').
+
+    Raises:
+        ValueError: If any resource has active clusters or managed jobs.
+    """
+
+    def get_all_clusters():
+        return global_user_state.get_clusters()
+
+    def get_all_managed_jobs():
+        # pylint: disable=import-outside-toplevel
+        from sky.jobs.server import core as managed_jobs_core
+        try:
+            return managed_jobs_core.queue(refresh=False,
+                                           skip_finished=True,
+                                           all_users=True)
+        except exceptions.ClusterNotUpError:
+            logger.warning('All jobs should be finished.')
+            return []
+
+    # Fetch both clusters and jobs in parallel
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        clusters_future = executor.submit(get_all_clusters)
+        jobs_future = executor.submit(get_all_managed_jobs)
+
+        all_clusters = clusters_future.result()
+        all_managed_jobs = jobs_future.result()
+
+    # Collect all error messages instead of raising immediately
+    error_messages = []
+
+    # Check each resource against the fetched data
+    for resource_name, operation in resource_operations:
+        resource_filter = filter_factory(resource_name)
+
+        # Filter clusters for this resource
+        resource_clusters = [
+            cluster for cluster in all_clusters if resource_filter(cluster)
+        ]
+
+        # Filter managed jobs for this resource
+        resource_active_jobs = [
+            job for job in all_managed_jobs if resource_filter(job)
+        ]
+
+        # Collect error messages for this resource
+        resource_errors = []
+
+        if resource_clusters:
+            active_cluster_names = [
+                cluster['name'] for cluster in resource_clusters
+            ]
+            cluster_list = ', '.join(active_cluster_names)
+            resource_errors.append(
+                f'{len(resource_clusters)} active cluster(s): {cluster_list}')
+
+        if resource_active_jobs:
+            job_names = [str(job['job_id']) for job in resource_active_jobs]
+            job_list = ', '.join(job_names)
+            resource_errors.append(
+                f'{len(resource_active_jobs)} active managed job(s): '
+                f'{job_list}')
+
+        # If this resource has issues, add to overall error messages
+        if resource_errors:
+            resource_error_summary = ' and '.join(resource_errors)
+            error_messages.append(
+                f'Cannot {operation} {resource_type} {resource_name!r} '
+                f'because it has {resource_error_summary}.')
+
+    # If we collected any errors, raise them all together
+    if error_messages:
+        if len(error_messages) == 1:
+            # Single resource error
+            full_message = error_messages[
+                0] + ' Please terminate these resources first.'
+        else:
+            # Multiple resource errors
+            full_message = (f'Cannot proceed due to active resources in '
+                            f'{len(error_messages)} {resource_type}(s):\n' +
+                            '\n'.join(f'• {msg}' for msg in error_messages) +
+                            '\nPlease terminate these resources first.')
+        raise ValueError(full_message)

sky/utils/resources_utils.py

@@ -273,10 +273,18 @@ def need_to_query_reservations() -> bool:
     clouds that do not use reservations.
     """
     for cloud_str in registry.CLOUD_REGISTRY.keys():
-        cloud_specific_reservations = skypilot_config.get_nested(
-            (cloud_str, 'specific_reservations'), None)
-        cloud_prioritize_reservations = skypilot_config.get_nested(
-            (cloud_str, 'prioritize_reservations'), False)
+        cloud_specific_reservations = (
+            skypilot_config.get_effective_region_config(
+                cloud=cloud_str,
+                region=None,
+                keys=('specific_reservations',),
+                default_value=None))
+        cloud_prioritize_reservations = (
+            skypilot_config.get_effective_region_config(
+                cloud=cloud_str,
+                region=None,
+                keys=('prioritize_reservations',),
+                default_value=False))
         if (cloud_specific_reservations is not None or
                 cloud_prioritize_reservations):
             return True