skypilot-nightly 1.0.0.dev20250524__py3-none-any.whl → 1.0.0.dev20250526__py3-none-any.whl

sky/server/requests/payloads.py

@@ -536,3 +536,20 @@ class UploadZipFileResponse(pydantic.BaseModel):
 class EnabledCloudsBody(RequestBody):
     """The request body for the enabled clouds endpoint."""
     workspace: Optional[str] = None
+
+
+class UpdateWorkspaceBody(RequestBody):
+    """The request body for updating a specific workspace configuration."""
+    workspace_name: str = ''  # Will be set from path parameter
+    config: Dict[str, Any]
+
+
+class CreateWorkspaceBody(RequestBody):
+    """The request body for creating a new workspace."""
+    workspace_name: str = ''  # Will be set from path parameter
+    config: Dict[str, Any]
+
+
+class DeleteWorkspaceBody(RequestBody):
+    """The request body for deleting a workspace."""
+    workspace_name: str

sky/server/server.py

@@ -34,7 +34,6 @@ from sky import execution
 from sky import global_user_state
 from sky import models
 from sky import sky_logging
-from sky import skypilot_config
 from sky.clouds import service_catalog
 from sky.data import storage_utils
 from sky.jobs.server import server as jobs_rest
@@ -59,6 +58,7 @@ from sky.utils import dag_utils
 from sky.utils import env_options
 from sky.utils import status_lib
 from sky.utils import subprocess_utils
+from sky.workspaces import server as workspaces_rest
 
 # pylint: disable=ungrouped-imports
 if sys.version_info >= (3, 10):
@@ -255,6 +255,9 @@ app.add_middleware(AuthProxyMiddleware)
 app.add_middleware(RequestIDMiddleware)
 app.include_router(jobs_rest.router, prefix='/jobs', tags=['jobs'])
 app.include_router(serve_rest.router, prefix='/serve', tags=['serve'])
+app.include_router(workspaces_rest.router,
+                   prefix='/workspaces',
+                   tags=['workspaces'])
 
 
 @app.get('/token')
@@ -323,18 +326,6 @@ async def enabled_clouds(request: fastapi.Request,
     )
 
 
-@app.get('/workspaces')
-async def get_workspace_config(request: fastapi.Request) -> None:
-    """Gets workspace config on the server."""
-    executor.schedule_request(
-        request_id=request.state.request_id,
-        request_name='workspaces',
-        request_body=payloads.RequestBody(),
-        func=skypilot_config.get_workspaces,
-        schedule_type=requests_lib.ScheduleType.SHORT,
-    )
-
-
 @app.post('/realtime_kubernetes_gpu_availability')
 async def realtime_kubernetes_gpu_availability(
     request: fastapi.Request,

sky/setup_files/dependencies.py

@@ -56,6 +56,7 @@ install_requires = [
     'aiofiles',
     'httpx',
     'setproctitle',
+    'sqlalchemy',
 ]
 
 local_ray = [

sky/skypilot_config.py

@@ -57,6 +57,8 @@ import threading
 import typing
 from typing import Any, Dict, Iterator, List, Optional, Tuple
 
+import filelock
+
 from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
@@ -66,6 +68,7 @@ from sky.utils import config_utils
 from sky.utils import context
 from sky.utils import schemas
 from sky.utils import ux_utils
+from sky.utils.kubernetes import config_map_utils
 
 if typing.TYPE_CHECKING:
     import yaml
@@ -120,10 +123,17 @@ class ConfigContext:
 
 
 # The global loaded config.
+_active_workspace_context = threading.local()
 _global_config_context = ConfigContext()
-_reload_config_lock = threading.Lock()
 
-_active_workspace_context = threading.local()
+SKYPILOT_CONFIG_LOCK_PATH = '~/.sky/locks/.skypilot_config.lock'
+
+
+def get_skypilot_config_lock_path() -> str:
+    """Get the path for the SkyPilot config lock file."""
+    lock_path = os.path.expanduser(SKYPILOT_CONFIG_LOCK_PATH)
+    os.makedirs(os.path.dirname(lock_path), exist_ok=True)
+    return lock_path
 
 
 def _get_config_context() -> ConfigContext:
@@ -389,7 +399,7 @@ def overlay_skypilot_config(
 
 def safe_reload_config() -> None:
     """Reloads the config, safe to be called concurrently."""
-    with _reload_config_lock:
+    with filelock.FileLock(get_skypilot_config_lock_path()):
         _reload_config()
 
 
@@ -691,9 +701,21 @@ def apply_cli_config(cli_config: Optional[List[str]]) -> Dict[str, Any]:
     return parsed_config
 
 
-def get_workspaces() -> Dict[str, Any]:
-    """Returns the workspace config."""
-    workspaces = get_nested(('workspaces',), default_value={})
-    if constants.SKYPILOT_DEFAULT_WORKSPACE not in workspaces:
-        workspaces[constants.SKYPILOT_DEFAULT_WORKSPACE] = {}
-    return workspaces
+def update_config_no_lock(config: config_utils.Config) -> None:
+    """Dumps the new config to a file and syncs to ConfigMap if in Kubernetes.
+
+    Args:
+        config: The config to save and sync.
+    """
+    global_config_path = os.path.expanduser(get_user_config_path())
+
+    # Always save to the local file (PVC in Kubernetes, local file otherwise)
+    common_utils.dump_yaml(global_config_path, dict(config))
+
+    if config_map_utils.is_running_in_kubernetes():
+        # In Kubernetes, sync the PVC config to ConfigMap for user convenience
+        # PVC file is the source of truth, ConfigMap is just a mirror for easy
+        # access
+        config_map_utils.patch_configmap_with_config(config, global_config_path)
+
+    _reload_config()

sky/utils/db_utils.py

@@ -1,9 +1,17 @@
 """Utils for sky databases."""
 import contextlib
+import enum
 import sqlite3
 import threading
+import typing
 from typing import Any, Callable, Optional
 
+import sqlalchemy
+from sqlalchemy import exc as sqlalchemy_exc
+
+if typing.TYPE_CHECKING:
+    from sqlalchemy.orm import Session
+
 # This parameter (passed to sqlite3.connect) controls how long we will wait to
 # obtains a database lock (not necessarily during connection, but whenever it is
 # needed). It is not a connection timeout.
@@ -21,6 +29,11 @@ from typing import Any, Callable, Optional
 _DB_TIMEOUT_S = 60
 
 
+class SQLAlchemyDialect(enum.Enum):
+    SQLITE = 'sqlite'
+    POSTGRESQL = 'postgresql'
+
+
 @contextlib.contextmanager
 def safe_cursor(db_path: str):
     """A newly created, auto-committing, auto-closing cursor."""
@@ -71,6 +84,46 @@ def add_column_to_table(
     conn.commit()
 
 
+def add_column_to_table_sqlalchemy(
+    session: 'Session',
+    table_name: str,
+    column_name: str,
+    column_type: str,
+    copy_from: Optional[str] = None,
+    value_to_replace_existing_entries: Optional[Any] = None,
+):
+    """Add a column to a table."""
+    dialect = session.bind.dialect
+    if dialect.name == SQLAlchemyDialect.SQLITE.value:
+        try:
+            session.execute(
+                sqlalchemy.text(f'ALTER TABLE {table_name} '
+                                f'ADD COLUMN {column_name} {column_type}'))
+            if copy_from is not None:
+                session.execute(
+                    sqlalchemy.text(f'UPDATE {table_name} '
+                                    f'SET {column_name} = {copy_from}'))
+            if value_to_replace_existing_entries is not None:
+                session.execute(
+                    sqlalchemy.text(f'UPDATE {table_name} '
+                                    f'SET {column_name} = :replacement_value '
+                                    f'WHERE {column_name} IS NULL'),
+                    {'replacement_value': value_to_replace_existing_entries})
+        except sqlalchemy_exc.OperationalError as e:
+            if 'duplicate column name' in str(e):
+                pass
+            else:
+                raise
+    elif dialect.name == SQLAlchemyDialect.POSTGRESQL.value:
+        # TODO(syang) support postgres dialect
+        session.rollback()
+        raise ValueError('Unsupported database dialect')
+    else:
+        session.rollback()
+        raise ValueError('Unsupported database dialect')
+    session.commit()
+
+
 def rename_column(
     cursor: 'sqlite3.Cursor',
     conn: 'sqlite3.Connection',

sky/utils/kubernetes/config_map_utils.py

@@ -0,0 +1,133 @@
+"""Utilities for Kubernetes ConfigMap operations in SkyPilot."""
+import os
+
+from sky import sky_logging
+from sky import skypilot_config
+from sky.adaptors import kubernetes
+from sky.utils import common_utils
+
+logger = sky_logging.init_logger(__name__)
+
+# Kubernetes ConfigMap sync constants
+_KUBE_SERVICE_ACCOUNT_PATH = '/var/run/secrets/kubernetes.io/serviceaccount'
+_CONFIGMAP_SYNC_TIMEOUT = 10  # seconds
+
+
+def is_running_in_kubernetes() -> bool:
+    """Check if we're running inside a Kubernetes pod."""
+    return os.path.exists(f'{_KUBE_SERVICE_ACCOUNT_PATH}/token')
+
+
+def _get_kubernetes_namespace() -> str:
+    """Get the current Kubernetes namespace from the service account."""
+    try:
+        namespace_file = f'{_KUBE_SERVICE_ACCOUNT_PATH}/namespace'
+        if os.path.exists(namespace_file):
+            with open(namespace_file, encoding='utf-8') as f:
+                return f.read().strip()
+    except (OSError, IOError):
+        pass
+    return 'default'
+
+
+def _get_configmap_name() -> str:
+    """Get the ConfigMap name for the SkyPilot config."""
+    release_name = (os.getenv('HELM_RELEASE_NAME') or
+                    os.getenv('SKYPILOT_RELEASE_NAME') or 'skypilot')
+    return f'{release_name}-config'
+
+
+def initialize_configmap_sync_on_startup(config_file_path: str) -> None:
+    """Initialize ConfigMap sync on API server startup.
+
+    This syncs existing PVC config to ConfigMap if ConfigMap doesn't exist.
+    This handles the upgrade scenario where an existing deployment has
+    workspace configs on PVC but no ConfigMap exists.
+
+    Args:
+        config_file_path: Path to the config file to sync.
+    """
+    config_file_path = os.path.expanduser(config_file_path)
+    if not is_running_in_kubernetes() or not os.path.exists(config_file_path):
+        return
+
+    try:
+        namespace = _get_kubernetes_namespace()
+        configmap_name = _get_configmap_name()
+
+        # Check if ConfigMap exists
+        try:
+            kubernetes.core_api().read_namespaced_config_map(
+                name=configmap_name, namespace=namespace)
+            # ConfigMap exists, don't overwrite it
+            logger.debug(f'ConfigMap {configmap_name} already exists')
+            return
+        except kubernetes.kubernetes.client.rest.ApiException as e:
+            if e.status != 404:
+                raise
+            # ConfigMap doesn't exist, create it
+
+        current_config = skypilot_config.parse_and_validate_config_file(
+            config_file_path)
+        config_yaml = common_utils.dump_yaml_str(dict(current_config))
+
+        configmap_body = {
+            'apiVersion': 'v1',
+            'kind': 'ConfigMap',
+            'metadata': {
+                'name': configmap_name,
+                'namespace': namespace,
+                'labels': {
+                    'app.kubernetes.io/name': 'skypilot',
+                    'app.kubernetes.io/component': 'config'
+                }
+            },
+            'data': {
+                'config.yaml': config_yaml
+            }
+        }
+
+        kubernetes.core_api().create_namespaced_config_map(
+            namespace=namespace,
+            body=configmap_body,
+            _request_timeout=_CONFIGMAP_SYNC_TIMEOUT)
+
+        logger.info(f'Synced PVC config to new ConfigMap {configmap_name}')
+
+    except Exception as e:  # pylint: disable=broad-except
+        logger.warning(f'Failed to initialize ConfigMap sync: {e}')
+
+
+def patch_configmap_with_config(config, config_file_path: str) -> None:
+    """Patch the Kubernetes ConfigMap with the updated config.
+
+    Args:
+        config: The updated config to sync to the ConfigMap.
+        config_file_path: Path to the config file for fallback sync.
+    """
+    if not is_running_in_kubernetes():
+        return
+
+    try:
+        namespace = _get_kubernetes_namespace()
+        configmap_name = _get_configmap_name()
+        config_yaml = common_utils.dump_yaml_str(dict(config))
+        patch_body = {'data': {'config.yaml': config_yaml}}
+
+        try:
+            kubernetes.core_api().patch_namespaced_config_map(
+                name=configmap_name,
+                namespace=namespace,
+                body=patch_body,
+                _request_timeout=_CONFIGMAP_SYNC_TIMEOUT)
+            logger.debug(f'Synced config to ConfigMap {configmap_name}')
+        except kubernetes.kubernetes.client.rest.ApiException as e:
+            if e.status == 404:
+                # ConfigMap doesn't exist, create it
+                logger.info(f'ConfigMap {configmap_name} not found, creating')
+                initialize_configmap_sync_on_startup(config_file_path)
+            else:
+                raise
+
+    except Exception as e:  # pylint: disable=broad-except
+        logger.warning(f'Failed to sync config to ConfigMap: {e}')

sky/utils/kubernetes/deploy_remote_cluster.py

@@ -624,6 +624,9 @@ def main():
     kubeconfig_path = os.path.expanduser(args.kubeconfig_path)
     global_use_ssh_config = args.use_ssh_config
 
+    failed_clusters = []
+    successful_clusters = []
+
     # Print cleanup mode marker if applicable
     if args.cleanup:
         print('SKYPILOT_CLEANUP_MODE: Cleanup mode activated')
@@ -793,10 +796,21 @@ def main():
                 print(
                     f'{GREEN}==== Completed deployment for cluster: {cluster_name} ====${NC}'
                 )
+                successful_clusters.append(cluster_name)
             except Exception as e:  # pylint: disable=broad-except
+                reason = str(e)
+                failed_clusters.append((cluster_name, reason))
                 print(
-                    f'{RED}Error deploying SSH Node Pool {cluster_name}: {e}{NC}'
-                )
+                    f'{RED}Error deploying SSH Node Pool {cluster_name}: {reason}{NC}'
+                )  # Print for internal logging
+
+    if failed_clusters:
+        action = 'clean' if args.cleanup else 'deploy'
+        msg = f'{GREEN}Successfully {action}ed {len(successful_clusters)} cluster(s) ({", ".join(successful_clusters)}). {NC}'
+        msg += f'{RED}Failed to {action} {len(failed_clusters)} cluster(s): {NC}'
+        for cluster_name, reason in failed_clusters:
+            msg += f'\n  {cluster_name}: {reason}'
+        raise RuntimeError(msg)
 
 
 def deploy_cluster(head_node,
@@ -847,8 +861,10 @@ def deploy_cluster(head_node,
         print_output=True)
     if result is None:
         with ux_utils.print_exception_no_traceback():
-            raise RuntimeError(f'Failed to SSH to head node ({head_node}). '
-                               f'Please check the SSH configuration.')
+            raise RuntimeError(
+                f'Failed to SSH to head node ({head_node}). '
+                f'Please check the SSH configuration and logs for more details.'
+            )
 
     # Checking history
     history_exists = (history_worker_nodes is not None and

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -26,6 +26,48 @@ logger = sky_logging.init_logger(__name__)
 DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
 
 
+def check_ssh_cluster_dependencies(
+        raise_error: bool = True) -> Optional[List[str]]:
+    """Checks if the dependencies for ssh cluster are installed.
+
+    Args:
+        raise_error: set to true when the dependency needs to be present.
+            set to false for `sky check`, where reason strings are compiled
+            at the end.
+
+    Returns: the reasons list if there are missing dependencies.
+    """
+    # error message
+    jq_message = ('`jq` is required to setup ssh cluster.')
+
+    # save
+    reasons = []
+    required_binaries = []
+
+    # Ensure jq is installed
+    try:
+        subprocess.run(['jq', '--version'],
+                       stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL,
+                       check=True)
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        required_binaries.append('jq')
+        reasons.append(jq_message)
+
+    if required_binaries:
+        reasons.extend([
+            'On Debian/Ubuntu, install the missing dependenc(ies) with:',
+            f'  $ sudo apt install {" ".join(required_binaries)}',
+            'On MacOS, install with: ',
+            f'  $ brew install {" ".join(required_binaries)}',
+        ])
+        if raise_error:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError('\n'.join(reasons))
+        return reasons
+    return None
+
+
 def deploy_ssh_cluster(cleanup: bool = False,
                        infra: Optional[str] = None,
                        kubeconfig_path: Optional[str] = None):
@@ -41,6 +83,8 @@ def deploy_ssh_cluster(cleanup: bool = False,
         kubeconfig_path: Path to save the Kubernetes configuration file.
             If None, the default ~/.kube/config will be used.
     """
+    check_ssh_cluster_dependencies()
+
     # Prepare command to call deploy_remote_cluster.py script
     # TODO(romilb): We should move this to a native python method/class call
     #  instead of invoking a script with subprocess.
@@ -81,9 +125,9 @@ def deploy_ssh_cluster(cleanup: bool = False,
             cmd=deploy_command,
             log_path=log_path,
             require_outputs=True,
-            stream_logs=False,  # TODO: Fixme to False after we fix the logging
+            stream_logs=False,
             line_processor=log_utils.SkySSHUpLineProcessor(log_path=log_path,
-                                                           is_local=True),
+                                                           is_local=False),
             cwd=cwd,
             env=env)
 
@@ -91,9 +135,9 @@ def deploy_ssh_cluster(cleanup: bool = False,
         success = True
     else:
         with ux_utils.print_exception_no_traceback():
-            log_hint = ux_utils.log_path_hint(log_path, is_local=True)
-            raise RuntimeError('Failed to deploy SkyPilot on SSH targets. '
-                               f'Full log: {log_hint}'
+            log_hint = ux_utils.log_path_hint(log_path, is_local=False)
+            raise RuntimeError('Failed to deploy SkyPilot on some Node Pools. '
+                               f'{log_hint}'
                                f'\nError: {stderr}')
 
     if success:

sky/utils/kubernetes/ssh-tunnel.sh

@@ -188,14 +188,17 @@ generate_credentials_json() {
     debug_log "Key data length: $(echo -n "$client_key_data" | wc -c) bytes"
     
     # Check if we can create proper JSON with `jq`
-    if command -v jq &>/dev/null; then
-      debug_log "Using jq for JSON formatting"
-      
-      # Create a temporary file for the JSON output to avoid shell escaping issues
-      local TEMP_JSON_FILE=$(mktemp)
-      
-      # Write the JSON to the temporary file using jq for proper JSON formatting
-      cat > "$TEMP_JSON_FILE" << EOL
+    if ! command -v jq &>/dev/null; then
+      echo "jq is not installed. Please install jq to use this script." >&2
+      exit 1
+    fi
+    debug_log "Using jq for JSON formatting"
+    
+    # Create a temporary file for the JSON output to avoid shell escaping issues
+    local TEMP_JSON_FILE=$(mktemp)
+    
+    # Write the JSON to the temporary file using jq for proper JSON formatting
+    cat > "$TEMP_JSON_FILE" << EOL
 {
   "apiVersion": "client.authentication.k8s.io/v1beta1",
   "kind": "ExecCredential",
@@ -207,25 +210,14 @@ generate_credentials_json() {
 }
 EOL
       
-      # Read the JSON from the file
-      local json_response=$(cat "$TEMP_JSON_FILE")
-      
-      # Clean up
-      rm -f "$TEMP_JSON_FILE"
-      
-      # Output the JSON
-      echo "$json_response"
-    else
-      debug_log "jq is not available, using simpler formatting method"
-      
-      # Alternative approach: encode with base64 and use the token field instead
-      # This works because kubectl will decode token data properly
-      local combined_data=$(echo -n "${client_cert_data}:${client_key_data}" | base64 | tr -d '\n')
-      
-      echo "{\"apiVersion\":\"client.authentication.k8s.io/v1beta1\",\"kind\":\"ExecCredential\",\"status\":{\"token\":\"$combined_data\",\"expirationTimestamp\":\"$expiration_time\"}}"
-      
-      debug_log "Sent certificate data as encoded token instead of direct certificate fields"
-    fi
+    # Read the JSON from the file
+    local json_response=$(cat "$TEMP_JSON_FILE")
+    
+    # Clean up
+    rm -f "$TEMP_JSON_FILE"
+    
+    # Output the JSON
+    echo "$json_response"
   else
     # Fallback to token-based credential for tunnel-only authentication
     echo "{\"apiVersion\":\"client.authentication.k8s.io/v1beta1\",\"kind\":\"ExecCredential\",\"status\":{\"token\":\"k8s-ssh-tunnel-token\",\"expirationTimestamp\":\"$expiration_time\"}}"
@@ -384,4 +376,4 @@ fi
 
 # Return valid credential format with certificates if available
 generate_credentials_json
-exit 0 
+exit 0