skypilot-nightly 1.0.0.dev20251012__py3-none-any.whl → 1.0.0.dev20251014__py3-none-any.whl

sky/provision/shadeform/instance.py

@@ -0,0 +1,351 @@
+"""Shadeform instance provisioning."""
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+import requests
+
+from sky import sky_logging
+from sky.provision import common
+from sky.provision.shadeform import shadeform_utils
+from sky.utils import status_lib
+
+POLL_INTERVAL = 10
+INSTANCE_READY_TIMEOUT = 3600
+
+logger = sky_logging.init_logger(__name__)
+
+# Status mapping from Shadeform to SkyPilot
+SHADEFORM_STATUS_MAP = {
+    'creating': status_lib.ClusterStatus.INIT,
+    'pending_provider': status_lib.ClusterStatus.INIT,
+    'pending': status_lib.ClusterStatus.INIT,
+    'active': status_lib.ClusterStatus.UP,
+    'deleted': status_lib.ClusterStatus.STOPPED,
+}
+
+
+def _get_cluster_instances(cluster_name_on_cloud: str) -> Dict[str, Any]:
+    """Get all instances belonging to a cluster."""
+    try:
+        response = shadeform_utils.get_instances()
+        instances = response.get('instances', [])
+
+        cluster_instances = {}
+        possible_names = [
+            f'{cluster_name_on_cloud}-head', f'{cluster_name_on_cloud}-worker'
+        ]
+
+        for instance in instances:
+            if instance.get('name') in possible_names:
+                cluster_instances[instance['id']] = instance
+
+        return cluster_instances
+    except (ValueError, KeyError, requests.exceptions.RequestException) as e:
+        logger.warning(f'Failed to get instances: {e}')
+        return {}
+
+
+def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
+    """Get the head instance ID from a list of instances."""
+    for instance_id, instance in instances.items():
+        if instance.get('name', '').endswith('-head'):
+            return instance_id
+    return None
+
+
+def _wait_for_instances_ready(cluster_name_on_cloud: str,
+                              expected_count: int,
+                              timeout: int = INSTANCE_READY_TIMEOUT) -> bool:
+    """Wait for instances to be ready (active state with SSH access)."""
+    start_time = time.time()
+
+    while time.time() - start_time < timeout:
+        instances = _get_cluster_instances(cluster_name_on_cloud)
+        ready_count = 0
+
+        for instance in instances.values():
+            if (instance.get('status') == 'active' and
+                    instance.get('ip') is not None and
+                    instance.get('ssh_port') is not None):
+                ready_count += 1
+
+        logger.info(f'Waiting for instances to be ready: '
+                    f'({ready_count}/{expected_count})')
+
+        if ready_count >= expected_count:
+            return True
+
+        time.sleep(POLL_INTERVAL)
+
+    return False
+
+
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
+                  config: common.ProvisionConfig) -> common.ProvisionRecord:
+    """Run instances for the given cluster."""
+    del cluster_name  # unused - we use cluster_name_on_cloud
+    logger.info(f'Running instances for cluster {cluster_name_on_cloud} '
+                f'in region {region}')
+    logger.debug(f'DEBUG: region type={type(region)}, value={region!r}')
+    logger.debug(f'DEBUG: config node_config={config.node_config}')
+
+    # Check existing instances
+    existing_instances = _get_cluster_instances(cluster_name_on_cloud)
+    head_instance_id = _get_head_instance_id(existing_instances)
+
+    # Filter active instances
+    active_instances = {
+        iid: inst
+        for iid, inst in existing_instances.items()
+        if inst.get('status') == 'active'
+    }
+
+    current_count = len(active_instances)
+    target_count = config.count
+
+    logger.info(f'Current instances: {current_count}, target: {target_count}')
+
+    if current_count >= target_count:
+        if head_instance_id is None:
+            raise RuntimeError(
+                f'Cluster {cluster_name_on_cloud} has no head node')
+        logger.info(f'Cluster already has {current_count} instances, '
+                    f'no need to start more')
+        return common.ProvisionRecord(
+            provider_name='shadeform',
+            cluster_name=cluster_name_on_cloud,
+            region=region,
+            zone=None,  # Shadeform doesn't use separate zones
+            head_instance_id=head_instance_id,
+            resumed_instance_ids=[],
+            created_instance_ids=[])
+
+    # Create new instances
+    to_create = target_count - current_count
+    created_instance_ids = []
+
+    for _ in range(to_create):
+        node_type = 'head' if head_instance_id is None else 'worker'
+        instance_name = f'{cluster_name_on_cloud}-{node_type}'
+
+        # Extract configuration from node_config
+
+        # The node_config contains instance specs including InstanceType
+        # which follows the format: {cloud_provider}_{instance_type}
+        # (e.g., "massedcompute_A6000_basex2")
+        node_config = config.node_config
+        assert 'InstanceType' in node_config, \
+            'InstanceType must be present in node_config'
+
+        # Parse the instance type to extract cloud provider and instance specs
+        # Expected format: "{cloud}_{instance_type}" where cloud is provider
+        # (massedcompute, scaleway, lambda, etc.)
+        instance_type_full = node_config['InstanceType']
+        assert (isinstance(instance_type_full, str) and
+                '_' in instance_type_full), \
+            f'InstanceType must be in format cloud_instance_type, got: ' \
+            f'{instance_type_full}'
+
+        instance_type_split = instance_type_full.split('_')
+        assert len(instance_type_split) >= 2, \
+            f'InstanceType must contain at least one underscore, got: ' \
+            f'{instance_type_full}'
+
+        # Extract cloud provider (first part) and instance type (remaining)
+        # Example: "massedcompute_A6000-basex2" -> cloud="massedcompute",
+        # instance_type="A6000-basex2"
+        cloud = instance_type_split[0]
+        instance_type = '_'.join(instance_type_split[1:])
+
+        # Shadeform uses underscores instead of hyphens
+        instance_type = instance_type.replace('-', '_')
+
+        if instance_type.endswith('B'):
+            instance_type = instance_type[:-1]
+
+        # Replace "GBx" with "Gx" (case sensitive)
+        if 'GBx' in instance_type:
+            instance_type = instance_type.replace('GBx', 'Gx')
+
+        assert cloud, 'Cloud provider cannot be empty'
+        assert instance_type, 'Instance type cannot be empty'
+
+        # Get SSH key ID for authentication - this is optional and may be None
+        ssh_key_id = config.authentication_config.get('ssh_key_id')
+
+        create_config = {
+            'cloud': cloud,
+            'region': region,
+            'shade_instance_type': instance_type,
+            'name': instance_name,
+            'ssh_key_id': ssh_key_id
+        }
+
+        try:
+            logger.info(f'Creating {node_type} instance: {instance_name}')
+            response = shadeform_utils.create_instance(create_config)
+            instance_id = response['id']
+            created_instance_ids.append(instance_id)
+
+            if head_instance_id is None:
+                head_instance_id = instance_id
+
+            logger.info(f'Created instance {instance_id} ({node_type})')
+
+        except Exception as e:
+            logger.error(f'Failed to create instance: {e}')
+            # Clean up any created instances
+            for iid in created_instance_ids:
+                try:
+                    shadeform_utils.delete_instance(iid)
+                except requests.exceptions.RequestException as cleanup_e:
+                    logger.warning(
+                        f'Failed to cleanup instance {iid}: {cleanup_e}')
+            raise
+
+    # Wait for all instances to be ready
+    logger.info('Waiting for instances to become ready...')
+    if not _wait_for_instances_ready(cluster_name_on_cloud, target_count):
+        raise RuntimeError('Timed out waiting for instances to be ready')
+
+    assert head_instance_id is not None, 'head_instance_id should not be None'
+
+    return common.ProvisionRecord(provider_name='shadeform',
+                                  cluster_name=cluster_name_on_cloud,
+                                  region=region,
+                                  zone=region,
+                                  head_instance_id=head_instance_id,
+                                  resumed_instance_ids=[],
+                                  created_instance_ids=created_instance_ids)
+
+
+def wait_instances(region: str, cluster_name_on_cloud: str,
+                   state: Optional[status_lib.ClusterStatus]) -> None:
+    """Wait for instances to reach the specified state."""
+    del region, cluster_name_on_cloud, state  # unused
+    # For Shadeform, instances are ready when they reach 'active' status
+    # This is already handled in run_instances
+
+
+def stop_instances(cluster_name_on_cloud: str,
+                   provider_config: Optional[Dict[str, Any]] = None,
+                   worker_only: bool = False) -> None:
+    """Stop instances (not supported by Shadeform)."""
+    del cluster_name_on_cloud, provider_config, worker_only  # unused
+    raise NotImplementedError(
+        'Stopping instances is not supported by Shadeform')
+
+
+def terminate_instances(cluster_name_on_cloud: str,
+                        provider_config: Optional[Dict[str, Any]] = None,
+                        worker_only: bool = False) -> None:
+    """Terminate instances."""
+    del provider_config  # unused
+    logger.info(f'Terminating instances for cluster {cluster_name_on_cloud}')
+
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        logger.info(f'No instances found for cluster {cluster_name_on_cloud}')
+        return
+
+    instances_to_delete = instances
+    if worker_only:
+        # Only delete worker nodes, not head
+        instances_to_delete = {
+            iid: inst
+            for iid, inst in instances.items()
+            if not inst.get('name', '').endswith('-head')
+        }
+
+    for instance_id, instance in instances_to_delete.items():
+        try:
+            logger.info(
+                f'Terminating instance {instance_id} ({instance.get("name")})')
+            shadeform_utils.delete_instance(instance_id)
+        except requests.exceptions.RequestException as e:
+            logger.warning(f'Failed to terminate instance {instance_id}: {e}')
+
+
+def get_cluster_info(
+        region: str,
+        cluster_name_on_cloud: str,
+        provider_config: Optional[Dict[str, Any]] = None) -> common.ClusterInfo:
+    """Get cluster information."""
+    del region, provider_config  # unused
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        return common.ClusterInfo(instances={},
+                                  head_instance_id=None,
+                                  provider_name='shadeform')
+
+    head_instance_id = _get_head_instance_id(instances)
+
+    # Convert instance format for ClusterInfo
+    cluster_instances = {}
+    for instance_id, instance in instances.items():
+        instance_info = common.InstanceInfo(
+            instance_id=instance_id,
+            internal_ip=instance.get('ip', ''),
+            external_ip=instance.get('ip', ''),
+            ssh_port=instance.get('ssh_port', 22),
+            tags={},
+        )
+        # ClusterInfo expects Dict[InstanceId, List[InstanceInfo]]
+        cluster_instances[instance_id] = [instance_info]
+
+    ssh_user = 'shadeform'  # default
+    if head_instance_id is not None:
+        ssh_user = instances.get(head_instance_id,
+                                 {}).get('ssh_user', 'shadeform')
+
+    return common.ClusterInfo(instances=cluster_instances,
+                              head_instance_id=head_instance_id,
+                              provider_name='shadeform',
+                              ssh_user=ssh_user)
+
+
+def query_instances(
+    cluster_name: str,
+    cluster_name_on_cloud: str,
+    provider_config: Optional[Dict[str, Any]] = None,
+    non_terminated_only: bool = True,
+) -> Dict[str, Tuple[Optional['status_lib.ClusterStatus'], Optional[str]]]:
+    """Query the status of instances."""
+    del cluster_name, provider_config  # unused
+    instances = _get_cluster_instances(cluster_name_on_cloud)
+
+    if not instances:
+        return {}
+
+    status_map: Dict[str, Tuple[Optional['status_lib.ClusterStatus'],
+                                Optional[str]]] = {}
+    for instance_id, instance in instances.items():
+        shadeform_status = instance.get('status', 'unknown')
+        sky_status = SHADEFORM_STATUS_MAP.get(shadeform_status,
+                                              status_lib.ClusterStatus.INIT)
+
+        if (non_terminated_only and
+                sky_status == status_lib.ClusterStatus.STOPPED):
+            continue
+
+        status_map[instance_id] = (sky_status, None)
+
+    return status_map
+
+
+def open_ports(cluster_name_on_cloud: str,
+               ports: List[str],
+               provider_config: Optional[Dict[str, Any]] = None) -> None:
+    """Open ports (not supported by Shadeform)."""
+    del cluster_name_on_cloud, ports, provider_config  # unused
+    raise NotImplementedError()
+
+
+def cleanup_ports(cluster_name_on_cloud: str,
+                  ports: List[str],
+                  provider_config: Optional[Dict[str, Any]] = None) -> None:
+    """Cleanup ports (not supported by Shadeform)."""
+    del cluster_name_on_cloud, ports, provider_config  # unused
+    # Nothing to cleanup since we don't support dynamic port opening

sky/provision/shadeform/shadeform_utils.py

@@ -0,0 +1,83 @@
+"""Shadeform API utilities."""
+
+import os
+from typing import Any, Dict
+
+from sky.adaptors import common
+
+# Lazy import to avoid dependency on external packages
+requests = common.LazyImport('requests')
+
+# Shadeform API configuration
+SHADEFORM_API_BASE = 'https://api.shadeform.ai/v1'
+SHADEFORM_API_KEY_PATH = '~/.shadeform/api_key'
+
+
+def get_api_key() -> str:
+    """Get Shadeform API key from file."""
+    api_key_path = os.path.expanduser(SHADEFORM_API_KEY_PATH)
+    if not os.path.exists(api_key_path):
+        raise FileNotFoundError(
+            f'Shadeform API key not found at {api_key_path}. '
+            'Please save your API key to this file.')
+
+    with open(api_key_path, 'r', encoding='utf-8') as f:
+        api_key = f.read().strip()
+
+    if not api_key:
+        raise ValueError(f'Shadeform API key is empty in {api_key_path}')
+
+    return api_key
+
+
+def make_request(method: str, endpoint: str, **kwargs) -> Any:
+    """Make a request to the Shadeform API."""
+    url = f'{SHADEFORM_API_BASE}/{endpoint.lstrip("/")}'
+    headers = {
+        'X-API-KEY': get_api_key(),
+        'Content-Type': 'application/json',
+    }
+
+    response = requests.request(method, url, headers=headers, **kwargs)
+    response.raise_for_status()
+
+    # Some APIs (like delete) return empty responses with just 200 status
+    if response.text.strip():
+        return response.json()
+    else:
+        # Return empty dict for empty responses (e.g., delete operations)
+        return {}
+
+
+def get_instances() -> Dict[str, Any]:
+    """Get all instances."""
+    return make_request('GET', '/instances')
+
+
+def get_instance_info(instance_id: str) -> Dict[str, Any]:
+    """Get information about a specific instance."""
+    return make_request('GET', f'/instances/{instance_id}/info')
+
+
+def create_instance(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Create a new instance."""
+    return make_request('POST', '/instances/create', json=config)
+
+
+def delete_instance(instance_id: str) -> Dict[str, Any]:
+    """Delete an instance.
+
+    Note: Shadeform delete API returns empty response with 200 status.
+    """
+    return make_request('POST', f'/instances/{instance_id}/delete')
+
+
+def get_ssh_keys() -> Dict[str, Any]:
+    """Get all SSH keys."""
+    return make_request('GET', '/sshkeys')
+
+
+def add_ssh_key(name: str, public_key: str) -> Dict[str, Any]:
+    """Add a new SSH key."""
+    config = {'name': name, 'public_key': public_key}
+    return make_request('POST', '/sshkeys/add', json=config)

sky/server/common.py

@@ -554,8 +554,8 @@ def _start_api_server(deploy: bool = False,
         # pylint: disable=import-outside-toplevel
         import sky.jobs.utils as job_utils
         max_memory = (server_constants.MIN_AVAIL_MEM_GB_CONSOLIDATION_MODE
-                      if job_utils.is_consolidation_mode() else
-                      server_constants.MIN_AVAIL_MEM_GB)
+                      if job_utils.is_consolidation_mode(on_api_restart=True)
+                      else server_constants.MIN_AVAIL_MEM_GB)
         if avail_mem_size_gb <= max_memory:
             logger.warning(
                 f'{colorama.Fore.YELLOW}Your SkyPilot API server machine only '
@@ -571,6 +571,8 @@ def _start_api_server(deploy: bool = False,
             args += [f'--host={host}']
         if metrics_port is not None:
             args += [f'--metrics-port={metrics_port}']
+        # Use this argument to disable the internal signal file check.
+        args += ['--start-with-python']
 
         if foreground:
             # Replaces the current process with the API server

sky/server/requests/executor.py

@@ -81,6 +81,26 @@ logger = sky_logging.init_logger(__name__)
 # platforms, including macOS.
 multiprocessing.set_start_method('spawn', force=True)
 
+# Max threads that is equivalent to the number of thread workers in the
+# default thread pool executor of event loop.
+_REQUEST_THREADS_LIMIT = min(32, (os.cpu_count() or 0) + 4)
+
+_REQUEST_THREAD_EXECUTOR_LOCK = threading.Lock()
+# A dedicated thread pool executor for synced requests execution in coroutine
+_REQUEST_THREAD_EXECUTOR: Optional[concurrent.futures.ThreadPoolExecutor] = None
+
+
+def get_request_thread_executor() -> concurrent.futures.ThreadPoolExecutor:
+    """Lazy init and return the request thread executor for current process."""
+    global _REQUEST_THREAD_EXECUTOR
+    if _REQUEST_THREAD_EXECUTOR is not None:
+        return _REQUEST_THREAD_EXECUTOR
+    with _REQUEST_THREAD_EXECUTOR_LOCK:
+        if _REQUEST_THREAD_EXECUTOR is None:
+            _REQUEST_THREAD_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
+                max_workers=_REQUEST_THREADS_LIMIT)
+        return _REQUEST_THREAD_EXECUTOR
+
 
 class RequestQueue:
     """The queue for the requests, either redis or multiprocessing.
@@ -404,6 +424,7 @@ def _request_execution_wrapper(request_id: str,
             os.close(original_stderr)
             original_stderr = None
 
+    request_name = None
     try:
         # As soon as the request is updated with the executor PID, we can
         # receive SIGTERM from cancellation. So, we update the request inside
@@ -495,7 +516,8 @@ def _request_execution_wrapper(request_id: str,
             annotations.clear_request_level_cache()
             with metrics_utils.time_it(name='release_memory', group='internal'):
                 common_utils.release_memory()
-            _record_memory_metrics(request_name, proc, rss_begin, peak_rss)
+            if request_name is not None:
+                _record_memory_metrics(request_name, proc, rss_begin, peak_rss)
         except Exception as e:  # pylint: disable=broad-except
             logger.error(f'Failed to record memory metrics: '
                          f'{common_utils.format_exception(e)}')
@@ -576,8 +598,8 @@ async def _execute_request_coroutine(request: api_requests.Request):
     # 1. skypilot config is not contextual
     # 2. envs that read directly from os.environ are not contextual
     ctx.override_envs(request_body.env_vars)
-    fut: asyncio.Future = context_utils.to_thread(func,
-                                                  **request_body.to_kwargs())
+    fut: asyncio.Future = context_utils.to_thread_with_executor(
+        get_request_thread_executor(), func, **request_body.to_kwargs())
 
     async def poll_task(request_id: str) -> bool:
         req_status = await api_requests.get_request_status_async(request_id)

sky/server/server.py

@@ -3,6 +3,7 @@
 import argparse
 import asyncio
 import base64
+from concurrent.futures import ThreadPoolExecutor
 import contextlib
 import datetime
 import hashlib
@@ -1731,9 +1732,9 @@ async def kubernetes_pod_ssh_proxy(websocket: fastapi.WebSocket,
     logger.info(f'WebSocket connection accepted for cluster: {cluster_name}')
 
     # Run core.status in another thread to avoid blocking the event loop.
-    cluster_records = await context_utils.to_thread(core.status,
-                                                    cluster_name,
-                                                    all_users=True)
+    with ThreadPoolExecutor(max_workers=1) as thread_pool_executor:
+        cluster_records = await context_utils.to_thread_with_executor(
+            thread_pool_executor, core.status, cluster_name, all_users=True)
     cluster_record = cluster_records[0]
     if cluster_record['status'] != status_lib.ClusterStatus.UP:
         raise fastapi.HTTPException(
@@ -1967,6 +1968,7 @@ if __name__ == '__main__':
     # Serve metrics on a separate port to isolate it from the application APIs:
     # metrics port will not be exposed to the public network typically.
     parser.add_argument('--metrics-port', default=9090, type=int)
+    parser.add_argument('--start-with-python', action='store_true')
     cmd_args = parser.parse_args()
     if cmd_args.port == cmd_args.metrics_port:
         logger.error('port and metrics-port cannot be the same, exiting.')
@@ -1981,6 +1983,10 @@ if __name__ == '__main__':
         logger.error(f'Port {cmd_args.port} is not available, exiting.')
         raise RuntimeError(f'Port {cmd_args.port} is not available')
 
+    if not cmd_args.start_with_python:
+        # Maybe touch the signal file on API server startup.
+        managed_job_utils.is_consolidation_mode(on_api_restart=True)
+
     # Show the privacy policy if it is not already shown. We place it here so
     # that it is shown only when the API server is started.
     usage_lib.maybe_show_privacy_policy()

sky/setup_files/dependencies.py

@@ -222,6 +222,7 @@ extras_require: Dict[str, List[str]] = {
     'hyperbolic': [],  # No dependencies needed for hyperbolic
     'seeweb': ['ecsapi>=0.2.0'],
     'server': server_dependencies,
+    'shadeform': [],  # No dependencies needed for shadeform
 }
 
 # Calculate which clouds should be included in the [all] installation.

sky/sky_logging.py

@@ -109,7 +109,6 @@ def _setup_logger():
     global _default_handler
     if _default_handler is None:
         _default_handler = EnvAwareHandler(sys.stdout)
-        _default_handler.flush = sys.stdout.flush  # type: ignore
         if env_options.Options.SHOW_DEBUG_INFO.get():
             _default_handler.setLevel(logging.DEBUG)
         else:
@@ -129,7 +128,6 @@ def _setup_logger():
         for logger_name in _SENSITIVE_LOGGER:
             logger = logging.getLogger(logger_name)
             handler_to_logger = EnvAwareHandler(sys.stdout, sensitive=True)
-            handler_to_logger.flush = sys.stdout.flush  # type: ignore
             logger.addHandler(handler_to_logger)
             logger.setLevel(logging.INFO)
             if _show_logging_prefix():

sky/skylet/constants.py

@@ -226,7 +226,9 @@ RAY_INSTALLATION_COMMANDS = (
     f'{SKY_UV_PIP_CMD} list | grep "ray " | '
     f'grep {SKY_REMOTE_RAY_VERSION} 2>&1 > /dev/null '
     f'|| {RAY_STATUS} || '
-    f'{SKY_UV_PIP_CMD} install -U ray[default]=={SKY_REMOTE_RAY_VERSION}; '  # pylint: disable=line-too-long
+    # The pydantic-core==2.41.3 for arm seems corrupted
+    # so we need to avoid that specific version.
+    f'{SKY_UV_PIP_CMD} install -U "ray[default]=={SKY_REMOTE_RAY_VERSION}" "pydantic-core==2.41.1"; '  # pylint: disable=line-too-long
     # In some envs, e.g. pip does not have permission to write under /opt/conda
     # ray package will be installed under ~/.local/bin. If the user's PATH does
     # not include ~/.local/bin (the pip install will have the output: `WARNING:
@@ -402,10 +404,25 @@ OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
 ]
 # When overriding the SkyPilot configs on the API server with the client one,
 # we skip the following keys because they are meant to be client-side configs.
-SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [('api_server',),
-                                                       ('allowed_clouds',),
-                                                       ('workspaces',), ('db',),
-                                                       ('daemons',)]
+# Also, we skip the consolidation mode config as those should be only set on
+# the API server side.
+SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [
+    ('api_server',),
+    ('allowed_clouds',),
+    ('workspaces',),
+    ('db',),
+    ('daemons',),
+    # TODO(kevin,tian): Override the whole controller config once our test
+    # infrastructure supports setting dynamic server side configs.
+    # Tests that are affected:
+    # - test_managed_jobs_ha_kill_starting
+    # - test_managed_jobs_ha_kill_running
+    # - all tests that use LOW_CONTROLLER_RESOURCE_ENV or
+    #   LOW_CONTROLLER_RESOURCE_OVERRIDE_CONFIG (won't cause test failure,
+    #   but the configs won't be applied)
+    ('jobs', 'controller', 'consolidation_mode'),
+    ('serve', 'controller', 'consolidation_mode'),
+]
 
 # Constants for Azure blob storage
 WAIT_FOR_STORAGE_ACCOUNT_CREATION = 60
@@ -471,7 +488,7 @@ CATALOG_DIR = '~/.sky/catalogs'
 ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
               'kubernetes', 'runpod', 'vast', 'vsphere', 'cudo', 'fluidstack',
               'paperspace', 'primeintellect', 'do', 'nebius', 'ssh',
-              'hyperbolic', 'seeweb')
+              'hyperbolic', 'seeweb', 'shadeform')
 # END constants used for service catalog.
 
 # The user ID of the SkyPilot system.

sky/skylet/log_lib.py

@@ -271,7 +271,6 @@ def run_with_log(
                 stdout, stderr = context_utils.pipe_and_wait_process(
                     ctx,
                     proc,
-                    cancel_callback=subprocess_utils.kill_children_processes,
                     stdout_stream_handler=stdout_stream_handler,
                     stderr_stream_handler=stderr_stream_handler)
             elif process_stream:

sky/skylet/log_lib.pyi

@@ -42,7 +42,7 @@ class _ProcessingArgs:
         ...
 
 
-def _get_context() -> Optional[context.Context]:
+def _get_context() -> Optional[context.SkyPilotContext]:
     ...