skypilot-nightly 1.0.0.dev20250919__py3-none-any.whl → 1.0.0.dev20250925__py3-none-any.whl

sky/utils/kubernetes/kubernetes_deploy_utils.py

@@ -4,6 +4,7 @@ import shlex
 import subprocess
 import sys
 import tempfile
+import textwrap
 from typing import List, Optional
 
 import colorama
@@ -24,6 +25,9 @@ logger = sky_logging.init_logger(__name__)
 
 # Default path for Kubernetes configuration file
 DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
+DEFAULT_LOCAL_CLUSTER_NAME = 'skypilot'
+LOCAL_CLUSTER_PORT_RANGE = 101
+LOCAL_CLUSTER_INTERNAL_PORT_START = 30000
 
 
 def check_ssh_cluster_dependencies(
@@ -252,7 +256,68 @@ def deploy_remote_cluster(ip_list: List[str],
                         is_local=True))
 
 
-def deploy_local_cluster(gpus: bool):
+def generate_kind_config(port_start: int,
+                         num_nodes: int = 1,
+                         gpus: bool = False) -> str:
+    """Generate a kind cluster config with ports mapped from host to container
+
+    Port range will be [port_start, port_start + LOCAL_CLUSTER_PORT_RANGE)
+    Internally, this will map to ports 30000 - 30100
+
+    Args:
+        path: Path to generate the config file at
+        port_start: Port range start for mappings
+        num_nodes: Number of nodes in the cluster
+        gpus: If true, initialize kind cluster with GPU support
+
+    Returns:
+        The kind cluster config
+    """
+    internal_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+    internal_end = internal_start + LOCAL_CLUSTER_PORT_RANGE - 1
+
+    config = textwrap.dedent(f"""
+    apiVersion: kind.x-k8s.io/v1alpha4
+    kind: Cluster
+    kubeadmConfigPatches:
+    - |
+      kind: ClusterConfiguration
+      apiServer:
+        extraArgs:
+          "service-node-port-range": {internal_start}-{internal_end}
+    nodes:
+    - role: control-plane
+      kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    """)
+    if gpus:
+        config += textwrap.indent(
+            textwrap.dedent("""
+        extraMounts:
+        - hostPath: /dev/null
+          containerPath: /var/run/nvidia-container-devices/all"""), ' ' * 2)
+    config += textwrap.indent(textwrap.dedent("""
+      extraPortMappings:"""), ' ' * 2)
+    for offset in range(LOCAL_CLUSTER_PORT_RANGE):
+        config += textwrap.indent(
+            textwrap.dedent(f"""
+        - containerPort: {internal_start + offset}
+          hostPort: {port_start + offset}
+          listenAddress: "0.0.0.0"
+          protocol: tcp
+        """), ' ' * 2)
+    if num_nodes > 1:
+        config += '- role: worker\n' * (num_nodes - 1)
+    return config
+
+
+def deploy_local_cluster(name: Optional[str], gpus: bool):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    context_name = f'kind-{name}'
     cluster_created = False
 
     # Check if GPUs are available on the host
@@ -262,41 +327,57 @@ def deploy_local_cluster(gpus: bool):
     # Check if ~/.kube/config exists:
     if os.path.exists(os.path.expanduser('~/.kube/config')):
         curr_context = kubernetes_utils.get_current_kube_config_context_name()
-        skypilot_context = 'kind-skypilot'
-        if curr_context is not None and curr_context != skypilot_context:
+        if curr_context is not None and curr_context != context_name:
             logger.info(
                 f'Current context in kube config: {curr_context}'
-                '\nWill automatically switch to kind-skypilot after the local '
-                'cluster is created.')
-    message_str = 'Creating local cluster{}...'
-    message_str = message_str.format((' with GPU support (this may take up '
-                                      'to 15 minutes)') if gpus else '')
-    path_to_package = os.path.dirname(__file__)
-    up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
-
-    # Get directory of script and run it from there
-    cwd = os.path.dirname(os.path.abspath(up_script_path))
-    run_command = up_script_path + ' --gpus' if gpus else up_script_path
-    run_command = shlex.split(run_command)
+                f'\nWill automatically switch to {context_name} after the '
+                'local cluster is created.')
+    message_str = 'Creating local cluster {}{}...'
+    message_str = message_str.format(
+        name,
+        ' with GPU support (this may take up to 15 minutes)' if gpus else '')
+
+    with tempfile.NamedTemporaryFile(mode='w+', suffix='.yaml',
+                                     delete=True) as f:
+        # Choose random port range to use on the host machine.
+        # Port range is port_start - port_start + 99 (exactly 100 ports).
+        # port_start = random.randint(300, 399) * 100
+        # TODO (kyuds): hard coding to pass smoketests. Need to figure out
+        # how to deal with this later.
+        port_start = LOCAL_CLUSTER_INTERNAL_PORT_START
+        port_end = port_start + LOCAL_CLUSTER_PORT_RANGE - 1
+        logger.debug(f'Using port range {port_start}-{port_end}')
+        f.write(generate_kind_config(port_start, gpus=gpus))
+        f.flush()
+
+        path_to_package = os.path.dirname(__file__)
+        up_script_path = os.path.join(path_to_package, 'create_cluster.sh')
+
+        # Get directory of script and run it from there
+        cwd = os.path.dirname(os.path.abspath(up_script_path))
+        run_command = f'{up_script_path} {name} {f.name}'
+        if gpus:
+            run_command += ' --gpus'
+        run_command = shlex.split(run_command)
 
-    # Setup logging paths
-    run_timestamp = sky_logging.get_run_timestamp()
-    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
-                            'local_up.log')
-    logger.info(message_str)
+        # Setup logging paths
+        run_timestamp = sky_logging.get_run_timestamp()
+        log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                                'local_up.log')
+        logger.info(message_str)
 
-    with rich_utils.safe_status(
-            ux_utils.spinner_message(message_str,
-                                     log_path=log_path,
-                                     is_local=True)):
-        returncode, _, stderr = log_lib.run_with_log(
-            cmd=run_command,
-            log_path=log_path,
-            require_outputs=True,
-            stream_logs=False,
-            line_processor=log_utils.SkyLocalUpLineProcessor(log_path=log_path,
-                                                             is_local=True),
-            cwd=cwd)
+        with rich_utils.safe_status(
+                ux_utils.spinner_message(message_str,
+                                         log_path=log_path,
+                                         is_local=True)):
+            returncode, _, stderr = log_lib.run_with_log(
+                cmd=run_command,
+                log_path=log_path,
+                require_outputs=True,
+                stream_logs=False,
+                line_processor=log_utils.SkyLocalUpLineProcessor(
+                    log_path=log_path, is_local=True),
+                cwd=cwd)
 
     # Kind always writes to stderr even if it succeeds.
     # If the failure happens after the cluster is created, we need
@@ -309,11 +390,11 @@ def deploy_local_cluster(gpus: bool):
     elif returncode == 100:
         logger.info(
             ux_utils.finishing_message(
-                'Local cluster already exists.\n',
+                f'Local cluster {name} already exists.\n',
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=
-                'If you want to delete it instead, run: sky local down'))
+                'If you want to delete it instead, run: `sky local down --name {name}`'))  # pylint: disable=line-too-long
     else:
         with ux_utils.print_exception_no_traceback():
             log_hint = ux_utils.log_path_hint(log_path, is_local=True)
@@ -339,7 +420,7 @@ def deploy_local_cluster(gpus: bool):
         if gpus:
             # Get GPU model by querying the node labels
             label_name_escaped = 'skypilot.co/accelerator'.replace('.', '\\.')
-            gpu_type_cmd = f'kubectl get node skypilot-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
+            gpu_type_cmd = f'kubectl get node {name}-control-plane -o jsonpath=\"{{.metadata.labels[\'{label_name_escaped}\']}}\"'  # pylint: disable=line-too-long
             try:
                 # Run the command and capture the output
                 gpu_count_output = subprocess.check_output(gpu_type_cmd,
@@ -375,8 +456,9 @@ def deploy_local_cluster(gpus: bool):
                         'This may cause issues with running tasks.')
         logger.info(
             ux_utils.finishing_message(
-                message=(f'Local Kubernetes cluster created successfully with '
-                         f'{num_cpus} CPUs{gpu_message}.'),
+                message=(
+                    f'Local Kubernetes cluster {name} created successfully '
+                    f'with {num_cpus} CPUs{gpu_message}.'),
                 log_path=log_path,
                 is_local=True,
                 follow_up_message=(
@@ -384,3 +466,54 @@ def deploy_local_cluster(gpus: bool):
                     'Hint: To change the number of CPUs, change your docker '
                     'runtime settings. See https://kind.sigs.k8s.io/docs/user/quick-start/#settings-for-docker-desktop for more info.'  # pylint: disable=line-too-long
                     f'{gpu_hint}')))
+
+
+def teardown_local_cluster(name: Optional[str] = None):
+    name = name or DEFAULT_LOCAL_CLUSTER_NAME
+    cluster_removed = False
+
+    path_to_package = os.path.dirname(__file__)
+    down_script_path = os.path.join(path_to_package, 'delete_cluster.sh')
+
+    cwd = os.path.dirname(os.path.abspath(down_script_path))
+    run_command = f'{down_script_path} {name}'
+    run_command = shlex.split(run_command)
+
+    # Setup logging paths
+    run_timestamp = sky_logging.get_run_timestamp()
+    log_path = os.path.join(constants.SKY_LOGS_DIRECTORY, run_timestamp,
+                            'local_down.log')
+
+    with rich_utils.safe_status(
+            ux_utils.spinner_message(f'Removing local cluster {name}',
+                                     log_path=log_path,
+                                     is_local=True)):
+
+        returncode, stdout, stderr = log_lib.run_with_log(cmd=run_command,
+                                                          log_path=log_path,
+                                                          require_outputs=True,
+                                                          stream_logs=False,
+                                                          cwd=cwd)
+        stderr = stderr.replace('No kind clusters found.\n', '')
+
+        if returncode == 0:
+            cluster_removed = True
+        elif returncode == 100:
+            logger.info(
+                ux_utils.error_message(f'Local cluster {name} does not exist.'))
+        else:
+            with ux_utils.print_exception_no_traceback():
+                raise RuntimeError(f'Failed to down local cluster {name}. '
+                                   f'Stdout: {stdout}'
+                                   f'\nError: {stderr}')
+    if cluster_removed:
+        # Run sky check
+        with rich_utils.safe_status(
+                ux_utils.spinner_message('Running sky check...')):
+            sky_check.check_capability(sky_cloud.CloudCapability.COMPUTE,
+                                       clouds=['kubernetes'],
+                                       quiet=True)
+        logger.info(
+            ux_utils.finishing_message(f'Local cluster {name} removed.',
+                                       log_path=log_path,
+                                       is_local=True))

sky/utils/kubernetes_enums.py

@@ -44,3 +44,8 @@ class KubernetesAutoscalerType(enum.Enum):
     KARPENTER = 'karpenter'
     COREWEAVE = 'coreweave'
     GENERIC = 'generic'
+
+    def emits_autoscale_event(self) -> bool:
+        """Returns whether specific autoscaler emits the event reason
+        TriggeredScaleUp."""
+        return self not in {self.KARPENTER}

sky/utils/ux_utils.py

@@ -1,11 +1,12 @@
 """Utility functions for UX."""
 import contextlib
 import enum
+import fnmatch
 import os
 import sys
 import traceback
 import typing
-from typing import Callable, Optional, Union
+from typing import Callable, Iterable, List, Optional, Union
 
 import colorama
 
@@ -288,3 +289,36 @@ def command_hint_messages(hint_type: CommandHintType,
                 f'{BOLD}sky jobs queue{RESET_BOLD}')
     else:
         raise ValueError(f'Invalid hint type: {hint_type}')
+
+
+def is_glob_pattern(pattern: str) -> bool:
+    """Checks if a string contains common glob pattern wildcards."""
+    glob_chars = {'*', '?', '[', ']'}
+    # Also check for '**' as a specific globstar pattern
+    if '**' in pattern:
+        return True
+    for char in pattern:
+        if char in glob_chars:
+            return True
+    return False
+
+
+def get_non_matched_query(query_clusters: Iterable[str],
+                          cluster_names: Iterable[str]) -> List[str]:
+    """Gets the non-matched query clusters."""
+    glob_query_clusters = []
+    non_glob_query_clusters = []
+    for cluster_name in query_clusters:
+        if is_glob_pattern(cluster_name):
+            glob_query_clusters.append(cluster_name)
+        else:
+            non_glob_query_clusters.append(cluster_name)
+    not_found_clusters = [
+        query_cluster for query_cluster in non_glob_query_clusters
+        if query_cluster not in cluster_names
+    ]
+    not_found_clusters.extend([
+        query_cluster for query_cluster in glob_query_clusters
+        if not fnmatch.filter(cluster_names, query_cluster)
+    ])
+    return not_found_clusters

sky/utils/yaml_utils.py

@@ -44,6 +44,15 @@ def read_yaml(path: Optional[str]) -> Dict[str, Any]:
     return config
 
 
+def read_yaml_str(yaml_str: str) -> Dict[str, Any]:
+    stream = io.StringIO(yaml_str)
+    parsed_yaml = safe_load(stream)
+    if not parsed_yaml:
+        # Empty dict
+        return {}
+    return parsed_yaml
+
+
 def read_yaml_all_str(yaml_str: str) -> List[Dict[str, Any]]:
     stream = io.StringIO(yaml_str)
     config = safe_load_all(stream)

sky/volumes/client/sdk.py

@@ -3,13 +3,16 @@ import json
 import typing
 from typing import Any, Dict, List
 
+from sky import exceptions
 from sky import sky_logging
 from sky.adaptors import common as adaptors_common
 from sky.server import common as server_common
+from sky.server import versions
 from sky.server.requests import payloads
 from sky.usage import usage_lib
 from sky.utils import annotations
 from sky.utils import context
+from sky.utils import ux_utils
 from sky.volumes import volume as volume_lib
 
 if typing.TYPE_CHECKING:
@@ -71,12 +74,44 @@ def apply(volume: volume_lib.Volume) -> server_common.RequestId[None]:
         config=volume.config,
         labels=volume.labels,
     )
-    response = requests.post(f'{server_common.get_server_url()}/volumes/apply',
-                             json=json.loads(body.model_dump_json()),
-                             cookies=server_common.get_api_cookie_jar())
+    response = server_common.make_authenticated_request(
+        'POST', '/volumes/apply', json=json.loads(body.model_dump_json()))
     return server_common.get_request_id(response)
 
 
+@context.contextual
+@usage_lib.entrypoint
+@server_common.check_server_healthy_or_start
+@annotations.client_api
+@versions.minimal_api_version(20)
+def validate(volume: volume_lib.Volume) -> None:
+    """Validates the volume.
+
+    All validation is done on the server side.
+
+    Args:
+        volume: The volume to validate.
+
+    Raises:
+        ValueError: If the volume is invalid.
+    """
+    body = payloads.VolumeValidateBody(
+        name=volume.name,
+        volume_type=volume.type,
+        infra=volume.infra,
+        resource_name=volume.resource_name,
+        size=volume.size,
+        config=volume.config,
+        labels=volume.labels,
+    )
+    response = server_common.make_authenticated_request(
+        'POST', '/volumes/validate', json=json.loads(body.model_dump_json()))
+    if response.status_code == 400:
+        with ux_utils.print_exception_no_traceback():
+            raise exceptions.deserialize_exception(
+                response.json().get('detail'))
+
+
 @context.contextual
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
@@ -87,8 +122,10 @@ def ls() -> server_common.RequestId[List[Dict[str, Any]]]:
     Returns:
         The request ID of the list request.
     """
-    response = requests.get(f'{server_common.get_server_url()}/volumes',
-                            cookies=server_common.get_api_cookie_jar())
+    response = server_common.make_authenticated_request(
+        'GET',
+        '/volumes',
+    )
     return server_common.get_request_id(response)
 
 
@@ -106,7 +143,6 @@ def delete(names: List[str]) -> server_common.RequestId[None]:
         The request ID of the delete request.
     """
     body = payloads.VolumeDeleteBody(names=names)
-    response = requests.post(f'{server_common.get_server_url()}/volumes/delete',
-                             json=json.loads(body.model_dump_json()),
-                             cookies=server_common.get_api_cookie_jar())
+    response = server_common.make_authenticated_request(
+        'POST', '/volumes/delete', json=json.loads(body.model_dump_json()))
     return server_common.get_request_id(response)

sky/volumes/server/server.py

@@ -3,12 +3,13 @@
 import fastapi
 
 from sky import clouds
+from sky import exceptions
 from sky import sky_logging
 from sky.server.requests import executor
 from sky.server.requests import payloads
 from sky.server.requests import requests as requests_lib
 from sky.utils import registry
-from sky.utils import volume
+from sky.utils import volume as volume_utils
 from sky.volumes.server import core
 
 logger = sky_logging.init_logger(__name__)
@@ -46,6 +47,31 @@ async def volume_delete(request: fastapi.Request,
     )
 
 
+@router.post('/validate')
+async def volume_validate(
+        _: fastapi.Request,
+        volume_validate_body: payloads.VolumeValidateBody) -> None:
+    """Validates a volume."""
+    # pylint: disable=import-outside-toplevel
+    from sky.volumes import volume as volume_lib
+
+    try:
+        volume_config = {
+            'name': volume_validate_body.name,
+            'type': volume_validate_body.volume_type,
+            'infra': volume_validate_body.infra,
+            'size': volume_validate_body.size,
+            'labels': volume_validate_body.labels,
+            'config': volume_validate_body.config,
+            'resource_name': volume_validate_body.resource_name,
+        }
+        volume = volume_lib.Volume.from_yaml_config(volume_config)
+        volume.validate()
+    except Exception as e:
+        raise fastapi.HTTPException(status_code=400,
+                                    detail=exceptions.serialize_exception(e))
+
+
 @router.post('/apply')
 async def volume_apply(request: fastapi.Request,
                        volume_apply_body: payloads.VolumeApplyBody) -> None:
@@ -55,7 +81,7 @@ async def volume_apply(request: fastapi.Request,
     volume_config = volume_apply_body.config
 
     supported_volume_types = [
-        volume_type.value for volume_type in volume.VolumeType
+        volume_type.value for volume_type in volume_utils.VolumeType
     ]
     if volume_type not in supported_volume_types:
         raise fastapi.HTTPException(
@@ -64,24 +90,24 @@ async def volume_apply(request: fastapi.Request,
     if cloud is None:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'Invalid cloud: {volume_cloud}')
-    if volume_type == volume.VolumeType.PVC.value:
+    if volume_type == volume_utils.VolumeType.PVC.value:
         if not cloud.is_same_cloud(clouds.Kubernetes()):
             raise fastapi.HTTPException(
                 status_code=400,
                 detail='PVC storage is only supported on Kubernetes')
         supported_access_modes = [
-            access_mode.value for access_mode in volume.VolumeAccessMode
+            access_mode.value for access_mode in volume_utils.VolumeAccessMode
         ]
         if volume_config is None:
             volume_config = {}
         access_mode = volume_config.get('access_mode')
         if access_mode is None:
-            volume_config[
-                'access_mode'] = volume.VolumeAccessMode.READ_WRITE_ONCE.value
+            volume_config['access_mode'] = (
+                volume_utils.VolumeAccessMode.READ_WRITE_ONCE.value)
         elif access_mode not in supported_access_modes:
             raise fastapi.HTTPException(
                 status_code=400, detail=f'Invalid access mode: {access_mode}')
-    elif volume_type == volume.VolumeType.RUNPOD_NETWORK_VOLUME.value:
+    elif volume_type == volume_utils.VolumeType.RUNPOD_NETWORK_VOLUME.value:
         if not cloud.is_same_cloud(clouds.RunPod()):
             raise fastapi.HTTPException(
                 status_code=400,

sky/volumes/volume.py

@@ -115,9 +115,6 @@ class Volume:
         self.region = infra_info.region
         self.zone = infra_info.zone
 
-        # Validate the volume config
-        self._validate_config()
-
     def _adjust_config(self) -> None:
         """Adjust the volume config (e.g., parse size)."""
         if self.size is None:
@@ -132,8 +129,28 @@ class Volume:
         except ValueError as e:
             raise ValueError(f'Invalid size {self.size}: {e}') from e
 
-    def _validate_config(self) -> None:
-        """Validate the volume config."""
+    def validate(self, skip_cloud_compatibility: bool = False) -> None:
+        """Validates the volume."""
+        self.validate_name()
+        self.validate_size()
+        if not skip_cloud_compatibility:
+            self.validate_cloud_compatibility()
+        # Extra, type-specific validations
+        self._validate_config_extra()
+
+    def validate_name(self) -> None:
+        """Validates if the volume name is set."""
+        assert self.name is not None, 'Volume name must be set'
+
+    def validate_size(self) -> None:
+        """Validates that size is specified for new volumes."""
+        if not self.resource_name and not self.size:
+            raise ValueError('Size is required for new volumes. '
+                             'Please specify the size in the YAML file or '
+                             'use the --size flag.')
+
+    def validate_cloud_compatibility(self) -> None:
+        """Validates that the specified cloud is compatible with volume type."""
         cloud_obj_from_type = VOLUME_TYPE_TO_CLOUD.get(
             volume_lib.VolumeType(self.type))
         if self.cloud:
@@ -150,25 +167,16 @@ class Volume:
         self.region, self.zone = cloud_obj.validate_region_zone(
             self.region, self.zone)
 
-        # Name must be set by factory before validation.
-        assert self.name is not None
         valid, err_msg = cloud_obj.is_volume_name_valid(self.name)
         if not valid:
             raise ValueError(f'Invalid volume name: {err_msg}')
 
-        if not self.resource_name and not self.size:
-            raise ValueError('Size is required for new volumes. '
-                             'Please specify the size in the YAML file or '
-                             'use the --size flag.')
         if self.labels:
             for key, value in self.labels.items():
                 valid, err_msg = cloud_obj.is_label_valid(key, value)
                 if not valid:
                     raise ValueError(f'{err_msg}')
 
-        # Extra, type-specific validations
-        self._validate_config_extra()
-
     # Hook methods for subclasses
     def _validate_config_extra(self) -> None:
         """Additional type-specific validation.