skypilot-nightly 1.0.0.dev20251203__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

sky/skylet/constants.py

@@ -25,6 +25,7 @@ SKY_RUNTIME_DIR_ENV_VAR_KEY = 'SKY_RUNTIME_DIR'
 # them be in $HOME makes it more convenient.
 SKY_LOGS_DIRECTORY = '~/sky_logs'
 SKY_REMOTE_WORKDIR = '~/sky_workdir'
+SKY_TEMPLATES_DIRECTORY = '~/sky_templates'
 SKY_IGNORE_FILE = '.skyignore'
 GIT_IGNORE_FILE = '.gitignore'
 
@@ -67,10 +68,23 @@ SKY_PIP_CMD = f'{SKY_PYTHON_CMD} -m pip'
 #   #!/opt/conda/bin/python3
 SKY_RAY_CMD = (f'{SKY_PYTHON_CMD} $([ -s {SKY_RAY_PATH_FILE} ] && '
                f'cat {SKY_RAY_PATH_FILE} 2> /dev/null || which ray)')
+
+# Use $(which env) to find env, falling back to /usr/bin/env if which is
+# unavailable. This works around a Slurm quirk where srun's execvp() doesn't
+# check execute permissions, failing when $HOME/.local/bin/env (non-executable,
+# from uv installation) shadows /usr/bin/env.
+SKY_SLURM_UNSET_PYTHONPATH = ('$(which env 2>/dev/null || echo /usr/bin/env) '
+                              '-u PYTHONPATH')
+SKY_SLURM_PYTHON_CMD = (f'{SKY_SLURM_UNSET_PYTHONPATH} '
+                        f'$({SKY_GET_PYTHON_PATH_CMD})')
+
 # Separate env for SkyPilot runtime dependencies.
 SKY_REMOTE_PYTHON_ENV_NAME = 'skypilot-runtime'
 SKY_REMOTE_PYTHON_ENV: str = f'{SKY_RUNTIME_DIR}/{SKY_REMOTE_PYTHON_ENV_NAME}'
 ACTIVATE_SKY_REMOTE_PYTHON_ENV = f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate'
+# Place the conda root in the runtime directory, as installing to $HOME
+# on an NFS takes too long (1-2m slower).
+SKY_CONDA_ROOT = f'{SKY_RUNTIME_DIR}/miniconda3'
 # uv is used for venv and pip, much faster than python implementations.
 SKY_UV_INSTALL_DIR = '"$HOME/.local/bin"'
 # set UV_SYSTEM_PYTHON to false in case the
@@ -162,6 +176,10 @@ DISABLE_GPU_ECC_COMMAND = (
     '{ sudo reboot || echo "Failed to reboot. ECC mode may not be disabled"; } '
     '|| true; ')
 
+SETUP_SKY_DIRS_COMMANDS = (f'mkdir -p ~/sky_workdir && '
+                           f'mkdir -p ~/.sky/sky_app && '
+                           f'mkdir -p {SKY_RUNTIME_DIR}/.sky;')
+
 # Install conda on the remote cluster if it is not already installed.
 # We use conda with python 3.10 to be consistent across multiple clouds with
 # best effort.
@@ -178,8 +196,9 @@ CONDA_INSTALLATION_COMMANDS = (
     # because for some images, conda is already installed, but not initialized.
     # In this case, we need to initialize conda and set auto_activate_base to
     # true.
-    '{ bash Miniconda3-Linux.sh -b || true; '
-    'eval "$(~/miniconda3/bin/conda shell.bash hook)" && conda init && '
+    '{ '
+    f'bash Miniconda3-Linux.sh -b -p "{SKY_CONDA_ROOT}" || true; '
+    f'eval "$({SKY_CONDA_ROOT}/bin/conda shell.bash hook)" && conda init && '
     # Caller should replace {conda_auto_activate} with either true or false.
     'conda config --set auto_activate_base {conda_auto_activate} && '
     'conda activate base; }; '
@@ -222,7 +241,7 @@ _sky_version = str(version.parse(sky.__version__))
 RAY_STATUS = f'RAY_ADDRESS=127.0.0.1:{SKY_REMOTE_RAY_PORT} {SKY_RAY_CMD} status'
 RAY_INSTALLATION_COMMANDS = (
     f'{SKY_UV_INSTALL_CMD};'
-    'mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app;'
+    f'{SETUP_SKY_DIRS_COMMANDS}'
     # Print the PATH in provision.log to help debug PATH issues.
     'echo PATH=$PATH; '
     # Install setuptools<=69.5.1 to avoid the issue with the latest setuptools
@@ -256,7 +275,7 @@ RAY_INSTALLATION_COMMANDS = (
     #
     # Here, we add ~/.local/bin to the end of the PATH to make sure the issues
     # mentioned above are resolved.
-    'export PATH=$PATH:$HOME/.local/bin; '
+    f'export PATH=$PATH:{SKY_RUNTIME_DIR}/.local/bin; '
     # Writes ray path to file if it does not exist or the file is empty.
     f'[ -s {SKY_RAY_PATH_FILE} ] || '
     f'{{ {SKY_UV_RUN_CMD} '
@@ -264,18 +283,23 @@ RAY_INSTALLATION_COMMANDS = (
 
 # Copy SkyPilot templates from the installed wheel to ~/sky_templates.
 # This must run after the skypilot wheel is installed.
+# Note: We remove ~/sky_templates first to avoid import conflicts where Python
+# would import from ~/sky_templates instead of site-packages (because
+# sky_templates itself is a package), leading to src == dst error when
+# launching on an existing cluster.
 COPY_SKYPILOT_TEMPLATES_COMMANDS = (
+    f'rm -rf {SKY_TEMPLATES_DIRECTORY}; '
     f'{ACTIVATE_SKY_REMOTE_PYTHON_ENV}; '
     f'{SKY_PYTHON_CMD} -c \''
     'import sky_templates, shutil, os; '
     'src = os.path.dirname(sky_templates.__file__); '
-    'dst = os.path.expanduser(\"~/sky_templates\"); '
+    f'dst = os.path.expanduser(\"{SKY_TEMPLATES_DIRECTORY}\"); '
     'print(f\"Copying templates from {src} to {dst}...\"); '
-    'shutil.copytree(src, dst, dirs_exist_ok=True); '
+    'shutil.copytree(src, dst); '
     'print(f\"Templates copied successfully\")\'; '
     # Make scripts executable.
-    'find ~/sky_templates -type f ! -name "*.py" ! -name "*.md" '
-    '-exec chmod +x {} \\; ')
+    f'find {SKY_TEMPLATES_DIRECTORY} -type f ! -name "*.py" ! -name "*.md" '
+    '-exec chmod +x {} + ; ')
 
 SKYPILOT_WHEEL_INSTALLATION_COMMANDS = (
     f'{SKY_UV_INSTALL_CMD};'
@@ -438,6 +462,7 @@ OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
     ('gcp', 'enable_gvnic'),
     ('gcp', 'enable_gpu_direct'),
     ('gcp', 'placement_policy'),
+    ('vast', 'secure_only'),
     ('active_workspace',),
 ]
 # When overriding the SkyPilot configs on the API server with the client one,
@@ -532,7 +557,7 @@ CATALOG_SCHEMA_VERSION = 'v8'
 CATALOG_DIR = '~/.sky/catalogs'
 ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
               'kubernetes', 'runpod', 'vast', 'vsphere', 'cudo', 'fluidstack',
-              'paperspace', 'primeintellect', 'do', 'nebius', 'ssh',
+              'paperspace', 'primeintellect', 'do', 'nebius', 'ssh', 'slurm',
               'hyperbolic', 'seeweb', 'shadeform')
 # END constants used for service catalog.
 

sky/skylet/events.py

@@ -236,7 +236,7 @@ class AutostopEvent(SkyletEvent):
                     RAY_PROVISIONER_SKYPILOT_TERMINATOR):
                 logger.info('Using new provisioner to stop the cluster.')
                 self._stop_cluster_with_new_provisioner(autostop_config, config,
-                                                        provider_name)
+                                                        provider_name, cloud)
                 return
             logger.info('Not using new provisioner to stop the cluster. '
                         f'Cloud of this cluster: {provider_name}')
@@ -314,7 +314,8 @@ class AutostopEvent(SkyletEvent):
             raise NotImplementedError
 
     def _stop_cluster_with_new_provisioner(self, autostop_config,
-                                           cluster_config, provider_name):
+                                           cluster_config, provider_name,
+                                           cloud):
         # pylint: disable=import-outside-toplevel
         from sky import provision as provision_lib
         autostop_lib.set_autostopping_started()
@@ -334,8 +335,13 @@ class AutostopEvent(SkyletEvent):
 
         # Stop the ray autoscaler to avoid scaling up, during
         # stopping/terminating of the cluster.
-        logger.info('Stopping the ray cluster.')
-        subprocess.run(f'{constants.SKY_RAY_CMD} stop', shell=True, check=True)
+        if not cloud.uses_ray():
+            logger.info('Skipping ray stop as cloud does not use Ray.')
+        else:
+            logger.info('Stopping the ray cluster.')
+            subprocess.run(f'{constants.SKY_RAY_CMD} stop',
+                           shell=True,
+                           check=True)
 
         operation_fn = provision_lib.stop_instances
         if autostop_config.down:

sky/skylet/executor/__init__.py

@@ -0,0 +1 @@
+"""Task Executors"""

sky/skylet/executor/slurm.py

@@ -0,0 +1,189 @@
+"""Slurm distributed task executor for SkyPilot.
+
+This module is invoked on each Slurm compute node via:
+    srun python -m sky.skylet.executor.slurm --script=... --log-dir=...
+"""
+import argparse
+import json
+import os
+import pathlib
+import socket
+import subprocess
+import sys
+import time
+
+import colorama
+
+from sky.skylet.log_lib import run_bash_command_with_log
+
+
+def _get_ip_address() -> str:
+    """Get the IP address of the current node."""
+    ip_result = subprocess.run(['hostname', '-I'],
+                               capture_output=True,
+                               text=True,
+                               check=False)
+    return ip_result.stdout.strip().split(
+    )[0] if ip_result.returncode == 0 else 'unknown'
+
+
+def _get_job_node_ips() -> str:
+    """Get IPs of all nodes in the current Slurm job."""
+    nodelist = os.environ.get('SLURM_JOB_NODELIST', '')
+    assert nodelist, 'SLURM_JOB_NODELIST is not set'
+
+    # Expand compressed nodelist (e.g., "node[1-3,5]"
+    # -> "node1\nnode2\nnode3\nnode5")
+    result = subprocess.run(['scontrol', 'show', 'hostnames', nodelist],
+                            capture_output=True,
+                            text=True,
+                            check=False)
+    if result.returncode != 0:
+        raise RuntimeError(f'Failed to get hostnames for: {nodelist}')
+
+    hostnames = result.stdout.strip().split('\n')
+    ips = []
+    for hostname in hostnames:
+        try:
+            ip = socket.gethostbyname(hostname)
+            ips.append(ip)
+        except socket.gaierror as e:
+            raise RuntimeError('Failed to get IP for hostname: '
+                               f'{hostname}') from e
+
+    return '\n'.join(ips)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='SkyPilot Slurm task runner for distributed execution')
+    parser.add_argument('--script', help='User script (inline, shell-quoted)')
+    parser.add_argument('--script-path',
+                        help='Path to script file (if too long for inline)')
+    parser.add_argument('--env-vars',
+                        default='{}',
+                        help='JSON-encoded environment variables')
+    parser.add_argument('--log-dir',
+                        required=True,
+                        help='Directory for log files')
+    parser.add_argument('--cluster-num-nodes',
+                        type=int,
+                        required=True,
+                        help='Total number of nodes in the cluster')
+    parser.add_argument('--cluster-ips',
+                        required=True,
+                        help='Comma-separated list of cluster node IPs')
+    parser.add_argument('--task-name',
+                        default=None,
+                        help='Task name for single-node log prefix')
+    parser.add_argument(
+        '--is-setup',
+        action='store_true',
+        help=
+        'Whether this is a setup command (affects logging prefix and filename)')
+    parser.add_argument('--alloc-signal-file',
+                        help='Path to allocation signal file')
+    parser.add_argument('--setup-done-signal-file',
+                        help='Path to setup-done signal file')
+    args = parser.parse_args()
+
+    assert args.script is not None or args.script_path is not None, (
+        'Either '
+        '--script or --script-path must be provided')
+
+    # Task rank, different from index of the node in the cluster.
+    rank = int(os.environ['SLURM_PROCID'])
+    num_nodes = int(os.environ.get('SLURM_NNODES', 1))
+    is_single_node_cluster = (args.cluster_num_nodes == 1)
+
+    # Determine node index from IP (like Ray's cluster_ips_to_node_id)
+    cluster_ips = args.cluster_ips.split(',')
+    ip_addr = _get_ip_address()
+    try:
+        node_idx = cluster_ips.index(ip_addr)
+    except ValueError as e:
+        raise RuntimeError(f'IP address {ip_addr} not found in '
+                           f'cluster IPs: {cluster_ips}') from e
+    node_name = 'head' if node_idx == 0 else f'worker{node_idx}'
+
+    # Log files are written to a shared filesystem, so each node must use a
+    # unique filename to avoid collisions.
+    if args.is_setup:
+        # TODO(kevin): This is inconsistent with other clouds, where it is
+        # simply called 'setup.log'. On Slurm that is obviously not possible,
+        # since the ~/sky_logs directory is shared by all nodes, so
+        # 'setup.log' will be overwritten by other nodes.
+        # Perhaps we should apply this naming convention to other clouds.
+        log_filename = f'setup-{node_name}.log'
+    elif is_single_node_cluster:
+        log_filename = 'run.log'
+    else:
+        log_filename = f'{rank}-{node_name}.log'
+    log_path = os.path.join(args.log_dir, log_filename)
+
+    if args.script_path:
+        with open(args.script_path, 'r', encoding='utf-8') as f:
+            script = f.read()
+    else:
+        script = args.script
+
+    # Parse env vars and add SKYPILOT environment variables
+    env_vars = json.loads(args.env_vars)
+    if not args.is_setup:
+        # For setup, env vars are set in CloudVmRayBackend._setup.
+        env_vars['SKYPILOT_NODE_RANK'] = str(rank)
+        env_vars['SKYPILOT_NUM_NODES'] = str(num_nodes)
+        env_vars['SKYPILOT_NODE_IPS'] = _get_job_node_ips()
+
+    # Signal file coordination for setup/run synchronization
+    # Rank 0 touches the allocation signal to indicate resources acquired
+    if args.alloc_signal_file is not None and rank == 0:
+        pathlib.Path(args.alloc_signal_file).touch()
+
+    # Wait for setup to complete.
+    while args.setup_done_signal_file is not None and not os.path.exists(
+            args.setup_done_signal_file):
+        time.sleep(0.1)
+
+    # Build log prefix
+    # For setup on head: (setup pid={pid})
+    # For setup on workers: (setup pid={pid}, ip=1.2.3.4)
+    # For single-node cluster: (task_name, pid={pid})
+    # For multi-node on head: (head, rank=0, pid={pid})
+    # For multi-node on workers: (worker1, rank=1, pid={pid}, ip=1.2.3.4)
+    # The {pid} placeholder will be replaced by run_with_log
+    if args.is_setup:
+        # Setup prefix: head (node_idx=0) shows no IP, workers show IP
+        if node_idx == 0:
+            prefix = (f'{colorama.Fore.CYAN}(setup pid={{pid}})'
+                      f'{colorama.Style.RESET_ALL} ')
+        else:
+            prefix = (f'{colorama.Fore.CYAN}(setup pid={{pid}}, ip={ip_addr})'
+                      f'{colorama.Style.RESET_ALL} ')
+    elif is_single_node_cluster:
+        # Single-node cluster: use task name
+        name_str = args.task_name if args.task_name else 'task'
+        prefix = (f'{colorama.Fore.CYAN}({name_str}, pid={{pid}})'
+                  f'{colorama.Style.RESET_ALL} ')
+    else:
+        # Multi-node cluster: head (node_idx=0) shows no IP, workers show IP
+        if node_idx == 0:
+            prefix = (
+                f'{colorama.Fore.CYAN}({node_name}, rank={rank}, pid={{pid}})'
+                f'{colorama.Style.RESET_ALL} ')
+        else:
+            prefix = (f'{colorama.Fore.CYAN}'
+                      f'({node_name}, rank={rank}, pid={{pid}}, ip={ip_addr})'
+                      f'{colorama.Style.RESET_ALL} ')
+
+    returncode = run_bash_command_with_log(script,
+                                           log_path,
+                                           env_vars=env_vars,
+                                           stream_logs=True,
+                                           streaming_prefix=prefix)
+
+    sys.exit(returncode)
+
+
+if __name__ == '__main__':
+    main()

sky/skylet/job_lib.py

@@ -1273,4 +1273,5 @@ class JobLibCodeGen:
     def _build(cls, code: List[str]) -> str:
         code = cls._PREFIX + code
         code = ';'.join(code)
-        return f'{constants.SKY_PYTHON_CMD} -u -c {shlex.quote(code)}'
+        return (f'{constants.ACTIVATE_SKY_REMOTE_PYTHON_ENV}; '
+                f'{constants.SKY_PYTHON_CMD} -u -c {shlex.quote(code)}')

sky/skylet/log_lib.py

@@ -172,7 +172,7 @@ def run_with_log(
     streaming_prefix: Optional[str] = None,
     log_cmd: bool = False,
     **kwargs,
-) -> Union[int, Tuple[int, str, str]]:
+) -> Union[int, Tuple[int, str, str], Tuple[int, int]]:
     """Runs a command and logs its output to a file.
 
     Args:
@@ -183,6 +183,8 @@ def run_with_log(
         process_stream: Whether to post-process the stdout/stderr of the
             command, such as replacing or skipping lines on the fly. If
             enabled, lines are printed only when '\r' or '\n' is found.
+        streaming_prefix: Optional prefix for each log line. Can contain {pid}
+            placeholder which will be replaced with the subprocess PID.
 
     Returns the returncode or returncode, stdout and stderr of the command.
       Note that the stdout and stderr is already decoded.
@@ -228,6 +230,13 @@ def run_with_log(
                 # For backward compatibility, do not specify use_kill_pg by
                 # default.
                 subprocess_utils.kill_process_daemon(proc.pid)
+
+            # Format streaming_prefix with subprocess PID if it contains {pid}
+            formatted_streaming_prefix = streaming_prefix
+            if streaming_prefix and '{pid}' in streaming_prefix:
+                formatted_streaming_prefix = streaming_prefix.format(
+                    pid=proc.pid)
+
             stdout = ''
             stderr = ''
             stdout_stream_handler = None
@@ -256,7 +265,7 @@ def run_with_log(
                     line_processor=line_processor,
                     # Replace CRLF when the output is logged to driver by ray.
                     replace_crlf=with_ray,
-                    streaming_prefix=streaming_prefix,
+                    streaming_prefix=formatted_streaming_prefix,
                 )
                 stdout_stream_handler = functools.partial(
                     _handle_io_stream,
@@ -349,7 +358,8 @@ def run_bash_command_with_log(bash_command: str,
                               log_path: str,
                               env_vars: Optional[Dict[str, str]] = None,
                               stream_logs: bool = False,
-                              with_ray: bool = False):
+                              with_ray: bool = False,
+                              streaming_prefix: Optional[str] = None):
     with tempfile.NamedTemporaryFile('w', prefix='sky_app_',
                                      delete=False) as fp:
         bash_command = make_task_bash_script(bash_command, env_vars=env_vars)
@@ -364,6 +374,7 @@ def run_bash_command_with_log(bash_command: str,
                             log_path,
                             stream_logs=stream_logs,
                             with_ray=with_ray,
+                            streaming_prefix=streaming_prefix,
                             shell=True)
 
 
@@ -372,9 +383,14 @@ def run_bash_command_with_log_and_return_pid(
         log_path: str,
         env_vars: Optional[Dict[str, str]] = None,
         stream_logs: bool = False,
-        with_ray: bool = False):
-    return_code = run_bash_command_with_log(bash_command, log_path, env_vars,
-                                            stream_logs, with_ray)
+        with_ray: bool = False,
+        streaming_prefix: Optional[str] = None):
+    return_code = run_bash_command_with_log(bash_command,
+                                            log_path,
+                                            env_vars,
+                                            stream_logs,
+                                            with_ray,
+                                            streaming_prefix=streaming_prefix)
     return {'return_code': return_code, 'pid': os.getpid()}
 
 

sky/skylet/log_lib.pyi

@@ -68,7 +68,7 @@ def run_with_log(cmd: Union[List[str], str],
                  process_stream: bool = ...,
                  line_processor: Optional[log_utils.LineProcessor] = ...,
                  streaming_prefix: Optional[str] = ...,
-                 ray_job_id: Optional[str] = ...,
+                 log_cmd: bool = ...,
                  **kwargs) -> int:
     ...
 
@@ -87,7 +87,7 @@ def run_with_log(cmd: Union[List[str], str],
                  process_stream: bool = ...,
                  line_processor: Optional[log_utils.LineProcessor] = ...,
                  streaming_prefix: Optional[str] = ...,
-                 ray_job_id: Optional[str] = ...,
+                 log_cmd: bool = ...,
                  **kwargs) -> Tuple[int, str, str]:
     ...
 
@@ -106,8 +106,8 @@ def run_with_log(cmd: Union[List[str], str],
                  process_stream: bool = ...,
                  line_processor: Optional[log_utils.LineProcessor] = ...,
                  streaming_prefix: Optional[str] = ...,
-                 ray_job_id: Optional[str] = ...,
-                 **kwargs) -> Union[int, Tuple[int, str, str]]:
+                 log_cmd: bool = ...,
+                 **kwargs) -> Tuple[int, int]:
     ...
 
 
@@ -125,7 +125,8 @@ def run_bash_command_with_log(bash_command: str,
                               log_path: str,
                               env_vars: Optional[Dict[str, str]] = ...,
                               stream_logs: bool = ...,
-                              with_ray: bool = ...):
+                              with_ray: bool = ...,
+                              streaming_prefix: Optional[str] = ...) -> int:
     ...
 
 
@@ -134,7 +135,8 @@ def run_bash_command_with_log_and_return_pid(
         log_path: str,
         env_vars: Optional[Dict[str, str]] = ...,
         stream_logs: bool = ...,
-        with_ray: bool = ...):
+        with_ray: bool = ...,
+        streaming_prefix: Optional[str] = ...) -> Dict[str, Union[int, str]]:
     ...
 
 

sky/skylet/skylet.py

@@ -48,8 +48,12 @@ def start_grpc_server(port: int = constants.SKYLET_GRPC_PORT) -> grpc.Server:
     # putting it here for visibility.
     # TODO(kevin): Determine the optimal max number of threads.
     max_workers = min(32, (os.cpu_count() or 1) + 4)
+    # There's only a single skylet process per cluster, so disable
+    # SO_REUSEPORT to raise an error if the port is already in use.
+    options = (('grpc.so_reuseport', 0),)
     server = grpc.server(
-        concurrent.futures.ThreadPoolExecutor(max_workers=max_workers))
+        concurrent.futures.ThreadPoolExecutor(max_workers=max_workers),
+        options=options)
 
     autostopv1_pb2_grpc.add_AutostopServiceServicer_to_server(
         services.AutostopServiceImpl(), server)

sky/skylet/subprocess_daemon.py

@@ -110,7 +110,8 @@ def kill_process_tree(process: psutil.Process,
 
 
 def main():
-    # daemonize()
+    daemonize()
+
     parser = argparse.ArgumentParser()
     parser.add_argument('--parent-pid', type=int, required=True)
     parser.add_argument('--proc-pid', type=int, required=True)

sky/ssh_node_pools/constants.py

@@ -0,0 +1,12 @@
+"""Constants for SSH Node Pools"""
+# pylint: disable=line-too-long
+import os
+
+DEFAULT_KUBECONFIG_PATH = os.path.expanduser('~/.kube/config')
+SSH_CONFIG_PATH = os.path.expanduser('~/.ssh/config')
+NODE_POOLS_INFO_DIR = os.path.expanduser('~/.sky/ssh_node_pools_info')
+NODE_POOLS_KEY_DIR = os.path.expanduser('~/.sky/ssh_keys')
+DEFAULT_SSH_NODE_POOLS_PATH = os.path.expanduser('~/.sky/ssh_node_pools.yaml')
+
+# TODO (kyuds): make this configurable?
+K3S_TOKEN = 'mytoken'  # Any string can be used as the token

sky/ssh_node_pools/core.py

@@ -1,10 +1,15 @@
 """SSH Node Pool management core functionality."""
 import os
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional, Tuple
 
 import yaml
 
+from sky import clouds
+from sky.ssh_node_pools import constants
+from sky.ssh_node_pools import deploy
+from sky.usage import usage_lib
+from sky.utils import common_utils
 from sky.utils import yaml_utils
 
 
@@ -12,8 +17,8 @@ class SSHNodePoolManager:
     """Manager for SSH Node Pool configurations."""
 
     def __init__(self):
-        self.config_path = Path.home() / '.sky' / 'ssh_node_pools.yaml'
-        self.keys_dir = Path.home() / '.sky' / 'ssh_keys'
+        self.config_path = Path(constants.DEFAULT_SSH_NODE_POOLS_PATH)
+        self.keys_dir = Path(constants.NODE_POOLS_KEY_DIR)
         self.keys_dir.mkdir(parents=True, exist_ok=True)
 
     def get_all_pools(self) -> Dict[str, Any]:
@@ -133,3 +138,35 @@ def list_ssh_keys() -> List[str]:
     """List available SSH keys."""
     manager = SSHNodePoolManager()
     return manager.list_ssh_keys()
+
+
+@usage_lib.entrypoint
+def ssh_up(infra: Optional[str] = None, cleanup: bool = False) -> None:
+    """Deploys or tears down a Kubernetes cluster on SSH targets.
+
+    Args:
+        infra: Name of the cluster configuration in ssh_node_pools.yaml.
+            If None, the first cluster in the file is used.
+        cleanup: If True, clean up the cluster instead of deploying.
+    """
+    deploy.run(cleanup=cleanup, infra=infra)
+
+
+@usage_lib.entrypoint
+def ssh_status(context_name: str) -> Tuple[bool, str]:
+    """Check the status of an SSH Node Pool context.
+
+    Args:
+        context_name: The SSH context name (e.g., 'ssh-my-cluster')
+
+    Returns:
+        Tuple[bool, str]: (is_ready, reason)
+            - is_ready: True if the SSH Node Pool is ready, False otherwise
+            - reason: Explanation of the status
+    """
+    try:
+        is_ready, reason = clouds.SSH.check_single_context(context_name)
+        return is_ready, reason
+    except Exception as e:  # pylint: disable=broad-except
+        return False, ('Failed to check SSH context: '
+                       f'{common_utils.format_exception(e)}')

sky/ssh_node_pools/deploy/__init__.py

@@ -0,0 +1,4 @@
+"""Module for Deploying SSH Node Pools"""
+from sky.ssh_node_pools.deploy.deploy import run
+
+__all__ = ['run']