skypilot-nightly 1.0.0.dev20241121__py3-none-any.whl → 1.0.0.dev20241122__py3-none-any.whl

sky/skylet/constants.py

@@ -39,6 +39,7 @@ SKY_GET_PYTHON_PATH_CMD = (f'[ -s {SKY_PYTHON_PATH_FILE} ] && '
                            'which python3')
 # Python executable, e.g., /opt/conda/bin/python3
 SKY_PYTHON_CMD = f'$({SKY_GET_PYTHON_PATH_CMD})'
+# Prefer SKY_UV_PIP_CMD, which is faster. TODO(cooper): remove all usages.
 SKY_PIP_CMD = f'{SKY_PYTHON_CMD} -m pip'
 # Ray executable, e.g., /opt/conda/bin/ray
 # We need to add SKY_PYTHON_CMD before ray executable because:
@@ -50,6 +51,10 @@ SKY_RAY_CMD = (f'{SKY_PYTHON_CMD} $([ -s {SKY_RAY_PATH_FILE} ] && '
 SKY_REMOTE_PYTHON_ENV_NAME = 'skypilot-runtime'
 SKY_REMOTE_PYTHON_ENV = f'~/{SKY_REMOTE_PYTHON_ENV_NAME}'
 ACTIVATE_SKY_REMOTE_PYTHON_ENV = f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate'
+# uv is used for venv and pip, much faster than python implementations.
+SKY_UV_INSTALL_DIR = '"$HOME/.local/bin"'
+SKY_UV_CMD = f'{SKY_UV_INSTALL_DIR}/uv'
+SKY_UV_PIP_CMD = f'VIRTUAL_ENV={SKY_REMOTE_PYTHON_ENV} {SKY_UV_CMD} pip'
 # Deleting the SKY_REMOTE_PYTHON_ENV_NAME from the PATH to deactivate the
 # environment. `deactivate` command does not work when conda is used.
 DEACTIVATE_SKY_REMOTE_PYTHON_ENV = (
@@ -148,31 +153,29 @@ CONDA_INSTALLATION_COMMANDS = (
     'echo "Creating conda env with Python 3.10" && '
     f'conda create -y -n {SKY_REMOTE_PYTHON_ENV_NAME} python=3.10 && '
     f'conda activate {SKY_REMOTE_PYTHON_ENV_NAME};'
+    # Install uv for venv management and pip installation.
+    'which uv >/dev/null 2>&1 || '
+    'curl -LsSf https://astral.sh/uv/install.sh '
+    f'| UV_INSTALL_DIR={SKY_UV_INSTALL_DIR} sh;'
     # Create a separate conda environment for SkyPilot dependencies.
     f'[ -d {SKY_REMOTE_PYTHON_ENV} ] || '
     # Do NOT use --system-site-packages here, because if users upgrade any
     # packages in the base env, they interfere with skypilot dependencies.
     # Reference: https://github.com/skypilot-org/skypilot/issues/4097
-    f'{SKY_PYTHON_CMD} -m venv {SKY_REMOTE_PYTHON_ENV};'
+    f'{SKY_UV_CMD} venv {SKY_REMOTE_PYTHON_ENV};'
     f'echo "$(echo {SKY_REMOTE_PYTHON_ENV})/bin/python" > {SKY_PYTHON_PATH_FILE};'
 )
 
 _sky_version = str(version.parse(sky.__version__))
 RAY_STATUS = f'RAY_ADDRESS=127.0.0.1:{SKY_REMOTE_RAY_PORT} {SKY_RAY_CMD} status'
-# Install ray and skypilot on the remote cluster if they are not already
-# installed. {var} will be replaced with the actual value in
-# backend_utils.write_cluster_config.
-RAY_SKYPILOT_INSTALLATION_COMMANDS = (
+RAY_INSTALLATION_COMMANDS = (
     'mkdir -p ~/sky_workdir && mkdir -p ~/.sky/sky_app;'
-    # Disable the pip version check to avoid the warning message, which makes
-    # the output hard to read.
-    'export PIP_DISABLE_PIP_VERSION_CHECK=1;'
     # Print the PATH in provision.log to help debug PATH issues.
     'echo PATH=$PATH; '
     # Install setuptools<=69.5.1 to avoid the issue with the latest setuptools
     # causing the error:
     #   ImportError: cannot import name 'packaging' from 'pkg_resources'"
-    f'{SKY_PIP_CMD} install "setuptools<70"; '
+    f'{SKY_UV_PIP_CMD} install "setuptools<70"; '
     # Backward compatibility for ray upgrade (#3248): do not upgrade ray if the
     # ray cluster is already running, to avoid the ray cluster being restarted.
     #
@@ -186,10 +189,10 @@ RAY_SKYPILOT_INSTALLATION_COMMANDS = (
     # latest ray port 6380, but those existing cluster launched before #1790
     # that has ray cluster on the default port 6379 will be upgraded and
     # restarted.
-    f'{SKY_PIP_CMD} list | grep "ray " | '
+    f'{SKY_UV_PIP_CMD} list | grep "ray " | '
     f'grep {SKY_REMOTE_RAY_VERSION} 2>&1 > /dev/null '
     f'|| {RAY_STATUS} || '
-    f'{SKY_PIP_CMD} install --exists-action w -U ray[default]=={SKY_REMOTE_RAY_VERSION}; '  # pylint: disable=line-too-long
+    f'{SKY_UV_PIP_CMD} install -U ray[default]=={SKY_REMOTE_RAY_VERSION}; '  # pylint: disable=line-too-long
     # In some envs, e.g. pip does not have permission to write under /opt/conda
     # ray package will be installed under ~/.local/bin. If the user's PATH does
     # not include ~/.local/bin (the pip install will have the output: `WARNING:
@@ -202,24 +205,31 @@ RAY_SKYPILOT_INSTALLATION_COMMANDS = (
     # Writes ray path to file if it does not exist or the file is empty.
     f'[ -s {SKY_RAY_PATH_FILE} ] || '
     f'{{ {ACTIVATE_SKY_REMOTE_PYTHON_ENV} && '
-    f'which ray > {SKY_RAY_PATH_FILE} || exit 1; }}; '
-    # END ray package check and installation
-    f'{{ {SKY_PIP_CMD} list | grep "skypilot " && '
+    f'which ray > {SKY_RAY_PATH_FILE} || exit 1; }}; ')
+
+SKYPILOT_WHEEL_INSTALLATION_COMMANDS = (
+    f'{{ {SKY_UV_PIP_CMD} list | grep "skypilot " && '
     '[ "$(cat ~/.sky/wheels/current_sky_wheel_hash)" == "{sky_wheel_hash}" ]; } || '  # pylint: disable=line-too-long
-    f'{{ {SKY_PIP_CMD} uninstall skypilot -y; '
-    f'{SKY_PIP_CMD} install "$(echo ~/.sky/wheels/{{sky_wheel_hash}}/'
+    f'{{ {SKY_UV_PIP_CMD} uninstall skypilot; '
+    f'{SKY_UV_PIP_CMD} install "$(echo ~/.sky/wheels/{{sky_wheel_hash}}/'
     f'skypilot-{_sky_version}*.whl)[{{cloud}}, remote]" && '
     'echo "{sky_wheel_hash}" > ~/.sky/wheels/current_sky_wheel_hash || '
-    'exit 1; }; '
-    # END SkyPilot package check and installation
+    'exit 1; }; ')
 
+# Install ray and skypilot on the remote cluster if they are not already
+# installed. {var} will be replaced with the actual value in
+# backend_utils.write_cluster_config.
+RAY_SKYPILOT_INSTALLATION_COMMANDS = (
+    f'{RAY_INSTALLATION_COMMANDS} '
+    f'{SKYPILOT_WHEEL_INSTALLATION_COMMANDS} '
     # Only patch ray when the ray version is the same as the expected version.
     # The ray installation above can be skipped due to the existing ray cluster
     # for backward compatibility. In this case, we should not patch the ray
     # files.
-    f'{SKY_PIP_CMD} list | grep "ray " | grep {SKY_REMOTE_RAY_VERSION} 2>&1 > /dev/null '
-    f'&& {{ {SKY_PYTHON_CMD} -c "from sky.skylet.ray_patches import patch; patch()" '
-    '|| exit 1; };')
+    f'{SKY_UV_PIP_CMD} list | grep "ray " | '
+    f'grep {SKY_REMOTE_RAY_VERSION} 2>&1 > /dev/null && '
+    f'{{ {SKY_PYTHON_CMD} -c '
+    '"from sky.skylet.ray_patches import patch; patch()" || exit 1; }; ')
 
 # The name for the environment variable that stores SkyPilot user hash, which
 # is mainly used to make sure sky commands runs on a VM launched by SkyPilot

sky/templates/kubernetes-ray.yml.j2

@@ -222,7 +222,9 @@ provider:
             - protocol: TCP
               port: 22
               targetPort: 22
-      # Service that maps to the head node of the Ray cluster.
+      # Service that maps to the head node of the Ray cluster, so that the
+      # worker nodes can find the head node using
+      # {{cluster_name_on_cloud}}-head.{{k8s_namespace}}.svc.cluster.local
       - apiVersion: v1
         kind: Service
         metadata:
@@ -235,18 +237,12 @@ provider:
             # names.
             name: {{cluster_name_on_cloud}}-head
         spec:
+            # Create a headless service so that the head node can be reached by
+            # the worker nodes with any port number.
+            clusterIP: None
             # This selector must match the head node pod's selector below.
             selector:
                 component: {{cluster_name_on_cloud}}-head
-            ports:
-                - name: client
-                  protocol: TCP
-                  port: 10001
-                  targetPort: 10001
-                - name: dashboard
-                  protocol: TCP
-                  port: 8265
-                  targetPort: 8265
 
 # Specify the pod type for the ray head node (as configured below).
 head_node_type: ray_head_default
@@ -280,7 +276,6 @@ available_node_types:
         # serviceAccountName: skypilot-service-account
         serviceAccountName: {{k8s_service_account_name}}
         automountServiceAccountToken: {{k8s_automount_sa_token}}
-
         restartPolicy: Never
 
         # Add node selector if GPU/TPUs are requested:
@@ -322,18 +317,147 @@ available_node_types:
         - name: ray-node
           imagePullPolicy: IfNotPresent
           image: {{image_id}}
+          env:
+            - name: SKYPILOT_POD_NODE_TYPE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['ray-node-type']
           # Do not change this command - it keeps the pod alive until it is
           # explicitly killed.
           command: ["/bin/bash", "-c", "--"]
           args: 
             - |
               # Helper function to conditionally use sudo
+              # TODO(zhwu): consolidate the two prefix_cmd and sudo replacements
               prefix_cmd() { if [ $(id -u) -ne 0 ]; then echo "sudo"; else echo ""; fi; }
+              [ $(id -u) -eq 0 ] && function sudo() { "$@"; } || true;
+
+              STEPS=("apt-ssh-setup" "runtime-setup" "env-setup")
 
-              # Run apt update in background and log to a file
+              # STEP 1: Run apt update, install missing packages, and set up ssh.
               (
+                (
                 DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get update > /tmp/apt-update.log 2>&1 || \
                 echo "Warning: apt-get update failed. Continuing anyway..." >> /tmp/apt-update.log
+                PACKAGES="rsync curl netcat gcc patch pciutils fuse openssh-server";
+
+                # Separate packages into two groups: packages that are installed first
+                # so that curl and rsync are available sooner to unblock the following
+                # conda installation and rsync.
+                set -e
+                INSTALL_FIRST="";
+                MISSING_PACKAGES="";
+                for pkg in $PACKAGES; do
+                  if [ "$pkg" == "netcat" ]; then
+                    if ! dpkg -l | grep -q "^ii  \(netcat\|netcat-openbsd\|netcat-traditional\) "; then
+                      INSTALL_FIRST="$INSTALL_FIRST netcat-openbsd";
+                    fi
+                  elif ! dpkg -l | grep -q "^ii  $pkg "; then
+                    if [ "$pkg" == "curl" ] || [ "$pkg" == "rsync" ]; then
+                      INSTALL_FIRST="$INSTALL_FIRST $pkg";
+                    else
+                      MISSING_PACKAGES="$MISSING_PACKAGES $pkg";
+                    fi
+                  fi
+                done;
+                if [ ! -z "$INSTALL_FIRST" ]; then
+                  echo "Installing core packages: $INSTALL_FIRST";
+                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $INSTALL_FIRST;
+                fi;
+                # SSH and other packages are not necessary, so we disable set -e
+                set +e
+                
+                if [ ! -z "$MISSING_PACKAGES" ]; then
+                  echo "Installing missing packages: $MISSING_PACKAGES";
+                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES;
+                fi;
+                $(prefix_cmd) mkdir -p /var/run/sshd;
+                $(prefix_cmd) sed -i "s/PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config;
+                $(prefix_cmd) sed "s@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g" -i /etc/pam.d/sshd;
+                cd /etc/ssh/ && $(prefix_cmd) ssh-keygen -A;
+                $(prefix_cmd) mkdir -p ~/.ssh;
+                $(prefix_cmd) chown -R $(whoami) ~/.ssh;
+                $(prefix_cmd) chmod 700 ~/.ssh;
+                $(prefix_cmd) cat /etc/secret-volume/ssh-publickey* > ~/.ssh/authorized_keys;
+                $(prefix_cmd) chmod 644 ~/.ssh/authorized_keys;
+                $(prefix_cmd) service ssh restart;
+                $(prefix_cmd) sed -i "s/mesg n/tty -s \&\& mesg n/" ~/.profile;
+                ) > /tmp/${STEPS[0]}.log 2>&1 || {
+                  echo "Error: ${STEPS[0]} failed. Continuing anyway..." > /tmp/${STEPS[0]}.failed
+                  cat /tmp/${STEPS[0]}.log
+                  exit 1
+                }
+              ) &
+
+              # STEP 2: Install conda, ray and skypilot (for dependencies); start
+              # ray cluster.
+              (
+                (
+                set -e
+                mkdir -p ~/.sky
+                # Wait for `curl` package to be installed before installing conda
+                # and ray.
+                until dpkg -l | grep -q "^ii  curl "; do
+                  sleep 0.1
+                  echo "Waiting for curl package to be installed..."
+                done
+                {{ conda_installation_commands }}
+                {{ ray_installation_commands }}
+                VIRTUAL_ENV=~/skypilot-runtime ~/.local/bin/uv pip install skypilot[kubernetes,remote]
+                touch /tmp/ray_skypilot_installation_complete
+                echo "=== Ray and skypilot installation completed ==="
+
+                # Disable set -e, as we have some commands that are ok to fail
+                # after the ray start.
+                # TODO(zhwu): this is a hack, we should fix the commands that are
+                # ok to fail.
+                if [ "$SKYPILOT_POD_NODE_TYPE" == "head" ]; then
+                  set +e
+                  {{ ray_head_start_command }}
+                else
+                  # Start ray worker on the worker pod.
+                  # Wait until the head pod is available with an IP address
+                  export SKYPILOT_RAY_HEAD_IP="{{cluster_name_on_cloud}}-head.{{k8s_namespace}}.svc.cluster.local"
+                  export SKYPILOT_RAY_PORT={{skypilot_ray_port}}
+                  # Wait until the ray cluster is started on the head pod
+                  until dpkg -l | grep -q "^ii  \(netcat\|netcat-openbsd\|netcat-traditional\) "; do
+                    sleep 0.1
+                    echo "Waiting for netcat package to be installed..."
+                  done
+                  until nc -z -w 1 ${SKYPILOT_RAY_HEAD_IP} ${SKYPILOT_RAY_PORT}; do
+                    sleep 0.1
+                  done
+
+                  set +e
+                  {{ ray_worker_start_command }}
+                fi
+                ) > /tmp/${STEPS[1]}.log 2>&1 || {
+                  echo "Error: ${STEPS[1]} failed. Continuing anyway..." > /tmp/${STEPS[1]}.failed
+                  cat /tmp/${STEPS[1]}.log
+                  exit 1
+                }
+              ) &
+
+
+              # STEP 3: Set up environment variables; this should be relatively fast.
+              (
+                (
+                set -e
+                if [ $(id -u) -eq 0 ]; then
+                  echo 'alias sudo=""' >> ~/.bashrc; echo succeed;
+                else
+                  if command -v sudo >/dev/null 2>&1; then
+                    timeout 2 sudo -l >/dev/null 2>&1 && echo succeed || { echo 52; exit 52; };
+                  else
+                    { echo 52; exit 52; };
+                  fi;
+                fi;
+                printenv | while IFS='=' read -r key value; do echo "export $key=\"$value\""; done > ~/container_env_var.sh && $(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh
+                ) > /tmp/${STEPS[2]}.log 2>&1 || {
+                  echo "Error: ${STEPS[2]} failed. Continuing anyway..." > /tmp/${STEPS[2]}.failed
+                  cat /tmp/${STEPS[2]}.log
+                  exit 1
+                }
               ) &
 
               function mylsof { p=$(for pid in /proc/{0..9}*; do i=$(basename "$pid"); for file in "$pid"/fd/*; do link=$(readlink -e "$file"); if [ "$link" = "$1" ]; then echo "$i"; fi; done; done); echo "$p"; };
@@ -441,42 +565,48 @@ setup_commands:
   # Line 'sudo grep ..': set the number of threads per process to unlimited to avoid ray job submit stucking issue when the number of running ray jobs increase.
   # Line 'mkdir -p ..': disable host key check
   # Line 'python3 -c ..': patch the buggy ray files and enable `-o allow_other` option for `goofys`
+  # Line 'for step in ..': check if any failure indicator exists for the setup done in pod args and print the error message. This is only a best effort, as the
+  # commands in pod args are asynchronous and we cannot guarantee the failure indicators are created before the setup commands finish.
   - |
-    PACKAGES="gcc patch pciutils rsync fuse curl";
-    MISSING_PACKAGES="";
-    for pkg in $PACKAGES; do
-      if ! dpkg -l | grep -q "^ii  $pkg "; then
-        MISSING_PACKAGES="$MISSING_PACKAGES $pkg";
-      fi
-    done;
-    if [ ! -z "$MISSING_PACKAGES" ]; then
-      echo "Installing missing packages: $MISSING_PACKAGES";
-      sudo DEBIAN_FRONTEND=noninteractive apt-get install -y $MISSING_PACKAGES;
-    fi;
     mkdir -p ~/.ssh; touch ~/.ssh/config;
     {%- for initial_setup_command in initial_setup_commands %}
     {{ initial_setup_command }}
     {%- endfor %}
-    {{ conda_installation_commands }}
-    {{ ray_skypilot_installation_commands }}
+    STEPS=("apt-ssh-setup" "runtime-setup" "env-setup")
+    start_epoch=$(date +%s);
+    echo "=== Logs for asynchronous ray and skypilot installation ===";
+    [ -f /tmp/ray_skypilot_installation_complete ] && cat /tmp/${STEPS[1]}.log || 
+    { tail -f -n +1 /tmp/${STEPS[1]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/ray_skypilot_installation_complete ]; do sleep 0.5; done; kill $TAIL_PID || true; };
+    [ -f /tmp/${STEPS[1]}.failed ] && { echo "Error: ${STEPS[1]} failed. Exiting."; exit 1; } || true;
+    end_epoch=$(date +%s);
+    echo "=== Ray and skypilot dependencies installation completed in $(($end_epoch - $start_epoch)) secs ===";
+    start_epoch=$(date +%s);
+    {{ skypilot_wheel_installation_commands }}
+    end_epoch=$(date +%s);
+    echo "=== Skypilot wheel installation completed in $(($end_epoch - $start_epoch)) secs ===";
+    start_epoch=$(date +%s);
     sudo touch ~/.sudo_as_admin_successful;
     sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
-    sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
+    sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf');
+    ulimit -n 1048576;
     mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
     [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf'); # This is needed for `-o allow_other` option for `goofys`;
-  {% if tpu_requested %}
-  # The /tmp/tpu_logs directory is where TPU-related logs, such as logs from
-  # the TPU runtime, are written. These capture runtime information about the
-  # TPU execution, including any warnings, errors, or general activity of
-  # the TPU driver. By default, the /tmp/tpu_logs directory is created with
-  # 755 permissions, and the user of the provisioned pod is not necessarily
-  # a root. Hence, we need to update the write permission so the logs can be
-  # properly written.
-  # TODO(Doyoung): Investigate to see why TPU workload fails to run without
-  # execution permission, such as granting 766 to log file. Check if it's a
-  # must and see if there's a workaround to grant minimum permission. 
-  - sudo chmod 777 /tmp/tpu_logs;
-  {% endif %}
+    end_epoch=$(date +%s);
+    echo "=== Setup system configs and fuse completed in $(($end_epoch - $start_epoch)) secs ===";
+    for step in $STEPS; do [ -f "/tmp/${step}.failed" ] && { echo "Error: /tmp/${step}.failed found:"; cat /tmp/${step}.log; exit 1; } || true; done;
+    {% if tpu_requested %}
+    # The /tmp/tpu_logs directory is where TPU-related logs, such as logs from
+    # the TPU runtime, are written. These capture runtime information about the
+    # TPU execution, including any warnings, errors, or general activity of
+    # the TPU driver. By default, the /tmp/tpu_logs directory is created with
+    # 755 permissions, and the user of the provisioned pod is not necessarily
+    # a root. Hence, we need to update the write permission so the logs can be
+    # properly written.
+    # TODO(Doyoung): Investigate to see why TPU workload fails to run without
+    # execution permission, such as granting 766 to log file. Check if it's a
+    # must and see if there's a workaround to grant minimum permission. 
+    sudo chmod 777 /tmp/tpu_logs;
+    {% endif %}
 
 # Format: `REMOTE_PATH : LOCAL_PATH`
 file_mounts: {

sky/utils/subprocess_utils.py

@@ -2,9 +2,10 @@
 from multiprocessing import pool
 import os
 import random
+import resource
 import subprocess
 import time
-from typing import Any, Callable, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import colorama
 import psutil
@@ -18,6 +19,8 @@ from sky.utils import ux_utils
 
 logger = sky_logging.init_logger(__name__)
 
+_fd_limit_warning_shown = False
+
 
 @timeline.event
 def run(cmd, **kwargs):
@@ -43,12 +46,54 @@ def run_no_outputs(cmd, **kwargs):
                **kwargs)
 
 
-def get_parallel_threads() -> int:
-    """Returns the number of idle CPUs."""
+def _get_thread_multiplier(cloud_str: Optional[str] = None) -> int:
+    # If using Kubernetes, we use 4x the number of cores.
+    if cloud_str and cloud_str.lower() == 'kubernetes':
+        return 4
+    return 1
+
+
+def get_max_workers_for_file_mounts(common_file_mounts: Dict[str, str],
+                                    cloud_str: Optional[str] = None) -> int:
+    global _fd_limit_warning_shown
+    fd_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+    # Raise warning for low fd_limit (only once)
+    if fd_limit < 1024 and not _fd_limit_warning_shown:
+        logger.warning(
+            f'Open file descriptor limit ({fd_limit}) is low. File sync to '
+            'remote clusters may be slow. Consider increasing the limit using '
+            '`ulimit -n <number>` or modifying system limits.')
+        _fd_limit_warning_shown = True
+
+    fd_per_rsync = 5
+    for src in common_file_mounts.values():
+        if os.path.isdir(src):
+            # Assume that each file/folder under src takes 5 file descriptors
+            # on average.
+            fd_per_rsync = max(fd_per_rsync, len(os.listdir(src)) * 5)
+
+    # Reserve some file descriptors for the system and other processes
+    fd_reserve = 100
+
+    max_workers = (fd_limit - fd_reserve) // fd_per_rsync
+    # At least 1 worker, and avoid too many workers overloading the system.
+    num_threads = get_parallel_threads(cloud_str)
+    max_workers = min(max(max_workers, 1), num_threads)
+    logger.debug(f'Using {max_workers} workers for file mounts.')
+    return max_workers
+
+
+def get_parallel_threads(cloud_str: Optional[str] = None) -> int:
+    """Returns the number of threads to use for parallel execution.
+
+    Args:
+        cloud_str: The cloud
+    """
     cpu_count = os.cpu_count()
     if cpu_count is None:
         cpu_count = 1
-    return max(4, cpu_count - 1)
+    return max(4, cpu_count - 1) * _get_thread_multiplier(cloud_str)
 
 
 def run_in_parallel(func: Callable,

{skypilot_nightly-1.0.0.dev20241121.dist-info → skypilot_nightly-1.0.0.dev20241122.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241121
+Version: 1.0.0.dev20241122
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20241121.dist-info → skypilot_nightly-1.0.0.dev20241122.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=SCQV-Q-BPSS_Xa37LmEJ5zO8u_tBijR-C7U6m9OAVAQ,5882
+sky/__init__.py,sha256=9rkn0Uy1cjIXtImEc_jmOOF1EEB5YgeBWSYeUlDzDmQ,5882
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=pAdCT60OxxiXI9KXDyP2lQ9u9vMc6aMtq5Xi2h_hbdw,20984
 sky/check.py,sha256=D3Y3saIFAYVvPxuBHnVgJEO0fUVDxgjwuMBaO-D778k,9472
@@ -10,7 +10,7 @@ sky/exceptions.py,sha256=E3C2Ejcc8RUDAUQn7ar_Jr97C_AxD2rKKMmJOfLJ9d0,8965
 sky/execution.py,sha256=k6_n-kY1w93UVJZncQdNjqnil1aFIz5WGVwALNUCMqo,27458
 sky/global_user_state.py,sha256=ob3jvtG_yMPGvLlVScgeJ9pqk3FP4jhfEixw8WzFwho,29682
 sky/optimizer.py,sha256=GjvKQIBtY3NlULzau_9tfa7V2KYVJRrmNrjKVIWCPIQ,59753
-sky/resources.py,sha256=usmB8p7HyzyWHcevQ8HV6eIlukYJ9BC0trFOaE2kNuw,69049
+sky/resources.py,sha256=260JnyzYz0uAYVH8lIfbHjSZUvHne8mfoWmjo_YgeEI,69106
 sky/sky_logging.py,sha256=oLmTmwkuucIto3LHXLJfMcyRpYSkmZAZa5XzQPA5IHk,4434
 sky/skypilot_config.py,sha256=E3g65cX3P3dT9b5N0GgFBG6yB0FXwIGpisKoozmJmWU,9094
 sky/status_lib.py,sha256=J7Jb4_Dz0v2T64ttOdyUgpokvl4S0sBJrMfH7Fvo51A,1457
@@ -30,8 +30,8 @@ sky/adaptors/runpod.py,sha256=4Nt_BfZhJAKQNA3wO8cxvvNI8x4NsDGHu_4EhRDlGYQ,225
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=wwfbrxPhjMPs6PSyy3tAHI8WJhl-xhgzWBsAZjmJJ6g,6249
-sky/backends/backend_utils.py,sha256=vese_H1lOFL40r-LPhjqqM8OlgICDgEW-YtNQ95cYs8,125608
-sky/backends/cloud_vm_ray_backend.py,sha256=aeb45nVxAqiku2AU66CRd9-UK4b9nqoZYSL8z2kYOnI,232706
+sky/backends/backend_utils.py,sha256=_Kx8fp_inwxL8xf4I6HFWUGOZRT9edDM-2osaJaMlP0,126074
+sky/backends/cloud_vm_ray_backend.py,sha256=BDpruXsj-u4wc3WYscLhIbSjjsNZ85iI7fkb-T8f2Bs,233321
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=0JL5m0YUgOmOL4aWEUe4tmt89dsxjk4_WXkPwgEKEis,16801
 sky/backends/wheel_utils.py,sha256=CUVOwlBtQjOMv-RSDGx2jMQ0M1D0w9ZPm0TDafJwBDI,8180
@@ -40,21 +40,21 @@ sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=eb-i6zYoo-Zkod-T9qtCu1FcYLw--Yyos1SyibUPZNE,26194
 sky/clouds/__init__.py,sha256=WuNIJEnZmBO72tU5awgaaL3rdvFRSkgaYNNeuY68dXo,1356
-sky/clouds/aws.py,sha256=um7-lam6BfYYgCvNAMsIY_Gty8wt8cOlCHsu3Ah3Od8,49616
-sky/clouds/azure.py,sha256=38eUcB1_lt5FvDWo-G_pKIIsT1c_bCU2AifEYo7KX9Y,30756
-sky/clouds/cloud.py,sha256=Y_9Hi2DhAbrqMLvb_NFPt--N5V6ua8BgbwV4xIc19KU,35216
+sky/clouds/aws.py,sha256=mGXgTWU-XU8tQ3jkzHN80gu39REWQikHWeeOkTcb5SA,49644
+sky/clouds/azure.py,sha256=KtnnNZn4ZEr7xndBHxX91v0YXSI1QWPgIefuM1zDUBA,30784
+sky/clouds/cloud.py,sha256=NIODnzroLwMA3VJq3DOFt1voUoZJB8aielpDkX61iD0,35240
 sky/clouds/cloud_registry.py,sha256=oLoYFjm_SDTgdHokY7b6A5Utq80HXRQNxV0fLjDdVsQ,2361
-sky/clouds/cudo.py,sha256=UiY273Sln7VOYDYx93yWiWH_RLlOKZ2cm7mA31ld4A8,13094
-sky/clouds/fluidstack.py,sha256=ufve4wXo_VmaEkxqTw2Jnt-DORBDRnqUPU1kB_mD89s,12383
-sky/clouds/gcp.py,sha256=BjCehW3s0IYkRDdEEDm0vYWXQDpOV8KU98OMVRPnQNg,54676
-sky/clouds/ibm.py,sha256=w8bo1EIY_YWYNu0fy-OpAyr6DZviU0tpIXUsiV01rVE,21423
-sky/clouds/kubernetes.py,sha256=EuHD_xl5pbW2_9lPNuNdxss2X4DPbHnnD0VoiereshI,29590
-sky/clouds/lambda_cloud.py,sha256=ExL_uixdFrF9qSL5JYXpaOXCZ9_eOA2q444kcmBHBXQ,12644
-sky/clouds/oci.py,sha256=KAu32FFM3rB1Ngp8vJmVxxwNpmAmgrzTDV0FIof3MKI,26525
-sky/clouds/paperspace.py,sha256=4cjNua6jpuxmfidvLY4tSRX1oj_QaaHDinPMunGplyU,10868
-sky/clouds/runpod.py,sha256=_4myVdGIvQshkka8fn6mBXHgz5TZqhrNhAEM2_HrCT8,11487
-sky/clouds/scp.py,sha256=NivPvzQxA90R37QR3fgTup8ScGfxKsXAhH0xklAj5QU,15817
-sky/clouds/vsphere.py,sha256=ZzlcQBzv0aaRYXwZOrdKIGFK94LaOfDSV3lJBg9xyc4,12256
+sky/clouds/cudo.py,sha256=mglcsoYhdWwgkVgFcoZLE0M_UCXOJkvW6nITufPd_BQ,13118
+sky/clouds/fluidstack.py,sha256=u2I6jXEtTqgqRWi2EafMsKqc8VkUq1cR6CSDUvk72_U,12407
+sky/clouds/gcp.py,sha256=RH3MMdgR3FWPiwm7rFrJ5oebtTcSUVoh7UfQMc_6U4A,54704
+sky/clouds/ibm.py,sha256=0ArRTQx1_DpTNGByFhukzFedEDzmVjBsGiiques1bQ0,21447
+sky/clouds/kubernetes.py,sha256=Kt2ymUvxE5FgoYx5V-shPHX28mCidj9uJtS1p3X0Yu4,31412
+sky/clouds/lambda_cloud.py,sha256=42AmcN2X_wdBMuAw606nR_pQCBAy5QFiAo711_WRqDE,12672
+sky/clouds/oci.py,sha256=OzGWoU3DiMbFujMQLXgCr94Oqb9EyP0CsM4gMYOeU9s,26553
+sky/clouds/paperspace.py,sha256=0UxOcv_NaLY5hrFoAA_ZienclZUOqzf0yxXXZu4jXG0,10896
+sky/clouds/runpod.py,sha256=UlHFPQY4wGGi0gLDO-vZoeJcgbQTCYXh4Pk8mKQBNUk,11515
+sky/clouds/scp.py,sha256=JHyMqkAAqr9lJq79IVjj3rU1g-ZCCGLZTJEzIhYsw7c,15845
+sky/clouds/vsphere.py,sha256=LzO-Mc-zDgpaDYZxNKGdEFa0eR5DHpTgKsPX60mPi10,12280
 sky/clouds/service_catalog/__init__.py,sha256=cFZ3HLdQVa42xOhK2XxuB_xPIX4X1UR89InR4y2y_78,14757
 sky/clouds/service_catalog/aws_catalog.py,sha256=j33lNC5GXWK6CiGWZORCnumGlRODmCAT2_lfWp0YtBc,13106
 sky/clouds/service_catalog/azure_catalog.py,sha256=5Q51x_WEKvQ2YSgJvZHRH3URlbwIstYuwpjaWW_wJlw,8149
@@ -107,10 +107,10 @@ sky/provision/__init__.py,sha256=rnuwL9x3qQGhX3EYW6KyZWfw_yBqGSiFBDz86T5xus4,633
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
 sky/provision/docker_utils.py,sha256=l4AMzwXGZd8RyNq8AwOaKV9bFSofLYfSyj2NBhkXYsY,19200
-sky/provision/instance_setup.py,sha256=UBuVHjQ7f80un7aLY-kCLR0CCNVx-qIq1bh06nxuwJk,23050
+sky/provision/instance_setup.py,sha256=8Pudbpke6ah0xufr2UwtsDnNZ64-aAYkz8M44ZA0huI,23218
 sky/provision/logging.py,sha256=yZWgejrFBhhRjAtvFu5N5bRXIMK5TuwNjp1vKQqz2pw,2103
 sky/provision/metadata_utils.py,sha256=LrxeV4wD2QPzNdXV_npj8q-pr35FatxBBjF_jSbpOT0,4013
-sky/provision/provisioner.py,sha256=mTvtBjS-Xz64LJcyeHx_-wdM8Gin8D49YRaV_TADaz4,25334
+sky/provision/provisioner.py,sha256=REh4ajMIzFMJu4vAj81Qo9qAbJ7-3_5_IpzjBjo_MH0,29154
 sky/provision/aws/__init__.py,sha256=mxq8PeWJqUtalDozTNpbtENErRZ1ktEs8uf2aG9UUgU,731
 sky/provision/aws/config.py,sha256=dbwulPxXGIJjKJddv85PbtlXOjwLemaD65j3DISNsK0,24214
 sky/provision/aws/instance.py,sha256=eCslJ2XfJo_pkQMnKFQqhGnUIRvwKiT12oxBY5-klss,40750
@@ -137,10 +137,10 @@ sky/provision/gcp/instance_utils.py,sha256=veRBr6Oziv0KaUdC4acuWeaOremNV0gMYCCHa
 sky/provision/gcp/mig_utils.py,sha256=oFpcFZoapHMILSE4iIm8V5bxP1RhbMHRF7cciqq8qAk,7883
 sky/provision/kubernetes/__init__.py,sha256=y6yVfii81WYG3ROxv4hiIj-ydinS5-xGxLvXnARVQoI,719
 sky/provision/kubernetes/config.py,sha256=WEKcFXXhe89bLGAvoMiBvTDxdxkpTIA6ezrj2vmzldc,29072
-sky/provision/kubernetes/instance.py,sha256=BWgIb1bsxFeMOwN7RtE7nDI0aFPU_WtQq7iEuyq51nc,47111
+sky/provision/kubernetes/instance.py,sha256=2zd_Z09amOsi0vPZjQYMJCkCWbN2YecMLL9HkmUuPrM,48414
 sky/provision/kubernetes/network.py,sha256=EpNjRQ131CXepqbdkoRKFu4szVrm0oKEpv1l8EgOkjU,12364
 sky/provision/kubernetes/network_utils.py,sha256=t1FS3K400fetH7cBuRgQJZl5_jEeMshsvsYmnMUcq8k,11399
-sky/provision/kubernetes/utils.py,sha256=4Gqn66jE0Nzs4UBjajfUM4E4mXWYnYzdhLz6SQG44Kg,101244
+sky/provision/kubernetes/utils.py,sha256=5nzyD2d5sroM1ajB6M40IycuZX5OYiwzpb4hd4U6eS8,101291
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
@@ -194,7 +194,7 @@ sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=JPDHmByuhoNYXSUHl-OnyeJUkOFWn7gDM1FrS7Kr3E8,4478
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=sQFbGSSiB4oAicMA_ctbtIo8FzaUSO1FGyQbMcuEnFA,14468
+sky/skylet/constants.py,sha256=MtE7Tj3m2Fn_IMxifyxOxIT3KTM6tanszqNnU_tlop8,14872
 sky/skylet/events.py,sha256=A09E7LmmwzcGrSG0n8K7d3EZ1ZJr1mmmzoGyhnArYJA,12303
 sky/skylet/job_lib.py,sha256=Lfk3h-NLoi9mLo5-BPur43fUHVKMpa2hR5wEOCZGsB4,43846
 sky/skylet/log_lib.py,sha256=tFJXRb-DClFhUh6wQ45cSBNtBkdOZCGlVfwDnqKFRgE,20394
@@ -228,7 +228,7 @@ sky/templates/jobs-controller.yaml.j2,sha256=Gu3ogFxFYr09VEXP-6zEbrCUOFo1aYxWEjA
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
 sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8wyVEbRNFHRsBuLsw,626
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=HlG7CPBBedCVBlL9qv0erW_eKm6Irj0LFyaAWuJW_lc,3148
-sky/templates/kubernetes-ray.yml.j2,sha256=w7KNEBjj3zbd8RqWJuKwZSpQwMCdUAfUb7I7jNBn0XY,20672
+sky/templates/kubernetes-ray.yml.j2,sha256=uymIxWyu7yHOL_G2XJZp0EiqHYtgbIJe4fZPXU6F1gc,28017
 sky/templates/kubernetes-ssh-jump.yml.j2,sha256=k5W5sOIMppU7dDkJMwPlqsUcb92y7L5_TVG3hkgMy8M,2747
 sky/templates/lambda-ray.yml.j2,sha256=HyvO_tX2vxwSsc4IFVSqGuIbjLMk0bevP9bcxb8ZQII,4498
 sky/templates/local-ray.yml.j2,sha256=FNHeyHF6nW9nU9QLIZceUWfvrFTTcO51KqhTnYCEFaA,1185
@@ -258,7 +258,7 @@ sky/utils/log_utils.py,sha256=oZYF45uC7GFjAqO-Je-aiX6zhtq91TP-KKaIbQNF-jY,14024
 sky/utils/resources_utils.py,sha256=Xqi7gxPYw2y5wl5okUI5zx5LEij0hJF_V3Zi8q7TXYg,7890
 sky/utils/rich_utils.py,sha256=hmnI1X5dKvRIQzB7EyNb34FT97qFNve-0QHqM5r0mVk,3066
 sky/utils/schemas.py,sha256=yz8IKUA2oWJartaranIc9MfDZmZcIybPZUGViw1Ii1Q,29475
-sky/utils/subprocess_utils.py,sha256=4pnt_QPvPuh3ylG5xlr18JlZeF6693h3fmG1uaD8qLo,8669
+sky/utils/subprocess_utils.py,sha256=iLOda3vfkD-sIUPlfkDGZs9HnJWLlLRvHVgca9DZH8s,10410
 sky/utils/timeline.py,sha256=ebHxKJK2HX0utGArrUgSezTPkcwav3VETa_AQS34t-E,3925
 sky/utils/ux_utils.py,sha256=CqyIFGDuSE8fQasPkna_loZMwtboC9KedR09WEQ7qz0,6502
 sky/utils/validator.py,sha256=cAFERCoC7jH0DFKepcU4x9SYmdrYL1iVmW9tXA18hvo,701
@@ -275,9 +275,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7Z
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=hyYDaYSNxYaNvzUQBzC8AidB7nDeojizjkzc_CTxycY,1077
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20241121.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241121.dist-info/METADATA,sha256=2P9MAo9kQQSy6-t-MITf4Az4j2ucbnnlN6rE1V4aVxs,20222
-skypilot_nightly-1.0.0.dev20241121.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-skypilot_nightly-1.0.0.dev20241121.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241121.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241121.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20241122.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20241122.dist-info/METADATA,sha256=zxomzD3shH_UFn3jG4yVym0RY1iIAqt0-9CYfrJauhU,20222
+skypilot_nightly-1.0.0.dev20241122.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+skypilot_nightly-1.0.0.dev20241122.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20241122.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20241122.dist-info/RECORD,,