skypilot-nightly 1.0.0.dev20250916__py3-none-any.whl → 1.0.0.dev20250919__py3-none-any.whl

sky/templates/kubernetes-ray.yml.j2

@@ -823,7 +823,8 @@ available_node_types:
                 fi
                 $(prefix_cmd) cp -p "$FUSERMOUNT_PATH" "${FUSERMOUNT_PATH}-original"
                 $(prefix_cmd) ln -sf {{k8s_fusermount_shared_dir}}/fusermount-shim "$FUSERMOUNT_PATH"
-                FUSERMOUNT3_PATH=$(which fusermount3)
+                # "|| true" because fusermount3 is not always available
+                FUSERMOUNT3_PATH=$(which fusermount3) || true
                 if [ -z "$FUSERMOUNT3_PATH" ]; then
                     FUSERMOUNT3_PATH="${FUSERMOUNT_PATH}3"
                 fi
@@ -872,11 +873,14 @@ available_node_types:
 
                 touch /tmp/apt_ssh_setup_complete
                 echo "=== SSH setup completed ==="
-                ) > /tmp/${STEPS[0]}.log 2>&1 || {
-                  echo "Error: ${STEPS[0]} failed. Continuing anyway..." > /tmp/${STEPS[0]}.failed
+                ) > /tmp/${STEPS[0]}.log 2>&1
+                if [ "$?" -ne "0" ]; then
+                  {
+                  echo "Error: ${STEPS[0]} failed. Continuing anyway..." > /tmp/${STEPS[0]}.failed 2>&1
                   cat /tmp/${STEPS[0]}.log
                   exit 1
-                }
+                  }
+                fi
               ) &
 
               # STEP 2: Install conda, ray and skypilot (for dependencies); start
@@ -931,11 +935,14 @@ available_node_types:
                   set +e
                   {{ ray_worker_start_command }}
                 fi
-                ) > /tmp/${STEPS[1]}.log 2>&1 || {
-                  echo "Error: ${STEPS[1]} failed. Continuing anyway..." > /tmp/${STEPS[1]}.failed
+                ) > /tmp/${STEPS[1]}.log 2>&1
+                if [ "$?" -ne "0" ]; then
+                  {
+                  echo "Error: ${STEPS[1]} failed. Continuing anyway..." > /tmp/${STEPS[1]}.failed 2>&1
                   cat /tmp/${STEPS[1]}.log
                   exit 1
-                }
+                  }
+                fi
               ) &
 
 
@@ -953,11 +960,14 @@ available_node_types:
                   fi;
                 fi;
                 export -p > ~/container_env_var.sh && $(prefix_cmd) mv ~/container_env_var.sh /etc/profile.d/container_env_var.sh
-                ) > /tmp/${STEPS[2]}.log 2>&1 || {
-                  echo "Error: ${STEPS[2]} failed. Continuing anyway..." > /tmp/${STEPS[2]}.failed
+                ) > /tmp/${STEPS[2]}.log 2>&1
+                if [ "$?" -ne "0" ]; then
+                  {
+                  echo "Error: ${STEPS[2]} failed. Continuing anyway..." > /tmp/${STEPS[2]}.failed 2>&1
                   cat /tmp/${STEPS[2]}.log
                   exit 1
-                }
+                  }
+                fi
               ) &
 
               function mylsof { p=$(for pid in /proc/{0..9}*; do i=$(basename "$pid"); for file in "$pid"/fd/*; do link=$(readlink -e "$file"); if [ "$link" = "$1" ]; then echo "$i"; fi; done; done); echo "$p"; };
@@ -1324,7 +1334,7 @@ setup_commands:
     if [ -f /tmp/apt_ssh_setup_started ]; then
       echo "=== Logs for asynchronous SSH setup ===";
       [ -f /tmp/apt_ssh_setup_complete ] && cat /tmp/${STEPS[0]}.log ||
-      { tail -f -n +1 /tmp/${STEPS[0]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/apt_ssh_setup_complete ]; do sleep 0.5; done; kill $TAIL_PID || true; };
+      { tail -f -n +1 /tmp/${STEPS[0]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/apt_ssh_setup_complete ] || [ -f /tmp/${STEPS[0]}.failed ]; do sleep 0.5; done; kill $TAIL_PID || true; };
       [ -f /tmp/${STEPS[0]}.failed ] && { echo "Error: ${STEPS[0]} failed. Exiting."; exit 1; } || true;
     fi
 
@@ -1332,7 +1342,7 @@ setup_commands:
     if [ -f /tmp/skypilot_is_nimbus ]; then
       echo "=== Logs for asynchronous ray and skypilot installation ===";
       [ -f /tmp/ray_skypilot_installation_complete ] && cat /tmp/${STEPS[1]}.log ||
-      { tail -f -n +1 /tmp/${STEPS[1]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/ray_skypilot_installation_complete ]; do sleep 0.5; done; kill $TAIL_PID || true; };
+      { tail -f -n +1 /tmp/${STEPS[1]}.log & TAIL_PID=$!; echo "Tail PID: $TAIL_PID"; until [ -f /tmp/ray_skypilot_installation_complete ] || [ -f /tmp/${STEPS[1]}.failed ]; do sleep 0.5; done; kill $TAIL_PID || true; };
       [ -f /tmp/${STEPS[1]}.failed ] && { echo "Error: ${STEPS[1]} failed. Exiting."; exit 1; } || true;
     fi
     end_epoch=$(date +%s);

sky/templates/primeintellect-ray.yml.j2

@@ -0,0 +1,71 @@
+cluster_name: {{cluster_name_on_cloud}}
+
+# The maximum number of workers nodes to launch in addition to the head node.
+max_workers: {{num_nodes - 1}}
+upscaling_speed: {{num_nodes - 1}}
+idle_timeout_minutes: 60
+
+provider:
+  type: external
+  module: sky.provision.primeintellect
+  region: "{{region}}"
+  zones: "{{zones}}"
+
+auth:
+  ssh_user: skypilot:ssh_user
+  ssh_private_key: {{ssh_private_key}}
+
+available_node_types:
+  ray_head_default:
+    resources: {}
+    node_config:
+      InstanceType: {{instance_type}}
+      DiskSize: {{disk_size}}
+      ImageId: {{image_id}}
+      PublicKey: |-
+        skypilot:ssh_public_key_content
+
+head_node_type: ray_head_default
+
+# Format: `REMOTE_PATH : LOCAL_PATH`
+file_mounts: {
+  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
+  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
+{%- for remote_path, local_path in credentials.items() %}
+  "{{remote_path}}": "{{local_path}}",
+  "~/.ssh/sky-cluster-key": "{{ssh_private_key}}",
+{%- endfor %}
+}
+
+rsync_exclude: []
+
+initialization_commands: []
+
+# List of shell commands to run to set up nodes.
+# NOTE: these are very performance-sensitive. Each new item opens/closes an SSH
+# connection, which is expensive. Try your best to co-locate commands into fewer
+# items!
+#
+# Increment the following for catching performance bugs easier:
+#   current num items (num SSH connections): 1
+setup_commands:
+  # Disable unattended-upgrades and handle apt-get locks
+  # Install patch utility for Ray
+  # Install conda and Ray
+  # Set system limits for Ray performance (nofile and TasksMax)
+  - {%- for initial_setup_command in initial_setup_commands %}
+    {{ initial_setup_command }}
+    {%- endfor %}
+    sudo systemctl stop unattended-upgrades || true;
+    sudo systemctl disable unattended-upgrades || true;
+    sudo sed -i 's/Unattended-Upgrade "1"/Unattended-Upgrade "0"/g' /etc/apt/apt.conf.d/20auto-upgrades || true;
+    sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
+    sudo pkill -9 apt-get;
+    sudo pkill -9 dpkg;
+    sudo dpkg --configure -a;
+    which patch > /dev/null || sudo apt install -y patch;
+    {{ conda_installation_commands }}
+    {{ ray_skypilot_installation_commands }}
+    sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
+    sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
+    {{ ssh_max_sessions_config }}

sky/utils/locks.py

@@ -11,6 +11,7 @@ import time
 from typing import Any, Optional
 
 import filelock
+import psycopg2
 import sqlalchemy
 
 from sky import global_user_state
@@ -197,6 +198,7 @@ class PostgresLock(DistributedLock):
         if engine.dialect.name != db_utils.SQLAlchemyDialect.POSTGRESQL.value:
             raise ValueError('PostgresLock requires PostgreSQL database. '
                              f'Current dialect: {engine.dialect.name}')
+        # Borrow a dedicated connection from the pool.
         return engine.raw_connection()
 
     def acquire(self, blocking: bool = True) -> AcquireReturnProxy:
@@ -233,9 +235,7 @@ class PostgresLock(DistributedLock):
                 time.sleep(self.poll_interval)
 
         except Exception:
-            if self._connection:
-                self._connection.close()
-                self._connection = None
+            self._close_connection()
             raise
 
     def release(self) -> None:
@@ -248,27 +248,58 @@ class PostgresLock(DistributedLock):
             cursor.execute('SELECT pg_advisory_unlock(%s)', (self._lock_key,))
             self._connection.commit()
             self._acquired = False
+        except psycopg2.OperationalError as e:
+            # Lost connection to the database, likely the lock is force unlocked
+            # by other routines.
+            logger.debug(f'Failed to release postgres lock {self.lock_id}: {e}')
         finally:
-            if self._connection:
-                self._connection.close()
-                self._connection = None
+            self._close_connection()
 
     def force_unlock(self) -> None:
         """Force unlock the postgres advisory lock."""
         try:
-            if not self._connection:
+            # The lock is held by current routine, gracefully unlock it
+            if self._acquired:
+                self.release()
+                return
+
+            # The lock is held by another routine, force unlock it.
+            if self._connection is None:
                 self._connection = self._get_connection()
             cursor = self._connection.cursor()
             cursor.execute('SELECT pg_advisory_unlock(%s)', (self._lock_key,))
-            self._connection.commit()
+            result = cursor.fetchone()[0]
+            if result:
+                # The lock is held by current routine and unlock suceed
+                self._connection.commit()
+                self._acquired = False
+                return
+            cursor.execute(
+                ('SELECT pid FROM pg_locks WHERE locktype = \'advisory\' '
+                 'AND ((classid::bigint << 32) | objid::bigint) = %s'),
+                (self._lock_key,))
+            row = cursor.fetchone()
+            if row:
+                # The lock is still held by another routine, false unlock it
+                # by killing the PG connection of that routine.
+                cursor.execute('SELECT pg_terminate_backend(%s)', (row[0],))
+                self._connection.commit()
+                return
         except Exception as e:
             raise RuntimeError(
                 f'Failed to force unlock postgres lock {self.lock_id}: {e}'
             ) from e
         finally:
-            if self._connection:
+            self._close_connection()
+
+    def _close_connection(self) -> None:
+        """Close the postgres connection."""
+        if self._connection:
+            try:
                 self._connection.close()
-                self._connection = None
+            except Exception as e:  # pylint: disable=broad-except
+                logger.debug(f'Failed to close postgres connection: {e}')
+            self._connection = None
 
     def is_locked(self) -> bool:
         """Check if the postgres advisory lock is acquired."""

{skypilot_nightly-1.0.0.dev20250916.dist-info → skypilot_nightly-1.0.0.dev20250919.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250916
+Version: 1.0.0.dev20250919
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -121,6 +121,7 @@ Provides-Extra: fluidstack
 Provides-Extra: cudo
 Requires-Dist: cudo-compute>=0.1.10; extra == "cudo"
 Provides-Extra: paperspace
+Provides-Extra: primeintellect
 Provides-Extra: do
 Requires-Dist: pydo>=0.3.0; extra == "do"
 Requires-Dist: azure-core>=1.24.0; extra == "do"
@@ -151,49 +152,49 @@ Requires-Dist: grpcio>=1.63.0; extra == "server"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "server"
 Requires-Dist: aiosqlite; extra == "server"
 Provides-Extra: all
-Requires-Dist: boto3>=1.26.1; extra == "all"
-Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: msgraph-sdk; extra == "all"
-Requires-Dist: pyjwt; extra == "all"
+Requires-Dist: docker; extra == "all"
 Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
-Requires-Dist: passlib; extra == "all"
-Requires-Dist: ecsapi>=0.2.0; extra == "all"
-Requires-Dist: azure-common; extra == "all"
-Requires-Dist: grpcio>=1.63.0; extra == "all"
-Requires-Dist: ibm-vpc; extra == "all"
-Requires-Dist: cudo-compute>=0.1.10; extra == "all"
-Requires-Dist: azure-core>=1.24.0; extra == "all"
-Requires-Dist: ibm-cos-sdk; extra == "all"
-Requires-Dist: azure-identity>=1.19.0; extra == "all"
+Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
+Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
 Requires-Dist: nebius>=0.2.47; extra == "all"
+Requires-Dist: azure-core>=1.24.0; extra == "all"
+Requires-Dist: ibm-cos-sdk; extra == "all"
+Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
+Requires-Dist: boto3>=1.26.1; extra == "all"
+Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
+Requires-Dist: azure-core>=1.31.0; extra == "all"
+Requires-Dist: passlib; extra == "all"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
-Requires-Dist: msrestazure; extra == "all"
 Requires-Dist: oci; extra == "all"
-Requires-Dist: azure-core>=1.31.0; extra == "all"
-Requires-Dist: ray[default]>=2.6.1; extra == "all"
+Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
 Requires-Dist: google-cloud-storage; extra == "all"
-Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
+Requires-Dist: sqlalchemy_adapter; extra == "all"
+Requires-Dist: runpod>=1.6.1; extra == "all"
+Requires-Dist: azure-common; extra == "all"
 Requires-Dist: ibm-cloud-sdk-core; extra == "all"
 Requires-Dist: awscli>=1.27.10; extra == "all"
-Requires-Dist: aiosqlite; extra == "all"
-Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: websockets; extra == "all"
-Requires-Dist: sqlalchemy_adapter; extra == "all"
-Requires-Dist: python-dateutil; extra == "all"
-Requires-Dist: azure-cli>=2.65.0; extra == "all"
-Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
-Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
-Requires-Dist: aiohttp; extra == "all"
-Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
+Requires-Dist: ray[default]>=2.6.1; extra == "all"
+Requires-Dist: msgraph-sdk; extra == "all"
+Requires-Dist: ecsapi>=0.2.0; extra == "all"
 Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
-Requires-Dist: pydo>=0.3.0; extra == "all"
-Requires-Dist: runpod>=1.6.1; extra == "all"
 Requires-Dist: botocore>=1.29.10; extra == "all"
+Requires-Dist: ibm-vpc; extra == "all"
+Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
+Requires-Dist: cudo-compute>=0.1.10; extra == "all"
+Requires-Dist: colorama<0.4.5; extra == "all"
+Requires-Dist: azure-cli>=2.65.0; extra == "all"
+Requires-Dist: python-dateutil; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: aiosqlite; extra == "all"
 Requires-Dist: anyio; extra == "all"
+Requires-Dist: pyjwt; extra == "all"
+Requires-Dist: websockets; extra == "all"
+Requires-Dist: azure-identity>=1.19.0; extra == "all"
+Requires-Dist: grpcio>=1.63.0; extra == "all"
+Requires-Dist: msrestazure; extra == "all"
 Requires-Dist: casbin; extra == "all"
+Requires-Dist: pydo>=0.3.0; extra == "all"
 Dynamic: author
 Dynamic: classifier
 Dynamic: description
@@ -291,12 +292,12 @@ SkyPilot supports your existing GPU, TPU, and CPU workloads, with no code change
 Install with pip:
 ```bash
 # Choose your clouds:
-pip install -U "skypilot[kubernetes,aws,gcp,azure,oci,nebius,lambda,runpod,fluidstack,paperspace,cudo,ibm,scp]"
+pip install -U "skypilot[kubernetes,aws,gcp,azure,oci,nebius,lambda,runpod,fluidstack,paperspace,cudo,ibm,scp,seeweb]"
 ```
 To get the latest features and fixes, use the nightly build or [install from source](https://docs.skypilot.co/en/latest/getting-started/installation.html):
 ```bash
 # Choose your clouds:
-pip install "skypilot-nightly[kubernetes,aws,gcp,azure,oci,nebius,lambda,runpod,fluidstack,paperspace,cudo,ibm,scp]"
+pip install "skypilot-nightly[kubernetes,aws,gcp,azure,oci,nebius,lambda,runpod,fluidstack,paperspace,cudo,ibm,scp,seeweb]"
 ```
 
 <p align="center">
@@ -305,7 +306,7 @@ pip install "skypilot-nightly[kubernetes,aws,gcp,azure,oci,nebius,lambda,runpod,
 
 Current supported infra: Kubernetes, AWS, GCP, Azure, OCI, Nebius, Lambda Cloud, RunPod, Fluidstack,
 Cudo, Digital Ocean, Paperspace, Cloudflare, Samsung, IBM, Vast.ai,
-VMware vSphere.
+VMware vSphere, Seeweb.
 <p align="center">
   <img alt="SkyPilot" src="https://raw.githubusercontent.com/skypilot-org/skypilot/master/docs/source/images/cloud-logos-light.png" width=85%>
 </p>