skypilot-nightly 1.0.0.dev20251210__py3-none-any.whl → 1.0.0.dev20260112__py3-none-any.whl

sky/volumes/server/core.py

@@ -30,6 +30,10 @@ def volume_refresh():
     volumes = volume_list(is_ephemeral=False)
     for volume in volumes:
         volume_name = volume.name
+        if volume.usedby_fetch_failed:
+            logger.info(f'Skipping status update for volume {volume_name} '
+                        f'due to failed usedby fetch')
+            continue
         usedby_pods = volume.usedby_pods
         with _volume_lock(volume_name):
             latest_volume = global_user_state.get_volume_by_name(volume_name)
@@ -55,6 +59,9 @@ def volume_list(
         is_ephemeral: Optional[bool] = None) -> List[responses.VolumeRecord]:
     """Gets the volumes.
 
+    Args:
+        is_ephemeral: Whether to include ephemeral volumes.
+
     Returns:
         [
             {
@@ -74,6 +81,7 @@ def volume_list(
                 'status': sky.VolumeStatus,
                 'usedby_pods': List[str],
                 'usedby_clusters': List[str],
+                'usedby_fetch_failed': bool,
                 'is_ephemeral': bool,
             }
         ]
@@ -93,11 +101,23 @@ def volume_list(
             cloud_to_configs[cloud].append(config)
 
         cloud_to_used_by_pods, cloud_to_used_by_clusters = {}, {}
+        cloud_to_failed_volume_names = {}
         for cloud, configs in cloud_to_configs.items():
-            used_by_pods, used_by_clusters = provision.get_all_volumes_usedby(
-                cloud, configs)
-            cloud_to_used_by_pods[cloud] = used_by_pods
-            cloud_to_used_by_clusters[cloud] = used_by_clusters
+            try:
+                used_by_pods, used_by_clusters, failed_volume_names = (
+                    provision.get_all_volumes_usedby(cloud, configs))
+                cloud_to_used_by_pods[cloud] = used_by_pods
+                cloud_to_used_by_clusters[cloud] = used_by_clusters
+                cloud_to_failed_volume_names[cloud] = failed_volume_names
+            except Exception as e:  # pylint: disable=broad-except
+                logger.warning(
+                    f'Failed to get usedby info for volumes on {cloud}: {e}')
+                cloud_to_used_by_pods[cloud] = {}
+                cloud_to_used_by_clusters[cloud] = {}
+                cloud_to_failed_volume_names[cloud] = {
+                    config.name for config in configs
+                }
+                continue
 
         all_users = global_user_state.get_all_users()
         user_map = {user.id: user.name for user in all_users}
@@ -114,6 +134,7 @@ def volume_list(
                 'last_use': volume.get('last_use'),
                 'usedby_pods': [],
                 'usedby_clusters': [],
+                'usedby_fetch_failed': False,
                 'is_ephemeral': volume.get('is_ephemeral', False),
             }
             status = volume.get('status')
@@ -126,12 +147,17 @@ def volume_list(
                 logger.warning(f'Volume {volume_name} has no handle.')
                 continue
             cloud = config.cloud
-            usedby_pods, usedby_clusters = provision.map_all_volumes_usedby(
-                cloud,
-                cloud_to_used_by_pods[cloud],
-                cloud_to_used_by_clusters[cloud],
-                config,
-            )
+            if volume_name in cloud_to_failed_volume_names[cloud]:
+                record['usedby_fetch_failed'] = True
+            else:
+                usedby_pods, usedby_clusters = provision.map_all_volumes_usedby(
+                    cloud,
+                    cloud_to_used_by_pods[cloud],
+                    cloud_to_used_by_clusters[cloud],
+                    config,
+                )
+                record['usedby_pods'] = usedby_pods
+                record['usedby_clusters'] = usedby_clusters
             record['type'] = config.type
             record['cloud'] = config.cloud
             record['region'] = config.region
@@ -139,18 +165,20 @@ def volume_list(
             record['size'] = config.size
             record['config'] = config.config
             record['name_on_cloud'] = config.name_on_cloud
-            record['usedby_pods'] = usedby_pods
-            record['usedby_clusters'] = usedby_clusters
             records.append(responses.VolumeRecord(**record))
         return records
 
 
-def volume_delete(names: List[str], ignore_not_found: bool = False) -> None:
+def volume_delete(names: List[str],
+                  ignore_not_found: bool = False,
+                  purge: bool = False) -> None:
     """Deletes volumes.
 
     Args:
         names: List of volume names to delete.
         ignore_not_found: If True, ignore volumes that are not found.
+        purge: If True, delete the volume from the database even if the
+          deletion API fails.
 
     Raises:
         ValueError: If the volume does not exist
@@ -167,22 +195,32 @@ def volume_delete(names: List[str], ignore_not_found: bool = False) -> None:
             if config is None:
                 raise ValueError(f'Volume {name} has no handle.')
             cloud = config.cloud
-            usedby_pods, usedby_clusters = provision.get_volume_usedby(
-                cloud, config)
-            if usedby_clusters:
-                usedby_clusters_str = ', '.join(usedby_clusters)
-                cluster_str = 'clusters' if len(
-                    usedby_clusters) > 1 else 'cluster'
-                raise ValueError(f'Volume {name} is used by {cluster_str}'
-                                 f' {usedby_clusters_str}.')
-            if usedby_pods:
-                usedby_pods_str = ', '.join(usedby_pods)
-                pod_str = 'pods' if len(usedby_pods) > 1 else 'pod'
-                raise ValueError(
-                    f'Volume {name} is used by {pod_str} {usedby_pods_str}.')
+            if not purge:
+                usedby_pods, usedby_clusters = provision.get_volume_usedby(
+                    cloud, config)
+                if usedby_clusters:
+                    usedby_clusters_str = ', '.join(usedby_clusters)
+                    cluster_str = 'clusters' if len(
+                        usedby_clusters) > 1 else 'cluster'
+                    raise ValueError(f'Volume {name} is used by {cluster_str}'
+                                     f' {usedby_clusters_str}.')
+                if usedby_pods:
+                    usedby_pods_str = ', '.join(usedby_pods)
+                    pod_str = 'pods' if len(usedby_pods) > 1 else 'pod'
+                    raise ValueError(
+                        f'Volume {name} is used by {pod_str} {usedby_pods_str}.'
+                    )
             logger.debug(f'Deleting volume {name} with config {config}')
             with _volume_lock(name):
-                provision.delete_volume(cloud, config)
+                try:
+                    provision.delete_volume(cloud, config)
+                except Exception as e:  # pylint: disable=broad-except
+                    if purge:
+                        logger.warning(f'Failed to delete volume {name} '
+                                       f'on {cloud}: {e}. Purging from '
+                                       'database.')
+                    else:
+                        raise
                 global_user_state.delete_volume(name)
         logger.info(f'Deleted volumes: {names}')
 

sky_templates/ray/start_cluster

@@ -77,14 +77,18 @@ if ! run_ray --version > /dev/null; then
 fi
 echo -e "${GREEN}Ray $(run_ray --version | cut -d' ' -f3) is installed.${NC}"
 
-RAY_ADDRESS="127.0.0.1:${RAY_HEAD_PORT}"
+LOCAL_RAY_ADDRESS="127.0.0.1:${RAY_HEAD_PORT}"
+RAY_ADDRESS=${LOCAL_RAY_ADDRESS}
 if [ "${SKYPILOT_NODE_RANK}" -ne 0 ]; then
     HEAD_IP=$(echo "${SKYPILOT_NODE_IPS}" | head -n1)
     RAY_ADDRESS="${HEAD_IP}:${RAY_HEAD_PORT}"
 fi
 
-# Check if user-space Ray is already running
-if run_ray status --address="${RAY_ADDRESS}" &> /dev/null; then
+# Check if user-space Ray is already running. Use local address to check, as
+# if we use the head node address, the check will succeed even if the Ray
+# cluster is started on the head node but not started on the current worker
+# node.
+if run_ray status --address="${LOCAL_RAY_ADDRESS}" &> /dev/null; then
     echo -e "${YELLOW}Ray cluster is already running.${NC}"
     run_ray status --address="${RAY_ADDRESS}"
     exit 0
@@ -140,7 +144,7 @@ if [ "${SKYPILOT_NODE_RANK}" -eq 0 ]; then
                 echo -e "${RED}Error: Timeout waiting for nodes.${NC}" >&2
                 exit 1
             fi
-            ready_nodes=$(run_ray list nodes --format=json | python3 -c "import sys, json; print(len(json.load(sys.stdin)))")
+            ready_nodes=$(run_ray list nodes --address="${RAY_ADDRESS}" --format=json | python3 -c "import sys, json; print(len(json.load(sys.stdin)))")
             if [ "${ready_nodes}" -ge "${SKYPILOT_NUM_NODES}" ]; then
                 break
             fi

{skypilot_nightly-1.0.0.dev20251210.dist-info → skypilot_nightly-1.0.0.dev20260112.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20251210
+Version: 1.0.0.dev20260112
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -73,7 +73,7 @@ Provides-Extra: aws
 Requires-Dist: awscli>=1.27.10; extra == "aws"
 Requires-Dist: botocore>=1.29.10; extra == "aws"
 Requires-Dist: boto3>=1.26.1; extra == "aws"
-Requires-Dist: colorama<0.4.5; extra == "aws"
+Requires-Dist: colorama<0.4.7; extra == "aws"
 Requires-Dist: casbin; extra == "aws"
 Requires-Dist: sqlalchemy_adapter; extra == "aws"
 Requires-Dist: passlib; extra == "aws"
@@ -161,7 +161,7 @@ Provides-Extra: cloudflare
 Requires-Dist: awscli>=1.27.10; extra == "cloudflare"
 Requires-Dist: botocore>=1.29.10; extra == "cloudflare"
 Requires-Dist: boto3>=1.26.1; extra == "cloudflare"
-Requires-Dist: colorama<0.4.5; extra == "cloudflare"
+Requires-Dist: colorama<0.4.7; extra == "cloudflare"
 Requires-Dist: casbin; extra == "cloudflare"
 Requires-Dist: sqlalchemy_adapter; extra == "cloudflare"
 Requires-Dist: passlib; extra == "cloudflare"
@@ -176,7 +176,7 @@ Provides-Extra: coreweave
 Requires-Dist: awscli>=1.27.10; extra == "coreweave"
 Requires-Dist: botocore>=1.29.10; extra == "coreweave"
 Requires-Dist: boto3>=1.26.1; extra == "coreweave"
-Requires-Dist: colorama<0.4.5; extra == "coreweave"
+Requires-Dist: colorama<0.4.7; extra == "coreweave"
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "coreweave"
 Requires-Dist: websockets; extra == "coreweave"
 Requires-Dist: python-dateutil; extra == "coreweave"
@@ -245,6 +245,7 @@ Requires-Dist: greenlet; extra == "ssh"
 Provides-Extra: runpod
 Requires-Dist: runpod>=1.6.1; extra == "runpod"
 Requires-Dist: tomli; extra == "runpod"
+Requires-Dist: pycares<5; extra == "runpod"
 Requires-Dist: casbin; extra == "runpod"
 Requires-Dist: sqlalchemy_adapter; extra == "runpod"
 Requires-Dist: passlib; extra == "runpod"
@@ -345,7 +346,7 @@ Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "nebius"
 Requires-Dist: awscli>=1.27.10; extra == "nebius"
 Requires-Dist: botocore>=1.29.10; extra == "nebius"
 Requires-Dist: boto3>=1.26.1; extra == "nebius"
-Requires-Dist: colorama<0.4.5; extra == "nebius"
+Requires-Dist: colorama<0.4.7; extra == "nebius"
 Requires-Dist: casbin; extra == "nebius"
 Requires-Dist: sqlalchemy_adapter; extra == "nebius"
 Requires-Dist: passlib; extra == "nebius"
@@ -391,6 +392,7 @@ Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "shadeform"
 Requires-Dist: aiosqlite; extra == "shadeform"
 Requires-Dist: greenlet; extra == "shadeform"
 Provides-Extra: slurm
+Requires-Dist: python-hostlist; extra == "slurm"
 Requires-Dist: casbin; extra == "slurm"
 Requires-Dist: sqlalchemy_adapter; extra == "slurm"
 Requires-Dist: passlib; extra == "slurm"
@@ -402,51 +404,53 @@ Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "slurm"
 Requires-Dist: aiosqlite; extra == "slurm"
 Requires-Dist: greenlet; extra == "slurm"
 Provides-Extra: all
-Requires-Dist: ibm-cloud-sdk-core; extra == "all"
-Requires-Dist: azure-core>=1.24.0; extra == "all"
-Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
-Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: sqlalchemy_adapter; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: tomli; extra == "all"
+Requires-Dist: ecsapi==0.4.0; extra == "all"
 Requires-Dist: msgraph-sdk; extra == "all"
-Requires-Dist: greenlet; extra == "all"
-Requires-Dist: oci; extra == "all"
-Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
-Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
-Requires-Dist: pyjwt; extra == "all"
-Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
+Requires-Dist: azure-cli>=2.65.0; extra == "all"
+Requires-Dist: python-dateutil; extra == "all"
 Requires-Dist: ray[default]>=2.6.1; extra == "all"
-Requires-Dist: runpod>=1.6.1; extra == "all"
-Requires-Dist: docker; extra == "all"
+Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
+Requires-Dist: pydo>=0.3.0; extra == "all"
+Requires-Dist: google-cloud-storage; extra == "all"
 Requires-Dist: azure-identity>=1.19.0; extra == "all"
-Requires-Dist: python-dateutil; extra == "all"
+Requires-Dist: grpcio>=1.63.0; extra == "all"
+Requires-Dist: colorama<0.4.7; extra == "all"
+Requires-Dist: boto3>=1.26.1; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: sqlalchemy_adapter; extra == "all"
+Requires-Dist: anyio; extra == "all"
+Requires-Dist: pyjwt; extra == "all"
+Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
+Requires-Dist: oci; extra == "all"
+Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
+Requires-Dist: websockets; extra == "all"
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
-Requires-Dist: azure-core>=1.31.0; extra == "all"
+Requires-Dist: ibm-cloud-sdk-core; extra == "all"
+Requires-Dist: runpod>=1.6.1; extra == "all"
+Requires-Dist: azure-core>=1.24.0; extra == "all"
 Requires-Dist: passlib; extra == "all"
-Requires-Dist: awscli>=1.27.10; extra == "all"
+Requires-Dist: ibm-vpc; extra == "all"
+Requires-Dist: nebius>=0.3.12; extra == "all"
 Requires-Dist: cudo-compute>=0.1.10; extra == "all"
-Requires-Dist: boto3>=1.26.1; extra == "all"
-Requires-Dist: botocore>=1.29.10; extra == "all"
-Requires-Dist: websockets; extra == "all"
-Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
-Requires-Dist: azure-cli>=2.65.0; extra == "all"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
+Requires-Dist: awscli>=1.27.10; extra == "all"
+Requires-Dist: pycares<5; extra == "all"
+Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
+Requires-Dist: greenlet; extra == "all"
+Requires-Dist: azure-core>=1.31.0; extra == "all"
 Requires-Dist: msrestazure; extra == "all"
+Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
+Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
 Requires-Dist: ibm-cos-sdk; extra == "all"
-Requires-Dist: grpcio>=1.63.0; extra == "all"
-Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
-Requires-Dist: azure-common; extra == "all"
-Requires-Dist: aiohttp; extra == "all"
-Requires-Dist: nebius>=0.3.12; extra == "all"
-Requires-Dist: ibm-vpc; extra == "all"
+Requires-Dist: python-hostlist; extra == "all"
+Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
+Requires-Dist: botocore>=1.29.10; extra == "all"
+Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
 Requires-Dist: casbin; extra == "all"
-Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
-Requires-Dist: tomli; extra == "all"
-Requires-Dist: ecsapi==0.4.0; extra == "all"
-Requires-Dist: pydo>=0.3.0; extra == "all"
-Requires-Dist: google-cloud-storage; extra == "all"
-Requires-Dist: anyio; extra == "all"
 Requires-Dist: aiosqlite; extra == "all"
+Requires-Dist: azure-common; extra == "all"
 Provides-Extra: remote
 Requires-Dist: grpcio>=1.63.0; extra == "remote"
 Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "remote"
@@ -496,7 +500,7 @@ Dynamic: summary
 </p>
 
 <h3 align="center">
-    Simplify & scale any AI infrastructure
+    Run AI on Any Infrastructure
 </h3>
 
 <div align="center">
@@ -506,10 +510,18 @@ Dynamic: summary
 </div>
 
 
+SkyPilot is a system to run, manage, and scale AI workloads on any AI infrastructure.
+
+SkyPilot gives **AI teams** a simple interface to run jobs on any infra.
+**Infra teams** get a unified control plane to manage any AI compute — with advanced scheduling, scaling, and orchestration.
+
+<img src="./docs/source/images/skypilot-abstractions-long-2.png" alt="SkyPilot Abstractions">
 
-----
+-----
 
 :fire: *News* :fire:
+- [Dec 2025] **SkyPilot v0.11** released: Multi-Cloud Pools, Fast Managed Jobs, Enterprise-Readiness at Large Scale, Programmability. [**Release notes**](https://github.com/skypilot-org/skypilot/releases/tag/v0.11.0)
+- [Dec 2025] **SkyPilot Pools** released: Run batch inference and other jobs on a managed pool of warm workers (across clouds or clusters). [**blog**](https://blog.skypilot.co/skypilot-pools-deepseek-ocr/), [**docs**](https://docs.skypilot.co/en/latest/examples/pools.html)
 - [Nov 2025] Serve **Kimi K2 Thinking** with reasoning capabilities on your Kubernetes or clouds: [**example**](./llm/kimi-k2-thinking/)
 - [Oct 2025] Run **RL training for LLMs** with SkyRL on your Kubernetes or clouds: [**example**](./llm/skyrl/)
 - [Oct 2025] Train and serve [Andrej Karpathy's](https://x.com/karpathy/status/1977755427569111362) **nanochat** - the best ChatGPT that $100 can buy: [**example**](./llm/nanochat)
@@ -518,22 +530,6 @@ Dynamic: summary
 - [Sep 2025] Network and Storage Benchmarks for LLM training on the cloud: [**blog**](https://maknee.github.io/blog/2025/Network-And-Storage-Training-Skypilot/)
 - [Aug 2025] Serve and finetune **OpenAI GPT-OSS models** (gpt-oss-120b, gpt-oss-20b) with one command on any infra: [**serve**](./llm/gpt-oss/) + [**LoRA and full finetuning**](./llm/gpt-oss-finetuning/)
 - [Jul 2025] Run distributed **RL training for LLMs** with Verl (PPO, GRPO) on any cloud: [**example**](./llm/verl/)
-- [Jul 2025] Finetune **Llama4** on any distributed cluster/cloud: [**example**](./llm/llama-4-finetuning/)
-- [Jul 2025] Two-part blog series, `The Evolution of AI Job Orchestration`: (1) [Running AI jobs on GPU Neoclouds](https://blog.skypilot.co/ai-job-orchestration-pt1-gpu-neoclouds/), (2) [The AI-Native Control Plane & Orchestration that Finally Works for ML](https://blog.skypilot.co/ai-job-orchestration-pt2-ai-control-plane/)
-- [Apr 2025] Spin up **Qwen3** on your cluster/cloud: [**example**](./llm/qwen/)
-
-
-
-**LLM Finetuning Cookbooks**: Finetuning Llama 2 / Llama 3.1 in your own cloud environment, privately: Llama 2 [**example**](./llm/vicuna-llama-2/) and [**blog**](https://blog.skypilot.co/finetuning-llama2-operational-guide/); Llama 3.1 [**example**](./llm/llama-3_1-finetuning/) and [**blog**](https://blog.skypilot.co/finetune-llama-3_1-on-your-infra/)
-
-----
-
-SkyPilot is a system to run, manage, and scale AI workloads on any AI infrastructure.
-
-SkyPilot gives **AI teams** a simple interface to run jobs on any infra.
-**Infra teams** get a unified control plane to manage any AI compute — with advanced scheduling, scaling, and orchestration.
-
-<img src="./docs/source/images/skypilot-abstractions-long-2.png" alt="SkyPilot Abstractions">
 
 ## Overview