skypilot-nightly 1.0.0.dev20250530__py3-none-any.whl → 1.0.0.dev20250531__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '86ea13eb24c6357ea885539fd3196d88295f5943'
+_SKYPILOT_COMMIT_SHA = '687edf041d231b7f5003d7bc24c12e3eddd69fae'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250530'
+__version__ = '1.0.0.dev20250531'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/backend_utils.py

@@ -813,9 +813,9 @@ def write_cluster_config(
 
     # Add kubernetes config fields from ~/.sky/config
     if isinstance(cloud, clouds.Kubernetes):
+        cluster_config_overrides = to_provision.cluster_config_overrides
         kubernetes_utils.combine_pod_config_fields(
-            tmp_yaml_path,
-            cluster_config_overrides=to_provision.cluster_config_overrides)
+            tmp_yaml_path, cluster_config_overrides=cluster_config_overrides)
         kubernetes_utils.combine_metadata_fields(tmp_yaml_path)
         yaml_obj = common_utils.read_yaml(tmp_yaml_path)
         pod_config: Dict[str, Any] = yaml_obj['available_node_types'][

sky/backends/cloud_vm_ray_backend.py

@@ -3950,11 +3950,12 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
                               job_id: Optional[int] = None,
                               job_name: Optional[str] = None,
                               controller: bool = False,
-                              follow: bool = True) -> int:
+                              follow: bool = True,
+                              tail: Optional[int] = None) -> int:
         # if job_name is not None, job_id should be None
         assert job_name is None or job_id is None, (job_name, job_id)
         code = managed_jobs.ManagedJobCodeGen.stream_logs(
-            job_name, job_id, follow, controller)
+            job_name, job_id, follow, controller, tail)
 
         # With the stdin=subprocess.DEVNULL, the ctrl-c will not directly
         # kill the process, so we need to handle it manually here.

sky/cli.py

@@ -415,6 +415,13 @@ _TASK_OPTIONS = [
                                    case_sensitive=False),
                  required=False,
                  help=resources_utils.DiskTier.cli_help_message()),
+    click.option('--network-tier',
+                 default=None,
+                 type=click.Choice(
+                     resources_utils.NetworkTier.supported_tiers(),
+                     case_sensitive=False),
+                 required=False,
+                 help=resources_utils.NetworkTier.cli_help_message()),
     click.option(
         '--use-spot/--no-use-spot',
         required=False,
@@ -696,6 +703,7 @@ def _parse_override_params(
         image_id: Optional[str] = None,
         disk_size: Optional[int] = None,
         disk_tier: Optional[str] = None,
+        network_tier: Optional[str] = None,
         ports: Optional[Tuple[str, ...]] = None,
         config_override: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
     """Parses the override parameters into a dictionary."""
@@ -749,6 +757,11 @@ def _parse_override_params(
             override_params['disk_tier'] = None
         else:
             override_params['disk_tier'] = disk_tier
+    if network_tier is not None:
+        if network_tier.lower() == 'none':
+            override_params['network_tier'] = None
+        else:
+            override_params['network_tier'] = network_tier
     if ports:
         if any(p.lower() == 'none' for p in ports):
             if len(ports) > 1:
@@ -857,6 +870,7 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
     image_id: Optional[str] = None,
     disk_size: Optional[int] = None,
     disk_tier: Optional[str] = None,
+    network_tier: Optional[str] = None,
     ports: Optional[Tuple[str, ...]] = None,
     env: Optional[List[Tuple[str, str]]] = None,
     field_to_ignore: Optional[List[str]] = None,
@@ -897,6 +911,7 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
                                              image_id=image_id,
                                              disk_size=disk_size,
                                              disk_tier=disk_tier,
+                                             network_tier=network_tier,
                                              ports=ports,
                                              config_override=config_override)
     if field_to_ignore is not None:
@@ -1235,6 +1250,7 @@ def launch(
         env: List[Tuple[str, str]],
         disk_size: Optional[int],
         disk_tier: Optional[str],
+        network_tier: Optional[str],
         ports: Tuple[str, ...],
         idle_minutes_to_autostop: Optional[int],
         down: bool,  # pylint: disable=redefined-outer-name
@@ -1288,6 +1304,7 @@ def launch(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         config_override=config_override,
     )
@@ -1405,6 +1422,7 @@ def exec(cluster: Optional[str],
          memory: Optional[str],
          disk_size: Optional[int],
          disk_tier: Optional[str],
+         network_tier: Optional[str],
          async_call: bool,
          config_override: Optional[Dict[str, Any]] = None):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
@@ -1500,6 +1518,7 @@ def exec(cluster: Optional[str],
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         field_to_ignore=['cpus', 'memory', 'disk_size', 'disk_tier', 'ports'],
         config_override=config_override,
@@ -4216,6 +4235,7 @@ def jobs_launch(
     env: List[Tuple[str, str]],
     disk_size: Optional[int],
     disk_tier: Optional[str],
+    network_tier: Optional[str],
     ports: Tuple[str],
     priority: Optional[int],
     detach_run: bool,
@@ -4262,6 +4282,7 @@ def jobs_launch(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         job_recovery=job_recovery,
         priority=priority,
@@ -4599,6 +4620,7 @@ def _generate_task_with_service(
     memory: Optional[str],
     disk_size: Optional[int],
     disk_tier: Optional[str],
+    network_tier: Optional[str],
     not_supported_cmd: str,
 ) -> sky.Task:
     """Generate a task with service section from a service YAML file."""
@@ -4625,6 +4647,7 @@ def _generate_task_with_service(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
     )
     if isinstance(task, sky.Dag):
@@ -4738,6 +4761,7 @@ def serve_up(
     memory: Optional[str],
     disk_size: Optional[int],
     disk_tier: Optional[str],
+    network_tier: Optional[str],
     yes: bool,
     async_call: bool,
 ):
@@ -4792,6 +4816,7 @@ def serve_up(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         not_supported_cmd='sky serve up',
     )
@@ -4836,16 +4861,16 @@ def serve_up(
               help='Skip confirmation prompt.')
 @timeline.event
 @usage_lib.entrypoint
-def serve_update(service_name: str, service_yaml: Tuple[str, ...],
-                 workdir: Optional[str], infra: Optional[str],
-                 cloud: Optional[str], region: Optional[str],
-                 zone: Optional[str], num_nodes: Optional[int],
-                 use_spot: Optional[bool], image_id: Optional[str],
-                 env_file: Optional[Dict[str, str]], env: List[Tuple[str, str]],
-                 gpus: Optional[str], instance_type: Optional[str],
-                 ports: Tuple[str], cpus: Optional[str], memory: Optional[str],
-                 disk_size: Optional[int], disk_tier: Optional[str], mode: str,
-                 yes: bool, async_call: bool):
+def serve_update(
+        service_name: str, service_yaml: Tuple[str, ...],
+        workdir: Optional[str], infra: Optional[str], cloud: Optional[str],
+        region: Optional[str], zone: Optional[str], num_nodes: Optional[int],
+        use_spot: Optional[bool], image_id: Optional[str],
+        env_file: Optional[Dict[str, str]], env: List[Tuple[str, str]],
+        gpus: Optional[str], instance_type: Optional[str], ports: Tuple[str],
+        cpus: Optional[str], memory: Optional[str], disk_size: Optional[int],
+        disk_tier: Optional[str], network_tier: Optional[str], mode: str,
+        yes: bool, async_call: bool):
     """Update a SkyServe service.
 
     service_yaml must point to a valid YAML file.
@@ -4895,6 +4920,7 @@ def serve_update(service_name: str, service_yaml: Tuple[str, ...],
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         not_supported_cmd='sky serve update',
     )

sky/client/cli.py

@@ -415,6 +415,13 @@ _TASK_OPTIONS = [
                                    case_sensitive=False),
                  required=False,
                  help=resources_utils.DiskTier.cli_help_message()),
+    click.option('--network-tier',
+                 default=None,
+                 type=click.Choice(
+                     resources_utils.NetworkTier.supported_tiers(),
+                     case_sensitive=False),
+                 required=False,
+                 help=resources_utils.NetworkTier.cli_help_message()),
     click.option(
         '--use-spot/--no-use-spot',
         required=False,
@@ -696,6 +703,7 @@ def _parse_override_params(
         image_id: Optional[str] = None,
         disk_size: Optional[int] = None,
         disk_tier: Optional[str] = None,
+        network_tier: Optional[str] = None,
         ports: Optional[Tuple[str, ...]] = None,
         config_override: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
     """Parses the override parameters into a dictionary."""
@@ -749,6 +757,11 @@ def _parse_override_params(
             override_params['disk_tier'] = None
         else:
             override_params['disk_tier'] = disk_tier
+    if network_tier is not None:
+        if network_tier.lower() == 'none':
+            override_params['network_tier'] = None
+        else:
+            override_params['network_tier'] = network_tier
     if ports:
         if any(p.lower() == 'none' for p in ports):
             if len(ports) > 1:
@@ -857,6 +870,7 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
     image_id: Optional[str] = None,
     disk_size: Optional[int] = None,
     disk_tier: Optional[str] = None,
+    network_tier: Optional[str] = None,
     ports: Optional[Tuple[str, ...]] = None,
     env: Optional[List[Tuple[str, str]]] = None,
     field_to_ignore: Optional[List[str]] = None,
@@ -897,6 +911,7 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
                                              image_id=image_id,
                                              disk_size=disk_size,
                                              disk_tier=disk_tier,
+                                             network_tier=network_tier,
                                              ports=ports,
                                              config_override=config_override)
     if field_to_ignore is not None:
@@ -1235,6 +1250,7 @@ def launch(
         env: List[Tuple[str, str]],
         disk_size: Optional[int],
         disk_tier: Optional[str],
+        network_tier: Optional[str],
         ports: Tuple[str, ...],
         idle_minutes_to_autostop: Optional[int],
         down: bool,  # pylint: disable=redefined-outer-name
@@ -1288,6 +1304,7 @@ def launch(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         config_override=config_override,
     )
@@ -1405,6 +1422,7 @@ def exec(cluster: Optional[str],
          memory: Optional[str],
          disk_size: Optional[int],
          disk_tier: Optional[str],
+         network_tier: Optional[str],
          async_call: bool,
          config_override: Optional[Dict[str, Any]] = None):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
@@ -1500,6 +1518,7 @@ def exec(cluster: Optional[str],
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         field_to_ignore=['cpus', 'memory', 'disk_size', 'disk_tier', 'ports'],
         config_override=config_override,
@@ -4216,6 +4235,7 @@ def jobs_launch(
     env: List[Tuple[str, str]],
     disk_size: Optional[int],
     disk_tier: Optional[str],
+    network_tier: Optional[str],
     ports: Tuple[str],
     priority: Optional[int],
     detach_run: bool,
@@ -4262,6 +4282,7 @@ def jobs_launch(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         job_recovery=job_recovery,
         priority=priority,
@@ -4599,6 +4620,7 @@ def _generate_task_with_service(
     memory: Optional[str],
     disk_size: Optional[int],
     disk_tier: Optional[str],
+    network_tier: Optional[str],
     not_supported_cmd: str,
 ) -> sky.Task:
     """Generate a task with service section from a service YAML file."""
@@ -4625,6 +4647,7 @@ def _generate_task_with_service(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
     )
     if isinstance(task, sky.Dag):
@@ -4738,6 +4761,7 @@ def serve_up(
     memory: Optional[str],
     disk_size: Optional[int],
     disk_tier: Optional[str],
+    network_tier: Optional[str],
     yes: bool,
     async_call: bool,
 ):
@@ -4792,6 +4816,7 @@ def serve_up(
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         not_supported_cmd='sky serve up',
     )
@@ -4836,16 +4861,16 @@ def serve_up(
               help='Skip confirmation prompt.')
 @timeline.event
 @usage_lib.entrypoint
-def serve_update(service_name: str, service_yaml: Tuple[str, ...],
-                 workdir: Optional[str], infra: Optional[str],
-                 cloud: Optional[str], region: Optional[str],
-                 zone: Optional[str], num_nodes: Optional[int],
-                 use_spot: Optional[bool], image_id: Optional[str],
-                 env_file: Optional[Dict[str, str]], env: List[Tuple[str, str]],
-                 gpus: Optional[str], instance_type: Optional[str],
-                 ports: Tuple[str], cpus: Optional[str], memory: Optional[str],
-                 disk_size: Optional[int], disk_tier: Optional[str], mode: str,
-                 yes: bool, async_call: bool):
+def serve_update(
+        service_name: str, service_yaml: Tuple[str, ...],
+        workdir: Optional[str], infra: Optional[str], cloud: Optional[str],
+        region: Optional[str], zone: Optional[str], num_nodes: Optional[int],
+        use_spot: Optional[bool], image_id: Optional[str],
+        env_file: Optional[Dict[str, str]], env: List[Tuple[str, str]],
+        gpus: Optional[str], instance_type: Optional[str], ports: Tuple[str],
+        cpus: Optional[str], memory: Optional[str], disk_size: Optional[int],
+        disk_tier: Optional[str], network_tier: Optional[str], mode: str,
+        yes: bool, async_call: bool):
     """Update a SkyServe service.
 
     service_yaml must point to a valid YAML file.
@@ -4895,6 +4920,7 @@ def serve_update(service_name: str, service_yaml: Tuple[str, ...],
         env=env,
         disk_size=disk_size,
         disk_tier=disk_tier,
+        network_tier=network_tier,
         ports=ports,
         not_supported_cmd='sky serve update',
     )

sky/clouds/cloud.py

@@ -45,6 +45,7 @@ class CloudImplementationFeatures(enum.Enum):
     DOCKER_IMAGE = 'docker_image'
     SPOT_INSTANCE = 'spot_instance'
     CUSTOM_DISK_TIER = 'custom_disk_tier'
+    CUSTOM_NETWORK_TIER = 'custom_network_tier'
     OPEN_PORTS = 'open_ports'
     STORAGE_MOUNTING = 'storage_mounting'
     HOST_CONTROLLERS = 'host_controllers'  # Can run jobs/serve controllers
@@ -139,6 +140,9 @@ class Cloud:
     _DEFAULT_DISK_TIER = resources_utils.DiskTier.MEDIUM
     _BEST_DISK_TIER = resources_utils.DiskTier.ULTRA
     _SUPPORTED_DISK_TIERS = {resources_utils.DiskTier.BEST}
+    _SUPPORTED_NETWORK_TIERS = {
+        resources_utils.NetworkTier.STANDARD, resources_utils.NetworkTier.BEST
+    }
     _SUPPORTS_SERVICE_ACCOUNT_ON_REMOTE = False
 
     # The version of provisioner and status query. This is used to determine
@@ -715,6 +719,22 @@ class Cloud:
                 raise exceptions.NotSupportedError(
                     f'{disk_tier} is not supported by {cls._REPR}.')
 
+    @classmethod
+    def check_network_tier_enabled(
+            cls, instance_type: Optional[str],
+            network_tier: resources_utils.NetworkTier) -> None:
+        """Errors out if the network tier is not supported by the
+        cloud provider.
+
+        Raises:
+            exceptions.NotSupportedError: If the network tier is not supported.
+        """
+        del instance_type  # unused
+        if network_tier not in cls._SUPPORTED_NETWORK_TIERS:
+            with ux_utils.print_exception_no_traceback():
+                raise exceptions.NotSupportedError(
+                    f'{network_tier} is not supported by {cls._REPR}.')
+
     @classmethod
     def _translate_disk_tier(
         cls, disk_tier: Optional[resources_utils.DiskTier]

sky/clouds/cudo.py

@@ -59,6 +59,8 @@ class Cudo(clouds.Cloud):
             ('Spot is not supported, as Cudo API does not implement spot.'),
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             ('Custom disk tier is currently not supported on Cudo Compute'),
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported on Cudo Compute'),
         clouds.CloudImplementationFeatures.IMAGE_ID:
             ('Image ID is currently not supported on Cudo. '),
         clouds.CloudImplementationFeatures.DOCKER_IMAGE:

sky/clouds/do.py

@@ -33,6 +33,9 @@ class DO(clouds.Cloud):
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             'Custom disk tiers'
             f' is not supported in {_REPR}.',
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
         clouds.CloudImplementationFeatures.HIGH_AVAILABILITY_CONTROLLERS:
             ('High availability controllers are not supported in '
              f'{_REPR}.'),

sky/clouds/fluidstack.py

@@ -53,6 +53,9 @@ class Fluidstack(clouds.Cloud):
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             'Custom disk tiers'
             f' is not supported in {_REPR}.',
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
         clouds.CloudImplementationFeatures.HOST_CONTROLLERS:
             'Host controllers'
             f' are not supported in {_REPR}.',

sky/clouds/gcp.py

@@ -119,6 +119,11 @@ _DEFAULT_GPU_K80_IMAGE_ID = 'skypilot:k80-debian-10'
 # Refer to https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/main/examples/machine-learning/a3-highgpu-8g/README.md#before-starting
 _DEFAULT_GPU_DIRECT_IMAGE_ID = 'skypilot:gpu-direct-cos'
 
+# From https://cloud.google.com/compute/docs/gpus/gpudirect
+# A specific image is used to ensure that the the GPU is configured with TCPX support.
+_NETWORK_GCP_IMAGE_ID = ('docker:us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/'
+                         'nccl-plugin-gpudirecttcpx')
+
 
 def _run_output(cmd):
     proc = subprocess.run(cmd,
@@ -505,6 +510,8 @@ class GCP(clouds.Cloud):
             False,
             override_configs=resources.cluster_config_overrides)
         resources_vars['enable_gpu_direct'] = enable_gpu_direct
+        network_tier = r.network_tier
+        resources_vars['network_tier'] = network_tier
         accelerators = r.accelerators
         if accelerators is not None:
             assert len(accelerators) == 1, r
@@ -539,8 +546,8 @@ class GCP(clouds.Cloud):
                     resources_vars['gpu'] = 'nvidia-tesla-{}'.format(
                         acc.lower())
                 resources_vars['gpu_count'] = acc_count
-                if enable_gpu_direct:
-                    image_id = _DEFAULT_GPU_DIRECT_IMAGE_ID
+                if enable_gpu_direct or network_tier == resources_utils.NetworkTier.BEST:
+                    image_id = _NETWORK_GCP_IMAGE_ID
                 else:
                     if acc == 'K80':
                         # Though the image is called cu113, it actually has later
@@ -630,7 +637,7 @@ class GCP(clouds.Cloud):
             ('gcp', 'placement_policy'),
             None,
             override_configs=resources.cluster_config_overrides)
-        if enable_gpu_direct:
+        if enable_gpu_direct or network_tier == resources_utils.NetworkTier.BEST:
             user_data += constants.GPU_DIRECT_TCPX_USER_DATA
             docker_run_options += constants.GPU_DIRECT_TCPX_SPECIFIC_OPTIONS
             if placement_policy is None:

sky/clouds/kubernetes.py

@@ -75,6 +75,9 @@ class Kubernetes(clouds.Cloud):
                                                              'tiers are not '
                                                              'supported in '
                                                              'Kubernetes.',
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
     }
 
     IMAGE_CPU = 'skypilot:custom-cpu-ubuntu-2004'
@@ -127,6 +130,12 @@ class Kubernetes(clouds.Cloud):
             if spot_label_key is not None:
                 unsupported_features.pop(
                     clouds.CloudImplementationFeatures.SPOT_INSTANCE, None)
+            # Allow custom network tier if supported by the cluster
+            # (e.g., Nebius clusters with high performance networking)
+            if cls._cluster_supports_high_performance_networking(context):
+                unsupported_features.pop(
+                    clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER,
+                    None)
         except exceptions.KubeAPIUnreachableError as e:
             cls._log_unreachable_context(context, str(e))
         return unsupported_features
@@ -471,6 +480,12 @@ class Kubernetes(clouds.Cloud):
         ssh_jump_image = service_catalog.get_image_id_from_tag(
             self.IMAGE_CPU, clouds='kubernetes')
 
+        # Set environment variables for the pod. Note that SkyPilot env vars
+        # are set separately when the task is run. These env vars are
+        # independent of the SkyPilot task to be run.
+        k8s_env_vars = {kubernetes.IN_CLUSTER_CONTEXT_NAME_ENV_VAR: context}
+
+        # Setup GPU/TPU labels and resource keys.
         k8s_acc_label_key = None
         k8s_acc_label_values = None
         k8s_topology_label_key = None
@@ -492,6 +507,17 @@ class Kubernetes(clouds.Cloud):
             else:
                 k8s_resource_key = kubernetes_utils.get_gpu_resource_key()
         else:
+            # If no GPUs are requested, we set NVIDIA_VISIBLE_DEVICES=none to
+            # maintain GPU isolation. This is to override the default behavior
+            # of Nvidia device plugin which would expose all GPUs to the pod
+            # when no GPUs are requested.
+            # Note that NVIDIA_VISIBLE_DEVICES is different from
+            # CUDA_VISIBLE_DEVICES - the latter is used to control which GPUs
+            # are visible to the application and is set inside the pod, while
+            # the former is used to control which GPUs are visible to the pod
+            # through the nvidia runtime.
+            # See: https://github.com/NVIDIA/k8s-device-plugin/issues/61
+            k8s_env_vars['NVIDIA_VISIBLE_DEVICES'] = 'none'
             avoid_label_keys = kubernetes_utils.get_accelerator_label_keys(
                 context)
             if len(avoid_label_keys) == 0:
@@ -552,10 +578,22 @@ class Kubernetes(clouds.Cloud):
             timeout,
             override_configs=resources.cluster_config_overrides)
 
-        # Set environment variables for the pod. Note that SkyPilot env vars
-        # are set separately when the task is run. These env vars are
-        # independent of the SkyPilot task to be run.
-        k8s_env_vars = {kubernetes.IN_CLUSTER_CONTEXT_NAME_ENV_VAR: context}
+        # Check if this cluster supports high performance networking and
+        # configure IPC_LOCK capability for clusters like Nebius that support it
+        k8s_ipc_lock_capability = False
+        if (resources.network_tier is not None and
+                resources.network_tier == resources_utils.NetworkTier.BEST):
+            # Only proceed if CUSTOM_NETWORK_TIER is supported by this cluster
+            unsupported_features = self._unsupported_features_for_resources(
+                resources)
+            if clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER \
+                not in unsupported_features:
+                k8s_ipc_lock_capability = True
+
+        if k8s_ipc_lock_capability:
+            k8s_env_vars['NCCL_IB_HCA'] = 'mlx5'
+            k8s_env_vars['UCX_NET_DEVICES'] = \
+                'mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1'  # pylint: disable=line-too-long
 
         # We specify object-store-memory to be 500MB to avoid taking up too
         # much memory on the head node. 'num-cpus' should be set to limit
@@ -621,6 +659,7 @@ class Kubernetes(clouds.Cloud):
             'k8s_high_availability_storage_class_name':
                 (k8s_ha_storage_class_name),
             'avoid_label_keys': avoid_label_keys,
+            'k8s_ipc_lock_capability': k8s_ipc_lock_capability,
         }
 
         # Add kubecontext if it is set. It may be None if SkyPilot is running
@@ -904,3 +943,30 @@ class Kubernetes(clouds.Cloud):
             f'{cls.canonical_name()}/{c}'
             for c in cls.existing_allowed_contexts(silent=True)
         ]
+
+    @classmethod
+    @annotations.lru_cache(scope='request', maxsize=10)
+    def _cluster_supports_high_performance_networking(cls,
+                                                      context: str) -> bool:
+        """Check if the cluster supports high performance networking.
+
+        Currently detects Nebius clusters by checking for nebius.com/ labels
+        on cluster nodes.
+
+        Args:
+            context: The Kubernetes context to check.
+
+        Returns:
+            True if the cluster supports high performance networking.
+        """
+        try:
+            nodes = kubernetes_utils.get_kubernetes_nodes(context=context)
+            for node in nodes:
+                if node.metadata.labels:
+                    for label_key in node.metadata.labels.keys():
+                        if label_key.startswith('nebius.com/'):
+                            return True
+        except exceptions.KubeAPIUnreachableError:
+            # If we can't reach the cluster, assume no high perf networking
+            return False
+        return False

sky/clouds/lambda_cloud.py

@@ -43,6 +43,9 @@ class Lambda(clouds.Cloud):
         clouds.CloudImplementationFeatures.SPOT_INSTANCE: f'Spot instances are not supported in {_REPR}.',
         clouds.CloudImplementationFeatures.IMAGE_ID: f'Specifying image ID is not supported in {_REPR}.',
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER: f'Custom disk tiers are not supported in {_REPR}.',
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
         clouds.CloudImplementationFeatures.HOST_CONTROLLERS: f'Host controllers are not supported in {_REPR}.',
         clouds.CloudImplementationFeatures.HIGH_AVAILABILITY_CONTROLLERS: f'High availability controllers are not supported on {_REPR}.',
     }

sky/clouds/nebius.py

@@ -56,6 +56,8 @@ class Nebius(clouds.Cloud):
             (f'Migrating disk is currently not supported on {_REPR}.'),
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             (f'Custom disk tier is currently not supported on {_REPR}.'),
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported on Nebius.'),
         clouds.CloudImplementationFeatures.HIGH_AVAILABILITY_CONTROLLERS:
             ('High availability controllers are not supported on Nebius.'),
     }

sky/clouds/paperspace.py

@@ -41,6 +41,9 @@ class Paperspace(clouds.Cloud):
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             'Custom disk tiers'
             f' is not supported in {_REPR}.',
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
         clouds.CloudImplementationFeatures.HIGH_AVAILABILITY_CONTROLLERS:
             (f'High availability controllers are not supported in {_REPR}.'),
     }

sky/clouds/runpod.py

@@ -30,6 +30,8 @@ class RunPod(clouds.Cloud):
              'are non-trivial on RunPod.'),
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             ('Customizing disk tier is not supported yet on RunPod.'),
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is not supported yet on RunPod.'),
         clouds.CloudImplementationFeatures.STORAGE_MOUNTING:
             ('Mounting object stores is not supported on RunPod. To read data '
              'from object stores on RunPod, use `mode: COPY` to copy the data '

sky/clouds/scp.py

@@ -56,6 +56,9 @@ class SCP(clouds.Cloud):
             (f'Spot instances are not supported in {_REPR}.'),
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             (f'Custom disk tiers are not supported in {_REPR}.'),
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
         clouds.CloudImplementationFeatures.OPEN_PORTS:
             (f'Opening ports is currently not supported on {_REPR}.'),
         clouds.CloudImplementationFeatures.HIGH_AVAILABILITY_CONTROLLERS:

sky/clouds/vast.py

@@ -25,6 +25,9 @@ class Vast(clouds.Cloud):
              'are non-trivial on Vast.'),
         clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
             ('Customizing disk tier is not supported yet on Vast.'),
+        clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER:
+            ('Custom network tier is currently not supported in '
+             f'{_REPR}.'),
         clouds.CloudImplementationFeatures.OPEN_PORTS:
             ('Opening ports is currently not supported on Vast.'),
         clouds.CloudImplementationFeatures.STORAGE_MOUNTING: