skypilot-nightly 1.0.0.dev20250530__py3-none-any.whl → 1.0.0.dev20250531__py3-none-any.whl

sky/jobs/server/core.py

@@ -521,8 +521,12 @@ def cancel(name: Optional[str] = None,
 
 
 @usage_lib.entrypoint
-def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
-              controller: bool, refresh: bool) -> int:
+def tail_logs(name: Optional[str],
+              job_id: Optional[int],
+              follow: bool,
+              controller: bool,
+              refresh: bool,
+              tail: Optional[int] = None) -> int:
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Tail logs of managed jobs.
 
@@ -565,7 +569,8 @@ def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
                                          job_id=job_id,
                                          job_name=name,
                                          follow=follow,
-                                         controller=controller)
+                                         controller=controller,
+                                         tail=tail)
 
 
 def start_dashboard_forwarding(refresh: bool = False) -> Tuple[int, int]:

sky/jobs/utils.py

@@ -13,7 +13,7 @@ import textwrap
 import time
 import traceback
 import typing
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Deque, Dict, List, Optional, Set, TextIO, Tuple, Union
 
 import colorama
 import filelock
@@ -546,7 +546,9 @@ def cancel_job_by_name(job_name: str,
     return f'{job_name!r} {msg}'
 
 
-def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
+def stream_logs_by_id(job_id: int,
+                      follow: bool = True,
+                      tail: Optional[int] = None) -> Tuple[str, int]:
     """Stream logs by job id.
 
     Returns:
@@ -583,7 +585,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
                     # Stream the logs to the console without reading the whole
                     # file into memory.
                     start_streaming = False
-                    for line in f:
+                    read_from: Union[TextIO, Deque[str]] = f
+                    if tail is not None:
+                        assert tail > 0
+                        # Read only the last 'tail' lines using deque
+                        read_from = collections.deque(f, maxlen=tail)
+                    for line in read_from:
                         if log_lib.LOG_FILE_START_STREAMING_AT in line:
                             start_streaming = True
                         if start_streaming:
@@ -644,10 +651,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
                     managed_job_state.ManagedJobStatus.RUNNING)
             assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
             status_display.stop()
+            tail_param = tail if tail is not None else 0
             returncode = backend.tail_logs(handle,
                                            job_id=None,
                                            managed_job_id=job_id,
-                                           follow=follow)
+                                           follow=follow,
+                                           tail=tail_param)
             if returncode in [rc.value for rc in exceptions.JobExitCode]:
                 # If the log tailing exits with a known exit code we can safely
                 # break the loop because it indicates the tailing process
@@ -784,7 +793,8 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
 def stream_logs(job_id: Optional[int],
                 job_name: Optional[str],
                 controller: bool = False,
-                follow: bool = True) -> Tuple[str, int]:
+                follow: bool = True,
+                tail: Optional[int] = None) -> Tuple[str, int]:
     """Stream logs by job id or job name.
 
     Returns:
@@ -855,7 +865,12 @@ def stream_logs(job_id: Optional[int],
         with open(controller_log_path, 'r', newline='', encoding='utf-8') as f:
             # Note: we do not need to care about start_stream_at here, since
             # that should be in the job log printed above.
-            for line in f:
+            read_from: Union[TextIO, Deque[str]] = f
+            if tail is not None:
+                assert tail > 0
+                # Read only the last 'tail' lines efficiently using deque
+                read_from = collections.deque(f, maxlen=tail)
+            for line in read_from:
                 print(line, end='')
             # Flush.
             print(end='', flush=True)
@@ -907,7 +922,7 @@ def stream_logs(job_id: Optional[int],
                 f'Multiple running jobs found with name {job_name!r}.')
         job_id = job_ids[0]
 
-    return stream_logs_by_id(job_id, follow)
+    return stream_logs_by_id(job_id, follow, tail)
 
 
 def dump_managed_job_queue() -> str:
@@ -1370,10 +1385,16 @@ class ManagedJobCodeGen:
                     job_name: Optional[str],
                     job_id: Optional[int],
                     follow: bool = True,
-                    controller: bool = False) -> str:
+                    controller: bool = False,
+                    tail: Optional[int] = None) -> str:
         code = textwrap.dedent(f"""\
-        result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
-                                follow={follow}, controller={controller})
+        if managed_job_version < 6:
+            # Versions before 5 did not support tail parameter
+            result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
+                                    follow={follow}, controller={controller})
+        else:
+            result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
+                                    follow={follow}, controller={controller}, tail={tail!r})
         if managed_job_version < 3:
             # Versions 2 and older did not return a retcode, so we just print
             # the result.

sky/provision/gcp/config.py

@@ -10,6 +10,7 @@ from sky.clouds.utils import gcp_utils
 from sky.provision import common
 from sky.provision.gcp import constants
 from sky.provision.gcp import instance_utils
+from sky.utils import resources_utils
 
 logger = logging.getLogger(__name__)
 
@@ -788,7 +789,8 @@ def _configure_subnet(region: str, cluster_name: str,
     default_interfaces = []
     enable_gpu_direct = config.provider_config.get('enable_gpu_direct', False)
     enable_gvnic = config.provider_config.get('enable_gvnic', False)
-    if enable_gpu_direct:
+    network_tier = config.provider_config.get('network_tier', 'standard')
+    if enable_gpu_direct or network_tier == resources_utils.NetworkTier.BEST:
         if not enable_gvnic:
             logger.warning(
                 'Enable GPU Direct requires gvnic to be enabled, enabling gvnic'

sky/provision/gcp/constants.py

@@ -84,6 +84,8 @@ GPU_DIRECT_TCPX_USER_DATA = textwrap.dedent("""
     echo "GPU Direct TCPX installed"
     """)
 
+# Some NCCL options are from the following link.
+# https://docs.nvidia.com/dgx-cloud/run-ai/latest/appendix-gcp.html
 GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
     '--cap-add=IPC_LOCK',
     '--userns=host',
@@ -103,6 +105,14 @@ GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
     '--device /dev/nvidia-uvm:/dev/nvidia-uvm',
     '--device /dev/nvidiactl:/dev/nvidiactl',
     '--env LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/tcpx/lib64',
+    '--env NCCL_GPUDIRECTTCPX_SOCKET_IFNAME=eth1,eth2,eth3,eth4',
+    '--env NCCL_GPUDIRECTTCPX_CTRL_DEV=eth0',
+    '--env NCCL_GPUDIRECTTCPX_TX_BINDINGS="eth1:8-21,112-125;eth2:8-21,112-125;eth3:60-73,164-177;eth4:60-73,164-177"',
+    '--env NCCL_GPUDIRECTTCPX_RX_BINDINGS="eth1:22-35,126-139;eth2:22-35,126-139;eth3:74-87,178-191;eth4:74-87,178-191"',
+    '--env NCCL_GPUDIRECTTCPX_PROGRAM_FLOW_STEERING_WAIT_MICROS=50000',
+    '--env NCCL_GPUDIRECTTCPX_UNIX_CLIENT_PREFIX="/run/tcpx"',
+    '--env NCCL_GPUDIRECTTCPX_FORCE_ACK=0',
+    '--env NCCL_SOCKET_IFNAME=eth0',
 ]
 
 PD_EXTREME_IOPS = 20000

sky/resources.py

@@ -98,7 +98,7 @@ class Resources:
     """
     # If any fields changed, increment the version. For backward compatibility,
     # modify the __setstate__ method to handle the old version.
-    _VERSION = 25
+    _VERSION = 26
 
     def __init__(
         self,
@@ -117,6 +117,7 @@ class Resources:
         image_id: Union[Dict[Optional[str], str], str, None] = None,
         disk_size: Optional[int] = None,
         disk_tier: Optional[Union[str, resources_utils.DiskTier]] = None,
+        network_tier: Optional[Union[str, resources_utils.NetworkTier]] = None,
         ports: Optional[Union[int, str, List[str], Tuple[str]]] = None,
         labels: Optional[Dict[str, str]] = None,
         autostop: Union[bool, int, Dict[str, Any], None] = None,
@@ -202,6 +203,8 @@ class Resources:
           disk_size: the size of the OS disk in GiB.
           disk_tier: the disk performance tier to use. If None, defaults to
             ``'medium'``.
+          network_tier: the network performance tier to use. If None, defaults to
+            ``'standard'``.
           ports: the ports to open on the instance.
           labels: the labels to apply to the instance. These are useful for
             assigning metadata that may be used by external tools.
@@ -309,6 +312,20 @@ class Resources:
             disk_tier = resources_utils.DiskTier(disk_tier_str)
         self._disk_tier = disk_tier
 
+        if isinstance(network_tier, str):
+            network_tier_str = str(network_tier).lower()
+            supported_tiers = [
+                tier.value for tier in resources_utils.NetworkTier
+            ]
+            if network_tier_str not in supported_tiers:
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError(
+                        f'Invalid network_tier {network_tier_str!r}. '
+                        f'Network tier must be one of '
+                        f'{", ".join(supported_tiers)}.')
+            network_tier = resources_utils.NetworkTier(network_tier_str)
+        self._network_tier = network_tier
+
         if ports is not None:
             if isinstance(ports, tuple):
                 ports = list(ports)
@@ -418,6 +435,10 @@ class Resources:
         if self.disk_tier is not None:
             disk_tier = f', disk_tier={self.disk_tier.value}'
 
+        network_tier = ''
+        if self.network_tier is not None:
+            network_tier = f', network_tier={self.network_tier.value}'
+
         disk_size = ''
         if self.disk_size != _DEFAULT_DISK_SIZE_GB:
             disk_size = f', disk_size={self.disk_size}'
@@ -437,7 +458,7 @@ class Resources:
         hardware_str = (
             f'{instance_type}{use_spot}'
             f'{cpus}{memory}{accelerators}{accelerator_args}{image_id}'
-            f'{disk_tier}{disk_size}{ports}')
+            f'{disk_tier}{network_tier}{disk_size}{ports}')
         # It may have leading ',' (for example, instance_type not set) or empty
         # spaces.  Remove them.
         while hardware_str and hardware_str[0] in (',', ' '):
@@ -567,6 +588,10 @@ class Resources:
     def disk_tier(self) -> Optional[resources_utils.DiskTier]:
         return self._disk_tier
 
+    @property
+    def network_tier(self) -> Optional[resources_utils.NetworkTier]:
+        return self._network_tier
+
     @property
     def ports(self) -> Optional[List[str]]:
         return self._ports
@@ -1223,7 +1248,6 @@ class Resources:
 
     def _try_validate_volumes(self) -> None:
         """Try to validate the volumes attribute.
-
         Raises:
             ValueError: if the attribute is invalid.
         """
@@ -1532,6 +1556,12 @@ class Resources:
                 if not (self.disk_tier <= other.disk_tier):  # pylint: disable=superfluous-parens
                     return False
 
+        if self.network_tier is not None:
+            if other.network_tier is None:
+                return False
+            if not self.network_tier <= other.network_tier:
+                return False
+
         if check_ports:
             if self.ports is not None:
                 if other.ports is None:
@@ -1586,6 +1616,7 @@ class Resources:
             not self._use_spot_specified,
             self._disk_size == _DEFAULT_DISK_SIZE_GB,
             self._disk_tier is None,
+            self._network_tier is None,
             self._image_id is None,
             self._ports is None,
             self._docker_login_config is None,
@@ -1629,6 +1660,7 @@ class Resources:
             zone=override.pop('zone', self.zone),
             image_id=override.pop('image_id', self.image_id),
             disk_tier=override.pop('disk_tier', self.disk_tier),
+            network_tier=override.pop('network_tier', self.network_tier),
             ports=override.pop('ports', self.ports),
             labels=override.pop('labels', self.labels),
             autostop=override.pop('autostop', current_autostop_config),
@@ -1667,6 +1699,9 @@ class Resources:
         if (self.disk_tier is not None and
                 self.disk_tier != resources_utils.DiskTier.BEST):
             features.add(clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER)
+        if (self.network_tier is not None and
+                self.network_tier == resources_utils.NetworkTier.BEST):
+            features.add(clouds.CloudImplementationFeatures.CUSTOM_NETWORK_TIER)
         if self.extract_docker_image() is not None:
             features.add(clouds.CloudImplementationFeatures.DOCKER_IMAGE)
         elif self.image_id is not None:
@@ -1845,6 +1880,7 @@ class Resources:
         resources_fields['disk_size'] = config.pop('disk_size', None)
         resources_fields['image_id'] = config.pop('image_id', None)
         resources_fields['disk_tier'] = config.pop('disk_tier', None)
+        resources_fields['network_tier'] = config.pop('network_tier', None)
         resources_fields['ports'] = config.pop('ports', None)
         resources_fields['labels'] = config.pop('labels', None)
         resources_fields['autostop'] = config.pop('autostop', None)
@@ -1897,6 +1933,8 @@ class Resources:
         add_if_not_none('image_id', self.image_id)
         if self.disk_tier is not None:
             config['disk_tier'] = self.disk_tier.value
+        if self.network_tier is not None:
+            config['network_tier'] = self.network_tier.value
         add_if_not_none('ports', self.ports)
         add_if_not_none('labels', self.labels)
         if self.volumes is not None:
@@ -2081,6 +2119,9 @@ class Resources:
             if isinstance(state.get('_cloud', None), clouds.Kubernetes):
                 _maybe_add_docker_prefix_to_image_id(state['_image_id'])
 
+        if version < 26:
+            self._network_tier = state.get('_network_tier', None)
+
         self.__dict__.update(state)
 
 

sky/server/constants.py

@@ -7,7 +7,7 @@ from sky.skylet import constants
 # API server version, whenever there is a change in API server that requires a
 # restart of the local API server or error out when the client does not match
 # the server version.
-API_VERSION = '6'
+API_VERSION = '7'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'

sky/server/requests/payloads.py

@@ -376,6 +376,7 @@ class JobsLogsBody(RequestBody):
     follow: bool = True
     controller: bool = False
     refresh: bool = False
+    tail: Optional[int] = None
 
 
 class RequestCancelBody(RequestBody):

sky/templates/kubernetes-ray.yml.j2

@@ -697,6 +697,13 @@ available_node_types:
               {{k8s_resource_key}}: {{accelerator_count}}
               {% endif %}
             {% endif %}
+          {% if k8s_ipc_lock_capability %}
+          securityContext:
+            capabilities:
+              add:
+                - IPC_LOCK
+          {% endif %}
+
 
       {% if high_availability %}
       pvc_spec:

sky/utils/resources_utils.py

@@ -50,6 +50,32 @@ class DiskTier(enum.Enum):
         return types.index(self) <= types.index(other)
 
 
+class NetworkTier(enum.Enum):
+    """All network tiers supported by SkyPilot."""
+    STANDARD = 'standard'
+    BEST = 'best'
+
+    @classmethod
+    def supported_tiers(cls) -> List[str]:
+        return [tier.value for tier in cls]
+
+    @classmethod
+    def cli_help_message(cls) -> str:
+        return (
+            f'Network tier. Could be one of {", ".join(cls.supported_tiers())}'
+            f'. Default: {cls.STANDARD.value}')
+
+    @classmethod
+    def from_str(cls, tier: str) -> 'NetworkTier':
+        if tier not in cls.supported_tiers():
+            raise ValueError(f'Invalid network tier: {tier}')
+        return cls(tier)
+
+    def __le__(self, other: 'NetworkTier') -> bool:
+        types = list(NetworkTier)
+        return types.index(self) <= types.index(other)
+
+
 class StorageType(enum.Enum):
     """Storage type."""
     # Durable network storage, e.g. GCP persistent disks

sky/utils/schemas.py

@@ -223,6 +223,9 @@ def _get_single_resources_schema():
             'disk_tier': {
                 'type': 'string',
             },
+            'network_tier': {
+                'type': 'string',
+            },
             'ports': {
                 'anyOf': [{
                     'type': 'string',

{skypilot_nightly-1.0.0.dev20250530.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250530
+Version: 1.0.0.dev20250531
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0