skypilot-nightly 1.0.0.dev20250926__py3-none-any.whl → 1.0.0.dev20251001__py3-none-any.whl

sky/skylet/services.py

@@ -1,15 +1,20 @@
 """gRPC service implementations for skylet."""
 
 import os
+from typing import List, Optional
 
 import grpc
 
+from sky import exceptions
 from sky import sky_logging
 from sky.jobs import state as managed_job_state
+from sky.jobs import utils as managed_job_utils
 from sky.schemas.generated import autostopv1_pb2
 from sky.schemas.generated import autostopv1_pb2_grpc
 from sky.schemas.generated import jobsv1_pb2
 from sky.schemas.generated import jobsv1_pb2_grpc
+from sky.schemas.generated import managed_jobsv1_pb2
+from sky.schemas.generated import managed_jobsv1_pb2_grpc
 from sky.schemas.generated import servev1_pb2
 from sky.schemas.generated import servev1_pb2_grpc
 from sky.serve import serve_rpc_utils
@@ -18,9 +23,14 @@ from sky.serve import serve_utils
 from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.skylet import job_lib
+from sky.skylet import log_lib
 
 logger = sky_logging.init_logger(__name__)
 
+# In the worst case, flush the log buffer every 50ms,
+# to ensure responsiveness.
+DEFAULT_LOG_CHUNK_FLUSH_INTERVAL = 0.05
+
 
 class AutostopServiceImpl(autostopv1_pb2_grpc.AutostopServiceServicer):
     """Implementation of the AutostopService gRPC service."""
@@ -275,8 +285,39 @@ class JobsServiceImpl(jobsv1_pb2_grpc.JobsServiceServicer):
             self,
             request: jobsv1_pb2.TailLogsRequest,  # type: ignore[return]
             context: grpc.ServicerContext):
-        # TODO(kevin): implement this
-        raise NotImplementedError('TailLogs is not implemented')
+        buffer = log_lib.LogBuffer()
+        try:
+            job_id = request.job_id if request.HasField(
+                'job_id') else job_lib.get_latest_job_id()
+            managed_job_id = request.managed_job_id if request.HasField(
+                'managed_job_id') else None
+            log_dir = job_lib.get_log_dir_for_job(job_id)
+            if log_dir is None:
+                run_timestamp = job_lib.get_run_timestamp(job_id)
+                log_dir = None if run_timestamp is None else os.path.join(
+                    constants.SKY_LOGS_DIRECTORY, run_timestamp)
+
+            for line in log_lib.buffered_iter_with_timeout(
+                    buffer,
+                    log_lib.tail_logs_iter(job_id, log_dir, managed_job_id,
+                                           request.follow, request.tail),
+                    DEFAULT_LOG_CHUNK_FLUSH_INTERVAL):
+                yield jobsv1_pb2.TailLogsResponse(log_line=line)
+
+            job_status = job_lib.get_status(job_id)
+            exit_code = exceptions.JobExitCode.from_job_status(job_status)
+            # Fix for dashboard: When follow=False and job is still running
+            # (NOT_FINISHED=101), exit with success (0) since fetching current
+            # logs is a successful operation.
+            # This prevents shell wrappers from printing "command terminated
+            # with exit code 101".
+            exit_code_int = 0 if not request.follow and int(
+                exit_code) == 101 else int(exit_code)
+            yield jobsv1_pb2.TailLogsResponse(exit_code=exit_code_int)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+        finally:
+            buffer.close()
 
     def GetJobStatus(  # type: ignore[return]
             self, request: jobsv1_pb2.GetJobStatusRequest,
@@ -343,3 +384,168 @@ class JobsServiceImpl(jobsv1_pb2_grpc.JobsServiceServicer):
                 job_log_dirs=job_log_dirs)
         except Exception as e:  # pylint: disable=broad-except
             context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+
+class ManagedJobsServiceImpl(managed_jobsv1_pb2_grpc.ManagedJobsServiceServicer
+                            ):
+    """Implementation of the ManagedJobsService gRPC service."""
+
+    def GetVersion(  # type: ignore[return]
+            self, request: managed_jobsv1_pb2.GetVersionRequest,
+            context: grpc.ServicerContext
+    ) -> managed_jobsv1_pb2.GetVersionResponse:
+        try:
+            return managed_jobsv1_pb2.GetVersionResponse(
+                controller_version=constants.SKYLET_VERSION)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def GetJobTable(  # type: ignore[return]
+        self, request: managed_jobsv1_pb2.GetJobTableRequest,
+        context: grpc.ServicerContext
+    ) -> managed_jobsv1_pb2.GetJobTableResponse:
+        try:
+            accessible_workspaces = list(request.accessible_workspaces)
+            job_ids = list(request.job_ids.ids) if request.job_ids else None
+            user_hashes: Optional[List[Optional[str]]] = None
+            if request.user_hashes:
+                user_hashes = list(request.user_hashes.hashes)
+                # For backwards compatibility, we show jobs that do not have a
+                # user_hash. TODO: Remove before 0.12.0.
+                if request.show_jobs_without_user_hash:
+                    user_hashes.append(None)
+            statuses = list(
+                request.statuses.statuses) if request.statuses else None
+
+            job_queue = managed_job_utils.get_managed_job_queue(
+                skip_finished=request.skip_finished,
+                accessible_workspaces=accessible_workspaces,
+                job_ids=job_ids,
+                workspace_match=request.workspace_match
+                if request.HasField('workspace_match') else None,
+                name_match=request.name_match
+                if request.HasField('name_match') else None,
+                pool_match=request.pool_match
+                if request.HasField('pool_match') else None,
+                page=request.page if request.HasField('page') else None,
+                limit=request.limit if request.HasField('limit') else None,
+                user_hashes=user_hashes,
+                statuses=statuses)
+            jobs = job_queue['jobs']
+            total = job_queue['total']
+            total_no_filter = job_queue['total_no_filter']
+            status_counts = job_queue['status_counts']
+
+            jobs_info = []
+            for job in jobs:
+                job_info = managed_jobsv1_pb2.ManagedJobInfo(
+                    job_id=job.get('job_id'),
+                    task_id=job.get('task_id'),
+                    job_name=job.get('job_name'),
+                    task_name=job.get('task_name'),
+                    job_duration=job.get('job_duration'),
+                    workspace=job.get('workspace'),
+                    status=managed_job_state.ManagedJobStatus(
+                        job.get('status')).to_protobuf(),
+                    schedule_state=managed_job_state.ManagedJobScheduleState(
+                        job.get('schedule_state')).to_protobuf(),
+                    resources=job.get('resources'),
+                    cluster_resources=job.get('cluster_resources'),
+                    cluster_resources_full=job.get('cluster_resources_full'),
+                    cloud=job.get('cloud'),
+                    region=job.get('region'),
+                    infra=job.get('infra'),
+                    accelerators=job.get('accelerators'),
+                    recovery_count=job.get('recovery_count'),
+                    details=job.get('details'),
+                    failure_reason=job.get('failure_reason'),
+                    user_name=job.get('user_name'),
+                    user_hash=job.get('user_hash'),
+                    submitted_at=job.get('submitted_at'),
+                    start_at=job.get('start_at'),
+                    end_at=job.get('end_at'),
+                    user_yaml=job.get('user_yaml'),
+                    entrypoint=job.get('entrypoint'),
+                    metadata={
+                        k: v
+                        for k, v in job.get('metadata', {}).items()
+                        if v is not None
+                    },
+                    pool=job.get('pool'),
+                    pool_hash=job.get('pool_hash'))
+                jobs_info.append(job_info)
+
+            return managed_jobsv1_pb2.GetJobTableResponse(
+                jobs=jobs_info,
+                total=total,
+                total_no_filter=total_no_filter,
+                status_counts=status_counts)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def GetAllJobIdsByName(  # type: ignore[return]
+        self, request: managed_jobsv1_pb2.GetAllJobIdsByNameRequest,
+        context: grpc.ServicerContext
+    ) -> managed_jobsv1_pb2.GetAllJobIdsByNameResponse:
+        try:
+            job_name = request.job_name if request.HasField(
+                'job_name') else None
+            job_ids = managed_job_state.get_all_job_ids_by_name(job_name)
+            return managed_jobsv1_pb2.GetAllJobIdsByNameResponse(
+                job_ids=job_ids)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def CancelJobs(  # type: ignore[return]
+            self, request: managed_jobsv1_pb2.CancelJobsRequest,
+            context: grpc.ServicerContext
+    ) -> managed_jobsv1_pb2.CancelJobsResponse:
+        try:
+            cancellation_criteria = request.WhichOneof('cancellation_criteria')
+            if cancellation_criteria is None:
+                context.abort(
+                    grpc.StatusCode.INVALID_ARGUMENT,
+                    'exactly one cancellation criteria must be specified.')
+
+            if cancellation_criteria == 'all_users':
+                user_hash = request.user_hash if request.HasField(
+                    'user_hash') else None
+                all_users = request.all_users
+                if not all_users and user_hash is None:
+                    context.abort(
+                        grpc.StatusCode.INVALID_ARGUMENT,
+                        'user_hash is required when all_users is False')
+                message = managed_job_utils.cancel_jobs_by_id(
+                    job_ids=None,
+                    all_users=all_users,
+                    current_workspace=request.current_workspace,
+                    user_hash=user_hash)
+            elif cancellation_criteria == 'job_ids':
+                job_ids = list(request.job_ids.ids)
+                message = managed_job_utils.cancel_jobs_by_id(
+                    job_ids=job_ids,
+                    current_workspace=request.current_workspace)
+            elif cancellation_criteria == 'job_name':
+                message = managed_job_utils.cancel_job_by_name(
+                    job_name=request.job_name,
+                    current_workspace=request.current_workspace)
+            elif cancellation_criteria == 'pool_name':
+                message = managed_job_utils.cancel_jobs_by_pool(
+                    pool_name=request.pool_name,
+                    current_workspace=request.current_workspace)
+            else:
+                context.abort(
+                    grpc.StatusCode.INVALID_ARGUMENT,
+                    f'invalid cancellation criteria: {cancellation_criteria}')
+            return managed_jobsv1_pb2.CancelJobsResponse(message=message)
+        except Exception as e:  # pylint: disable=broad-except
+            context.abort(grpc.StatusCode.INTERNAL, str(e))
+
+    def StreamLogs(
+            self,
+            request: managed_jobsv1_pb2.
+        StreamLogsRequest,  # type: ignore[return]
+            context: grpc.ServicerContext):
+        # TODO(kevin): implement this
+        context.abort(grpc.StatusCode.UNIMPLEMENTED,
+                      'StreamLogs is not implemented')

sky/skylet/skylet.py

@@ -10,6 +10,7 @@ import sky
 from sky import sky_logging
 from sky.schemas.generated import autostopv1_pb2_grpc
 from sky.schemas.generated import jobsv1_pb2_grpc
+from sky.schemas.generated import managed_jobsv1_pb2_grpc
 from sky.schemas.generated import servev1_pb2_grpc
 from sky.skylet import constants
 from sky.skylet import events
@@ -55,6 +56,8 @@ def start_grpc_server(port: int = constants.SKYLET_GRPC_PORT) -> grpc.Server:
         services.JobsServiceImpl(), server)
     servev1_pb2_grpc.add_ServeServiceServicer_to_server(
         services.ServeServiceImpl(), server)
+    managed_jobsv1_pb2_grpc.add_ManagedJobsServiceServicer_to_server(
+        services.ManagedJobsServiceImpl(), server)
 
     listen_addr = f'127.0.0.1:{port}'
     server.add_insecure_port(listen_addr)

sky/task.py

@@ -649,6 +649,10 @@ class Task:
             config['workdir'] = _fill_in_env_vars(config['workdir'],
                                                   env_and_secrets)
 
+        if config.get('volumes') is not None:
+            config['volumes'] = _fill_in_env_vars(config['volumes'],
+                                                  env_and_secrets)
+
         task = Task(
             config.pop('name', None),
             run=config.pop('run', None),

sky/utils/cluster_utils.py

@@ -193,11 +193,29 @@ class SSHConfigHelper(object):
         proxy_command = auth_config.get('ssh_proxy_command', None)
 
         docker_proxy_command_generator = None
+        proxy_command_for_nodes = proxy_command
         if docker_user is not None:
-            docker_proxy_command_generator = lambda ip, port: ' '.join(
-                ['ssh'] + command_runner.ssh_options_list(
-                    key_path, ssh_control_name=None, port=port) +
-                ['-W', '%h:%p', f'{auth_config["ssh_user"]}@{ip}'])
+
+            def _docker_proxy_cmd(ip: str, port: int) -> str:
+                inner_proxy = proxy_command
+                inner_port = port or 22
+                if inner_proxy is not None:
+                    inner_proxy = inner_proxy.replace('%h', ip)
+                    inner_proxy = inner_proxy.replace('%p', str(inner_port))
+                return ' '.join(['ssh'] + command_runner.ssh_options_list(
+                    key_path,
+                    ssh_control_name=None,
+                    ssh_proxy_command=inner_proxy,
+                    port=inner_port,
+                    # ProxyCommand (ssh -W) is a forwarding tunnel, not an
+                    # interactive session. ControlMaster would cache these
+                    # processes, causing them to hang and block subsequent
+                    # connections. Each ProxyCommand should be ephemeral.
+                    disable_control_master=True
+                ) + ['-W', '%h:%p', f'{auth_config["ssh_user"]}@{ip}'])
+
+            docker_proxy_command_generator = _docker_proxy_cmd
+            proxy_command_for_nodes = None
 
         codegen = ''
         # Add the nodes to the codegen
@@ -212,7 +230,7 @@ class SSHConfigHelper(object):
             # TODO(romilb): Update port number when k8s supports multinode
             codegen += cls._get_generated_config(
                 sky_autogen_comment, node_name, ip, username,
-                key_path_for_config, proxy_command, port,
+                key_path_for_config, proxy_command_for_nodes, port,
                 docker_proxy_command) + '\n'
 
         cluster_config_path = os.path.expanduser(

sky/utils/command_runner.py

@@ -652,15 +652,31 @@ class SSHCommandRunner(CommandRunner):
         if docker_user is not None:
             assert port is None or port == 22, (
                 f'port must be None or 22 for docker_user, got {port}.')
-            # Already checked in resources
-            assert ssh_proxy_command is None, (
-                'ssh_proxy_command is not supported when using docker.')
+            # When connecting via docker, the outer SSH hop points to the
+            # container's sshd (localhost). Preserve the user proxy for the
+            # inner hop that reaches the host VM, and clear the outer proxy to
+            # avoid forwarding localhost through the jump host.
+            inner_proxy_command = ssh_proxy_command
+            inner_proxy_port = port or 22
+            self._ssh_proxy_command = None
             self.ip = 'localhost'
             self.ssh_user = docker_user
             self.port = constants.DEFAULT_DOCKER_PORT
+            if inner_proxy_command is not None:
+                # Replace %h/%p placeholders with actual host values, since the
+                # final destination from the perspective of the user proxy is
+                # the host VM (ip, inner_proxy_port).
+                inner_proxy_command = inner_proxy_command.replace('%h', ip)
+                inner_proxy_command = inner_proxy_command.replace(
+                    '%p', str(inner_proxy_port))
             self._docker_ssh_proxy_command = lambda ssh: ' '.join(
-                ssh + ssh_options_list(ssh_private_key, None
-                                      ) + ['-W', '%h:%p', f'{ssh_user}@{ip}'])
+                ssh + ssh_options_list(ssh_private_key,
+                                       None,
+                                       ssh_proxy_command=inner_proxy_command,
+                                       port=inner_proxy_port,
+                                       disable_control_master=self.
+                                       disable_control_master) +
+                ['-W', '%h:%p', f'{ssh_user}@{ip}'])
         else:
             self.ip = ip
             self.ssh_user = ssh_user

sky/utils/command_runner.pyi

@@ -142,8 +142,10 @@ class SSHCommandRunner(CommandRunner):
         ssh_user: str,
         ssh_private_key: str,
         ssh_control_name: Optional[str] = ...,
+        ssh_proxy_command: Optional[str] = ...,
         docker_user: Optional[str] = ...,
         disable_control_master: Optional[bool] = ...,
+        port_forward_execute_remote_command: Optional[bool] = ...,
     ) -> None:
         ...
 
@@ -198,6 +200,15 @@ class SSHCommandRunner(CommandRunner):
             **kwargs) -> Union[Tuple[int, str, str], int]:
         ...
 
+    def ssh_base_command(
+        self,
+        *,
+        ssh_mode: SshMode,
+        port_forward: Optional[List[Tuple[int, int]]],
+        connect_timeout: Optional[int],
+    ) -> List[str]:
+        ...
+
     def rsync(self,
               source: str,
               target: str,

sky/utils/volume.py

@@ -26,6 +26,11 @@ class VolumeType(enum.Enum):
     PVC = 'k8s-pvc'
     RUNPOD_NETWORK_VOLUME = 'runpod-network-volume'
 
+    @classmethod
+    def supported_types(cls) -> list:
+        """Return list of supported volume type values."""
+        return [vt.value for vt in cls]
+
 
 class VolumeMount:
     """Volume mount specification."""

{skypilot_nightly-1.0.0.dev20250926.dist-info → skypilot_nightly-1.0.0.dev20251001.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250926
+Version: 1.0.0.dev20251001
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -154,51 +154,51 @@ Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "server"
 Requires-Dist: aiosqlite; extra == "server"
 Requires-Dist: greenlet; extra == "server"
 Provides-Extra: all
-Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
-Requires-Dist: azure-core>=1.31.0; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: ibm-cos-sdk; extra == "all"
-Requires-Dist: azure-identity>=1.19.0; extra == "all"
-Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
-Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
 Requires-Dist: azure-cli>=2.65.0; extra == "all"
-Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
-Requires-Dist: boto3>=1.26.1; extra == "all"
-Requires-Dist: msrestazure; extra == "all"
-Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
-Requires-Dist: aiohttp; extra == "all"
-Requires-Dist: colorama<0.4.5; extra == "all"
-Requires-Dist: websockets; extra == "all"
-Requires-Dist: ecsapi>=0.2.0; extra == "all"
-Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
-Requires-Dist: tomli; python_version < "3.11" and extra == "all"
 Requires-Dist: pydo>=0.3.0; extra == "all"
+Requires-Dist: azure-core>=1.24.0; extra == "all"
+Requires-Dist: anyio; extra == "all"
 Requires-Dist: casbin; extra == "all"
+Requires-Dist: colorama<0.4.5; extra == "all"
+Requires-Dist: ecsapi>=0.2.0; extra == "all"
+Requires-Dist: google-cloud-storage; extra == "all"
+Requires-Dist: google-api-python-client>=2.69.0; extra == "all"
 Requires-Dist: ray[default]>=2.6.1; extra == "all"
-Requires-Dist: azure-core>=1.24.0; extra == "all"
-Requires-Dist: cudo-compute>=0.1.10; extra == "all"
-Requires-Dist: sqlalchemy_adapter; extra == "all"
 Requires-Dist: ibm-vpc; extra == "all"
-Requires-Dist: runpod>=1.6.1; extra == "all"
+Requires-Dist: awscli>=1.27.10; extra == "all"
+Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
 Requires-Dist: passlib; extra == "all"
-Requires-Dist: anyio; extra == "all"
-Requires-Dist: grpcio>=1.63.0; extra == "all"
-Requires-Dist: python-dateutil; extra == "all"
-Requires-Dist: nebius>=0.2.47; extra == "all"
+Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
+Requires-Dist: ibm-cos-sdk; extra == "all"
+Requires-Dist: azure-mgmt-compute>=33.0.0; extra == "all"
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
-Requires-Dist: oci; extra == "all"
-Requires-Dist: pyjwt; extra == "all"
-Requires-Dist: msgraph-sdk; extra == "all"
-Requires-Dist: azure-common; extra == "all"
+Requires-Dist: nebius>=0.2.47; extra == "all"
 Requires-Dist: azure-mgmt-network>=27.0.0; extra == "all"
 Requires-Dist: vastai-sdk>=0.1.12; extra == "all"
-Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
+Requires-Dist: pyjwt; extra == "all"
+Requires-Dist: azure-core>=1.31.0; extra == "all"
+Requires-Dist: msrestazure; extra == "all"
+Requires-Dist: azure-common; extra == "all"
+Requires-Dist: runpod>=1.6.1; extra == "all"
+Requires-Dist: tomli; python_version < "3.11" and extra == "all"
+Requires-Dist: websockets; extra == "all"
+Requires-Dist: python-dateutil; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: greenlet; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: cudo-compute>=0.1.10; extra == "all"
+Requires-Dist: sqlalchemy_adapter; extra == "all"
+Requires-Dist: msgraph-sdk; extra == "all"
+Requires-Dist: botocore>=1.29.10; extra == "all"
 Requires-Dist: ibm-cloud-sdk-core; extra == "all"
+Requires-Dist: boto3>=1.26.1; extra == "all"
+Requires-Dist: azure-identity>=1.19.0; extra == "all"
+Requires-Dist: pyopenssl<24.3.0,>=23.2.0; extra == "all"
+Requires-Dist: azure-storage-blob>=12.23.1; extra == "all"
+Requires-Dist: grpcio>=1.63.0; extra == "all"
 Requires-Dist: aiosqlite; extra == "all"
-Requires-Dist: awscli>=1.27.10; extra == "all"
-Requires-Dist: botocore>=1.29.10; extra == "all"
-Requires-Dist: google-cloud-storage; extra == "all"
-Requires-Dist: greenlet; extra == "all"
+Requires-Dist: ibm-platform-services>=0.48.0; extra == "all"
+Requires-Dist: oci; extra == "all"
 Dynamic: author
 Dynamic: classifier
 Dynamic: description