skypilot-nightly 1.0.0.dev20250804__py3-none-any.whl → 1.0.0.dev20250807__py3-none-any.whl

sky/serve/server/core.py

@@ -1,9 +1,6 @@
 """SkyServe core APIs."""
-import pathlib
-import signal
-import threading
 import typing
-from typing import Any, Dict, List, Optional, Set, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from sky import backends
 from sky import exceptions
@@ -12,11 +9,8 @@ from sky.backends import backend_utils
 from sky.serve import serve_utils
 from sky.serve.server import impl
 from sky.usage import usage_lib
-from sky.utils import command_runner
 from sky.utils import controller_utils
-from sky.utils import rich_utils
 from sky.utils import subprocess_utils
-from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     import sky
@@ -24,42 +18,6 @@ if typing.TYPE_CHECKING:
 logger = sky_logging.init_logger(__name__)
 
 
-def _get_all_replica_targets(
-    service_name: str, backend: backends.CloudVmRayBackend,
-    handle: backends.CloudVmRayResourceHandle
-) -> Set[serve_utils.ServiceComponentTarget]:
-    """Helper function to get targets for all live replicas."""
-    code = serve_utils.ServeCodeGen.get_service_status([service_name],
-                                                       pool=False)
-    returncode, serve_status_payload, stderr = backend.run_on_head(
-        handle,
-        code,
-        require_outputs=True,
-        stream_logs=False,
-        separate_stderr=True)
-
-    try:
-        subprocess_utils.handle_returncode(returncode,
-                                           code,
-                                           'Failed to fetch services',
-                                           stderr,
-                                           stream_logs=True)
-    except exceptions.CommandError as e:
-        raise RuntimeError(e.error_msg) from e
-
-    service_records = serve_utils.load_service_status(serve_status_payload)
-    if not service_records:
-        raise ValueError(f'Service {service_name!r} not found.')
-    assert len(service_records) == 1
-    service_record = service_records[0]
-
-    return {
-        serve_utils.ServiceComponentTarget(serve_utils.ServiceComponent.REPLICA,
-                                           replica_info['replica_id'])
-        for replica_info in service_record['replica_info']
-    }
-
-
 @usage_lib.entrypoint
 def up(
     task: 'sky.Task',
@@ -277,59 +235,12 @@ def tail_logs(
         sky.exceptions.ClusterNotUpError: the sky serve controller is not up.
         ValueError: arguments not valid, or failed to tail the logs.
     """
-    if isinstance(target, str):
-        target = serve_utils.ServiceComponent(target)
-    if not isinstance(target, serve_utils.ServiceComponent):
-        with ux_utils.print_exception_no_traceback():
-            raise ValueError(f'`target` must be a string or '
-                             f'sky.serve.ServiceComponent, got {type(target)}.')
-
-    if target == serve_utils.ServiceComponent.REPLICA:
-        if replica_id is None:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError(
-                    '`replica_id` must be specified when using target=REPLICA.')
-    else:
-        if replica_id is not None:
-            with ux_utils.print_exception_no_traceback():
-                raise ValueError('`replica_id` must be None when using '
-                                 'target=CONTROLLER/LOAD_BALANCER.')
-
-    controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
-    handle = backend_utils.is_controller_accessible(
-        controller=controller_type,
-        stopped_message=controller_type.value.default_hint_if_non_existent)
-
-    backend = backend_utils.get_backend_from_handle(handle)
-    assert isinstance(backend, backends.CloudVmRayBackend), backend
-
-    if target != serve_utils.ServiceComponent.REPLICA:
-        code = serve_utils.ServeCodeGen.stream_serve_process_logs(
-            service_name,
-            stream_controller=(
-                target == serve_utils.ServiceComponent.CONTROLLER),
-            follow=follow,
-            tail=tail)
-    else:
-        assert replica_id is not None, service_name
-        code = serve_utils.ServeCodeGen.stream_replica_logs(service_name,
-                                                            replica_id,
-                                                            follow,
-                                                            tail=tail)
-
-    # With the stdin=subprocess.DEVNULL, the ctrl-c will not directly
-    # kill the process, so we need to handle it manually here.
-    if threading.current_thread() is threading.main_thread():
-        signal.signal(signal.SIGINT, backend_utils.interrupt_handler)
-        signal.signal(signal.SIGTSTP, backend_utils.stop_handler)
-
-    # Refer to the notes in
-    # sky/backends/cloud_vm_ray_backend.py::CloudVmRayBackend::tail_logs.
-    backend.run_on_head(handle,
-                        code,
-                        stream_logs=True,
-                        process_stream=False,
-                        ssh_mode=command_runner.SshMode.INTERACTIVE)
+    return impl.tail_logs(service_name,
+                          target=target,
+                          replica_id=replica_id,
+                          follow=follow,
+                          tail=tail,
+                          pool=False)
 
 
 @usage_lib.entrypoint
@@ -374,104 +285,9 @@ def sync_down_logs(
         sky.exceptions.ClusterNotUpError: If the controller is not up.
         ValueError: Arguments not valid.
     """
-    # Step 0) get the controller handle
-    with rich_utils.safe_status(
-            ux_utils.spinner_message('Checking service status...')):
-        controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
-        handle = backend_utils.is_controller_accessible(
-            controller=controller_type,
-            stopped_message=controller_type.value.default_hint_if_non_existent)
-        backend: backends.CloudVmRayBackend = (
-            backend_utils.get_backend_from_handle(handle))
-
-    requested_components: Set[serve_utils.ServiceComponent] = set()
-    if not targets:
-        # No targets specified -> request all components
-        requested_components = {
-            serve_utils.ServiceComponent.CONTROLLER,
-            serve_utils.ServiceComponent.LOAD_BALANCER,
-            serve_utils.ServiceComponent.REPLICA
-        }
-    else:
-        # Parse provided targets
-        if isinstance(targets, (str, serve_utils.ServiceComponent)):
-            requested_components = {serve_utils.ServiceComponent(targets)}
-        else:  # list
-            requested_components = {
-                serve_utils.ServiceComponent(t) for t in targets
-            }
-
-    normalized_targets: Set[serve_utils.ServiceComponentTarget] = set()
-    if serve_utils.ServiceComponent.CONTROLLER in requested_components:
-        normalized_targets.add(
-            serve_utils.ServiceComponentTarget(
-                serve_utils.ServiceComponent.CONTROLLER))
-    if serve_utils.ServiceComponent.LOAD_BALANCER in requested_components:
-        normalized_targets.add(
-            serve_utils.ServiceComponentTarget(
-                serve_utils.ServiceComponent.LOAD_BALANCER))
-    if serve_utils.ServiceComponent.REPLICA in requested_components:
-        with rich_utils.safe_status(
-                ux_utils.spinner_message('Getting live replica infos...')):
-            replica_targets = _get_all_replica_targets(service_name, backend,
-                                                       handle)
-        if not replica_ids:
-            # Replica target requested but no specific IDs
-            # -> Get all replica logs
-            normalized_targets.update(replica_targets)
-        else:
-            # Replica target requested with specific IDs
-            requested_replica_targets = [
-                serve_utils.ServiceComponentTarget(
-                    serve_utils.ServiceComponent.REPLICA, rid)
-                for rid in replica_ids
-            ]
-            for target in requested_replica_targets:
-                if target not in replica_targets:
-                    logger.warning(f'Replica ID {target.replica_id} not found '
-                                   f'for {service_name}. Skipping...')
-                else:
-                    normalized_targets.add(target)
-
-    def sync_down_logs_by_target(target: serve_utils.ServiceComponentTarget):
-        component = target.component
-        # We need to set one side of the pipe to a logs stream, and the other
-        # side to a file.
-        log_path = str(pathlib.Path(local_dir) / f'{target}.log')
-        stream_logs_code: str
-
-        if component == serve_utils.ServiceComponent.CONTROLLER:
-            stream_logs_code = (
-                serve_utils.ServeCodeGen.stream_serve_process_logs(
-                    service_name,
-                    stream_controller=True,
-                    follow=False,
-                    tail=tail))
-        elif component == serve_utils.ServiceComponent.LOAD_BALANCER:
-            stream_logs_code = (
-                serve_utils.ServeCodeGen.stream_serve_process_logs(
-                    service_name,
-                    stream_controller=False,
-                    follow=False,
-                    tail=tail))
-        elif component == serve_utils.ServiceComponent.REPLICA:
-            replica_id = target.replica_id
-            assert replica_id is not None, service_name
-            stream_logs_code = serve_utils.ServeCodeGen.stream_replica_logs(
-                service_name, replica_id, follow=False, tail=tail)
-        else:
-            assert False, component
-
-        # Refer to the notes in
-        # sky/backends/cloud_vm_ray_backend.py::CloudVmRayBackend::tail_logs.
-        backend.run_on_head(handle,
-                            stream_logs_code,
-                            stream_logs=False,
-                            process_stream=False,
-                            ssh_mode=command_runner.SshMode.INTERACTIVE,
-                            log_path=log_path)
-
-    subprocess_utils.run_in_parallel(sync_down_logs_by_target,
-                                     list(normalized_targets))
-
-    return local_dir
+    return impl.sync_down_logs(service_name,
+                               local_dir=local_dir,
+                               targets=targets,
+                               replica_ids=replica_ids,
+                               tail=tail,
+                               pool=False)

sky/serve/server/impl.py

@@ -1,7 +1,12 @@
 """Implementation of the SkyServe core APIs."""
+import pathlib
 import re
+import shlex
+import signal
 import tempfile
-from typing import Any, Dict, List, Optional, Tuple, Union
+import threading
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+import uuid
 
 import colorama
 import filelock
@@ -21,6 +26,7 @@ from sky.serve import serve_state
 from sky.serve import serve_utils
 from sky.skylet import constants
 from sky.utils import admin_policy_utils
+from sky.utils import command_runner
 from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import controller_utils
@@ -102,10 +108,10 @@ def up(
     pool: bool = False,
 ) -> Tuple[str, str]:
     """Spins up a service or a pool."""
-    if pool and not serve_utils.is_consolidation_mode():
+    if pool and not serve_utils.is_consolidation_mode(pool):
         raise ValueError(
             'Pool is only supported in consolidation mode. To fix, set '
-            '`serve.controller.consolidation_mode: true` in SkyPilot config.')
+            '`jobs.controller.consolidation_mode: true` in SkyPilot config.')
     task.validate()
     serve_utils.validate_service_task(task, pool=pool)
     assert task.service is not None
@@ -174,7 +180,8 @@ def up(
             prefix=f'controller-task-{service_name}-',
             mode='w',
     ) as controller_file:
-        controller_name = common.SKY_SERVE_CONTROLLER_NAME
+        controller = controller_utils.get_controller_for_pool(pool)
+        controller_name = controller.value.cluster_name
         task_config = task.to_yaml_config()
         common_utils.dump_yaml(service_file.name, task_config)
         remote_tmp_task_yaml_path = (
@@ -187,8 +194,13 @@ def up(
             controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
             task_resources=task.resources)
         controller_job_id = None
-        if serve_utils.is_consolidation_mode():
-            controller_job_id = 0
+        if serve_utils.is_consolidation_mode(pool):
+            # We need a unique integer per sky.serve.up call to avoid name
+            # conflict. Originally in non-consolidation mode, this is the ray
+            # job id; now we use the request id hash instead. Here we also
+            # make sure it is a 63-bit integer to avoid overflow on sqlalchemy.
+            rid = common_utils.get_current_request_id()
+            controller_job_id = hash(uuid.UUID(rid).int) & 0x7FFFFFFFFFFFFFFF
 
         vars_to_fill = {
             'remote_task_yaml_path': remote_tmp_task_yaml_path,
@@ -200,6 +212,7 @@ def up(
             'modified_catalogs':
                 service_catalog_common.get_modified_catalog_file_mounts(),
             'consolidation_mode_job_id': controller_job_id,
+            'entrypoint': shlex.quote(common_utils.get_current_command()),
             **tls_template_vars,
             **controller_utils.shared_controller_vars_to_fill(
                 controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
@@ -238,7 +251,7 @@ def up(
         # for the first time; otherwise it is a name conflict.
         # Since the controller may be shared among multiple users, launch the
         # controller with the API server's user hash.
-        if not serve_utils.is_consolidation_mode():
+        if not serve_utils.is_consolidation_mode(pool):
             print(f'{colorama.Fore.YELLOW}Launching controller for '
                   f'{service_name!r}...{colorama.Style.RESET_ALL}')
             with common.with_server_user():
@@ -251,9 +264,9 @@ def up(
                         _disable_controller_check=True,
                     )
         else:
+            controller_type = controller_utils.get_controller_for_pool(pool)
             controller_handle = backend_utils.is_controller_accessible(
-                controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
-                stopped_message='')
+                controller=controller_type, stopped_message='')
             backend = backend_utils.get_backend_from_handle(controller_handle)
             assert isinstance(backend, backends.CloudVmRayBackend)
             backend.sync_file_mounts(
@@ -270,10 +283,8 @@ def up(
             ]
             run_script = '\n'.join(env_cmds + [run_script])
             # Dump script for high availability recovery.
-            # if controller_utils.high_availability_specified(
-            #         controller_name):
-            #     managed_job_state.set_ha_recovery_script(
-            #         consolidation_mode_job_id, run_script)
+            if controller_utils.high_availability_specified(controller_name):
+                serve_state.set_ha_recovery_script(service_name, run_script)
             backend.run_on_head(controller_handle, run_script)
 
         style = colorama.Style
@@ -289,7 +300,7 @@ def up(
             # and return the endpoint if the job id matches. Otherwise it will
             # return None.
             code = serve_utils.ServeCodeGen.wait_service_registration(
-                service_name, controller_job_id)
+                service_name, controller_job_id, pool)
             backend = backend_utils.get_backend_from_handle(controller_handle)
             assert isinstance(backend, backends.CloudVmRayBackend)
             assert isinstance(controller_handle,
@@ -304,7 +315,7 @@ def up(
                 returncode, code, f'Failed to wait for {noun} initialization',
                 lb_port_payload)
         except exceptions.CommandError:
-            if serve_utils.is_consolidation_mode():
+            if serve_utils.is_consolidation_mode(pool):
                 with ux_utils.print_exception_no_traceback():
                     raise RuntimeError(
                         f'Failed to wait for {noun} initialization. '
@@ -339,7 +350,7 @@ def up(
         else:
             lb_port = serve_utils.load_service_initialization_result(
                 lb_port_payload)
-            if not serve_utils.is_consolidation_mode():
+            if not serve_utils.is_consolidation_mode(pool):
                 socket_endpoint = backend_utils.get_endpoints(
                     controller_handle.cluster_name,
                     lb_port,
@@ -442,8 +453,9 @@ def update(
                        'effect. To update TLS keyfile and certfile, please '
                        'tear down the service and spin up a new one.')
 
+    controller_type = controller_utils.get_controller_for_pool(pool)
     handle = backend_utils.is_controller_accessible(
-        controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
+        controller=controller_type,
         stopped_message=
         'Service controller is stopped. There is no service to update. '
         f'To spin up a new service, use {ux_utils.BOLD}'
@@ -572,9 +584,9 @@ def apply(
     """Applies the config to the service or pool."""
     with filelock.FileLock(serve_utils.get_service_filelock_path(service_name)):
         try:
+            controller_type = controller_utils.get_controller_for_pool(pool)
             handle = backend_utils.is_controller_accessible(
-                controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
-                stopped_message='')
+                controller=controller_type, stopped_message='')
             backend = backend_utils.get_backend_from_handle(handle)
             assert isinstance(backend, backends.CloudVmRayBackend)
             service_record = _get_service_record(service_name, pool, handle,
@@ -598,8 +610,9 @@ def down(
         service_names = []
     if isinstance(service_names, str):
         service_names = [service_names]
+    controller_type = controller_utils.get_controller_for_pool(pool)
     handle = backend_utils.is_controller_accessible(
-        controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
+        controller=controller_type,
         stopped_message=f'All {noun}s should have terminated.')
 
     service_names_str = ','.join(service_names)
@@ -624,7 +637,7 @@ def down(
     except exceptions.FetchClusterInfoError as e:
         raise RuntimeError(
             'Failed to fetch controller IP. Please refresh controller status '
-            f'by `sky status -r {common.SKY_SERVE_CONTROLLER_NAME}` '
+            f'by `sky status -r {controller_type.value.cluster_name}` '
             'and try again.') from e
 
     try:
@@ -654,7 +667,7 @@ def status(
             raise RuntimeError(f'Failed to refresh {noun}s status '
                                'due to network error.') from e
 
-    controller_type = controller_utils.Controllers.SKY_SERVE_CONTROLLER
+    controller_type = controller_utils.get_controller_for_pool(pool)
     handle = backend_utils.is_controller_accessible(
         controller=controller_type,
         stopped_message=controller_type.value.default_hint_if_non_existent.
@@ -690,7 +703,7 @@ def status(
         if service_record['load_balancer_port'] is not None:
             try:
                 lb_port = service_record['load_balancer_port']
-                if not serve_utils.is_consolidation_mode():
+                if not serve_utils.is_consolidation_mode(pool):
                     endpoint = backend_utils.get_endpoints(
                         cluster=common.SKY_SERVE_CONTROLLER_NAME,
                         port=lb_port).get(lb_port, None)
@@ -707,3 +720,228 @@ def status(
                 service_record['endpoint'] = f'{protocol}://{endpoint}'
 
     return service_records
+
+
+ServiceComponentOrStr = Union[str, serve_utils.ServiceComponent]
+
+
+def tail_logs(
+    service_name: str,
+    *,
+    target: ServiceComponentOrStr,
+    replica_id: Optional[int] = None,
+    follow: bool = True,
+    tail: Optional[int] = None,
+    pool: bool = False,
+) -> None:
+    """Tail logs of a service or pool."""
+    if isinstance(target, str):
+        target = serve_utils.ServiceComponent(target)
+
+    if pool and target == serve_utils.ServiceComponent.LOAD_BALANCER:
+        raise ValueError(f'Target {target} is not supported for pool.')
+
+    if target == serve_utils.ServiceComponent.REPLICA:
+        if replica_id is None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    '`replica_id` must be specified when using target=REPLICA.')
+    else:
+        if replica_id is not None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError('`replica_id` must be None when using '
+                                 'target=CONTROLLER/LOAD_BALANCER.')
+
+    controller_type = controller_utils.get_controller_for_pool(pool)
+    handle = backend_utils.is_controller_accessible(
+        controller=controller_type,
+        stopped_message=controller_type.value.default_hint_if_non_existent)
+
+    backend = backend_utils.get_backend_from_handle(handle)
+    assert isinstance(backend, backends.CloudVmRayBackend), backend
+
+    if target != serve_utils.ServiceComponent.REPLICA:
+        code = serve_utils.ServeCodeGen.stream_serve_process_logs(
+            service_name,
+            stream_controller=(
+                target == serve_utils.ServiceComponent.CONTROLLER),
+            follow=follow,
+            tail=tail,
+            pool=pool)
+    else:
+        assert replica_id is not None, service_name
+        code = serve_utils.ServeCodeGen.stream_replica_logs(service_name,
+                                                            replica_id,
+                                                            follow,
+                                                            tail=tail,
+                                                            pool=pool)
+
+    # With the stdin=subprocess.DEVNULL, the ctrl-c will not directly
+    # kill the process, so we need to handle it manually here.
+    if threading.current_thread() is threading.main_thread():
+        signal.signal(signal.SIGINT, backend_utils.interrupt_handler)
+        signal.signal(signal.SIGTSTP, backend_utils.stop_handler)
+
+    # Refer to the notes in
+    # sky/backends/cloud_vm_ray_backend.py::CloudVmRayBackend::tail_logs.
+    backend.run_on_head(handle,
+                        code,
+                        stream_logs=True,
+                        process_stream=False,
+                        ssh_mode=command_runner.SshMode.INTERACTIVE)
+
+
+def _get_all_replica_targets(
+        service_name: str, backend: backends.CloudVmRayBackend,
+        handle: backends.CloudVmRayResourceHandle,
+        pool: bool) -> Set[serve_utils.ServiceComponentTarget]:
+    """Helper function to get targets for all live replicas."""
+    code = serve_utils.ServeCodeGen.get_service_status([service_name],
+                                                       pool=pool)
+    returncode, serve_status_payload, stderr = backend.run_on_head(
+        handle,
+        code,
+        require_outputs=True,
+        stream_logs=False,
+        separate_stderr=True)
+
+    try:
+        subprocess_utils.handle_returncode(returncode,
+                                           code,
+                                           'Failed to fetch services',
+                                           stderr,
+                                           stream_logs=True)
+    except exceptions.CommandError as e:
+        raise RuntimeError(e.error_msg) from e
+
+    service_records = serve_utils.load_service_status(serve_status_payload)
+    if not service_records:
+        raise ValueError(f'Service {service_name!r} not found.')
+    assert len(service_records) == 1
+    service_record = service_records[0]
+
+    return {
+        serve_utils.ServiceComponentTarget(serve_utils.ServiceComponent.REPLICA,
+                                           replica_info['replica_id'])
+        for replica_info in service_record['replica_info']
+    }
+
+
+def sync_down_logs(
+    service_name: str,
+    *,
+    local_dir: str,
+    targets: Union[ServiceComponentOrStr, List[ServiceComponentOrStr],
+                   None] = None,
+    replica_ids: Optional[List[int]] = None,
+    tail: Optional[int] = None,
+    pool: bool = False,
+) -> str:
+    """Sync down logs of a service or pool."""
+    noun = 'pool' if pool else 'service'
+    repnoun = 'worker' if pool else 'replica'
+    caprepnoun = repnoun.capitalize()
+
+    # Step 0) get the controller handle
+    with rich_utils.safe_status(
+            ux_utils.spinner_message(f'Checking {noun} status...')):
+        controller_type = controller_utils.get_controller_for_pool(pool)
+        handle = backend_utils.is_controller_accessible(
+            controller=controller_type,
+            stopped_message=controller_type.value.default_hint_if_non_existent)
+        backend: backends.CloudVmRayBackend = (
+            backend_utils.get_backend_from_handle(handle))
+
+    requested_components: Set[serve_utils.ServiceComponent] = set()
+    if not targets:
+        # No targets specified -> request all components
+        requested_components = {
+            serve_utils.ServiceComponent.CONTROLLER,
+            serve_utils.ServiceComponent.LOAD_BALANCER,
+            serve_utils.ServiceComponent.REPLICA
+        }
+    else:
+        # Parse provided targets
+        if isinstance(targets, (str, serve_utils.ServiceComponent)):
+            requested_components = {serve_utils.ServiceComponent(targets)}
+        else:  # list
+            requested_components = {
+                serve_utils.ServiceComponent(t) for t in targets
+            }
+
+    normalized_targets: Set[serve_utils.ServiceComponentTarget] = set()
+    if serve_utils.ServiceComponent.CONTROLLER in requested_components:
+        normalized_targets.add(
+            serve_utils.ServiceComponentTarget(
+                serve_utils.ServiceComponent.CONTROLLER))
+    if serve_utils.ServiceComponent.LOAD_BALANCER in requested_components:
+        normalized_targets.add(
+            serve_utils.ServiceComponentTarget(
+                serve_utils.ServiceComponent.LOAD_BALANCER))
+    if serve_utils.ServiceComponent.REPLICA in requested_components:
+        with rich_utils.safe_status(
+                ux_utils.spinner_message(f'Getting live {repnoun} infos...')):
+            replica_targets = _get_all_replica_targets(service_name, backend,
+                                                       handle, pool)
+        if not replica_ids:
+            # Replica target requested but no specific IDs
+            # -> Get all replica logs
+            normalized_targets.update(replica_targets)
+        else:
+            # Replica target requested with specific IDs
+            requested_replica_targets = [
+                serve_utils.ServiceComponentTarget(
+                    serve_utils.ServiceComponent.REPLICA, rid)
+                for rid in replica_ids
+            ]
+            for target in requested_replica_targets:
+                if target not in replica_targets:
+                    logger.warning(f'{caprepnoun} ID {target.replica_id} not '
+                                   f'found for {service_name}. Skipping...')
+                else:
+                    normalized_targets.add(target)
+
+    def sync_down_logs_by_target(target: serve_utils.ServiceComponentTarget):
+        component = target.component
+        # We need to set one side of the pipe to a logs stream, and the other
+        # side to a file.
+        log_path = str(pathlib.Path(local_dir) / f'{target}.log')
+        stream_logs_code: str
+
+        if component == serve_utils.ServiceComponent.CONTROLLER:
+            stream_logs_code = (
+                serve_utils.ServeCodeGen.stream_serve_process_logs(
+                    service_name,
+                    stream_controller=True,
+                    follow=False,
+                    tail=tail,
+                    pool=pool))
+        elif component == serve_utils.ServiceComponent.LOAD_BALANCER:
+            stream_logs_code = (
+                serve_utils.ServeCodeGen.stream_serve_process_logs(
+                    service_name,
+                    stream_controller=False,
+                    follow=False,
+                    tail=tail,
+                    pool=pool))
+        elif component == serve_utils.ServiceComponent.REPLICA:
+            replica_id = target.replica_id
+            assert replica_id is not None, service_name
+            stream_logs_code = serve_utils.ServeCodeGen.stream_replica_logs(
+                service_name, replica_id, follow=False, tail=tail, pool=pool)
+        else:
+            assert False, component
+
+        # Refer to the notes in
+        # sky/backends/cloud_vm_ray_backend.py::CloudVmRayBackend::tail_logs.
+        backend.run_on_head(handle,
+                            stream_logs_code,
+                            stream_logs=False,
+                            process_stream=False,
+                            ssh_mode=command_runner.SshMode.INTERACTIVE,
+                            log_path=log_path)
+
+    subprocess_utils.run_in_parallel(sync_down_logs_by_target,
+                                     list(normalized_targets))
+
+    return local_dir