skypilot-nightly 1.0.0.dev20250114__py3-none-any.whl → 1.0.0.dev20250124__py3-none-any.whl

sky/serve/controller.py

@@ -67,9 +67,16 @@ class SkyServeController:
             try:
                 replica_infos = serve_state.get_replica_infos(
                     self._service_name)
+                # Use the active versions set by replica manager to make
+                # sure we only scale down the outdated replicas that are
+                # not used by the load balancer.
+                record = serve_state.get_service_from_name(self._service_name)
+                assert record is not None, ('No service record found for '
+                                            f'{self._service_name}')
+                active_versions = record['active_versions']
                 logger.info(f'All replica info: {replica_infos}')
-                scaling_options = self._autoscaler.evaluate_scaling(
-                    replica_infos)
+                scaling_options = self._autoscaler.generate_scaling_decisions(
+                    replica_infos, active_versions)
                 for scaling_option in scaling_options:
                     logger.info(f'Scaling option received: {scaling_option}')
                     if (scaling_option.operator ==
@@ -77,15 +84,10 @@ class SkyServeController:
                         assert (scaling_option.target is None or isinstance(
                             scaling_option.target, dict)), scaling_option
                         self._replica_manager.scale_up(scaling_option.target)
-                    elif (scaling_option.operator ==
-                          autoscalers.AutoscalerDecisionOperator.SCALE_DOWN):
+                    else:
                         assert isinstance(scaling_option.target,
                                           int), scaling_option
                         self._replica_manager.scale_down(scaling_option.target)
-                    else:
-                        with ux_utils.enable_traceback():
-                            logger.error('Error in scaling_option.operator: '
-                                         f'{scaling_option.operator}')
             except Exception as e:  # pylint: disable=broad-except
                 # No matter what error happens, we should keep the
                 # monitor running.

sky/serve/core.py

@@ -1,6 +1,9 @@
 """SkyServe core APIs."""
 import re
+import signal
+import subprocess
 import tempfile
+import threading
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import colorama
@@ -18,6 +21,7 @@ from sky.serve import serve_utils
 from sky.skylet import constants
 from sky.usage import usage_lib
 from sky.utils import admin_policy_utils
+from sky.utils import command_runner
 from sky.utils import common_utils
 from sky.utils import controller_utils
 from sky.utils import resources_utils
@@ -91,6 +95,38 @@ def _validate_service_task(task: 'sky.Task') -> None:
                     'Please specify the same port instead.')
 
 
+def _rewrite_tls_credential_paths_and_get_tls_env_vars(
+        service_name: str, task: 'sky.Task') -> Dict[str, Any]:
+    """Rewrite the paths of TLS credentials in the task.
+
+    Args:
+        service_name: Name of the service.
+        task: sky.Task to rewrite.
+
+    Returns:
+        The generated template variables for TLS.
+    """
+    service_spec = task.service
+    # Already checked by _validate_service_task
+    assert service_spec is not None
+    if service_spec.tls_credential is None:
+        return {'use_tls': False}
+    remote_tls_keyfile = (
+        serve_utils.generate_remote_tls_keyfile_name(service_name))
+    remote_tls_certfile = (
+        serve_utils.generate_remote_tls_certfile_name(service_name))
+    tls_template_vars = {
+        'use_tls': True,
+        'remote_tls_keyfile': remote_tls_keyfile,
+        'remote_tls_certfile': remote_tls_certfile,
+        'local_tls_keyfile': service_spec.tls_credential.keyfile,
+        'local_tls_certfile': service_spec.tls_credential.certfile,
+    }
+    service_spec.tls_credential = serve_utils.TLSCredential(
+        remote_tls_keyfile, remote_tls_certfile)
+    return tls_template_vars
+
+
 @usage_lib.entrypoint
 def up(
     task: 'sky.Task',
@@ -136,6 +172,9 @@ def up(
         controller_utils.maybe_translate_local_file_mounts_and_sync_up(
             task, path='serve')
 
+    tls_template_vars = _rewrite_tls_credential_paths_and_get_tls_env_vars(
+        service_name, task)
+
     with tempfile.NamedTemporaryFile(
             prefix=f'service-task-{service_name}-',
             mode='w',
@@ -164,6 +203,7 @@ def up(
             'remote_user_config_path': remote_config_yaml_path,
             'modified_catalogs':
                 service_catalog_common.get_modified_catalog_file_mounts(),
+            **tls_template_vars,
             **controller_utils.shared_controller_vars_to_fill(
                 controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
                 remote_user_config_path=remote_config_yaml_path,
@@ -269,10 +309,16 @@ def up(
         else:
             lb_port = serve_utils.load_service_initialization_result(
                 lb_port_payload)
-            endpoint = backend_utils.get_endpoints(
+            socket_endpoint = backend_utils.get_endpoints(
                 controller_handle.cluster_name, lb_port,
                 skip_status_check=True).get(lb_port)
-            assert endpoint is not None, 'Did not get endpoint for controller.'
+            assert socket_endpoint is not None, (
+                'Did not get endpoint for controller.')
+            # Already checked by _validate_service_task
+            assert task.service is not None
+            protocol = ('http'
+                        if task.service.tls_credential is None else 'https')
+            endpoint = f'{protocol}://{socket_endpoint}'
 
         sky_logging.print(
             f'{fore.CYAN}Service name: '
@@ -321,6 +367,7 @@ def update(
         service_name: Name of the service.
     """
     _validate_service_task(task)
+
     # Always apply the policy again here, even though it might have been applied
     # in the CLI. This is to ensure that we apply the policy to the final DAG
     # and get the mutated config.
@@ -329,6 +376,14 @@ def update(
     dag, _ = admin_policy_utils.apply(
         task, use_mutated_config_in_current_request=False)
     task = dag.tasks[0]
+
+    assert task.service is not None
+    if task.service.tls_credential is not None:
+        logger.warning('Updating TLS keyfile and certfile is not supported. '
+                       'Any updates to the keyfile and certfile will not take '
+                       'effect. To update TLS keyfile and certfile, please '
+                       'tear down the service and spin up a new one.')
+
     handle = backend_utils.is_controller_accessible(
         controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
         stopped_message=
@@ -596,6 +651,7 @@ def status(
             'requested_resources_str': (str) str representation of
               requested resources,
             'load_balancing_policy': (str) load balancing policy name,
+            'tls_encrypted': (bool) whether the service is TLS encrypted,
             'replica_info': (List[Dict[str, Any]]) replica information,
         }
 
@@ -731,8 +787,29 @@ def tail_logs(
 
     backend = backend_utils.get_backend_from_handle(handle)
     assert isinstance(backend, backends.CloudVmRayBackend), backend
-    backend.tail_serve_logs(handle,
-                            service_name,
-                            target,
-                            replica_id,
-                            follow=follow)
+
+    if target != serve_utils.ServiceComponent.REPLICA:
+        code = serve_utils.ServeCodeGen.stream_serve_process_logs(
+            service_name,
+            stream_controller=(
+                target == serve_utils.ServiceComponent.CONTROLLER),
+            follow=follow)
+    else:
+        assert replica_id is not None, service_name
+        code = serve_utils.ServeCodeGen.stream_replica_logs(
+            service_name, replica_id, follow)
+
+    # With the stdin=subprocess.DEVNULL, the ctrl-c will not directly
+    # kill the process, so we need to handle it manually here.
+    if threading.current_thread() is threading.main_thread():
+        signal.signal(signal.SIGINT, backend_utils.interrupt_handler)
+        signal.signal(signal.SIGTSTP, backend_utils.stop_handler)
+
+    # Refer to the notes in
+    # sky/backends/cloud_vm_ray_backend.py::CloudVmRayBackend::tail_logs.
+    backend.run_on_head(handle,
+                        code,
+                        stream_logs=True,
+                        process_stream=False,
+                        ssh_mode=command_runner.SshMode.INTERACTIVE,
+                        stdin=subprocess.DEVNULL)

sky/serve/load_balancer.py

@@ -27,10 +27,12 @@ class SkyServeLoadBalancer:
     policy.
     """
 
-    def __init__(self,
-                 controller_url: str,
-                 load_balancer_port: int,
-                 load_balancing_policy_name: Optional[str] = None) -> None:
+    def __init__(
+            self,
+            controller_url: str,
+            load_balancer_port: int,
+            load_balancing_policy_name: Optional[str] = None,
+            tls_credential: Optional[serve_utils.TLSCredential] = None) -> None:
         """Initialize the load balancer.
 
         Args:
@@ -38,6 +40,8 @@ class SkyServeLoadBalancer:
             load_balancer_port: The port where the load balancer listens to.
             load_balancing_policy_name: The name of the load balancing policy
                 to use. Defaults to None.
+            tls_credentials: The TLS credentials for HTTPS endpoint. Defaults
+                to None.
         """
         self._app = fastapi.FastAPI()
         self._controller_url: str = controller_url
@@ -49,6 +53,8 @@ class SkyServeLoadBalancer:
                     f'{load_balancing_policy_name}.')
         self._request_aggregator: serve_utils.RequestsAggregator = (
             serve_utils.RequestTimestamp())
+        self._tls_credential: Optional[serve_utils.TLSCredential] = (
+            tls_credential)
         # TODO(tian): httpx.Client has a resource limit of 100 max connections
         # for each client. We should wait for feedback on the best max
         # connections.
@@ -231,15 +237,25 @@ class SkyServeLoadBalancer:
             # Register controller synchronization task
             asyncio.create_task(self._sync_with_controller())
 
+        uvicorn_tls_kwargs = ({} if self._tls_credential is None else
+                              self._tls_credential.dump_uvicorn_kwargs())
+
+        protocol = 'https' if self._tls_credential is not None else 'http'
+
         logger.info('SkyServe Load Balancer started on '
-                    f'http://0.0.0.0:{self._load_balancer_port}')
+                    f'{protocol}://0.0.0.0:{self._load_balancer_port}')
 
-        uvicorn.run(self._app, host='0.0.0.0', port=self._load_balancer_port)
+        uvicorn.run(self._app,
+                    host='0.0.0.0',
+                    port=self._load_balancer_port,
+                    **uvicorn_tls_kwargs)
 
 
-def run_load_balancer(controller_addr: str,
-                      load_balancer_port: int,
-                      load_balancing_policy_name: Optional[str] = None) -> None:
+def run_load_balancer(
+        controller_addr: str,
+        load_balancer_port: int,
+        load_balancing_policy_name: Optional[str] = None,
+        tls_credential: Optional[serve_utils.TLSCredential] = None) -> None:
     """ Run the load balancer.
 
     Args:
@@ -251,7 +267,8 @@ def run_load_balancer(controller_addr: str,
     load_balancer = SkyServeLoadBalancer(
         controller_url=controller_addr,
         load_balancer_port=load_balancer_port,
-        load_balancing_policy_name=load_balancing_policy_name)
+        load_balancing_policy_name=load_balancing_policy_name,
+        tls_credential=tls_credential)
     load_balancer.run()
 
 

sky/serve/replica_managers.py

@@ -998,9 +998,7 @@ class SkyPilotReplicaManager(ReplicaManager):
                 # Re-raise the exception if it is not preempted.
                 raise
             job_status = list(job_statuses.values())[0]
-            if job_status in [
-                    job_lib.JobStatus.FAILED, job_lib.JobStatus.FAILED_SETUP
-            ]:
+            if job_status in job_lib.JobStatus.user_code_failure_states():
                 info.status_property.user_app_failed = True
                 serve_state.add_or_update_replica(self._service_name,
                                                   info.replica_id, info)

sky/serve/serve_state.py

@@ -79,6 +79,9 @@ db_utils.add_column_to_table(_DB.cursor, _DB.conn, 'services',
                              f'TEXT DEFAULT {json.dumps([])!r}')
 db_utils.add_column_to_table(_DB.cursor, _DB.conn, 'services',
                              'load_balancing_policy', 'TEXT DEFAULT NULL')
+# Whether the service's load balancer is encrypted with TLS.
+db_utils.add_column_to_table(_DB.cursor, _DB.conn, 'services', 'tls_encrypted',
+                             'INTEGER DEFAULT 0')
 _UNIQUE_CONSTRAINT_FAILED_ERROR_MSG = 'UNIQUE constraint failed: services.name'
 
 
@@ -245,7 +248,7 @@ _SERVICE_STATUS_TO_COLOR = {
 
 def add_service(name: str, controller_job_id: int, policy: str,
                 requested_resources_str: str, load_balancing_policy: str,
-                status: ServiceStatus) -> bool:
+                status: ServiceStatus, tls_encrypted: bool) -> bool:
     """Add a service in the database.
 
     Returns:
@@ -258,10 +261,11 @@ def add_service(name: str, controller_job_id: int, policy: str,
                 """\
                 INSERT INTO services
                 (name, controller_job_id, status, policy,
-                requested_resources_str, load_balancing_policy)
-                VALUES (?, ?, ?, ?, ?, ?)""",
+                requested_resources_str, load_balancing_policy, tls_encrypted)
+                VALUES (?, ?, ?, ?, ?, ?, ?)""",
                 (name, controller_job_id, status.value, policy,
-                 requested_resources_str, load_balancing_policy))
+                 requested_resources_str, load_balancing_policy,
+                 int(tls_encrypted)))
 
     except sqlite3.IntegrityError as e:
         if str(e) != _UNIQUE_CONSTRAINT_FAILED_ERROR_MSG:
@@ -328,7 +332,7 @@ def set_service_load_balancer_port(service_name: str,
 def _get_service_from_row(row) -> Dict[str, Any]:
     (current_version, name, controller_job_id, controller_port,
      load_balancer_port, status, uptime, policy, _, _, requested_resources_str,
-     _, active_versions, load_balancing_policy) = row[:14]
+     _, active_versions, load_balancing_policy, tls_encrypted) = row[:15]
     if load_balancing_policy is None:
         # This entry in database was added in #4439, and it will always be set
         # to a str value. If it is None, it means it is an legacy entry and is
@@ -351,6 +355,7 @@ def _get_service_from_row(row) -> Dict[str, Any]:
         'active_versions': json.loads(active_versions),
         'requested_resources_str': requested_resources_str,
         'load_balancing_policy': load_balancing_policy,
+        'tls_encrypted': bool(tls_encrypted),
     }
 
 

sky/serve/serve_utils.py

@@ -1,6 +1,7 @@
 """User interface with the SkyServe."""
 import base64
 import collections
+import dataclasses
 import enum
 import os
 import pathlib
@@ -92,6 +93,19 @@ class UpdateMode(enum.Enum):
     BLUE_GREEN = 'blue_green'
 
 
+@dataclasses.dataclass
+class TLSCredential:
+    """TLS credential for the service."""
+    keyfile: str
+    certfile: str
+
+    def dump_uvicorn_kwargs(self) -> Dict[str, str]:
+        return {
+            'ssl_keyfile': os.path.expanduser(self.keyfile),
+            'ssl_certfile': os.path.expanduser(self.certfile),
+        }
+
+
 DEFAULT_UPDATE_MODE = UpdateMode.ROLLING
 
 _SIGNAL_TO_ERROR = {
@@ -243,6 +257,18 @@ def generate_replica_log_file_name(service_name: str, replica_id: int) -> str:
     return os.path.join(dir_name, f'replica_{replica_id}.log')
 
 
+def generate_remote_tls_keyfile_name(service_name: str) -> str:
+    dir_name = generate_remote_service_dir_name(service_name)
+    # Don't expand here since it is used for remote machine.
+    return os.path.join(dir_name, 'tls_keyfile')
+
+
+def generate_remote_tls_certfile_name(service_name: str) -> str:
+    dir_name = generate_remote_service_dir_name(service_name)
+    # Don't expand here since it is used for remote machine.
+    return os.path.join(dir_name, 'tls_certfile')
+
+
 def generate_replica_cluster_name(service_name: str, replica_id: int) -> str:
     return f'{service_name}-{replica_id}'
 
@@ -799,7 +825,8 @@ def get_endpoint(service_record: Dict[str, Any]) -> str:
     if endpoint is None:
         return '-'
     assert isinstance(endpoint, str), endpoint
-    return endpoint
+    protocol = 'https' if service_record['tls_encrypted'] else 'http'
+    return f'{protocol}://{endpoint}'
 
 
 def format_service_table(service_records: List[Dict[str, Any]],

sky/serve/service.py

@@ -151,7 +151,8 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
         policy=service_spec.autoscaling_policy_str(),
         requested_resources_str=backend_utils.get_task_resources_str(task),
         load_balancing_policy=service_spec.load_balancing_policy,
-        status=serve_state.ServiceStatus.CONTROLLER_INIT)
+        status=serve_state.ServiceStatus.CONTROLLER_INIT,
+        tls_encrypted=service_spec.tls_credential is not None)
     # Directly throw an error here. See sky/serve/api.py::up
     # for more details.
     if not success:
@@ -214,7 +215,6 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
             serve_state.set_service_controller_port(service_name,
                                                     controller_port)
 
-            # TODO(tian): Support HTTPS.
             controller_addr = f'http://{controller_host}:{controller_port}'
 
             load_balancer_port = common_utils.find_free_port(
@@ -231,7 +231,8 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int):
                 target=ux_utils.RedirectOutputForProcess(
                     load_balancer.run_load_balancer,
                     load_balancer_log_file).run,
-                args=(controller_addr, load_balancer_port, policy_name))
+                args=(controller_addr, load_balancer_port, policy_name,
+                      service_spec.tls_credential))
             load_balancer_process.start()
             serve_state.set_service_load_balancer_port(service_name,
                                                        load_balancer_port)

sky/serve/service_spec.py

@@ -9,6 +9,7 @@ import yaml
 from sky import serve
 from sky.serve import constants
 from sky.serve import load_balancing_policies as lb_policies
+from sky.serve import serve_utils
 from sky.utils import common_utils
 from sky.utils import schemas
 from sky.utils import ux_utils
@@ -26,6 +27,7 @@ class SkyServiceSpec:
         max_replicas: Optional[int] = None,
         target_qps_per_replica: Optional[float] = None,
         post_data: Optional[Dict[str, Any]] = None,
+        tls_credential: Optional[serve_utils.TLSCredential] = None,
         readiness_headers: Optional[Dict[str, str]] = None,
         dynamic_ondemand_fallback: Optional[bool] = None,
         base_ondemand_fallback_replicas: Optional[int] = None,
@@ -72,6 +74,8 @@ class SkyServiceSpec:
         self._max_replicas: Optional[int] = max_replicas
         self._target_qps_per_replica: Optional[float] = target_qps_per_replica
         self._post_data: Optional[Dict[str, Any]] = post_data
+        self._tls_credential: Optional[serve_utils.TLSCredential] = (
+            tls_credential)
         self._readiness_headers: Optional[Dict[str, str]] = readiness_headers
         self._dynamic_ondemand_fallback: Optional[
             bool] = dynamic_ondemand_fallback
@@ -163,6 +167,14 @@ class SkyServiceSpec:
 
         service_config['load_balancing_policy'] = config.get(
             'load_balancing_policy', None)
+
+        tls_section = config.get('tls', None)
+        if tls_section is not None:
+            service_config['tls_credential'] = serve_utils.TLSCredential(
+                keyfile=tls_section.get('keyfile', None),
+                certfile=tls_section.get('certfile', None),
+            )
+
         return SkyServiceSpec(**service_config)
 
     @staticmethod
@@ -223,6 +235,9 @@ class SkyServiceSpec:
                         self.downscale_delay_seconds)
         add_if_not_none('load_balancing_policy', None,
                         self._load_balancing_policy)
+        if self.tls_credential is not None:
+            add_if_not_none('tls', 'keyfile', self.tls_credential.keyfile)
+            add_if_not_none('tls', 'certfile', self.tls_credential.certfile)
         return config
 
     def probe_str(self):
@@ -267,12 +282,19 @@ class SkyServiceSpec:
                 f'replica{max_plural} (target QPS per replica: '
                 f'{self.target_qps_per_replica})')
 
+    def tls_str(self):
+        if self.tls_credential is None:
+            return 'No TLS Enabled'
+        return (f'Keyfile: {self.tls_credential.keyfile}, '
+                f'Certfile: {self.tls_credential.certfile}')
+
     def __repr__(self) -> str:
         return textwrap.dedent(f"""\
             Readiness probe method:           {self.probe_str()}
             Readiness initial delay seconds:  {self.initial_delay_seconds}
             Readiness probe timeout seconds:  {self.readiness_timeout_seconds}
             Replica autoscaling policy:       {self.autoscaling_policy_str()}
+            TLS Certificates:                 {self.tls_str()}
             Spot Policy:                      {self.spot_policy_str()}
             Load Balancing Policy:            {self.load_balancing_policy}
         """)
@@ -306,6 +328,15 @@ class SkyServiceSpec:
     def post_data(self) -> Optional[Dict[str, Any]]:
         return self._post_data
 
+    @property
+    def tls_credential(self) -> Optional[serve_utils.TLSCredential]:
+        return self._tls_credential
+
+    @tls_credential.setter
+    def tls_credential(self,
+                       value: Optional[serve_utils.TLSCredential]) -> None:
+        self._tls_credential = value
+
     @property
     def readiness_headers(self) -> Optional[Dict[str, str]]:
         return self._readiness_headers

sky/skylet/constants.py

@@ -86,7 +86,7 @@ TASK_ID_LIST_ENV_VAR = 'SKYPILOT_TASK_IDS'
 # cluster yaml is updated.
 #
 # TODO(zongheng,zhanghao): make the upgrading of skylet automatic?
-SKYLET_VERSION = '9'
+SKYLET_VERSION = '10'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.

sky/skylet/events.py

@@ -13,6 +13,8 @@ from sky import clouds
 from sky import sky_logging
 from sky.backends import cloud_vm_ray_backend
 from sky.clouds import cloud_registry
+from sky.jobs import scheduler as managed_job_scheduler
+from sky.jobs import state as managed_job_state
 from sky.jobs import utils as managed_job_utils
 from sky.serve import serve_utils
 from sky.skylet import autostop_lib
@@ -67,12 +69,13 @@ class JobSchedulerEvent(SkyletEvent):
         job_lib.scheduler.schedule_step(force_update_jobs=True)
 
 
-class ManagedJobUpdateEvent(SkyletEvent):
-    """Skylet event for updating managed job status."""
+class ManagedJobEvent(SkyletEvent):
+    """Skylet event for updating and scheduling managed jobs."""
     EVENT_INTERVAL_SECONDS = 300
 
     def _run(self):
         managed_job_utils.update_managed_job_status()
+        managed_job_scheduler.maybe_schedule_next_jobs()
 
 
 class ServiceUpdateEvent(SkyletEvent):
@@ -116,7 +119,8 @@ class AutostopEvent(SkyletEvent):
             logger.debug('autostop_config not set. Skipped.')
             return
 
-        if job_lib.is_cluster_idle():
+        if (job_lib.is_cluster_idle() and
+                not managed_job_state.get_num_alive_jobs()):
             idle_minutes = (time.time() -
                             autostop_lib.get_last_active_time()) // 60
             logger.debug(

sky/skylet/job_lib.py

@@ -10,9 +10,8 @@ import pathlib
 import shlex
 import signal
 import sqlite3
-import subprocess
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Sequence
 
 import colorama
 import filelock
@@ -23,6 +22,7 @@ from sky.skylet import constants
 from sky.utils import common_utils
 from sky.utils import db_utils
 from sky.utils import log_utils
+from sky.utils import subprocess_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -162,13 +162,17 @@ class JobStatus(enum.Enum):
     def nonterminal_statuses(cls) -> List['JobStatus']:
         return [cls.INIT, cls.SETTING_UP, cls.PENDING, cls.RUNNING]
 
-    def is_terminal(self):
+    def is_terminal(self) -> bool:
         return self not in self.nonterminal_statuses()
 
-    def __lt__(self, other):
+    @classmethod
+    def user_code_failure_states(cls) -> Sequence['JobStatus']:
+        return (cls.FAILED, cls.FAILED_SETUP)
+
+    def __lt__(self, other: 'JobStatus') -> bool:
         return list(JobStatus).index(self) < list(JobStatus).index(other)
 
-    def colored_str(self):
+    def colored_str(self) -> str:
         color = _JOB_STATUS_TO_COLOR[self]
         return f'{color}{self.value}{colorama.Style.RESET_ALL}'
 
@@ -205,31 +209,7 @@ class JobScheduler:
         _CURSOR.execute((f'UPDATE pending_jobs SET submit={int(time.time())} '
                          f'WHERE job_id={job_id!r}'))
         _CONN.commit()
-        # Use nohup to ensure the job driver process is a separate process tree,
-        # instead of being a child of the current process. This is important to
-        # avoid a chain of driver processes (job driver can call schedule_step()
-        # to submit new jobs, and the new job can also call schedule_step()
-        # recursively).
-        #
-        # echo $! will output the PID of the last background process started
-        # in the current shell, so we can retrieve it and record in the DB.
-        #
-        # TODO(zhwu): A more elegant solution is to use another daemon process
-        # to be in charge of starting these driver processes, instead of
-        # starting them in the current process.
-        wrapped_cmd = (f'nohup bash -c {shlex.quote(run_cmd)} '
-                       '</dev/null >/dev/null 2>&1 & echo $!')
-        proc = subprocess.run(wrapped_cmd,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE,
-                              stdin=subprocess.DEVNULL,
-                              start_new_session=True,
-                              check=True,
-                              shell=True,
-                              text=True)
-        # Get the PID of the detached process
-        pid = int(proc.stdout.strip())
-
+        pid = subprocess_utils.launch_new_process_tree(run_cmd)
         # TODO(zhwu): Backward compatibility, remove this check after 0.10.0.
         # This is for the case where the job is submitted with SkyPilot older
         # than #4318, using ray job submit.

sky/skylet/log_lib.py

@@ -25,9 +25,9 @@ from sky.utils import log_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
 
-_SKY_LOG_WAITING_GAP_SECONDS = 1
-_SKY_LOG_WAITING_MAX_RETRY = 5
-_SKY_LOG_TAILING_GAP_SECONDS = 0.2
+SKY_LOG_WAITING_GAP_SECONDS = 1
+SKY_LOG_WAITING_MAX_RETRY = 5
+SKY_LOG_TAILING_GAP_SECONDS = 0.2
 # Peek the head of the lines to check if we need to start
 # streaming when tail > 0.
 PEEK_HEAD_LINES_FOR_START_STREAM = 20
@@ -336,7 +336,7 @@ def _follow_job_logs(file,
             ]:
                 if wait_last_logs:
                     # Wait all the logs are printed before exit.
-                    time.sleep(1 + _SKY_LOG_TAILING_GAP_SECONDS)
+                    time.sleep(1 + SKY_LOG_TAILING_GAP_SECONDS)
                     wait_last_logs = False
                     continue
                 status_str = status.value if status is not None else 'None'
@@ -345,7 +345,7 @@ def _follow_job_logs(file,
                         f'Job finished (status: {status_str}).'))
                 return
 
-            time.sleep(_SKY_LOG_TAILING_GAP_SECONDS)
+            time.sleep(SKY_LOG_TAILING_GAP_SECONDS)
             status = job_lib.get_status_no_lock(job_id)
 
 
@@ -426,15 +426,15 @@ def tail_logs(job_id: Optional[int],
         retry_cnt += 1
         if os.path.exists(log_path) and status != job_lib.JobStatus.INIT:
             break
-        if retry_cnt >= _SKY_LOG_WAITING_MAX_RETRY:
+        if retry_cnt >= SKY_LOG_WAITING_MAX_RETRY:
             print(
                 f'{colorama.Fore.RED}ERROR: Logs for '
                 f'{job_str} (status: {status.value}) does not exist '
                 f'after retrying {retry_cnt} times.{colorama.Style.RESET_ALL}')
             return
-        print(f'INFO: Waiting {_SKY_LOG_WAITING_GAP_SECONDS}s for the logs '
+        print(f'INFO: Waiting {SKY_LOG_WAITING_GAP_SECONDS}s for the logs '
               'to be written...')
-        time.sleep(_SKY_LOG_WAITING_GAP_SECONDS)
+        time.sleep(SKY_LOG_WAITING_GAP_SECONDS)
         status = job_lib.update_job_status([job_id], silent=True)[0]
 
     start_stream_at = LOG_FILE_START_STREAMING_AT