skypilot-nightly 1.0.0.dev2024053101__py3-none-any.whl → 1.0.0.dev2025022801__py3-none-any.whl

sky/skylet/log_lib.py

@@ -2,6 +2,7 @@
 
 This is a remote utility module that provides logging functionality.
 """
+import collections
 import copy
 import io
 import multiprocessing.pool
@@ -12,7 +13,8 @@ import sys
 import tempfile
 import textwrap
 import time
-from typing import Dict, Iterator, List, Optional, Tuple, Union
+from typing import (Deque, Dict, Iterable, Iterator, List, Optional, TextIO,
+                    Tuple, Union)
 
 import colorama
 
@@ -21,13 +23,19 @@ from sky.skylet import constants
 from sky.skylet import job_lib
 from sky.utils import log_utils
 from sky.utils import subprocess_utils
+from sky.utils import ux_utils
 
-_SKY_LOG_WAITING_GAP_SECONDS = 1
-_SKY_LOG_WAITING_MAX_RETRY = 5
-_SKY_LOG_TAILING_GAP_SECONDS = 0.2
+SKY_LOG_WAITING_GAP_SECONDS = 1
+SKY_LOG_WAITING_MAX_RETRY = 5
+SKY_LOG_TAILING_GAP_SECONDS = 0.2
+# Peek the head of the lines to check if we need to start
+# streaming when tail > 0.
+PEEK_HEAD_LINES_FOR_START_STREAM = 20
 
 logger = sky_logging.init_logger(__name__)
 
+LOG_FILE_START_STREAMING_AT = 'Waiting for task resources on '
+
 
 class _ProcessingArgs:
     """Arguments for processing logs."""
@@ -170,53 +178,19 @@ def run_with_log(
     if process_stream:
         stdout_arg = subprocess.PIPE
         stderr_arg = subprocess.PIPE if not with_ray else subprocess.STDOUT
+    # Use stdin=subprocess.DEVNULL by default, as allowing inputs will mess up
+    # the terminal output when typing in the terminal that starts the API
+    # server.
+    stdin = kwargs.pop('stdin', subprocess.DEVNULL)
     with subprocess.Popen(cmd,
                           stdout=stdout_arg,
                           stderr=stderr_arg,
                           start_new_session=True,
                           shell=shell,
+                          stdin=stdin,
                           **kwargs) as proc:
         try:
-            # The proc can be defunct if the python program is killed. Here we
-            # open a new subprocess to gracefully kill the proc, SIGTERM
-            # and then SIGKILL the process group.
-            # Adapted from ray/dashboard/modules/job/job_manager.py#L154
-            parent_pid = os.getpid()
-            daemon_script = os.path.join(
-                os.path.dirname(os.path.abspath(job_lib.__file__)),
-                'subprocess_daemon.py')
-            if not hasattr(constants, 'SKY_GET_PYTHON_PATH_CMD'):
-                # Backward compatibility: for cluster started before #3326, this
-                # constant does not exist. Since we generate the job script
-                # in backends.cloud_vm_ray_backend with inspect, so the
-                # the lates `run_with_log` will be used, but the `constants` is
-                # not updated. We fallback to `python3` in this case.
-                # TODO(zhwu): remove this after 0.7.0.
-                python_path = 'python3'
-            else:
-                python_path = subprocess.check_output(
-                    constants.SKY_GET_PYTHON_PATH_CMD,
-                    shell=True,
-                    stderr=subprocess.DEVNULL,
-                    encoding='utf-8').strip()
-            daemon_cmd = [
-                python_path,
-                daemon_script,
-                '--parent-pid',
-                str(parent_pid),
-                '--proc-pid',
-                str(proc.pid),
-            ]
-
-            subprocess.Popen(
-                daemon_cmd,
-                start_new_session=True,
-                # Suppress output
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
-                # Disable input
-                stdin=subprocess.DEVNULL,
-            )
+            subprocess_utils.kill_process_daemon(proc.pid)
             stdout = ''
             stderr = ''
 
@@ -263,6 +237,9 @@ def make_task_bash_script(codegen: str,
     # set -a is used for exporting all variables functions to the environment
     # so that bash `user_script` can access `conda activate`. Detail: #436.
     # Reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html # pylint: disable=line-too-long
+    # DEACTIVATE_SKY_REMOTE_PYTHON_ENV: Deactivate the SkyPilot runtime env, as
+    # the ray cluster is started within the runtime env, which may cause the
+    # user program to run in that env as well.
     # PYTHONUNBUFFERED is used to disable python output buffering.
     script = [
         textwrap.dedent(f"""\
@@ -271,6 +248,7 @@ def make_task_bash_script(codegen: str,
             set -a
             . $(conda info --base 2> /dev/null)/etc/profile.d/conda.sh > /dev/null 2>&1 || true
             set +a
+            {constants.DEACTIVATE_SKY_REMOTE_PYTHON_ENV}
             export PYTHONUNBUFFERED=1
             cd {constants.SKY_REMOTE_WORKDIR}"""),
     ]
@@ -316,21 +294,16 @@ def run_bash_command_with_log(bash_command: str,
         # Need this `-i` option to make sure `source ~/.bashrc` work.
         inner_command = f'/bin/bash -i {script_path}'
 
-        subprocess_cmd: Union[str, List[str]]
-        subprocess_cmd = inner_command
-
-        return run_with_log(
-            subprocess_cmd,
-            log_path,
-            stream_logs=stream_logs,
-            with_ray=with_ray,
-            # Disable input to avoid blocking.
-            stdin=subprocess.DEVNULL,
-            shell=True)
+        return run_with_log(inner_command,
+                            log_path,
+                            stream_logs=stream_logs,
+                            with_ray=with_ray,
+                            shell=True)
 
 
 def _follow_job_logs(file,
                      job_id: int,
+                     start_streaming: bool,
                      start_streaming_at: str = '') -> Iterator[str]:
     """Yield each line from a file as they are written.
 
@@ -339,7 +312,6 @@ def _follow_job_logs(file,
     # No need to lock the status here, as the while loop can handle
     # the older status.
     status = job_lib.get_status_no_lock(job_id)
-    start_streaming = False
     wait_last_logs = True
     while True:
         tmp = file.readline()
@@ -366,21 +338,58 @@ def _follow_job_logs(file,
             ]:
                 if wait_last_logs:
                     # Wait all the logs are printed before exit.
-                    time.sleep(1 + _SKY_LOG_TAILING_GAP_SECONDS)
+                    time.sleep(1 + SKY_LOG_TAILING_GAP_SECONDS)
                     wait_last_logs = False
                     continue
                 status_str = status.value if status is not None else 'None'
-                print(f'INFO: Job finished (status: {status_str}).')
+                print(ux_utils.finishing_message(
+                    f'Job finished (status: {status_str}).'),
+                      flush=True)
                 return
 
-            time.sleep(_SKY_LOG_TAILING_GAP_SECONDS)
+            time.sleep(SKY_LOG_TAILING_GAP_SECONDS)
             status = job_lib.get_status_no_lock(job_id)
 
 
+def _peek_head_lines(log_file: TextIO) -> List[str]:
+    """Peek the head of the file."""
+    lines = [
+        log_file.readline() for _ in range(PEEK_HEAD_LINES_FOR_START_STREAM)
+    ]
+    # Reset the file pointer to the beginning
+    log_file.seek(0, os.SEEK_SET)
+    return [line for line in lines if line]
+
+
+def _should_stream_the_whole_tail_lines(head_lines_of_log_file: List[str],
+                                        tail_lines: Deque[str],
+                                        start_stream_at: str) -> bool:
+    """Check if the entire tail lines should be streamed."""
+    # See comment:
+    # https://github.com/skypilot-org/skypilot/pull/4241#discussion_r1833611567
+    # for more details.
+    # Case 1: If start_stream_at is found at the head of the tail lines,
+    # we should not stream the whole tail lines.
+    for index, line in enumerate(tail_lines):
+        if index >= PEEK_HEAD_LINES_FOR_START_STREAM:
+            break
+        if start_stream_at in line:
+            return False
+    # Case 2: If start_stream_at is found at the head of log file, but not at
+    # the tail lines, we need to stream the whole tail lines.
+    for line in head_lines_of_log_file:
+        if start_stream_at in line:
+            return True
+    # Case 3: If start_stream_at is not at the head, and not found at the tail
+    # lines, we should not stream the whole tail lines.
+    return False
+
+
 def tail_logs(job_id: Optional[int],
               log_dir: Optional[str],
               managed_job_id: Optional[int] = None,
-              follow: bool = True) -> None:
+              follow: bool = True,
+              tail: int = 0) -> None:
     """Tail the logs of a job.
 
     Args:
@@ -389,6 +398,8 @@ def tail_logs(job_id: Optional[int],
         managed_job_id: The managed job id (for logging info only to avoid
             confusion).
         follow: Whether to follow the logs or print the logs so far and exit.
+        tail: The number of lines to display from the end of the log file,
+            if 0, print all lines.
     """
     if job_id is None:
         # This only happens when job_lib.get_latest_job_id() returns None,
@@ -405,8 +416,6 @@ def tail_logs(job_id: Optional[int],
         return
     logger.debug(f'Tailing logs for job, real job_id {job_id}, managed_job_id '
                  f'{managed_job_id}.')
-    logger.info(f'{colorama.Fore.YELLOW}Start streaming logs for {job_str}.'
-                f'{colorama.Style.RESET_ALL}')
     log_path = os.path.join(log_dir, 'run.log')
     log_path = os.path.expanduser(log_path)
 
@@ -419,18 +428,20 @@ def tail_logs(job_id: Optional[int],
         retry_cnt += 1
         if os.path.exists(log_path) and status != job_lib.JobStatus.INIT:
             break
-        if retry_cnt >= _SKY_LOG_WAITING_MAX_RETRY:
+        if retry_cnt >= SKY_LOG_WAITING_MAX_RETRY:
             print(
                 f'{colorama.Fore.RED}ERROR: Logs for '
                 f'{job_str} (status: {status.value}) does not exist '
                 f'after retrying {retry_cnt} times.{colorama.Style.RESET_ALL}')
             return
-        print(f'INFO: Waiting {_SKY_LOG_WAITING_GAP_SECONDS}s for the logs '
+        print(f'INFO: Waiting {SKY_LOG_WAITING_GAP_SECONDS}s for the logs '
               'to be written...')
-        time.sleep(_SKY_LOG_WAITING_GAP_SECONDS)
+        time.sleep(SKY_LOG_WAITING_GAP_SECONDS)
         status = job_lib.update_job_status([job_id], silent=True)[0]
 
-    start_stream_at = 'INFO: Tip: use Ctrl-C to exit log'
+    start_stream_at = LOG_FILE_START_STREAMING_AT
+    # Explicitly declare the type to avoid mypy warning.
+    lines: Iterable[str] = []
     if follow and status in [
             job_lib.JobStatus.SETTING_UP,
             job_lib.JobStatus.PENDING,
@@ -441,19 +452,48 @@ def tail_logs(job_id: Optional[int],
         with open(log_path, 'r', newline='', encoding='utf-8') as log_file:
             # Using `_follow` instead of `tail -f` to streaming the whole
             # log and creating a new process for tail.
+            start_streaming = False
+            if tail > 0:
+                head_lines_of_log_file = _peek_head_lines(log_file)
+                lines = collections.deque(log_file, maxlen=tail)
+                start_streaming = _should_stream_the_whole_tail_lines(
+                    head_lines_of_log_file, lines, start_stream_at)
+                for line in lines:
+                    if start_stream_at in line:
+                        start_streaming = True
+                    if start_streaming:
+                        print(line, end='')
+                # Flush the last n lines
+                print(end='', flush=True)
+            # Now, the cursor is at the end of the last lines
+            # if tail > 0
             for line in _follow_job_logs(log_file,
                                          job_id=job_id,
+                                         start_streaming=start_streaming,
                                          start_streaming_at=start_stream_at):
                 print(line, end='', flush=True)
     else:
         try:
-            start_stream = False
-            with open(log_path, 'r', encoding='utf-8') as f:
-                for line in f.readlines():
+            start_streaming = False
+            with open(log_path, 'r', encoding='utf-8') as log_file:
+                if tail > 0:
+                    # If tail > 0, we need to read the last n lines.
+                    # We use double ended queue to rotate the last n lines.
+                    head_lines_of_log_file = _peek_head_lines(log_file)
+                    lines = collections.deque(log_file, maxlen=tail)
+                    start_streaming = _should_stream_the_whole_tail_lines(
+                        head_lines_of_log_file, lines, start_stream_at)
+                else:
+                    lines = log_file
+                for line in lines:
                     if start_stream_at in line:
-                        start_stream = True
-                    if start_stream:
+                        start_streaming = True
+                    if start_streaming:
                         print(line, end='', flush=True)
+                status_str = status.value if status is not None else 'None'
+                print(ux_utils.finishing_message(
+                    f'Job finished (status: {status_str}).'),
+                      flush=True)
         except FileNotFoundError:
             print(f'{colorama.Fore.RED}ERROR: Logs for job {job_id} (status:'
                   f' {status.value}) does not exist.{colorama.Style.RESET_ALL}')

sky/skylet/log_lib.pyi

@@ -13,6 +13,12 @@ from sky.skylet import constants as constants
 from sky.skylet import job_lib as job_lib
 from sky.utils import log_utils as log_utils
 
+SKY_LOG_WAITING_GAP_SECONDS: int = ...
+SKY_LOG_WAITING_MAX_RETRY: int = ...
+SKY_LOG_TAILING_GAP_SECONDS: float = ...
+LOG_FILE_START_STREAMING_AT: str = ...
+
+
 class _ProcessingArgs:
     log_path: str
     stream_logs: bool

sky/skylet/providers/command_runner.py

@@ -25,7 +25,7 @@ def docker_start_cmds(
     docker_cmd,
 ):
     """Generating docker start command without --rm.
-    
+
     The code is borrowed from `ray.autoscaler._private.docker`.
 
     Changes we made:
@@ -65,8 +65,8 @@ def docker_start_cmds(
         '--cap-add=SYS_ADMIN',
         '--device=/dev/fuse',
         '--security-opt=apparmor:unconfined',
+        '--entrypoint=/bin/bash',
         image,
-        'bash',
     ]
     return ' '.join(docker_run)
 
@@ -159,19 +159,17 @@ class SkyDockerCommandRunner(DockerCommandRunner):
             return True
 
         # SkyPilot: Docker login if user specified a private docker registry.
-        if "docker_login_config" in self.docker_config:
+        if 'docker_login_config' in self.docker_config:
             # TODO(tian): Maybe support a command to get the login password?
-            docker_login_config: docker_utils.DockerLoginConfig = self.docker_config[
-                "docker_login_config"]
+            docker_login_config: docker_utils.DockerLoginConfig = (
+                self.docker_config['docker_login_config'])
             self._run_with_retry(
                 f'{self.docker_cmd} login --username '
                 f'{docker_login_config.username} --password '
                 f'{docker_login_config.password} {docker_login_config.server}')
             # We automatically add the server prefix to the image name if
             # the user did not add it.
-            server_prefix = f'{docker_login_config.server}/'
-            if not specific_image.startswith(server_prefix):
-                specific_image = f'{server_prefix}{specific_image}'
+            specific_image = docker_login_config.format_image(specific_image)
 
         if self.docker_config.get('pull_before_run', True):
             assert specific_image, ('Image must be included in config if '

sky/skylet/providers/ibm/node_provider.py

@@ -377,7 +377,7 @@ class IBMVPCNodeProvider(NodeProvider):
                 node["id"], nic_id
             ).get_result()
             floating_ips = res["floating_ips"]
-            if len(floating_ips) == 0:
+            if not floating_ips:
                 # not adding a node that's yet/failed to
                 # to get a floating ip provisioned
                 continue
@@ -485,7 +485,7 @@ class IBMVPCNodeProvider(NodeProvider):
         """Returns instance (node) information matching the specified name"""
 
         instances_data = self.ibm_vpc_client.list_instances(name=name).get_result()
-        if len(instances_data["instances"]) > 0:
+        if instances_data["instances"]:
             return instances_data["instances"][0]
         return None
 

sky/skylet/providers/scp/config.py

@@ -107,20 +107,28 @@ class ZoneConfig:
                 for item in subnet_contents
                 if item['subnetState'] == 'ACTIVE' and item["vpcId"] == vpc
             ]
-            if len(subnet_list) > 0:
+            if subnet_list:
                 vpc_subnets[vpc] = subnet_list
 
         return vpc_subnets
 
     def _get_vm_init_script(self, ssh_public_key):
 
+        import subprocess
         init_script_content = self._get_default_config_cmd(
         ) + self._get_ssh_key_gen_cmd(ssh_public_key)
+        init_script_content_string = f'"{init_script_content}"'
+        command = f'echo {init_script_content_string} | base64'
+        result = subprocess.run(command,
+                                shell=True,
+                                capture_output=True,
+                                text=True)
+        init_script_content_base64 = result.stdout
         return {
-            "encodingType": "plain",
+            "encodingType": "base64",
             "initialScriptShell": "bash",
             "initialScriptType": "text",
-            "initialScriptContent": init_script_content
+            "initialScriptContent": init_script_content_base64
         }
 
     def _get_ssh_key_gen_cmd(self, ssh_public_key):

sky/skylet/providers/scp/node_provider.py

@@ -180,7 +180,7 @@ class SCPNodeProvider(NodeProvider):
             metadata['tags'] = instance_info['tags']
         # TODO(ewzeng): The internal ip is hard to get, so set it to the
         # external ip as a hack. This should be changed in the future.
-        #   https://docs.lambdalabs.com/cloud/learn-private-ip-address/
+        #   https://docs.lambdalabs.com/public-cloud/on-demand/getting-started/#learn-your-instances-private-ip-address
         metadata['internal_ip'] = vm['ip']
         metadata['external_ip'] = vm['external_ip']
         return metadata
@@ -259,7 +259,7 @@ class SCPNodeProvider(NodeProvider):
                     for sg in sg_contents
                     if sg["securityGroupId"] == sg_id
                 ]
-                if len(sg) != 0 and sg[0] == "ACTIVE":
+                if sg and sg[0] == "ACTIVE":
                     break
                 time.sleep(5)
 
@@ -282,16 +282,16 @@ class SCPNodeProvider(NodeProvider):
                 for sg in sg_contents
                 if sg["securityGroupId"] == sg_id
             ]
-            if len(sg) == 0:
+            if not sg:
                 break
 
     def _refresh_security_group(self, vms):
-        if len(vms) > 0:
+        if vms:
             return
         # remove security group if vm does not exist
         keys = self.metadata.keys()
         security_group_id = self.metadata[
-            keys[0]]['creation']['securityGroupId'] if len(keys) > 0 else None
+            keys[0]]['creation']['securityGroupId'] if keys else None
         if security_group_id:
             try:
                 self._del_security_group(security_group_id)
@@ -308,7 +308,7 @@ class SCPNodeProvider(NodeProvider):
                 for vm in vm_contents
                 if vm["virtualServerId"] == vm_id
             ]
-            if len(vms) == 0:
+            if not vms:
                 break
 
     def _del_firwall_rules(self, firewall_id, rule_ids):
@@ -391,7 +391,7 @@ class SCPNodeProvider(NodeProvider):
             return None, None, None, None
 
     def _undo_funcs(self, undo_func_list):
-        while len(undo_func_list) > 0:
+        while undo_func_list:
             func = undo_func_list.pop()
             func()
 
@@ -468,7 +468,7 @@ class SCPNodeProvider(NodeProvider):
 
             zone_config = ZoneConfig(self.scp_client, node_config)
             vpc_subnets = zone_config.get_vcp_subnets()
-            if (len(vpc_subnets) == 0):
+            if not vpc_subnets:
                 raise SCPError("This region/zone does not have available VPCs.")
 
             instance_config = zone_config.bootstrap_instance_config(node_config)

sky/skylet/skylet.py

@@ -20,11 +20,13 @@ EVENTS = [
     # The managed job update event should be after the job update event.
     # Otherwise, the abnormal managed job status update will be delayed
     # until the next job update event.
-    events.ManagedJobUpdateEvent(),
+    events.ManagedJobEvent(),
     # This is for monitoring controller job status. If it becomes
     # unhealthy, this event will correctly update the controller
     # status to CONTROLLER_FAILED.
     events.ServiceUpdateEvent(),
+    # Report usage heartbeat every 10 minutes.
+    events.UsageHeartbeatReportEvent(),
 ]
 
 while True:

sky/skylet/subprocess_daemon.py

@@ -1,20 +1,57 @@
 """Sky subprocess daemon.
-
 Wait for parent_pid to exit, then SIGTERM (or SIGKILL if needed) the child
 processes of proc_pid.
 """
-
 import argparse
+import os
 import sys
 import time
 
 import psutil
 
-if __name__ == '__main__':
 
+def daemonize():
+    """Detaches the process from its parent process with double-forking.
+
+    This detachment is crucial in the context of SkyPilot and Ray job. When
+    'sky cancel' is executed, it uses Ray's stop job API to terminate the job.
+    Without daemonization, this subprocess_daemon process will still be a child
+    of the parent process which would be terminated along with the parent
+    process, ray::task or the cancel request for jobs, which is launched with
+    Ray job. Daemonization ensures this process survives the 'sky cancel'
+    command, allowing it to prevent orphaned processes of Ray job.
+    """
+    # First fork: Creates a child process identical to the parent
+    if os.fork() > 0:
+        # Parent process exits, allowing the child to run independently
+        sys.exit()
+
+    # Continues to run from first forked child process.
+    # Detach from parent environment.
+    os.setsid()
+
+    # Second fork: Creates a grandchild process
+    if os.fork() > 0:
+        # First child exits, orphaning the grandchild
+        sys.exit()
+    # Continues execution in the grandchild process
+    # This process is now fully detached from the original parent and terminal
+
+
+if __name__ == '__main__':
+    daemonize()
     parser = argparse.ArgumentParser()
     parser.add_argument('--parent-pid', type=int, required=True)
     parser.add_argument('--proc-pid', type=int, required=True)
+    parser.add_argument(
+        '--initial-children',
+        type=str,
+        default='',
+        help=(
+            'Comma-separated list of initial children PIDs. This is to guard '
+            'against the case where the target process has already terminated, '
+            'while the children are still running.'),
+    )
     args = parser.parse_args()
 
     process = None
@@ -25,32 +62,47 @@ if __name__ == '__main__':
     except psutil.NoSuchProcess:
         pass
 
-    if process is None:
-        sys.exit()
+    # Initialize children list from arguments
+    children = []
+    if args.initial_children:
+        for pid in args.initial_children.split(','):
+            try:
+                child = psutil.Process(int(pid))
+                children.append(child)
+            except (psutil.NoSuchProcess, ValueError):
+                pass
 
-    if parent_process is not None:
-        # Wait for either parent or target process to exit.
+    if process is not None and parent_process is not None:
+        # Wait for either parent or target process to exit
         while process.is_running() and parent_process.is_running():
+            try:
+                tmp_children = process.children(recursive=True)
+                if tmp_children:
+                    children = tmp_children
+            except psutil.NoSuchProcess:
+                pass
             time.sleep(1)
 
-    try:
-        children = process.children(recursive=True)
-        children.append(process)
-    except psutil.NoSuchProcess:
+    if process is not None:
+        # Kill the target process first to avoid having more children, or fail
+        # the process due to the children being defunct.
+        children = [process] + children
+
+    if not children:
         sys.exit()
 
-    for pid in children:
+    for child in children:
         try:
-            pid.terminate()
+            child.terminate()
         except psutil.NoSuchProcess:
-            pass
+            continue
 
     # Wait 30s for the processes to exit gracefully.
     time.sleep(30)
 
     # SIGKILL if they're still running.
-    for pid in children:
+    for child in children:
         try:
-            pid.kill()
+            child.kill()
         except psutil.NoSuchProcess:
-            pass
+            continue