skypilot-nightly 1.0.0.dev20250922__py3-none-any.whl → 1.0.0.dev20250926__py3-none-any.whl

sky/jobs/controller.py

@@ -3,6 +3,7 @@
 import asyncio
 import logging
 import os
+import pathlib
 import resource
 import shutil
 import sys
@@ -17,6 +18,7 @@ import sky
 from sky import core
 from sky import exceptions
 from sky import sky_logging
+from sky import skypilot_config
 from sky.backends import backend_utils
 from sky.backends import cloud_vm_ray_backend
 from sky.data import data_utils
@@ -56,6 +58,7 @@ async def create_background_task(coro: typing.Coroutine) -> None:
     async with _background_tasks_lock:
         task = asyncio.create_task(coro)
         _background_tasks.add(task)
+        # TODO(cooperc): Discard needs a lock?
         task.add_done_callback(_background_tasks.discard)
 
 
@@ -896,6 +899,9 @@ class Controller:
             # some data here.
             raise error
 
+    # Use context.contextual to enable per-job output redirection and env var
+    # isolation.
+    @context.contextual
     async def run_job_loop(self,
                            job_id: int,
                            dag_yaml: str,
@@ -904,13 +910,9 @@ class Controller:
                            env_file_path: Optional[str] = None,
                            pool: Optional[str] = None):
         """Background task that runs the job loop."""
-        # Replace os.environ with ContextualEnviron to enable per-job
-        # environment isolation. This allows each job to have its own
-        # environment variables without affecting other jobs or the main
-        # process.
-        context.initialize()
         ctx = context.get()
-        ctx.redirect_log(log_file)  # type: ignore
+        assert ctx is not None, 'Context is not initialized'
+        ctx.redirect_log(pathlib.Path(log_file))
 
         # Load and apply environment variables from the job's environment file
         if env_file_path and os.path.exists(env_file_path):
@@ -921,13 +923,15 @@ class Controller:
                                 f'{list(env_vars.keys())}')
 
                 # Apply environment variables to the job's context
-                ctx = context.get()
                 if ctx is not None:
                     for key, value in env_vars.items():
                         if value is not None:
                             ctx.override_envs({key: value})
                             job_logger.debug(
                                 f'Set environment variable: {key}={value}')
+                    # Reload the skypilot config for this context to make sure
+                    # the latest config is used.
+                    skypilot_config.reload_config()
                 else:
                     job_logger.error(
                         'Context is None, cannot set environment variables')

sky/jobs/server/core.py

@@ -281,8 +281,7 @@ def launch(
         # Check whether cached jobs controller cluster is accessible
         cluster_name = (
             controller_utils.Controllers.JOBS_CONTROLLER.value.cluster_name)
-        record = global_user_state.get_cluster_from_name(cluster_name)
-        if record is not None:
+        if global_user_state.cluster_with_name_exists(cluster_name):
             # there is a cached jobs controller cluster
             try:
                 # TODO: do something with returned status?
@@ -369,6 +368,8 @@ def launch(
                 'priority': priority,
                 'consolidation_mode_job_id': consolidation_mode_job_id,
                 'pool': pool,
+                'job_controller_indicator_file':
+                    managed_job_constants.JOB_CONTROLLER_INDICATOR_FILE,
                 **controller_utils.shared_controller_vars_to_fill(
                     controller,
                     remote_user_config_path=remote_user_config_path,
@@ -959,9 +960,10 @@ def pool_apply(
     task: 'sky.Task',
     pool_name: str,
     mode: serve_utils.UpdateMode = serve_utils.DEFAULT_UPDATE_MODE,
+    workers: Optional[int] = None,
 ) -> None:
     """Apply a config to a pool."""
-    return impl.apply(task, pool_name, mode, pool=True)
+    return impl.apply(task, workers, pool_name, mode, pool=True)
 
 
 @usage_lib.entrypoint

sky/jobs/server/server.py

@@ -94,23 +94,27 @@ async def logs(
     request: fastapi.Request, jobs_logs_body: payloads.JobsLogsBody,
     background_tasks: fastapi.BackgroundTasks
 ) -> fastapi.responses.StreamingResponse:
-    executor.schedule_request(
+    schedule_type = api_requests.ScheduleType.SHORT
+    if jobs_logs_body.refresh:
+        # When refresh is specified, the job controller might be restarted,
+        # which takes longer time to finish. We schedule it to long executor.
+        schedule_type = api_requests.ScheduleType.LONG
+    request_task = executor.prepare_request(
         request_id=request.state.request_id,
         request_name='jobs.logs',
         request_body=jobs_logs_body,
         func=core.tail_logs,
-        # TODO(aylei): We have tail logs scheduled as SHORT request, because it
-        # should be responsive. However, it can be long running if the user's
-        # job keeps running, and we should avoid it taking the SHORT worker
-        # indefinitely.
-        # When refresh is True we schedule it as LONG because a controller
-        # restart might be needed.
-        schedule_type=api_requests.ScheduleType.LONG
-        if jobs_logs_body.refresh else api_requests.ScheduleType.SHORT,
+        schedule_type=schedule_type,
         request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
-    request_task = await api_requests.get_request_async(request.state.request_id
-                                                       )
+    if schedule_type == api_requests.ScheduleType.LONG:
+        executor.schedule_prepared_request(request_task)
+    else:
+        # For short request, run in the coroutine to avoid blocking
+        # short workers.
+        task = executor.execute_request_in_coroutine(request_task)
+        # Cancel the coroutine after the request is done or client disconnects
+        background_tasks.add_task(task.cancel)
 
     return stream_utils.stream_response(
         request_id=request_task.request_id,

sky/jobs/utils.py

@@ -156,7 +156,7 @@ def _validate_consolidation_mode_config(
     if current_is_consolidation_mode:
         controller_cn = (
             controller_utils.Controllers.JOBS_CONTROLLER.value.cluster_name)
-        if global_user_state.get_cluster_from_name(controller_cn) is not None:
+        if global_user_state.cluster_with_name_exists(controller_cn):
             with ux_utils.print_exception_no_traceback():
                 raise exceptions.InconsistentConsolidationModeError(
                     f'{colorama.Fore.RED}Consolidation mode for jobs is '

sky/logs/agent.py

@@ -34,7 +34,8 @@ class FluentbitAgent(LoggingAgent):
     def get_setup_command(self,
                           cluster_name: resources_utils.ClusterName) -> str:
         install_cmd = (
-            'if ! command -v fluent-bit >/dev/null 2>&1; then '
+            # pylint: disable=line-too-long
+            'if ! command -v fluent-bit >/dev/null 2>&1 && [ ! -f /opt/fluent-bit/bin/fluent-bit ]; then '
             'sudo apt-get update; sudo apt-get install -y gnupg; '
             # pylint: disable=line-too-long
             'sudo sh -c \'curl https://packages.fluentbit.io/fluentbit.key | gpg --dearmor > /usr/share/keyrings/fluentbit-keyring.gpg\'; '
@@ -51,14 +52,32 @@ class FluentbitAgent(LoggingAgent):
         cfg_path = os.path.join(constants.LOGGING_CONFIG_DIR, 'fluentbit.yaml')
         config_cmd = (f'mkdir -p {constants.LOGGING_CONFIG_DIR} && '
                       f'echo {shlex.quote(cfg)} > {cfg_path}')
+        kill_prior_cmd = (
+            'if [ -f "/tmp/fluentbit.pid" ]; then '
+            # pylint: disable=line-too-long
+            'echo "Killing prior fluent-bit process $(cat /tmp/fluentbit.pid)"; '
+            'kill "$(cat /tmp/fluentbit.pid)" || true; '
+            'fi')
         start_cmd = ('nohup $(command -v fluent-bit || '
                      'echo "/opt/fluent-bit/bin/fluent-bit") '
-                     f'-c {cfg_path} > /tmp/fluentbit.log 2>&1 &')
-        return f'set -e; {install_cmd}; {config_cmd}; {start_cmd}'
+                     f'-c {cfg_path} > /tmp/fluentbit.log 2>&1 & '
+                     'echo $! > /tmp/fluentbit.pid')
+        return ('set -e; '
+                f'{install_cmd}; '
+                f'{config_cmd}; '
+                f'{kill_prior_cmd}; '
+                f'{start_cmd}')
 
     def fluentbit_config(self,
                          cluster_name: resources_utils.ClusterName) -> str:
         cfg_dict = {
+            'parsers': [{
+                'name': 'sky-ray-parser',
+                'format': 'regex',
+                # pylint: disable=line-too-long
+                'regex': r'(?:\x1b\[[\d;]+m)?\((?<worker_name>[^,]+)(?:,\s*rank=(?<rank>\d+))?(?:,\s*pid=(?<pid>\d+))(?:,\s*ip=(?<ip>[\d.]+))?\)(?:\x1b\[[\d;]+m)?\s*(?<log_line>.*)',
+                'types': 'rank:integer pid:integer',
+            }],
             'pipeline': {
                 'inputs': [{
                     'name': 'tail',
@@ -70,6 +89,14 @@ class FluentbitAgent(LoggingAgent):
                     # right after the job completion.
                     'refresh_interval': 1,
                 }],
+                'filters': [{
+                    'name': 'parser',
+                    'match': '*',
+                    'key_name': 'log',
+                    'parser': 'sky-ray-parser',
+                    'preserve_key': 'on',  # preserve field for backwards compat
+                    'reserve_data': 'on',
+                }],
                 'outputs': [self.fluentbit_output_config(cluster_name)],
             }
         }

sky/logs/aws.py

@@ -5,7 +5,6 @@ from typing import Any, Dict, Optional
 import pydantic
 
 from sky.logs.agent import FluentbitAgent
-from sky.skylet import constants
 from sky.utils import resources_utils
 from sky.utils import yaml_utils
 
@@ -176,6 +175,8 @@ class CloudwatchLoggingAgent(FluentbitAgent):
         Returns:
             The Fluent Bit configuration as a YAML string.
         """
+        cfg_dict = yaml_utils.read_yaml_str(
+            super().fluentbit_config(cluster_name))
         display_name = cluster_name.display_name
         unique_name = cluster_name.name_on_cloud
         # Build tags for the log stream
@@ -197,24 +198,13 @@ class CloudwatchLoggingAgent(FluentbitAgent):
                 'value': value
             })
 
-        cfg_dict = {
-            'pipeline': {
-                'inputs': [{
-                    'name': 'tail',
-                    'path': f'{constants.SKY_LOGS_DIRECTORY}/*/*.log',
-                    'path_key': 'log_path',
-                    # Shorten the refresh interval from 60s to 1s since every
-                    # job creates a new log file and we must be responsive
-                    # for this: the VM might be autodown within a minute
-                    # right after the job completion.
-                    'refresh_interval': 1,
-                    'processors': {
-                        'logs': log_processors,
-                    }
-                }],
-                'outputs': [self.fluentbit_output_config(cluster_name)],
-            }
-        }
+        # Add log processors to config
+        processors_config = cfg_dict['pipeline']['inputs'][0].get(
+            'processors', {})
+        processors_logs_config = processors_config.get('logs', [])
+        processors_logs_config.extend(log_processors)
+        processors_config['logs'] = processors_logs_config
+        cfg_dict['pipeline']['inputs'][0]['processors'] = processors_config
 
         return yaml_utils.dump_yaml_str(cfg_dict)
 

sky/provision/__init__.py

@@ -168,7 +168,8 @@ def map_all_volumes_usedby(
 
 
 @_route_to_cloud_impl
-def run_instances(provider_name: str, region: str, cluster_name_on_cloud: str,
+def run_instances(provider_name: str, region: str, cluster_name: str,
+                  cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Start instances with bootstrapped configuration."""
     raise NotImplementedError

sky/provision/aws/instance.py

@@ -311,9 +311,10 @@ def _get_head_instance_id(instances: List) -> Optional[str]:
     return head_instance_id
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     ec2 = _default_ec2_resource(region)
     # NOTE: We set max_attempts=0 for fast failing when the resource is not
     # available (although the doc says it will only retry for network

sky/provision/azure/instance.py

@@ -362,9 +362,10 @@ def _create_instances(compute_client: 'azure_compute.ComputeManagementClient',
     return instances
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     # TODO(zhwu): This function is too long. We should refactor it.
     provider_config = config.provider_config
     resource_group = provider_config['resource_group']

sky/provision/cudo/instance.py

@@ -40,10 +40,10 @@ def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     return head_instance_id
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
-
+    del cluster_name  # unused
     pending_status = ['pend', 'init', 'prol', 'boot']
 
     while True:

sky/provision/do/instance.py

@@ -26,10 +26,10 @@ def _get_head_instance(
     return None
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
-
+    del cluster_name  # unused
     pending_status = ['new']
     newly_started_instances = utils.filter_instances(cluster_name_on_cloud,
                                                      pending_status + ['off'])

sky/provision/docker_utils.py

@@ -3,7 +3,7 @@
 import dataclasses
 import shlex
 import time
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from sky import sky_logging
 from sky.skylet import constants
@@ -192,12 +192,16 @@ class DockerInitializer:
         self.docker_cmd = 'podman' if use_podman else 'docker'
         self.log_path = log_path
 
-    def _run(self,
-             cmd,
-             run_env='host',
-             wait_for_docker_daemon: bool = False,
-             separate_stderr: bool = False,
-             log_err_when_fail: bool = True) -> str:
+    def _run(
+        self,
+        cmd,
+        run_env='host',
+        wait_for_docker_daemon: bool = False,
+        separate_stderr: bool = False,
+        log_err_when_fail: bool = True,
+        flock_name: Optional[str] = None,
+        flock_args: Optional[str] = None,
+    ) -> str:
 
         if run_env == 'docker':
             cmd = self._docker_expand_user(cmd, any_char=True)
@@ -206,8 +210,13 @@ class DockerInitializer:
             # an error: `the input device is not a TTY`, and it works without
             # `-it` flag.
             # TODO(zhwu): ray use the `-it` flag, we need to check why.
-            cmd = (f'{self.docker_cmd} exec {self.container_name} /bin/bash -c'
-                   f' {shlex.quote(cmd)} ')
+            cmd = (f'{self.docker_cmd} exec -u 0 {self.container_name}'
+                   f' /bin/bash -c {shlex.quote(cmd)} ')
+
+        if flock_name is not None:
+            flock_args = flock_args or ''
+            cmd = (f'flock {flock_args} /tmp/{flock_name} '
+                   f'-c {shlex.quote(cmd)}')
 
         logger.debug(f'+ {cmd}')
         start = time.time()
@@ -259,7 +268,10 @@ class DockerInitializer:
         if self._check_container_exited():
             self.initialized = True
             self._run(f'{self.docker_cmd} start {self.container_name}')
-            self._run('sudo service ssh start', run_env='docker')
+            self._run('sudo service ssh start',
+                      run_env='docker',
+                      flock_name=f'{self.container_name}.sky.lifecycle.lock',
+                      flock_args='-s -w 1')
             return self._run('whoami', run_env='docker')
 
         # SkyPilot: Docker login if user specified a private docker registry.
@@ -358,7 +370,9 @@ class DockerInitializer:
                     self._auto_configure_shm(user_docker_run_options)),
                 self.docker_cmd,
             )
-            self._run(f'{remove_container_cmd}; {start_command}')
+            self._run(f'{remove_container_cmd} && {start_command}',
+                      flock_name=f'{self.container_name}.sky.lifecycle.lock',
+                      flock_args='-x -w 10')
 
         # SkyPilot: Setup Commands.
         # TODO(zhwu): the following setups should be aligned with the kubernetes
@@ -376,14 +390,18 @@ class DockerInitializer:
             'echo "export DEBIAN_FRONTEND=noninteractive" >> ~/.bashrc;',
             run_env='docker')
         # Install dependencies.
-        self._run(
-            'sudo apt-get update; '
+        cmd = (
+            'bash -lc \''
+            'exec 200>/var/tmp/sky_apt.lock; '
+            'flock -x -w 120 200 || exit 1; '
+            'export DEBIAN_FRONTEND=noninteractive; '
+            'apt-get -yq update && '
             # Our mount script will install gcsfuse without fuse package.
             # We need to install fuse package first to enable storage mount.
             # The dpkg option is to suppress the prompt for fuse installation.
-            'sudo apt-get -o DPkg::Options::="--force-confnew" install -y '
-            'rsync curl wget patch openssh-server python3-pip fuse;',
-            run_env='docker')
+            'apt-get -o DPkg::Options::=--force-confnew install -y '
+            'rsync curl wget patch openssh-server python3-pip fuse\'')
+        self._run(cmd, run_env='docker')
 
         # Copy local authorized_keys to docker container.
         # Stop and disable jupyter service. This is to avoid port conflict on
@@ -459,9 +477,13 @@ class DockerInitializer:
         user_pos = string.find('~')
         if user_pos > -1:
             if self.home_dir is None:
-                cmd = (f'{self.docker_cmd} exec {self.container_name} '
-                       'printenv HOME')
-                self.home_dir = self._run(cmd, separate_stderr=True)
+                cmd = (f'{self.docker_cmd} exec {self.container_name}'
+                       ' printenv HOME')
+                self.home_dir = self._run(
+                    cmd,
+                    separate_stderr=True,
+                    flock_name=f'{self.container_name}.sky.lifecycle.lock',
+                    flock_args='-s -w 1')
                 # Check for unexpected newline in home directory, which can be
                 # a common issue when the output is mixed with stderr.
                 assert '\n' not in self.home_dir, (

sky/provision/fluidstack/instance.py

@@ -78,10 +78,10 @@ def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     return head_instance_id
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """Runs instances for the given cluster."""
-
+    del cluster_name  # unused
     pending_status = ['pending', 'provisioning']
     while True:
         instances = _filter_instances(cluster_name_on_cloud, pending_status)

sky/provision/gcp/instance.py

@@ -360,9 +360,10 @@ def _run_instances(region: str, cluster_name_on_cloud: str,
                                   created_instance_ids=created_instance_ids)
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
     """See sky/provision/__init__.py"""
+    del cluster_name  # unused
     try:
         return _run_instances(region, cluster_name_on_cloud, config)
     except gcp.http_error_exception() as e:

sky/provision/hyperbolic/instance.py

@@ -64,8 +64,9 @@ def _get_head_instance_id(instances: Dict[str, Any]) -> Optional[str]:
     return next(iter(instances.keys()))
 
 
-def run_instances(region: str, cluster_name_on_cloud: str,
+def run_instances(region: str, cluster_name: str, cluster_name_on_cloud: str,
                   config: common.ProvisionConfig) -> common.ProvisionRecord:
+    del cluster_name  # unused
     logger.info(f'Starting run_instances with region={region}, '
                 f'cluster={cluster_name_on_cloud}')
     logger.debug(f'Config: {config}')

sky/provision/instance_setup.py

@@ -84,7 +84,7 @@ def _set_usage_run_id_cmd() -> str:
     latest one when the function is called.
     """
     return (
-        f'cat {usage_constants.USAGE_RUN_ID_FILE} || '
+        f'cat {usage_constants.USAGE_RUN_ID_FILE} 2> /dev/null || '
         # The run id is retrieved locally for the current run, so that the
         # remote cluster will be set with the same run id as the initial
         # launch operation.