skypilot-nightly 1.0.0.dev20250730__py3-none-any.whl → 1.0.0.dev20250801__py3-none-any.whl

sky/skylet/constants.py

@@ -382,7 +382,8 @@ OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
 # we skip the following keys because they are meant to be client-side configs.
 SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [('api_server',),
                                                        ('allowed_clouds',),
-                                                       ('workspaces',), ('db',)]
+                                                       ('workspaces',), ('db',),
+                                                       ('daemons',)]
 
 # Constants for Azure blob storage
 WAIT_FOR_STORAGE_ACCOUNT_CREATION = 60

sky/skylet/events.py

@@ -75,7 +75,16 @@ class ManagedJobEvent(SkyletEvent):
     EVENT_INTERVAL_SECONDS = 300
 
     def _run(self):
+        logger.info('=== Updating managed job status ===')
         managed_job_utils.update_managed_jobs_statuses()
+
+
+class ManagedJobSchedulingEvent(SkyletEvent):
+    """Skylet event for scheduling managed jobs."""
+    EVENT_INTERVAL_SECONDS = 20
+
+    def _run(self):
+        logger.info('=== Scheduling next jobs ===')
         managed_job_scheduler.maybe_schedule_next_jobs()
 
 

sky/skypilot_config.py

@@ -495,6 +495,12 @@ def parse_and_validate_config_file(config_path: str) -> config_utils.Config:
     try:
         config_dict = common_utils.read_yaml(config_path)
         config = config_utils.Config.from_dict(config_dict)
+        # pop the db url from the config, and set it to the env var.
+        # this is to avoid db url (considered a sensitive value)
+        # being printed with the rest of the config.
+        db_url = config.pop_nested(('db',), None)
+        if db_url:
+            os.environ[constants.ENV_VAR_DB_CONNECTION_URI] = db_url
         if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
             logger.debug(f'Config loaded from {config_path}:\n'
                          f'{common_utils.dump_yaml_str(dict(config))}')
@@ -556,21 +562,16 @@ def _reload_config_as_server() -> None:
     _set_loaded_config_path(None)
 
     server_config_path = _resolve_server_config_path()
-    db_url_from_env = os.environ.get(constants.ENV_VAR_DB_CONNECTION_URI)
     server_config = _get_config_from_path(server_config_path)
-    if db_url_from_env:
-        server_config.set_nested(('db',), db_url_from_env)
-
-    if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
-        logger.debug(f'server config: \n'
-                     f'{common_utils.dump_yaml_str(dict(server_config))}')
-
-    db_url = server_config.get_nested(('db',), None)
-    if db_url and len(server_config.keys()) > 1:
-        raise ValueError(
-            'if db config is specified, no other config is allowed')
+    # Get the db url from the env var. _get_config_from_path should have moved
+    # the db url specified in config file to the env var.
+    db_url = os.environ.get(constants.ENV_VAR_DB_CONNECTION_URI)
 
     if db_url:
+        if len(server_config.keys()) > 1:
+            raise ValueError(
+                'If db config is specified, no other config is allowed')
+        logger.debug('retrieving config from database')
         with _DB_USE_LOCK:
             sqlalchemy_engine = sqlalchemy.create_engine(db_url,
                                                          poolclass=NullPool)
@@ -591,14 +592,13 @@ def _reload_config_as_server() -> None:
 
             db_config = _get_config_yaml_from_db(API_SERVER_CONFIG_KEY)
             if db_config:
-                if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
-                    logger.debug(
-                        f'Config loaded from db:\n'
-                        f'{common_utils.dump_yaml_str(dict(db_config))}')
                 server_config = overlay_skypilot_config(server_config,
                                                         db_config)
             # Close the engine to avoid connection leaks
             sqlalchemy_engine.dispose()
+    if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+        logger.debug(f'server config: \n'
+                     f'{common_utils.dump_yaml_str(dict(server_config))}')
     _set_loaded_config(server_config)
     _set_loaded_config_path(server_config_path)
 
@@ -681,6 +681,10 @@ def override_skypilot_config(
 
     disallowed_diff_keys = []
     for key in constants.SKIPPED_CLIENT_OVERRIDE_KEYS:
+        if key == ('db',):
+            # since db key is popped out of server config, the key is expected
+            # to be different between client and server.
+            continue
         value = override_configs.pop_nested(key, default_value=None)
         if (value is not None and
                 value != original_config.get_nested(key, default_value=None)):
@@ -855,11 +859,11 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
 
     db_updated = False
     if os.environ.get(constants.ENV_VAR_IS_SKYPILOT_SERVER) is not None:
-        existing_db_url = get_nested(('db',), None)
+        existing_db_url = os.environ.get(constants.ENV_VAR_DB_CONNECTION_URI)
+        new_db_url = config.pop_nested(('db',), None)
+        if new_db_url and new_db_url != existing_db_url:
+            raise ValueError('Cannot change db url while server is running')
         if existing_db_url:
-            new_db_url = config.get_nested(('db',), None)
-            if new_db_url and new_db_url != existing_db_url:
-                raise ValueError('Cannot change db url while server is running')
             with _DB_USE_LOCK:
                 sqlalchemy_engine = sqlalchemy.create_engine(existing_db_url,
                                                              poolclass=NullPool)
@@ -869,7 +873,6 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
                 def _set_config_yaml_to_db(key: str,
                                            config: config_utils.Config):
                     assert sqlalchemy_engine is not None
-                    config.pop_nested(('db',), None)
                     config_str = common_utils.dump_yaml_str(dict(config))
                     with orm.Session(sqlalchemy_engine) as session:
                         if (sqlalchemy_engine.dialect.name ==

sky/task.py

@@ -256,6 +256,7 @@ class Task:
         file_mounts_mapping: Optional[Dict[str, str]] = None,
         volume_mounts: Optional[List[volume_lib.VolumeMount]] = None,
         metadata: Optional[Dict[str, Any]] = None,
+        _user_specified_yaml: Optional[str] = None,
     ):
         """Initializes a Task.
 
@@ -381,6 +382,8 @@ class Task:
         if dag is not None:
             dag.add(self)
 
+        self._user_specified_yaml = _user_specified_yaml
+
     def validate(self,
                  skip_file_mounts: bool = False,
                  skip_workdir: bool = False):
@@ -525,6 +528,8 @@ class Task:
         env_overrides: Optional[List[Tuple[str, str]]] = None,
         secrets_overrides: Optional[List[Tuple[str, str]]] = None,
     ) -> 'Task':
+        user_specified_yaml = config.pop('_user_specified_yaml',
+                                         common_utils.dump_yaml_str(config))
         # More robust handling for 'envs': explicitly convert keys and values to
         # str, since users may pass '123' as keys/values which will get parsed
         # as int causing validate_schema() to fail.
@@ -590,19 +595,23 @@ class Task:
 
         # Fill in any Task.envs into file_mounts (src/dst paths, storage
         # name/source).
+        env_vars = config.get('envs', {})
+        secrets = config.get('secrets', {})
+        env_and_secrets = env_vars.copy()
+        env_and_secrets.update(secrets)
         if config.get('file_mounts') is not None:
             config['file_mounts'] = _fill_in_env_vars(config['file_mounts'],
-                                                      config.get('envs', {}))
+                                                      env_and_secrets)
 
         # Fill in any Task.envs into service (e.g. MODEL_NAME).
         if config.get('service') is not None:
             config['service'] = _fill_in_env_vars(config['service'],
-                                                  config.get('envs', {}))
+                                                  env_and_secrets)
 
         # Fill in any Task.envs into workdir
         if config.get('workdir') is not None:
             config['workdir'] = _fill_in_env_vars(config['workdir'],
-                                                  config.get('envs', {}))
+                                                  env_and_secrets)
 
         task = Task(
             config.pop('name', None),
@@ -616,6 +625,7 @@ class Task:
             file_mounts_mapping=config.pop('file_mounts_mapping', None),
             volumes=config.pop('volumes', None),
             metadata=config.pop('_metadata', None),
+            _user_specified_yaml=user_specified_yaml,
         )
 
         # Create lists to store storage objects inlined in file_mounts.
@@ -736,9 +746,19 @@ class Task:
         task.set_resources(sky.Resources.from_yaml_config(resources_config))
 
         service = config.pop('service', None)
+        pool = config.pop('pool', None)
+        if service is not None and pool is not None:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    'Cannot set both service and pool in the same task.')
+
         if service is not None:
             service = service_spec.SkyServiceSpec.from_yaml_config(service)
-        task.set_service(service)
+            task.set_service(service)
+        elif pool is not None:
+            pool['pool'] = True
+            pool = service_spec.SkyServiceSpec.from_yaml_config(pool)
+            task.set_service(pool)
 
         volume_mounts = config.pop('volume_mounts', None)
         if volume_mounts is not None:
@@ -773,7 +793,8 @@ class Task:
             # TODO(zongheng): use
             #  https://github.com/yaml/pyyaml/issues/165#issuecomment-430074049
             # to raise errors on duplicate keys.
-            config = yaml.safe_load(f)
+            user_specified_yaml = f.read()
+            config = yaml.safe_load(user_specified_yaml)
 
         if isinstance(config, str):
             with ux_utils.print_exception_no_traceback():
@@ -782,6 +803,7 @@ class Task:
 
         if config is None:
             config = {}
+        config['_user_specified_yaml'] = user_specified_yaml
         return Task.from_yaml_config(config)
 
     def resolve_and_validate_volumes(self) -> None:
@@ -1537,11 +1559,22 @@ class Task:
                 d[k] = v
         return d
 
-    def to_yaml_config(self, redact_secrets: bool = False) -> Dict[str, Any]:
+    def to_yaml_config(self,
+                       use_user_specified_yaml: bool = False) -> Dict[str, Any]:
         """Returns a yaml-style dict representation of the task.
 
         INTERNAL: this method is internal-facing.
         """
+        if use_user_specified_yaml:
+            if self._user_specified_yaml is None:
+                return self._to_yaml_config(redact_secrets=True)
+            config = yaml.safe_load(self._user_specified_yaml)
+            if config.get('secrets') is not None:
+                config['secrets'] = {k: '<redacted>' for k in config['secrets']}
+            return config
+        return self._to_yaml_config()
+
+    def _to_yaml_config(self, redact_secrets: bool = False) -> Dict[str, Any]:
         config = {}
 
         def add_if_not_none(key, value, no_empty: bool = False):
@@ -1586,13 +1619,9 @@ class Task:
         # Add envs without redaction
         add_if_not_none('envs', self.envs, no_empty=True)
 
-        # Add secrets with redaction if requested
         secrets = self.secrets
         if secrets and redact_secrets:
-            secrets = {
-                k: '<redacted>' if isinstance(v, str) else v
-                for k, v in secrets.items()
-            }
+            secrets = {k: '<redacted>' for k in secrets}
         add_if_not_none('secrets', secrets, no_empty=True)
 
         add_if_not_none('file_mounts', {})
@@ -1615,6 +1644,7 @@ class Task:
             ]
         # we manually check if its empty to not clog up the generated yaml
         add_if_not_none('_metadata', self._metadata if self._metadata else None)
+        add_if_not_none('_user_specified_yaml', self._user_specified_yaml)
         return config
 
     def get_required_cloud_features(

sky/templates/jobs-controller.yaml.j2

@@ -57,6 +57,9 @@ run: |
     --job-id $SKYPILOT_INTERNAL_JOB_ID \
     {%- endif %}
     --env-file {{remote_env_file_path}} \
+    {%- if pool is not none %}
+    --pool {{pool}} \
+    {%- endif %}
     --priority {{priority}}
 
 

sky/templates/sky-serve-controller.yaml.j2

@@ -45,13 +45,29 @@ file_mounts:
 run: |
   # Activate the Python environment, so that cloud SDKs can be found in the
   # PATH.
+  {%- if consolidation_mode_job_id is none %}
   {{ sky_activate_python_env }}
+  {%- endif %}
   # Start sky serve service.
-  python -u -m sky.serve.service \
+  {%- if consolidation_mode_job_id is not none %}
+  {{sky_python_cmd}} \
+  {%- else %}
+  python \
+  {%- endif %}
+   -u -m sky.serve.service \
     --service-name {{service_name}} \
     --task-yaml {{remote_task_yaml_path}} \
+    {%- if consolidation_mode_job_id is not none %}
+    --job-id {{consolidation_mode_job_id}} \
+    {%- else %}
     --job-id $SKYPILOT_INTERNAL_JOB_ID \
-    >> {{controller_log_file}} 2>&1
+    {%- endif %}
+    >> {{controller_log_file}} 2>&1 \
+    {%- if consolidation_mode_job_id is not none %}
+    &
+    {%- endif %}
+    # For consolidation mode, we need to run the service in the background so
+    # that it can immediately return in serve.core.up().
 
 envs:
 {%- for env_name, env_value in controller_envs.items() %}

sky/users/server.py

@@ -414,7 +414,7 @@ async def get_service_account_tokens(
 
 def _generate_service_account_user_id() -> str:
     """Generate a unique user ID for a service account."""
-    random_suffix = secrets.token_hex(16)  # 16 character hex string
+    random_suffix = secrets.token_hex(8)  # 16 character hex string
     service_account_id = (f'sa-{random_suffix}')
     return service_account_id
 

sky/utils/command_runner.py

@@ -201,6 +201,7 @@ class CommandRunner:
         separate_stderr: bool,
         skip_num_lines: int,
         source_bashrc: bool = False,
+        use_login: bool = True,
     ) -> str:
         """Returns the command to run."""
         if isinstance(cmd, list):
@@ -211,7 +212,7 @@ class CommandRunner:
             '/bin/bash',
             '--login',
             '-c',
-        ]
+        ] if use_login else ['/bin/bash', '-c']
         if source_bashrc:
             command += [
                 # Need this `-i` option to make sure `source ~/.bashrc` work.
@@ -1124,7 +1125,8 @@ class LocalProcessCommandRunner(CommandRunner):
                                                process_stream,
                                                separate_stderr,
                                                skip_num_lines=skip_num_lines,
-                                               source_bashrc=source_bashrc)
+                                               source_bashrc=source_bashrc,
+                                               use_login=False)
 
         log_dir = os.path.expanduser(os.path.dirname(log_path))
         os.makedirs(log_dir, exist_ok=True)

sky/utils/controller_utils.py

@@ -5,7 +5,7 @@ import enum
 import os
 import tempfile
 import typing
-from typing import Any, Dict, Iterable, List, Optional, Set
+from typing import Any, Callable, Dict, Iterable, List, Optional, Set
 import uuid
 
 import colorama
@@ -64,7 +64,7 @@ class _ControllerSpec:
     controller_type: str
     name: str
     cluster_name: str
-    in_progress_hint: str
+    in_progress_hint: Callable[[bool], str]
     decline_cancel_hint: str
     _decline_down_when_failed_to_fetch_status_hint: str
     decline_down_for_dirty_controller_hint: str
@@ -94,9 +94,9 @@ class Controllers(enum.Enum):
         controller_type='jobs',
         name='managed jobs controller',
         cluster_name=common.JOB_CONTROLLER_NAME,
-        in_progress_hint=(
-            '* {job_info}To see all managed jobs: '
-            f'{colorama.Style.BRIGHT}sky jobs queue{colorama.Style.RESET_ALL}'),
+        in_progress_hint=lambda _:
+        ('* {job_info}To see all managed jobs: '
+         f'{colorama.Style.BRIGHT}sky jobs queue{colorama.Style.RESET_ALL}'),
         decline_cancel_hint=(
             'Cancelling the jobs controller\'s jobs is not allowed.\nTo cancel '
             f'managed jobs, use: {colorama.Style.BRIGHT}sky jobs cancel '
@@ -126,8 +126,11 @@ class Controllers(enum.Enum):
         name='serve controller',
         cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
         in_progress_hint=(
-            f'* To see detailed service status: {colorama.Style.BRIGHT}'
-            f'sky serve status -v{colorama.Style.RESET_ALL}'),
+            lambda pool:
+            (f'* To see detailed pool status: {colorama.Style.BRIGHT}'
+             f'sky jobs pool status -v{colorama.Style.RESET_ALL}') if pool else
+            (f'* To see detailed service status: {colorama.Style.BRIGHT}'
+             f'sky serve status -v{colorama.Style.RESET_ALL}')),
         decline_cancel_hint=(
             'Cancelling the sky serve controller\'s jobs is not allowed.'),
         _decline_down_when_failed_to_fetch_status_hint=(
@@ -391,10 +394,11 @@ def check_cluster_name_not_controller(
 
 
 # Internal only:
-def download_and_stream_latest_job_log(
+def download_and_stream_job_log(
         backend: 'cloud_vm_ray_backend.CloudVmRayBackend',
         handle: 'cloud_vm_ray_backend.CloudVmRayResourceHandle',
-        local_dir: str) -> Optional[str]:
+        local_dir: str,
+        job_ids: Optional[List[str]] = None) -> Optional[str]:
     """Downloads and streams the latest job log.
 
     This function is only used by jobs controller and sky serve controller.
@@ -412,7 +416,7 @@ def download_and_stream_latest_job_log(
             # multi-node cluster is preempted, and we recover the managed job
             # on the existing cluster, which leads to a larger job_id. Those
             # job_ids all represent the same logical managed job.
-            job_ids=None,
+            job_ids=job_ids,
             local_dir=local_dir)
     except Exception as e:  # pylint: disable=broad-except
         # We want to avoid crashing the controller. sync_down_logs() is pretty

sky/utils/dag_utils.py

@@ -148,7 +148,7 @@ def load_chain_dag_from_yaml_str(
 
 
 def dump_chain_dag_to_yaml_str(dag: dag_lib.Dag,
-                               redact_secrets: bool = False) -> str:
+                               use_user_specified_yaml: bool = False) -> str:
     """Dumps a chain DAG to a YAML string.
 
     Args:
@@ -161,7 +161,9 @@ def dump_chain_dag_to_yaml_str(dag: dag_lib.Dag,
     assert dag.is_chain(), dag
     configs = [{'name': dag.name}]
     for task in dag.tasks:
-        configs.append(task.to_yaml_config(redact_secrets=redact_secrets))
+        configs.append(
+            task.to_yaml_config(
+                use_user_specified_yaml=use_user_specified_yaml))
     return common_utils.dump_yaml_str(configs)
 
 

sky/utils/db/migration_utils.py

@@ -12,7 +12,6 @@ import filelock
 import sqlalchemy
 
 from sky import sky_logging
-from sky import skypilot_config
 from sky.skylet import constants
 
 logger = sky_logging.init_logger(__name__)
@@ -24,16 +23,15 @@ GLOBAL_USER_STATE_VERSION = '001'
 GLOBAL_USER_STATE_LOCK_PATH = '~/.sky/locks/.state_db.lock'
 
 SPOT_JOBS_DB_NAME = 'spot_jobs_db'
-SPOT_JOBS_VERSION = '001'
+SPOT_JOBS_VERSION = '002'
 SPOT_JOBS_LOCK_PATH = '~/.sky/locks/.spot_jobs_db.lock'
 
 
 def get_engine(db_name: str):
     conn_string = None
     if os.environ.get(constants.ENV_VAR_IS_SKYPILOT_SERVER) is not None:
-        conn_string = skypilot_config.get_nested(('db',), None)
+        conn_string = os.environ.get(constants.ENV_VAR_DB_CONNECTION_URI)
     if conn_string:
-        logger.debug(f'using db URI from {conn_string}')
         engine = sqlalchemy.create_engine(conn_string,
                                           poolclass=sqlalchemy.NullPool)
     else:

sky/utils/schemas.py

@@ -605,7 +605,6 @@ def get_service_schema():
     return {
         '$schema': 'https://json-schema.org/draft/2020-12/schema',
         'type': 'object',
-        'required': ['readiness_probe'],
         'additionalProperties': False,
         'properties': {
             'readiness_probe': {
@@ -641,6 +640,9 @@ def get_service_schema():
                     }
                 }]
             },
+            'pool': {
+                'type': 'boolean',
+            },
             'replica_policy': {
                 'type': 'object',
                 'required': ['min_replicas'],
@@ -688,6 +690,9 @@ def get_service_schema():
             'replicas': {
                 'type': 'integer',
             },
+            'workers': {
+                'type': 'integer',
+            },
             'load_balancing_policy': {
                 'type': 'string',
                 'case_insensitive_enum': list(
@@ -825,6 +830,9 @@ def get_task_schema():
             'service': {
                 'type': 'object',
             },
+            'pool': {
+                'type': 'object',
+            },
             'setup': {
                 'type': 'string',
             },
@@ -1128,6 +1136,7 @@ _CONTEXT_CONFIG_SCHEMA_KUBERNETES = {
 
 def get_config_schema():
     # pylint: disable=import-outside-toplevel
+    from sky.server import daemons
 
     resources_schema = {
         k: v
@@ -1137,21 +1146,7 @@ def get_config_schema():
     }
     resources_schema['properties'].pop('ports')
 
-    def _get_controller_schema(add_consolidation_mode: bool = False):
-        controller_properties = {
-            'resources': resources_schema,
-            'high_availability': {
-                'type': 'boolean',
-                'default': False,
-            },
-            'autostop': _AUTOSTOP_SCHEMA,
-        }
-        if add_consolidation_mode:
-            controller_properties['consolidation_mode'] = {
-                'type': 'boolean',
-                'default': False,
-            }
-
+    def _get_controller_schema():
         return {
             'type': 'object',
             'required': [],
@@ -1161,7 +1156,18 @@ def get_config_schema():
                     'type': 'object',
                     'required': [],
                     'additionalProperties': False,
-                    'properties': controller_properties,
+                    'properties': {
+                        'resources': resources_schema,
+                        'high_availability': {
+                            'type': 'boolean',
+                            'default': False,
+                        },
+                        'autostop': _AUTOSTOP_SCHEMA,
+                        'consolidation_mode': {
+                            'type': 'boolean',
+                            'default': False,
+                        }
+                    },
                 },
                 'bucket': {
                     'type': 'string',
@@ -1474,6 +1480,27 @@ def get_config_schema():
         }
     }
 
+    daemon_config = {
+        'type': 'object',
+        'required': [],
+        'properties': {
+            'log_level': {
+                'type': 'string',
+                'case_insensitive_enum': ['DEBUG', 'INFO', 'WARNING'],
+            },
+        }
+    }
+
+    daemon_schema = {
+        'type': 'object',
+        'required': [],
+        'additionalProperties': False,
+        'properties': {}
+    }
+
+    for daemon in daemons.INTERNAL_REQUEST_DAEMONS:
+        daemon_schema['properties'][daemon.id] = daemon_config
+
     api_server = {
         'type': 'object',
         'required': [],
@@ -1707,8 +1734,8 @@ def get_config_schema():
             'db': {
                 'type': 'string',
             },
-            'jobs': _get_controller_schema(add_consolidation_mode=True),
-            'serve': _get_controller_schema(add_consolidation_mode=False),
+            'jobs': _get_controller_schema(),
+            'serve': _get_controller_schema(),
             'allowed_clouds': allowed_clouds,
             'admin_policy': admin_policy_schema,
             'docker': docker_configs,
@@ -1719,6 +1746,7 @@ def get_config_schema():
             'provision': provision_configs,
             'rbac': rbac_schema,
             'logs': logs_schema,
+            'daemons': daemon_schema,
             **cloud_configs,
         },
     }

{skypilot_nightly-1.0.0.dev20250730.dist-info → skypilot_nightly-1.0.0.dev20250801.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250730
+Version: 1.0.0.dev20250801
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0