skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/skylet/autostop_lib.py

@@ -1,6 +1,8 @@
 """Autostop utilities."""
+import enum
 import pickle
 import shlex
+import subprocess
 import time
 import typing
 from typing import List, Optional
@@ -10,11 +12,17 @@ from sky.adaptors import common as adaptors_common
 from sky.skylet import configs
 from sky.skylet import constants
 from sky.utils import message_utils
+from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     import psutil
+
+    from sky.schemas.generated import autostopv1_pb2
 else:
     psutil = adaptors_common.LazyImport('psutil')
+    # To avoid requiring protobuf to be installed on the client side.
+    autostopv1_pb2 = adaptors_common.LazyImport(
+        'sky.schemas.generated.autostopv1_pb2')
 
 logger = sky_logging.init_logger(__name__)
 
@@ -30,6 +38,83 @@ _AUTOSTOP_LAST_ACTIVE_TIME = 'autostop_last_active_time'
 _AUTOSTOP_INDICATOR = 'autostop_indicator'
 
 
+class AutostopWaitFor(enum.Enum):
+    """Enum for the Autostop behaviour.
+
+    JOBS: Wait for jobs to finish.
+    JOBS_AND_SSH: Wait for jobs to finish and all SSH sessions to be closed.
+    NONE: Unconditionally stop the cluster after the idle time.
+    """
+    JOBS_AND_SSH = 'jobs_and_ssh'
+    JOBS = 'jobs'
+    NONE = 'none'
+
+    @classmethod
+    def supported_modes(cls) -> List[str]:
+        return [mode.value for mode in cls]
+
+    @classmethod
+    def cli_help_message(cls, pair: str) -> str:
+        return f"""\
+Determines the condition for resetting the idleness timer.
+This option works in conjunction with ``--{pair}``. Options:
+
+\b
+1. ``jobs_and_ssh`` (default): Wait for in-progress jobs and SSH connections to finish.
+2. ``jobs``: Only wait for in-progress jobs.
+3. ``none``: Wait for nothing; autostop right after ``{pair}``."""
+
+    @classmethod
+    def from_str(cls, mode: str) -> 'AutostopWaitFor':
+        """Returns the enum value for the given string."""
+        if mode.lower() == cls.JOBS.value:
+            return cls.JOBS
+        elif mode.lower() == cls.JOBS_AND_SSH.value:
+            return cls.JOBS_AND_SSH
+        elif mode.lower() == cls.NONE.value:
+            return cls.NONE
+        else:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(f'Unsupported autostop wait mode: '
+                                 f'{mode}. The mode must be either '
+                                 f'\'{cls.JOBS_AND_SSH.value}\', '
+                                 f'\'{cls.JOBS.value}\', or '
+                                 f'\'{cls.NONE.value}\'. ')
+
+    @classmethod
+    def from_protobuf(
+        cls, protobuf_value: 'autostopv1_pb2.AutostopWaitFor'
+    ) -> Optional['AutostopWaitFor']:
+        """Convert protobuf AutostopWaitFor enum to Python enum value."""
+        protobuf_to_enum = {
+            autostopv1_pb2.AUTOSTOP_WAIT_FOR_JOBS_AND_SSH: cls.JOBS_AND_SSH,
+            autostopv1_pb2.AUTOSTOP_WAIT_FOR_JOBS: cls.JOBS,
+            autostopv1_pb2.AUTOSTOP_WAIT_FOR_NONE: cls.NONE,
+            autostopv1_pb2.AUTOSTOP_WAIT_FOR_UNSPECIFIED: None,
+        }
+        if protobuf_value not in protobuf_to_enum:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    f'Unknown protobuf AutostopWaitFor value: {protobuf_value}')
+        return protobuf_to_enum[protobuf_value]
+
+    def to_protobuf(self) -> 'autostopv1_pb2.AutostopWaitFor':
+        """Convert this Python enum value to protobuf enum value."""
+        enum_to_protobuf = {
+            AutostopWaitFor.JOBS_AND_SSH:
+                autostopv1_pb2.AUTOSTOP_WAIT_FOR_JOBS_AND_SSH,
+            AutostopWaitFor.JOBS: autostopv1_pb2.AUTOSTOP_WAIT_FOR_JOBS,
+            AutostopWaitFor.NONE: autostopv1_pb2.AUTOSTOP_WAIT_FOR_NONE,
+        }
+        if self not in enum_to_protobuf:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(f'Unknown AutostopWaitFor value: {self}')
+        return enum_to_protobuf[self]
+
+
+DEFAULT_AUTOSTOP_WAIT_FOR: AutostopWaitFor = AutostopWaitFor.JOBS_AND_SSH
+
+
 class AutostopConfig:
     """Autostop configuration."""
 
@@ -37,12 +122,14 @@ class AutostopConfig:
                  autostop_idle_minutes: int,
                  boot_time: float,
                  backend: Optional[str],
+                 wait_for: AutostopWaitFor,
                  down: bool = False):
         assert autostop_idle_minutes < 0 or backend is not None, (
             autostop_idle_minutes, backend)
         self.autostop_idle_minutes = autostop_idle_minutes
         self.boot_time = boot_time
         self.backend = backend
+        self.wait_for = wait_for
         self.down = down
 
     def __setstate__(self, state: dict):
@@ -53,15 +140,18 @@ class AutostopConfig:
 def get_autostop_config() -> AutostopConfig:
     config_str = configs.get_config(_AUTOSTOP_CONFIG_KEY)
     if config_str is None:
-        return AutostopConfig(-1, -1, None)
+        return AutostopConfig(-1, -1, None, DEFAULT_AUTOSTOP_WAIT_FOR)
     return pickle.loads(config_str)
 
 
-def set_autostop(idle_minutes: int, backend: Optional[str], down: bool) -> None:
+def set_autostop(idle_minutes: int, backend: Optional[str],
+                 wait_for: AutostopWaitFor, down: bool) -> None:
     boot_time = psutil.boot_time()
-    autostop_config = AutostopConfig(idle_minutes, boot_time, backend, down)
+    autostop_config = AutostopConfig(idle_minutes, boot_time, backend, wait_for,
+                                     down)
     configs.set_config(_AUTOSTOP_CONFIG_KEY, pickle.dumps(autostop_config))
-    logger.debug(f'set_autostop(): idle_minutes {idle_minutes}, down {down}.')
+    logger.debug(f'set_autostop(): idle_minutes {idle_minutes}, down {down}, '
+                 f'wait_for {wait_for.value}.')
     # Reset timer whenever an autostop setting is submitted, i.e. the idle
     # time will be counted from now.
     set_last_active_time_to_now()
@@ -107,6 +197,28 @@ def set_last_active_time_to_now() -> None:
     configs.set_config(_AUTOSTOP_LAST_ACTIVE_TIME, str(time.time()))
 
 
+def has_active_ssh_sessions() -> bool:
+    """Returns True if there are any active SSH sessions on the node."""
+    try:
+        # /dev/pts is a virtual filesystem that contains the pseudo-terminal
+        # devices. ptmx is the pseudo-terminal multiplexer, which is the
+        # "master" device that creates new pseudo-terminal devices, so we
+        # exclude it from the count.
+        proc = subprocess.run('ls /dev/pts | grep -v ptmx | wc -l',
+                              capture_output=True,
+                              text=True,
+                              check=False,
+                              shell=True)
+        if proc.returncode != 0:
+            logger.warning(f'SSH session check command failed with return code '
+                           f'{proc.returncode}.')
+            return False
+        return int(proc.stdout.strip()) > 0
+    except Exception as e:  # pylint: disable=broad-except
+        logger.warning(f'Error checking active SSH sessions: {e}.')
+        return False
+
+
 class AutostopCodeGen:
     """Code generator for autostop utility functions.
 
@@ -114,13 +226,22 @@ class AutostopCodeGen:
 
       >> codegen = AutostopCodeGen.set_autostop(...)
     """
-    _PREFIX = ['from sky.skylet import autostop_lib']
+    _PREFIX = ['from sky.skylet import autostop_lib, constants']
 
     @classmethod
-    def set_autostop(cls, idle_minutes: int, backend: str, down: bool) -> str:
+    def set_autostop(cls,
+                     idle_minutes: int,
+                     backend: str,
+                     wait_for: Optional[AutostopWaitFor],
+                     down: bool = False) -> str:
+        if wait_for is None:
+            wait_for = DEFAULT_AUTOSTOP_WAIT_FOR
         code = [
-            f'autostop_lib.set_autostop({idle_minutes}, {backend!r},'
-            f' {down})',
+            f'\nif getattr(constants, "SKYLET_LIB_VERSION", 1) < 4: '
+            f'\n autostop_lib.set_autostop({idle_minutes}, {backend!r}, {down})'
+            f'\nelse: '
+            f'\n autostop_lib.set_autostop({idle_minutes}, {backend!r}, '
+            f'autostop_lib.{wait_for}, {down})',
         ]
         return cls._build(code)
 

sky/skylet/configs.py

@@ -2,17 +2,16 @@
 import functools
 import os
 import pathlib
+import threading
 from typing import Callable, Optional, Union
 
-from sky.utils import db_utils
+from sky.utils.db import db_utils
 
-_DB_PATH = os.path.expanduser('~/.sky/skylet_config.db')
-os.makedirs(pathlib.Path(_DB_PATH).parents[0], exist_ok=True)
+_DB_PATH = None
+_db_init_lock = threading.Lock()
 
-_table_created = False
 
-
-def ensure_table(func: Callable):
+def init_db(func: Callable):
     """Ensure the table exists before calling the function.
 
     Since this module will be imported whenever `sky` is imported (due to
@@ -24,25 +23,32 @@ def ensure_table(func: Callable):
 
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        global _table_created
-        if not _table_created:
-            with db_utils.safe_cursor(
-                    _DB_PATH) as c:  # Call it 'c' to avoid pylint complaining.
-                # Use WAL mode to avoid locking problem in #1507.
-                # Reference: https://stackoverflow.com/a/39265148
-                c.execute('PRAGMA journal_mode=WAL')
-                c.execute("""\
-                    CREATE TABLE IF NOT EXISTS config (
-                        key TEXT PRIMARY KEY,
-                        value TEXT)""")
-        _table_created = True
+        global _DB_PATH
+        if _DB_PATH is not None:
+            return func(*args, **kwargs)
+
+        with _db_init_lock:
+            if _DB_PATH is None:
+                _DB_PATH = os.path.expanduser('~/.sky/skylet_config.db')
+                os.makedirs(pathlib.Path(_DB_PATH).parents[0], exist_ok=True)
+                with db_utils.safe_cursor(
+                        _DB_PATH
+                ) as c:  # Call it 'c' to avoid pylint complaining.
+                    # Use WAL mode to avoid locking problem in #1507.
+                    # Reference: https://stackoverflow.com/a/39265148
+                    c.execute('PRAGMA journal_mode=WAL')
+                    c.execute("""\
+                        CREATE TABLE IF NOT EXISTS config (
+                            key TEXT PRIMARY KEY,
+                            value TEXT)""")
         return func(*args, **kwargs)
 
     return wrapper
 
 
-@ensure_table
+@init_db
 def get_config(key: str) -> Optional[bytes]:
+    assert _DB_PATH is not None
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         rows = cursor.execute('SELECT value FROM config WHERE key = ?', (key,))
         for (value,) in rows:
@@ -50,8 +56,9 @@ def get_config(key: str) -> Optional[bytes]:
         return None
 
 
-@ensure_table
+@init_db
 def set_config(key: str, value: Union[bytes, str]) -> None:
+    assert _DB_PATH is not None
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         cursor.execute(
             """\

sky/skylet/constants.py

@@ -1,4 +1,5 @@
 """Constants for SkyPilot."""
+import os
 from typing import List, Tuple
 
 from packaging import version
@@ -28,6 +29,7 @@ SKY_REMOTE_RAY_PORT_FILE = '~/.sky/ray_port.json'
 SKY_REMOTE_RAY_TEMPDIR = '/tmp/ray_skypilot'
 SKY_REMOTE_RAY_VERSION = '2.9.3'
 
+SKY_UNSET_PYTHONPATH = 'env -u PYTHONPATH'
 # We store the absolute path of the python executable (/opt/conda/bin/python3)
 # in this file, so that any future internal commands that need to use python
 # can use this path. This is useful for the case where the user has a custom
@@ -39,7 +41,7 @@ SKY_GET_PYTHON_PATH_CMD = (f'[ -s {SKY_PYTHON_PATH_FILE} ] && '
                            f'cat {SKY_PYTHON_PATH_FILE} 2> /dev/null || '
                            'which python3')
 # Python executable, e.g., /opt/conda/bin/python3
-SKY_PYTHON_CMD = f'$({SKY_GET_PYTHON_PATH_CMD})'
+SKY_PYTHON_CMD = f'{SKY_UNSET_PYTHONPATH} $({SKY_GET_PYTHON_PATH_CMD})'
 # Prefer SKY_UV_PIP_CMD, which is faster.
 # TODO(cooperc): remove remaining usage (GCP TPU setup).
 SKY_PIP_CMD = f'{SKY_PYTHON_CMD} -m pip'
@@ -55,20 +57,29 @@ SKY_REMOTE_PYTHON_ENV: str = f'~/{SKY_REMOTE_PYTHON_ENV_NAME}'
 ACTIVATE_SKY_REMOTE_PYTHON_ENV = f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate'
 # uv is used for venv and pip, much faster than python implementations.
 SKY_UV_INSTALL_DIR = '"$HOME/.local/bin"'
-SKY_UV_CMD = f'UV_SYSTEM_PYTHON=false {SKY_UV_INSTALL_DIR}/uv'
+# set UV_SYSTEM_PYTHON to false in case the
+# user provided docker image set it to true.
+# unset PYTHONPATH in case the user provided docker image set it.
+SKY_UV_CMD = ('UV_SYSTEM_PYTHON=false '
+              f'{SKY_UNSET_PYTHONPATH} {SKY_UV_INSTALL_DIR}/uv')
 # This won't reinstall uv if it's already installed, so it's safe to re-run.
 SKY_UV_INSTALL_CMD = (f'{SKY_UV_CMD} -V >/dev/null 2>&1 || '
                       'curl -LsSf https://astral.sh/uv/install.sh '
                       f'| UV_INSTALL_DIR={SKY_UV_INSTALL_DIR} sh')
 SKY_UV_PIP_CMD: str = (f'VIRTUAL_ENV={SKY_REMOTE_PYTHON_ENV} {SKY_UV_CMD} pip')
-# Deleting the SKY_REMOTE_PYTHON_ENV_NAME from the PATH to deactivate the
-# environment. `deactivate` command does not work when conda is used.
+SKY_UV_RUN_CMD: str = (f'VIRTUAL_ENV={SKY_REMOTE_PYTHON_ENV} {SKY_UV_CMD} run '
+                       '--no-project --no-config')
+# Deleting the SKY_REMOTE_PYTHON_ENV_NAME from the PATH and unsetting relevant
+# VIRTUAL_ENV envvars to deactivate the environment. `deactivate` command does
+# not work when conda is used.
 DEACTIVATE_SKY_REMOTE_PYTHON_ENV = (
     'export PATH='
-    f'$(echo $PATH | sed "s|$(echo ~)/{SKY_REMOTE_PYTHON_ENV_NAME}/bin:||")')
+    f'$(echo $PATH | sed "s|$(echo ~)/{SKY_REMOTE_PYTHON_ENV_NAME}/bin:||") && '
+    'unset VIRTUAL_ENV && unset VIRTUAL_ENV_PROMPT')
 
 # Prefix for SkyPilot environment variables
 SKYPILOT_ENV_VAR_PREFIX = 'SKYPILOT_'
+SKYPILOT_SERVER_ENV_VAR_PREFIX = 'SKYPILOT_SERVER_'
 
 # The name for the environment variable that stores the unique ID of the
 # current task. This will stay the same across multiple recoveries of the
@@ -89,17 +100,14 @@ TASK_ID_LIST_ENV_VAR = f'{SKYPILOT_ENV_VAR_PREFIX}TASK_IDS'
 # cluster yaml is updated.
 #
 # TODO(zongheng,zhanghao): make the upgrading of skylet automatic?
-SKYLET_VERSION = '12'
+SKYLET_VERSION = '25'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.
-SKYLET_LIB_VERSION = 3
+SKYLET_LIB_VERSION = 4
 SKYLET_VERSION_FILE = '~/.sky/skylet_version'
-
-# `sky jobs dashboard`-related
-#
-# Port on the remote jobs controller that the dashboard is running on.
-SPOT_DASHBOARD_REMOTE_PORT = 5000
+SKYLET_GRPC_PORT = 46590
+SKYLET_GRPC_TIMEOUT_SECONDS = 10
 
 # Docker default options
 DEFAULT_DOCKER_CONTAINER_NAME = 'sky_container'
@@ -151,7 +159,7 @@ CONDA_INSTALLATION_COMMANDS = (
     # because for some images, conda is already installed, but not initialized.
     # In this case, we need to initialize conda and set auto_activate_base to
     # true.
-    '{ bash Miniconda3-Linux.sh -b; '
+    '{ bash Miniconda3-Linux.sh -b || true; '
     'eval "$(~/miniconda3/bin/conda shell.bash hook)" && conda init && '
     # Caller should replace {conda_auto_activate} with either true or false.
     'conda config --set auto_activate_base {conda_auto_activate} && '
@@ -218,7 +226,9 @@ RAY_INSTALLATION_COMMANDS = (
     f'{SKY_UV_PIP_CMD} list | grep "ray " | '
     f'grep {SKY_REMOTE_RAY_VERSION} 2>&1 > /dev/null '
     f'|| {RAY_STATUS} || '
-    f'{SKY_UV_PIP_CMD} install -U ray[default]=={SKY_REMOTE_RAY_VERSION}; '  # pylint: disable=line-too-long
+    # The pydantic-core==2.41.3 for arm seems corrupted
+    # so we need to avoid that specific version.
+    f'{SKY_UV_PIP_CMD} install -U "ray[default]=={SKY_REMOTE_RAY_VERSION}" "pydantic-core==2.41.1"; '  # pylint: disable=line-too-long
     # In some envs, e.g. pip does not have permission to write under /opt/conda
     # ray package will be installed under ~/.local/bin. If the user's PATH does
     # not include ~/.local/bin (the pip install will have the output: `WARNING:
@@ -230,7 +240,7 @@ RAY_INSTALLATION_COMMANDS = (
     'export PATH=$PATH:$HOME/.local/bin; '
     # Writes ray path to file if it does not exist or the file is empty.
     f'[ -s {SKY_RAY_PATH_FILE} ] || '
-    f'{{ {ACTIVATE_SKY_REMOTE_PYTHON_ENV} && '
+    f'{{ {SKY_UV_RUN_CMD} '
     f'which ray > {SKY_RAY_PATH_FILE} || exit 1; }}; ')
 
 SKYPILOT_WHEEL_INSTALLATION_COMMANDS = (
@@ -323,6 +333,14 @@ FILE_MOUNTS_LOCAL_TMP_BASE_PATH = '~/.sky/tmp/'
 # controller_utils.translate_local_file_mounts_to_two_hop().
 FILE_MOUNTS_CONTROLLER_TMP_BASE_PATH = '~/.sky/tmp/controller'
 
+# For passing in CPU and memory limits to the controller pod when running
+# in k8s. Right now, we only use this for the jobs controller, but we may
+# use this for the serve controller as well in the future.
+# These files are written to disk by the skylet, who reads it from env vars
+# passed by the backend when starting the skylet (start_skylet_on_head_node).
+CONTROLLER_K8S_CPU_FILE = '~/.sky/_internal_k8s_pod_cpu'
+CONTROLLER_K8S_MEMORY_FILE = '~/.sky/_internal_k8s_pod_memory'
+
 # Used when an managed jobs are created and
 # files are synced up to the cloud.
 FILE_MOUNTS_WORKDIR_SUBPATH = 'job-{run_id}/workdir'
@@ -346,6 +364,11 @@ API_SERVER_CREATION_LOCK_PATH = '~/.sky/api_server/.creation.lock'
 # API server.
 SKY_API_SERVER_URL_ENV_VAR = f'{SKYPILOT_ENV_VAR_PREFIX}API_SERVER_ENDPOINT'
 
+# The name for the environment variable that stores the SkyPilot service
+# account token on client side.
+SERVICE_ACCOUNT_TOKEN_ENV_VAR = (
+    f'{SKYPILOT_ENV_VAR_PREFIX}SERVICE_ACCOUNT_TOKEN')
+
 # SkyPilot environment variables
 SKYPILOT_NUM_NODES = f'{SKYPILOT_ENV_VAR_PREFIX}NUM_NODES'
 SKYPILOT_NODE_IPS = f'{SKYPILOT_ENV_VAR_PREFIX}NODE_IPS'
@@ -358,7 +381,7 @@ SKY_SSH_USER_PLACEHOLDER = 'skypilot:ssh_user'
 
 RCLONE_CONFIG_DIR = '~/.config/rclone'
 RCLONE_CONFIG_PATH = f'{RCLONE_CONFIG_DIR}/rclone.conf'
-RCLONE_LOG_DIR = '~/.sky/rclone_log'
+RCLONE_MOUNT_CACHED_LOG_DIR = '~/.sky/rclone_log'
 RCLONE_CACHE_DIR = '~/.cache/rclone'
 RCLONE_CACHE_REFRESH_INTERVAL = 10
 
@@ -367,15 +390,41 @@ RCLONE_CACHE_REFRESH_INTERVAL = 10
 OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
     ('docker', 'run_options'),
     ('nvidia_gpus', 'disable_ecc'),
+    ('ssh', 'pod_config'),
+    ('kubernetes', 'custom_metadata'),
     ('kubernetes', 'pod_config'),
     ('kubernetes', 'provision_timeout'),
+    ('kubernetes', 'dws'),
+    ('kubernetes', 'kueue'),
     ('gcp', 'managed_instance_group'),
+    ('gcp', 'enable_gvnic'),
+    ('gcp', 'enable_gpu_direct'),
+    ('gcp', 'placement_policy'),
+    ('active_workspace',),
 ]
 # When overriding the SkyPilot configs on the API server with the client one,
 # we skip the following keys because they are meant to be client-side configs.
-SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [('admin_policy',),
-                                                       ('api_server',),
-                                                       ('allowed_clouds',)]
+# Also, we skip the consolidation mode config as those should be only set on
+# the API server side.
+SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [
+    ('api_server',),
+    ('allowed_clouds',),
+    ('workspaces',),
+    ('db',),
+    ('daemons',),
+    # TODO(kevin,tian): Override the whole controller config once our test
+    # infrastructure supports setting dynamic server side configs.
+    # Tests that are affected:
+    # - test_managed_jobs_ha_kill_starting
+    # - test_managed_jobs_ha_kill_running
+    # - all tests that use LOW_CONTROLLER_RESOURCE_ENV or
+    #   LOW_CONTROLLER_RESOURCE_OVERRIDE_CONFIG (won't cause test failure,
+    #   but the configs won't be applied)
+    ('jobs', 'controller', 'consolidation_mode'),
+    ('serve', 'controller', 'consolidation_mode'),
+    ('jobs', 'controller', 'controller_logs_gc_retention_hours'),
+    ('jobs', 'controller', 'task_logs_gc_retention_hours'),
+]
 
 # Constants for Azure blob storage
 WAIT_FOR_STORAGE_ACCOUNT_CREATION = 60
@@ -392,6 +441,12 @@ ROLE_ASSIGNMENT_FAILURE_ERROR_MSG = (
 # persistent through PVC. See kubernetes-ray.yml.j2.
 PERSISTENT_SETUP_SCRIPT_PATH = '~/.sky/.controller_recovery_setup_commands.sh'
 PERSISTENT_RUN_SCRIPT_DIR = '~/.sky/.controller_recovery_task_run'
+# Signal file to indicate that the controller is recovering from a failure.
+# See sky/jobs/utils.py::update_managed_jobs_statuses for more details.
+PERSISTENT_RUN_RESTARTING_SIGNAL_FILE = (
+    '~/.sky/.controller_recovery_restarting_signal')
+
+HA_PERSISTENT_RECOVERY_LOG_PATH = '/tmp/{}ha_recovery.log'
 
 # The placeholder for the local skypilot config path in file mounts for
 # controllers.
@@ -400,5 +455,102 @@ LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER = 'skypilot:local_skypilot_config_path'
 # Path to the generated cluster config yamls and ssh configs.
 SKY_USER_FILE_PATH = '~/.sky/generated'
 
+# TODO(cooperc): Update all env vars to begin with SKYPILOT_ or SKYPILOT_SERVER_
 # Environment variable that is set to 'true' if this is a skypilot server.
 ENV_VAR_IS_SKYPILOT_SERVER = 'IS_SKYPILOT_SERVER'
+OVERRIDE_CONSOLIDATION_MODE = 'IS_SKYPILOT_JOB_CONTROLLER'
+
+# Environment variable that is set to 'true' if metrics are enabled.
+ENV_VAR_SERVER_METRICS_ENABLED = 'SKY_API_SERVER_METRICS_ENABLED'
+
+# If set, overrides the header that we can use to get the user name.
+ENV_VAR_SERVER_AUTH_USER_HEADER = f'{SKYPILOT_ENV_VAR_PREFIX}AUTH_USER_HEADER'
+
+# Environment variable that is used as the DB connection string for the
+# skypilot server.
+ENV_VAR_DB_CONNECTION_URI = (f'{SKYPILOT_ENV_VAR_PREFIX}DB_CONNECTION_URI')
+
+# Environment variable that is set to 'true' if basic
+# authentication is enabled in the API server.
+ENV_VAR_ENABLE_BASIC_AUTH = 'ENABLE_BASIC_AUTH'
+SKYPILOT_INITIAL_BASIC_AUTH = 'SKYPILOT_INITIAL_BASIC_AUTH'
+SKYPILOT_INGRESS_BASIC_AUTH_ENABLED = 'SKYPILOT_INGRESS_BASIC_AUTH_ENABLED'
+ENV_VAR_ENABLE_SERVICE_ACCOUNTS = 'ENABLE_SERVICE_ACCOUNTS'
+
+# Enable debug logging for requests.
+ENV_VAR_ENABLE_REQUEST_DEBUG_LOGGING = (
+    f'{SKYPILOT_SERVER_ENV_VAR_PREFIX}ENABLE_REQUEST_DEBUG_LOGGING')
+
+SKYPILOT_DEFAULT_WORKSPACE = 'default'
+
+# BEGIN constants used for service catalog.
+HOSTED_CATALOG_DIR_URL = 'https://raw.githubusercontent.com/skypilot-org/skypilot-catalog/master/catalogs'  # pylint: disable=line-too-long
+HOSTED_CATALOG_DIR_URL_S3_MIRROR = 'https://skypilot-catalog.s3.us-east-1.amazonaws.com/catalogs'  # pylint: disable=line-too-long
+CATALOG_SCHEMA_VERSION = 'v8'
+CATALOG_DIR = '~/.sky/catalogs'
+ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
+              'kubernetes', 'runpod', 'vast', 'vsphere', 'cudo', 'fluidstack',
+              'paperspace', 'primeintellect', 'do', 'nebius', 'ssh',
+              'hyperbolic', 'seeweb', 'shadeform')
+# END constants used for service catalog.
+
+# The user ID of the SkyPilot system.
+SKYPILOT_SYSTEM_USER_ID = 'skypilot-system'
+
+# The directory to store the logging configuration.
+LOGGING_CONFIG_DIR = '~/.sky/logging'
+
+# Resources constants
+TIME_UNITS = {
+    'm': 1,
+    'h': 60,
+    'd': 24 * 60,
+    'w': 7 * 24 * 60,
+}
+
+TIME_PATTERN: str = ('^[0-9]+('
+                     f'{"|".join([unit.lower() for unit in TIME_UNITS])}|'
+                     f'{"|".join([unit.upper() for unit in TIME_UNITS])}|'
+                     ')?$')
+
+MEMORY_SIZE_UNITS = {
+    'kb': 2**10,
+    'ki': 2**10,
+    'mb': 2**20,
+    'mi': 2**20,
+    'gb': 2**30,
+    'gi': 2**30,
+    'tb': 2**40,
+    'ti': 2**40,
+    'pb': 2**50,
+    'pi': 2**50,
+}
+
+MEMORY_SIZE_PATTERN = (
+    '^[0-9]+('
+    f'{"|".join([unit.lower() for unit in MEMORY_SIZE_UNITS])}|'
+    f'{"|".join([unit.upper() for unit in MEMORY_SIZE_UNITS])}|'
+    f'{"|".join([unit[0].upper() + unit[1:] for unit in MEMORY_SIZE_UNITS if len(unit) > 1])}'  # pylint: disable=line-too-long
+    ')?$')
+
+LAST_USE_TRUNC_LENGTH = 25
+USED_BY_TRUNC_LENGTH = 25
+
+MIN_PRIORITY = -1000
+MAX_PRIORITY = 1000
+DEFAULT_PRIORITY = 0
+
+GRACE_PERIOD_SECONDS_ENV_VAR = SKYPILOT_ENV_VAR_PREFIX + 'GRACE_PERIOD_SECONDS'
+COST_REPORT_DEFAULT_DAYS = 30
+
+# The directory for file locks.
+SKY_LOCKS_DIR = os.path.expanduser('~/.sky/locks')
+
+ENV_VAR_LOOP_LAG_THRESHOLD_MS = (SKYPILOT_ENV_VAR_PREFIX +
+                                 'DEBUG_LOOP_LAG_THRESHOLD_MS')
+
+ARM64_ARCH = 'arm64'
+X86_64_ARCH = 'x86_64'
+
+SSH_DISABLE_LATENCY_MEASUREMENT_ENV_VAR = (
+    f'{SKYPILOT_ENV_VAR_PREFIX}SSH_DISABLE_LATENCY_MEASUREMENT')