skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/server/config.py

@@ -2,9 +2,11 @@
 
 import dataclasses
 import enum
+from typing import Optional
 
 from sky import sky_logging
 from sky.server import constants as server_constants
+from sky.server import daemons
 from sky.utils import common_utils
 
 # Constants based on profiling the peak memory usage while serving various
@@ -18,8 +20,9 @@ from sky.utils import common_utils
 # TODO(aylei): maintaining these constants is error-prone, we may need to
 # automatically tune parallelism at runtime according to system usage stats
 # in the future.
-_LONG_WORKER_MEM_GB = 0.4
-_SHORT_WORKER_MEM_GB = 0.25
+# TODO(luca): The future is now! ^^^
+LONG_WORKER_MEM_GB = 0.4
+SHORT_WORKER_MEM_GB = 0.3
 # To control the number of long workers.
 _CPU_MULTIPLIER_FOR_LONG_WORKERS = 2
 # Limit the number of long workers of local API server, since local server is
@@ -34,9 +37,8 @@ _MAX_LONG_WORKERS_LOCAL = 4
 _MAX_MEM_PERCENT_FOR_BLOCKING = 0.6
 # Minimal number of long workers to ensure responsiveness.
 _MIN_LONG_WORKERS = 1
-# Minimal number of short workers, there is a daemon task running on short
-# workers so at least 2 workers are needed to ensure responsiveness.
-_MIN_SHORT_WORKERS = 2
+# Minimal number of idle short workers to ensure responsiveness.
+_MIN_IDLE_SHORT_WORKERS = 1
 
 # Default number of burstable workers for local API server. A heuristic number
 # that is large enough for most local cases.
@@ -61,6 +63,7 @@ class QueueBackend(enum.Enum):
 class WorkerConfig:
     garanteed_parallelism: int
     burstable_parallelism: int
+    num_db_connections_per_worker: int
 
 
 @dataclasses.dataclass
@@ -68,10 +71,15 @@ class ServerConfig:
     num_server_workers: int
     long_worker_config: WorkerConfig
     short_worker_config: WorkerConfig
+    num_db_connections_per_worker: int
     queue_backend: QueueBackend
 
 
-def compute_server_config(deploy: bool) -> ServerConfig:
+def compute_server_config(
+        deploy: bool,
+        max_db_connections: Optional[int] = None,
+        quiet: bool = False,
+        reserved_memory_mb: Optional[float] = None) -> ServerConfig:
     """Compute the server config based on environment.
 
     We have different assumptions for the resources in different deployment
@@ -105,7 +113,11 @@ def compute_server_config(deploy: bool) -> ServerConfig:
     process after API server was introduced.
     """
     cpu_count = common_utils.get_cpu_count()
+    logger.debug(f'CPU count: {cpu_count}')
     mem_size_gb = common_utils.get_mem_size_gb()
+    if reserved_memory_mb is not None:
+        mem_size_gb -= (reserved_memory_mb / 1024)
+    logger.debug(f'Memory size: {mem_size_gb}GB')
     max_parallel_for_long = _max_long_worker_parallism(cpu_count,
                                                        mem_size_gb,
                                                        local=not deploy)
@@ -114,7 +126,17 @@ def compute_server_config(deploy: bool) -> ServerConfig:
     queue_backend = QueueBackend.MULTIPROCESSING
     burstable_parallel_for_long = 0
     burstable_parallel_for_short = 0
+    # if num_db_connections_per_worker is 0, server will use NullPool
+    # to conserve the number of concurrent db connections.
+    # This could lead to performance degradation.
+    num_db_connections_per_worker = 0
     num_server_workers = cpu_count
+
+    # +1 for the event loop running the main process
+    # and gc daemons in the '__main__' body of sky/server/server.py
+    max_parallel_all_workers = (max_parallel_for_long + max_parallel_for_short +
+                                num_server_workers + 1)
+
     if not deploy:
         # For local mode, use local queue backend since we only run 1 uvicorn
         # worker in local mode and no multiprocessing is needed.
@@ -125,7 +147,12 @@ def compute_server_config(deploy: bool) -> ServerConfig:
         burstable_parallel_for_short = _BURSTABLE_WORKERS_FOR_LOCAL
         # Runs in low resource mode if the available memory is less than
         # server_constants.MIN_AVAIL_MEM_GB.
-        if not deploy and mem_size_gb < server_constants.MIN_AVAIL_MEM_GB:
+        # pylint: disable=import-outside-toplevel
+        import sky.jobs.utils as job_utils
+        max_memory = (server_constants.MIN_AVAIL_MEM_GB_CONSOLIDATION_MODE
+                      if job_utils.is_consolidation_mode() else
+                      server_constants.MIN_AVAIL_MEM_GB)
+        if not deploy and mem_size_gb < max_memory:
             # Permanent worker process may have significant memory consumption
             # (~350MB per worker) after running commands like `sky check`, so we
             # don't start any permanent workers in low resource local mode. This
@@ -136,24 +163,41 @@ def compute_server_config(deploy: bool) -> ServerConfig:
             # permanently because it never exits.
             max_parallel_for_long = 0
             max_parallel_for_short = 0
-            logger.warning(
-                'SkyPilot API server will run in low resource mode because '
-                'the available memory is less than '
-                f'{server_constants.MIN_AVAIL_MEM_GB}GB.')
-    logger.info(
-        f'SkyPilot API server will start {num_server_workers} server processes '
-        f'with {max_parallel_for_long} background workers for long requests '
-        f'and will allow at max {max_parallel_for_short} short requests in '
-        f'parallel.')
+            if not quiet:
+                logger.warning(
+                    'SkyPilot API server will run in low resource mode because '
+                    'the available memory is less than '
+                    f'{server_constants.MIN_AVAIL_MEM_GB}GB.')
+    elif max_db_connections is not None:
+        if max_parallel_all_workers > max_db_connections:
+            if not quiet:
+                logger.warning(
+                    f'Max parallel all workers ({max_parallel_all_workers}) '
+                    'is greater than max db connections '
+                    f'({max_db_connections}). Increase the number of max db '
+                    f'connections to at least {max_parallel_all_workers} for '
+                    'optimal performance.')
+        else:
+            num_db_connections_per_worker = 1
+
+    if not quiet:
+        logger.info(
+            f'SkyPilot API server will start {num_server_workers} server '
+            f'processes with {max_parallel_for_long} background workers for '
+            f'long requests and will allow at max {max_parallel_for_short} '
+            'short requests in parallel.')
     return ServerConfig(
         num_server_workers=num_server_workers,
         queue_backend=queue_backend,
         long_worker_config=WorkerConfig(
             garanteed_parallelism=max_parallel_for_long,
-            burstable_parallelism=burstable_parallel_for_long),
+            burstable_parallelism=burstable_parallel_for_long,
+            num_db_connections_per_worker=num_db_connections_per_worker),
         short_worker_config=WorkerConfig(
             garanteed_parallelism=max_parallel_for_short,
-            burstable_parallelism=burstable_parallel_for_short),
+            burstable_parallelism=burstable_parallel_for_short,
+            num_db_connections_per_worker=num_db_connections_per_worker),
+        num_db_connections_per_worker=num_db_connections_per_worker,
     )
 
 
@@ -162,10 +206,15 @@ def _max_long_worker_parallism(cpu_count: int,
                                local=False) -> int:
     """Max parallelism for long workers."""
     # Reserve min available memory to avoid OOM.
-    available_mem = max(0, mem_size_gb - server_constants.MIN_AVAIL_MEM_GB)
+    # pylint: disable=import-outside-toplevel
+    import sky.jobs.utils as job_utils
+    max_memory = (server_constants.MIN_AVAIL_MEM_GB_CONSOLIDATION_MODE
+                  if job_utils.is_consolidation_mode() else
+                  server_constants.MIN_AVAIL_MEM_GB)
+    available_mem = max(0, mem_size_gb - max_memory)
     cpu_based_max_parallel = cpu_count * _CPU_MULTIPLIER_FOR_LONG_WORKERS
     mem_based_max_parallel = int(available_mem * _MAX_MEM_PERCENT_FOR_BLOCKING /
-                                 _LONG_WORKER_MEM_GB)
+                                 LONG_WORKER_MEM_GB)
     n = max(_MIN_LONG_WORKERS,
             min(cpu_based_max_parallel, mem_based_max_parallel))
     if local:
@@ -173,12 +222,25 @@ def _max_long_worker_parallism(cpu_count: int,
     return n
 
 
+def _get_min_short_workers() -> int:
+    """Min number of short workers."""
+    daemon_count = 0
+    for daemon in daemons.INTERNAL_REQUEST_DAEMONS:
+        if not daemon.should_skip():
+            daemon_count += 1
+    return _MIN_IDLE_SHORT_WORKERS + daemon_count
+
+
 def _max_short_worker_parallism(mem_size_gb: float,
                                 long_worker_parallism: int) -> int:
     """Max parallelism for short workers."""
     # Reserve memory for long workers and min available memory.
-    reserved_mem = server_constants.MIN_AVAIL_MEM_GB + (long_worker_parallism *
-                                                        _LONG_WORKER_MEM_GB)
+    # pylint: disable=import-outside-toplevel
+    import sky.jobs.utils as job_utils
+    max_memory = (server_constants.MIN_AVAIL_MEM_GB_CONSOLIDATION_MODE
+                  if job_utils.is_consolidation_mode() else
+                  server_constants.MIN_AVAIL_MEM_GB)
+    reserved_mem = max_memory + (long_worker_parallism * LONG_WORKER_MEM_GB)
     available_mem = max(0, mem_size_gb - reserved_mem)
-    n = max(_MIN_SHORT_WORKERS, int(available_mem / _SHORT_WORKER_MEM_GB))
+    n = max(_get_min_short_workers(), int(available_mem / SHORT_WORKER_MEM_GB))
     return n

sky/server/constants.py

@@ -4,17 +4,37 @@ import os
 
 from sky.skylet import constants
 
-# API server version, whenever there is a change in API server that requires a
-# restart of the local API server or error out when the client does not match
-# the server version.
-API_VERSION = '5'
+# pylint: disable=line-too-long
+# The SkyPilot API version that the code currently use.
+# Bump this version when the API is changed and special compatibility handling
+# based on version info is needed.
+# For more details and code guidelines, refer to:
+# https://docs.skypilot.co/en/latest/developers/CONTRIBUTING.html#backward-compatibility-guidelines
+API_VERSION = 24
+
+# The minimum peer API version that the code should still work with.
+# Notes (dev):
+# - This value is maintained by the CI pipeline, DO NOT EDIT this manually.
+# - Compatibility code for versions lower than this can be safely removed.
+# Refer to API_VERSION for more details.
+MIN_COMPATIBLE_API_VERSION = 11
+
+# The semantic version of the minimum compatible API version.
+# Refer to MIN_COMPATIBLE_API_VERSION for more details.
+# Note (dev): DO NOT EDIT this constant manually.
+MIN_COMPATIBLE_VERSION = '0.10.0'
+
+# The HTTP header name for the API version of the sender.
+API_VERSION_HEADER = 'X-SkyPilot-API-Version'
+
+# The HTTP header name for the SkyPilot version of the sender.
+VERSION_HEADER = 'X-SkyPilot-Version'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'
-# The user ID of the SkyPilot system.
-SKYPILOT_SYSTEM_USER_ID = 'skypilot-system'
 # The memory (GB) that SkyPilot tries to not use to prevent OOM.
 MIN_AVAIL_MEM_GB = 2
+MIN_AVAIL_MEM_GB_CONSOLIDATION_MODE = 4
 # Default encoder/decoder handler name.
 DEFAULT_HANDLER_NAME = 'default'
 # The path to the API request database.
@@ -24,9 +44,27 @@ API_SERVER_REQUEST_DB_PATH = '~/.sky/api_server/requests.db'
 # background.
 CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS = 60
 
+# The interval (seconds) for the volume status to be refreshed in the
+# background.
+VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS = 60
+
 # Environment variable for a file path to the API cookie file.
+# Keep in sync with websocket_proxy.py
 API_COOKIE_FILE_ENV_VAR = f'{constants.SKYPILOT_ENV_VAR_PREFIX}API_COOKIE_FILE'
+# Default file if unset.
+# Keep in sync with websocket_proxy.py
+API_COOKIE_FILE_DEFAULT_LOCATION = '~/.sky/cookies.txt'
 
 # The path to the dashboard build output
 DASHBOARD_DIR = os.path.join(os.path.dirname(__file__), '..', 'dashboard',
                              'out')
+
+# The interval (seconds) for the event to be restarted in the background.
+DAEMON_RESTART_INTERVAL_SECONDS = 20
+
+# Cookie header for stream request id.
+STREAM_REQUEST_HEADER = 'X-SkyPilot-Stream-Request-ID'
+
+# Valid empty values for pickled fields (base64-encoded pickled None)
+# base64.b64encode(pickle.dumps(None)).decode('utf-8')
+EMPTY_PICKLED_VALUE = 'gAROLg=='

sky/server/daemons.py

@@ -0,0 +1,295 @@
+"""Internal server daemons that run in the background."""
+import atexit
+import dataclasses
+import os
+import time
+import typing
+from typing import Callable
+
+from sky import sky_logging
+from sky import skypilot_config
+from sky.adaptors import common as adaptors_common
+from sky.server import constants as server_constants
+from sky.server.requests import request_names
+from sky.skylet import constants
+from sky.utils import annotations
+from sky.utils import common_utils
+from sky.utils import env_options
+from sky.utils import locks
+from sky.utils import subprocess_utils
+from sky.utils import timeline
+from sky.utils import ux_utils
+
+if typing.TYPE_CHECKING:
+    import pathlib
+else:
+    pathlib = adaptors_common.LazyImport('pathlib')
+
+logger = sky_logging.init_logger(__name__)
+
+
+def _default_should_skip():
+    return False
+
+
+@dataclasses.dataclass
+class InternalRequestDaemon:
+    """Internal daemon that runs an event in the background."""
+
+    id: str
+    name: request_names.RequestName
+    event_fn: Callable[[], None]
+    default_log_level: str = 'INFO'
+    should_skip: Callable[[], bool] = _default_should_skip
+
+    def refresh_log_level(self) -> int:
+        # pylint: disable=import-outside-toplevel
+        import logging
+
+        try:
+            # Refresh config within the while loop.
+            # Since this is a long running daemon,
+            # reload_for_new_request()
+            # is not called in between the event runs.
+            # We don't need to grab the lock here because each of the daemons
+            # run in their own process and thus have their own request context.
+            skypilot_config.reload_config()
+            # Get the configured log level for the daemon inside the event loop
+            # in case the log level changes after the API server is started.
+            level_str = skypilot_config.get_nested(
+                ('daemons', self.id, 'log_level'), self.default_log_level)
+            return getattr(logging, level_str.upper())
+        except AttributeError:
+            # Bad level should be rejected by
+            # schema validation, just in case.
+            logger.warning(f'Invalid log level: {level_str}, using DEBUG')
+            return logging.DEBUG
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception(f'Error refreshing log level for {self.id}: {e}')
+            return logging.DEBUG
+
+    def run_event(self):
+        """Run the event."""
+
+        # Disable logging for periodic refresh to avoid the usage message being
+        # sent multiple times.
+        os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
+
+        level = self.refresh_log_level()
+        while True:
+            try:
+                with ux_utils.enable_traceback(), \
+                    sky_logging.set_sky_logging_levels(level):
+                    sky_logging.reload_logger()
+                    level = self.refresh_log_level()
+                    self.event_fn()
+            except Exception:  # pylint: disable=broad-except
+                # It is OK to fail to run the event, as the event is not
+                # critical, but we should log the error.
+                logger.exception(
+                    f'Error running {self.name} event. '
+                    f'Restarting in '
+                    f'{server_constants.DAEMON_RESTART_INTERVAL_SECONDS} '
+                    'seconds...')
+                time.sleep(server_constants.DAEMON_RESTART_INTERVAL_SECONDS)
+            finally:
+                # Clear request level cache after each run to avoid
+                # using too much memory.
+                annotations.clear_request_level_cache()
+                timeline.save_timeline()
+                # Kill all children processes related to this request.
+                # Each executor handles a single request, so we can safely
+                # kill all children processes related to this request.
+                subprocess_utils.kill_children_processes()
+                common_utils.release_memory()
+
+
+def refresh_cluster_status_event():
+    """Periodically refresh the cluster status."""
+    # pylint: disable=import-outside-toplevel
+    from sky.backends import backend_utils
+
+    logger.info('=== Refreshing cluster status ===')
+    # This periodically refresh will hold the lock for the cluster being
+    # refreshed, but it is OK because other operations will just wait for
+    # the lock and get the just refreshed status without refreshing again.
+    backend_utils.refresh_cluster_records()
+    logger.info('Status refreshed. Sleeping '
+                f'{server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS}'
+                ' seconds for the next refresh...\n')
+    time.sleep(server_constants.CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS)
+
+
+def refresh_volume_status_event():
+    """Periodically refresh the volume status."""
+    # pylint: disable=import-outside-toplevel
+    from sky.volumes.server import core
+
+    # Disable logging for periodic refresh to avoid the usage message being
+    # sent multiple times.
+    os.environ[env_options.Options.DISABLE_LOGGING.env_key] = '1'
+
+    logger.info('=== Refreshing volume status ===')
+    core.volume_refresh()
+    logger.info('Volume status refreshed. Sleeping '
+                f'{server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS}'
+                ' seconds for the next refresh...\n')
+    time.sleep(server_constants.VOLUME_REFRESH_DAEMON_INTERVAL_SECONDS)
+
+
+_managed_job_consolidation_mode_lock = None
+
+
+# Attempt to gracefully release the lock when the process exits.
+# If this fails, it's okay, the lock will be released when the process dies.
+def _release_managed_job_consolidation_mode_lock() -> None:
+    global _managed_job_consolidation_mode_lock
+    if _managed_job_consolidation_mode_lock is not None:
+        _managed_job_consolidation_mode_lock.release()
+        _managed_job_consolidation_mode_lock = None
+
+
+atexit.register(_release_managed_job_consolidation_mode_lock)
+
+
+def managed_job_status_refresh_event():
+    """Refresh the managed job status for controller consolidation mode."""
+    # pylint: disable=import-outside-toplevel
+    from sky.jobs import constants as managed_job_constants
+    from sky.jobs import utils as managed_job_utils
+
+    global _managed_job_consolidation_mode_lock
+    if _managed_job_consolidation_mode_lock is None:
+        _managed_job_consolidation_mode_lock = locks.get_lock(
+            managed_job_constants.CONSOLIDATION_MODE_LOCK_ID)
+
+    # Touch the signal file here to avoid conflict with
+    # update_managed_jobs_statuses. Although we run
+    # ha_recovery_for_consolidation_mode before checking the job statuses
+    # (events.ManagedJobEvent), update_managed_jobs_statuses is also called in
+    # cancel_jobs_by_id.
+    # We also need to make sure that new controllers are not started until we
+    # acquire the consolidation mode lock, since if we have controllers on both
+    # the new and old API server during a rolling update, calling
+    # update_managed_jobs_statuses on the old API server could lead to
+    # FAILED_CONTROLLER.
+    signal_file = pathlib.Path(
+        constants.PERSISTENT_RUN_RESTARTING_SIGNAL_FILE).expanduser()
+    try:
+        signal_file.touch()
+
+        # Make sure the lock is acquired for this process before proceeding to
+        # do recovery. This will block if another API server is still running,
+        # but should proceed once it is terminated and releases the lock.
+        if not _managed_job_consolidation_mode_lock.is_locked():
+            logger.info('Acquiring the consolidation mode lock: '
+                        f'{_managed_job_consolidation_mode_lock}')
+            _managed_job_consolidation_mode_lock.acquire()
+            logger.info('Lock acquired!')
+        # We don't explicitly release the lock until the process exits.
+        # Even if _release_managed_job_consolidation_mode_lock is not called,
+        # the lock should be released when the process dies (either due to the
+        # advisory file lock being released or the postgres session dying).
+
+        # We run the recovery logic before checking the job statuses as those
+        # two are conflicting. Check PERSISTENT_RUN_RESTARTING_SIGNAL_FILE for
+        # details.
+        managed_job_utils.ha_recovery_for_consolidation_mode()
+    finally:
+        # Now, we should be sure that this is the only API server, we have
+        # started the new controllers and unclaimed all the jobs, and we are
+        # ready to update the job statuses.
+        signal_file.unlink()
+
+    # After recovery, we start the event loop.
+    from sky.skylet import events
+    refresh_event = events.ManagedJobEvent()
+    logger.info('=== Running managed job event ===')
+    refresh_event.run()
+    time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
+
+
+def should_skip_managed_job_status_refresh():
+    """Check if the managed job status refresh event should be skipped."""
+    # pylint: disable=import-outside-toplevel
+    from sky.jobs import utils as managed_job_utils
+    return not managed_job_utils.is_consolidation_mode()
+
+
+def _serve_status_refresh_event(pool: bool):
+    """Refresh the sky serve status for controller consolidation mode."""
+    # pylint: disable=import-outside-toplevel
+    from sky.serve import serve_utils
+
+    # We run the recovery logic before starting the event loop as those two are
+    # conflicting. Check PERSISTENT_RUN_RESTARTING_SIGNAL_FILE for details.
+    serve_utils.ha_recovery_for_consolidation_mode(pool=pool)
+
+    # After recovery, we start the event loop.
+    from sky.skylet import events
+    event = events.ServiceUpdateEvent(pool=pool)
+    noun = 'pool' if pool else 'serve'
+    logger.info(f'=== Running {noun} status refresh event ===')
+    event.run()
+    time.sleep(events.EVENT_CHECKING_INTERVAL_SECONDS)
+
+
+def _should_skip_serve_status_refresh_event(pool: bool):
+    """Check if the serve status refresh event should be skipped."""
+    # pylint: disable=import-outside-toplevel
+    from sky.serve import serve_utils
+    return not serve_utils.is_consolidation_mode(pool=pool)
+
+
+def sky_serve_status_refresh_event():
+    _serve_status_refresh_event(pool=False)
+
+
+def should_skip_sky_serve_status_refresh():
+    return _should_skip_serve_status_refresh_event(pool=False)
+
+
+def pool_status_refresh_event():
+    _serve_status_refresh_event(pool=True)
+
+
+def should_skip_pool_status_refresh():
+    return _should_skip_serve_status_refresh_event(pool=True)
+
+
+# Register the events to run in the background.
+INTERNAL_REQUEST_DAEMONS = [
+    # This status refresh daemon can cause the autostopp'ed/autodown'ed cluster
+    # set to updated status automatically, without showing users the hint of
+    # cluster being stopped or down when `sky status -r` is called.
+    InternalRequestDaemon(
+        id='skypilot-status-refresh-daemon',
+        name=request_names.RequestName.REQUEST_DAEMON_STATUS_REFRESH,
+        event_fn=refresh_cluster_status_event,
+        default_log_level='DEBUG'),
+    # Volume status refresh daemon to update the volume status periodically.
+    InternalRequestDaemon(
+        id='skypilot-volume-status-refresh-daemon',
+        name=request_names.RequestName.REQUEST_DAEMON_VOLUME_REFRESH,
+        event_fn=refresh_volume_status_event),
+    InternalRequestDaemon(id='managed-job-status-refresh-daemon',
+                          name=request_names.RequestName.
+                          REQUEST_DAEMON_MANAGED_JOB_STATUS_REFRESH,
+                          event_fn=managed_job_status_refresh_event,
+                          should_skip=should_skip_managed_job_status_refresh),
+    InternalRequestDaemon(
+        id='sky-serve-status-refresh-daemon',
+        name=request_names.RequestName.REQUEST_DAEMON_SKY_SERVE_STATUS_REFRESH,
+        event_fn=sky_serve_status_refresh_event,
+        should_skip=should_skip_sky_serve_status_refresh),
+    InternalRequestDaemon(
+        id='pool-status-refresh-daemon',
+        name=request_names.RequestName.REQUEST_DAEMON_POOL_STATUS_REFRESH,
+        event_fn=pool_status_refresh_event,
+        should_skip=should_skip_pool_status_refresh),
+]
+
+
+def is_daemon_request_id(request_id: str) -> bool:
+    """Returns whether a specific request_id is an internal daemon."""
+    return any([d.id == request_id for d in INTERNAL_REQUEST_DAEMONS])