skypilot-nightly 1.0.0.dev20250831__py3-none-any.whl → 1.0.0.dev20250902__py3-none-any.whl

sky/global_user_state.py

@@ -29,6 +29,7 @@ from sqlalchemy.ext import declarative
 from sky import models
 from sky import sky_logging
 from sky import skypilot_config
+from sky.server import metrics as metrics_lib
 from sky.skylet import constants
 from sky.utils import common_utils
 from sky.utils import context_utils
@@ -329,6 +330,7 @@ def _init_db(func):
 
 
 @_init_db
+@metrics_lib.time_me
 def add_or_update_user(user: models.User,
                        allow_duplicate_name: bool = True) -> bool:
     """Store the mapping from user hash to user name for display purposes.
@@ -423,6 +425,7 @@ def add_or_update_user(user: models.User,
 
 
 @_init_db
+@metrics_lib.time_me
 def get_user(user_id: str) -> Optional[models.User]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -436,6 +439,7 @@ def get_user(user_id: str) -> Optional[models.User]:
 
 
 @_init_db
+@metrics_lib.time_me
 def _get_users(user_ids: Set[str]) -> Dict[str, models.User]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -450,6 +454,7 @@ def _get_users(user_ids: Set[str]) -> Dict[str, models.User]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_user_by_name(username: str) -> List[models.User]:
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
         rows = session.query(user_table).filter_by(name=username).all()
@@ -464,6 +469,7 @@ def get_user_by_name(username: str) -> List[models.User]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_user_by_name_match(username_match: str) -> List[models.User]:
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
         rows = session.query(user_table).filter(
@@ -475,6 +481,7 @@ def get_user_by_name_match(username_match: str) -> List[models.User]:
 
 
 @_init_db
+@metrics_lib.time_me
 def delete_user(user_id: str) -> None:
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
         session.query(user_table).filter_by(id=user_id).delete()
@@ -482,6 +489,7 @@ def delete_user(user_id: str) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_all_users() -> List[models.User]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -495,6 +503,7 @@ def get_all_users() -> List[models.User]:
 
 
 @_init_db
+@metrics_lib.time_me
 def add_or_update_cluster(cluster_name: str,
                           cluster_handle: 'backends.ResourceHandle',
                           requested_resources: Optional[Set[Any]],
@@ -693,6 +702,7 @@ def add_or_update_cluster(cluster_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def add_cluster_event(cluster_name: str,
                       new_status: Optional[status_lib.ClusterStatus],
                       reason: str,
@@ -903,6 +913,7 @@ def _get_user_hash_or_current_user(user_hash: Optional[str]) -> str:
 
 
 @_init_db
+@metrics_lib.time_me
 def update_cluster_handle(cluster_name: str,
                           cluster_handle: 'backends.ResourceHandle'):
     assert _SQLALCHEMY_ENGINE is not None
@@ -914,6 +925,7 @@ def update_cluster_handle(cluster_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def update_last_use(cluster_name: str):
     """Updates the last used command for the cluster."""
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -923,6 +935,7 @@ def update_last_use(cluster_name: str):
 
 
 @_init_db
+@metrics_lib.time_me
 def remove_cluster(cluster_name: str, terminate: bool) -> None:
     """Removes cluster_name mapping."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -971,6 +984,7 @@ def remove_cluster(cluster_name: str, terminate: bool) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_handle_from_cluster_name(
         cluster_name: str) -> Optional['backends.ResourceHandle']:
     assert _SQLALCHEMY_ENGINE is not None
@@ -983,6 +997,7 @@ def get_handle_from_cluster_name(
 
 
 @_init_db
+@metrics_lib.time_me
 def get_glob_cluster_names(cluster_name: str) -> List[str]:
     assert _SQLALCHEMY_ENGINE is not None
     assert cluster_name is not None, 'cluster_name cannot be None'
@@ -1002,6 +1017,7 @@ def get_glob_cluster_names(cluster_name: str) -> List[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def set_cluster_status(cluster_name: str,
                        status: status_lib.ClusterStatus) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1019,6 +1035,7 @@ def set_cluster_status(cluster_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def set_cluster_autostop_value(cluster_name: str, idle_minutes: int,
                                to_down: bool) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1035,6 +1052,7 @@ def set_cluster_autostop_value(cluster_name: str, idle_minutes: int,
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_launch_time(cluster_name: str) -> Optional[int]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1045,6 +1063,7 @@ def get_cluster_launch_time(cluster_name: str) -> Optional[int]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_info(cluster_name: str) -> Optional[Dict[str, Any]]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1055,6 +1074,7 @@ def get_cluster_info(cluster_name: str) -> Optional[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_provision_log_path(cluster_name: str) -> Optional[str]:
     """Returns provision_log_path from clusters table, if recorded."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -1066,6 +1086,7 @@ def get_cluster_provision_log_path(cluster_name: str) -> Optional[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_history_provision_log_path(cluster_name: str) -> Optional[str]:
     """Returns provision_log_path from cluster_history for this name.
 
@@ -1107,6 +1128,7 @@ def get_cluster_history_provision_log_path(cluster_name: str) -> Optional[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def set_cluster_info(cluster_name: str, metadata: Dict[str, Any]) -> None:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1120,6 +1142,7 @@ def set_cluster_info(cluster_name: str, metadata: Dict[str, Any]) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_storage_mounts_metadata(
         cluster_name: str) -> Optional[Dict[str, Any]]:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1131,6 +1154,7 @@ def get_cluster_storage_mounts_metadata(
 
 
 @_init_db
+@metrics_lib.time_me
 def set_cluster_storage_mounts_metadata(
         cluster_name: str, storage_mounts_metadata: Dict[str, Any]) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1147,6 +1171,7 @@ def set_cluster_storage_mounts_metadata(
 
 
 @_init_db
+@metrics_lib.time_me
 def _get_cluster_usage_intervals(
         cluster_hash: Optional[str]
 ) -> Optional[List[Tuple[int, Optional[int]]]]:
@@ -1188,6 +1213,7 @@ def _get_cluster_duration(cluster_hash: str) -> int:
 
 
 @_init_db
+@metrics_lib.time_me
 def _set_cluster_usage_intervals(
         cluster_hash: str, usage_intervals: List[Tuple[int,
                                                        Optional[int]]]) -> None:
@@ -1205,6 +1231,7 @@ def _set_cluster_usage_intervals(
 
 
 @_init_db
+@metrics_lib.time_me
 def set_owner_identity_for_cluster(cluster_name: str,
                                    owner_identity: Optional[List[str]]) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1222,6 +1249,7 @@ def set_owner_identity_for_cluster(cluster_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def _get_hash_for_existing_cluster(cluster_name: str) -> Optional[str]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1232,6 +1260,7 @@ def _get_hash_for_existing_cluster(cluster_name: str) -> Optional[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_launched_resources_from_cluster_hash(
         cluster_hash: str) -> Optional[Tuple[int, Any]]:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1278,6 +1307,7 @@ def _load_storage_mounts_metadata(
 
 
 @_init_db
+@metrics_lib.time_me
 @context_utils.cancellation_guard
 def get_cluster_from_name(
         cluster_name: Optional[str]) -> Optional[Dict[str, Any]]:
@@ -1321,6 +1351,7 @@ def get_cluster_from_name(
 
 
 @_init_db
+@metrics_lib.time_me
 def get_clusters(
     *,  # keyword only separator
     exclude_managed_clusters: bool = False,
@@ -1416,6 +1447,7 @@ def get_clusters(
 
 
 @_init_db
+@metrics_lib.time_me
 def get_clusters_from_history(
         days: Optional[int] = None) -> List[Dict[str, Any]]:
     """Get cluster reports from history.
@@ -1541,6 +1573,7 @@ def get_clusters_from_history(
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_names_start_with(starts_with: str) -> List[str]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1550,6 +1583,7 @@ def get_cluster_names_start_with(starts_with: str) -> List[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cached_enabled_clouds(cloud_capability: 'cloud.CloudCapability',
                               workspace: str) -> List['clouds.Cloud']:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1575,6 +1609,7 @@ def get_cached_enabled_clouds(cloud_capability: 'cloud.CloudCapability',
 
 
 @_init_db
+@metrics_lib.time_me
 def set_enabled_clouds(enabled_clouds: List[str],
                        cloud_capability: 'cloud.CloudCapability',
                        workspace: str) -> None:
@@ -1604,6 +1639,7 @@ def _get_enabled_clouds_key(cloud_capability: 'cloud.CloudCapability',
 
 
 @_init_db
+@metrics_lib.time_me
 def get_allowed_clouds(workspace: str) -> List[str]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1615,6 +1651,7 @@ def get_allowed_clouds(workspace: str) -> List[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def set_allowed_clouds(allowed_clouds: List[str], workspace: str) -> None:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1641,6 +1678,7 @@ def _get_allowed_clouds_key(workspace: str) -> str:
 
 
 @_init_db
+@metrics_lib.time_me
 def add_or_update_storage(storage_name: str,
                           storage_handle: 'Storage.StorageMetadata',
                           storage_status: status_lib.StorageStatus):
@@ -1683,6 +1721,7 @@ def add_or_update_storage(storage_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def remove_storage(storage_name: str):
     """Removes Storage from Database"""
     assert _SQLALCHEMY_ENGINE is not None
@@ -1692,6 +1731,7 @@ def remove_storage(storage_name: str):
 
 
 @_init_db
+@metrics_lib.time_me
 def set_storage_status(storage_name: str,
                        status: status_lib.StorageStatus) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1705,6 +1745,7 @@ def set_storage_status(storage_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def get_storage_status(storage_name: str) -> Optional[status_lib.StorageStatus]:
     assert _SQLALCHEMY_ENGINE is not None
     assert storage_name is not None, 'storage_name cannot be None'
@@ -1716,6 +1757,7 @@ def get_storage_status(storage_name: str) -> Optional[status_lib.StorageStatus]:
 
 
 @_init_db
+@metrics_lib.time_me
 def set_storage_handle(storage_name: str,
                        handle: 'Storage.StorageMetadata') -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1730,6 +1772,7 @@ def set_storage_handle(storage_name: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def get_handle_from_storage_name(
         storage_name: Optional[str]) -> Optional['Storage.StorageMetadata']:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1743,6 +1786,7 @@ def get_handle_from_storage_name(
 
 
 @_init_db
+@metrics_lib.time_me
 def get_glob_storage_name(storage_name: str) -> List[str]:
     assert _SQLALCHEMY_ENGINE is not None
     assert storage_name is not None, 'storage_name cannot be None'
@@ -1762,6 +1806,7 @@ def get_glob_storage_name(storage_name: str) -> List[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_storage_names_start_with(starts_with: str) -> List[str]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1771,6 +1816,7 @@ def get_storage_names_start_with(starts_with: str) -> List[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_storage() -> List[Dict[str, Any]]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1789,6 +1835,7 @@ def get_storage() -> List[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_volume_names_start_with(starts_with: str) -> List[str]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1798,6 +1845,7 @@ def get_volume_names_start_with(starts_with: str) -> List[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_volumes() -> List[Dict[str, Any]]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1818,6 +1866,7 @@ def get_volumes() -> List[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_volume_by_name(name: str) -> Optional[Dict[str, Any]]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1837,6 +1886,7 @@ def get_volume_by_name(name: str) -> Optional[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def add_volume(name: str, config: models.VolumeConfig,
                status: status_lib.VolumeStatus) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1871,6 +1921,7 @@ def add_volume(name: str, config: models.VolumeConfig,
 
 
 @_init_db
+@metrics_lib.time_me
 def update_volume(name: str, last_attached_at: int,
                   status: status_lib.VolumeStatus) -> None:
     assert _SQLALCHEMY_ENGINE is not None
@@ -1883,6 +1934,7 @@ def update_volume(name: str, last_attached_at: int,
 
 
 @_init_db
+@metrics_lib.time_me
 def update_volume_status(name: str, status: status_lib.VolumeStatus) -> None:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1893,6 +1945,7 @@ def update_volume_status(name: str, status: status_lib.VolumeStatus) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def delete_volume(name: str) -> None:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1901,6 +1954,7 @@ def delete_volume(name: str) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_ssh_keys(user_hash: str) -> Tuple[str, str, bool]:
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1912,6 +1966,7 @@ def get_ssh_keys(user_hash: str) -> Tuple[str, str, bool]:
 
 
 @_init_db
+@metrics_lib.time_me
 def set_ssh_keys(user_hash: str, ssh_public_key: str, ssh_private_key: str):
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -1938,6 +1993,7 @@ def set_ssh_keys(user_hash: str, ssh_public_key: str, ssh_private_key: str):
 
 
 @_init_db
+@metrics_lib.time_me
 def add_service_account_token(token_id: str,
                               token_name: str,
                               token_hash: str,
@@ -1971,6 +2027,7 @@ def add_service_account_token(token_id: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def get_service_account_token(token_id: str) -> Optional[Dict[str, Any]]:
     """Get a service account token by token_id."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -1992,6 +2049,7 @@ def get_service_account_token(token_id: str) -> Optional[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_user_service_account_tokens(user_hash: str) -> List[Dict[str, Any]]:
     """Get all service account tokens for a user (as creator)."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -2011,6 +2069,7 @@ def get_user_service_account_tokens(user_hash: str) -> List[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def update_service_account_token_last_used(token_id: str) -> None:
     """Update the last_used_at timestamp for a service account token."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -2024,6 +2083,7 @@ def update_service_account_token_last_used(token_id: str) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def delete_service_account_token(token_id: str) -> bool:
     """Delete a service account token.
 
@@ -2039,6 +2099,7 @@ def delete_service_account_token(token_id: str) -> bool:
 
 
 @_init_db
+@metrics_lib.time_me
 def rotate_service_account_token(token_id: str,
                                  new_token_hash: str,
                                  new_expires_at: Optional[int] = None) -> None:
@@ -2069,6 +2130,7 @@ def rotate_service_account_token(token_id: str,
 
 
 @_init_db
+@metrics_lib.time_me
 def get_cluster_yaml_str(cluster_yaml_path: Optional[str]) -> Optional[str]:
     """Get the cluster yaml from the database or the local file system.
     If the cluster yaml is not in the database, check if it exists on the
@@ -2155,6 +2217,7 @@ def get_cluster_yaml_dict_multiple(
 
 
 @_init_db
+@metrics_lib.time_me
 def set_cluster_yaml(cluster_name: str, yaml_str: str) -> None:
     """Set the cluster yaml in the database."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -2177,6 +2240,7 @@ def set_cluster_yaml(cluster_name: str, yaml_str: str) -> None:
 
 
 @_init_db
+@metrics_lib.time_me
 def remove_cluster_yaml(cluster_name: str):
     assert _SQLALCHEMY_ENGINE is not None
     with orm.Session(_SQLALCHEMY_ENGINE) as session:
@@ -2186,6 +2250,7 @@ def remove_cluster_yaml(cluster_name: str):
 
 
 @_init_db
+@metrics_lib.time_me
 def get_all_service_account_tokens() -> List[Dict[str, Any]]:
     """Get all service account tokens across all users (for admin access)."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -2204,6 +2269,7 @@ def get_all_service_account_tokens() -> List[Dict[str, Any]]:
 
 
 @_init_db
+@metrics_lib.time_me
 def get_system_config(config_key: str) -> Optional[str]:
     """Get a system configuration value by key."""
     assert _SQLALCHEMY_ENGINE is not None
@@ -2216,6 +2282,7 @@ def get_system_config(config_key: str) -> Optional[str]:
 
 
 @_init_db
+@metrics_lib.time_me
 def set_system_config(config_key: str, config_value: str) -> None:
     """Set a system configuration value."""
     assert _SQLALCHEMY_ENGINE is not None

sky/jobs/server/server.py

@@ -79,7 +79,8 @@ async def logs(
         if jobs_logs_body.refresh else api_requests.ScheduleType.SHORT,
         request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
-    request_task = api_requests.get_request(request.state.request_id)
+    request_task = await api_requests.get_request_async(request.state.request_id
+                                                       )
 
     return stream_utils.stream_response(
         request_id=request_task.request_id,

sky/serve/server/server.py

@@ -107,7 +107,8 @@ async def tail_logs(
         request_cluster_name=common.SKY_SERVE_CONTROLLER_NAME,
     )
 
-    request_task = api_requests.get_request(request.state.request_id)
+    request_task = await api_requests.get_request_async(request.state.request_id
+                                                       )
 
     return stream_utils.stream_response(
         request_id=request_task.request_id,

sky/server/auth/oauth2_proxy.py

@@ -12,9 +12,11 @@ import aiohttp
 import fastapi
 import starlette.middleware.base
 
+from sky import global_user_state
 from sky import models
 from sky import sky_logging
 from sky.server.auth import authn
+from sky.users import permission
 from sky.utils import common_utils
 
 logger = sky_logging.init_logger(__name__)
@@ -149,6 +151,10 @@ class OAuth2ProxyMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
                                 'return user info, check your oauth2-proxy'
                                 'setup.'
                         })
+                newly_added = global_user_state.add_or_update_user(auth_user)
+                if newly_added:
+                    permission.permission_service.add_user_if_not_exists(
+                        auth_user.id)
                 request.state.auth_user = auth_user
                 await authn.override_user_info_in_request_body(
                     request, auth_user)

sky/server/common.py

@@ -648,14 +648,16 @@ def _set_metrics_env_var(env: Union[Dict[str, str], os._Environ], metrics: bool,
         deploy: Whether the server is running in deploy mode, which means
             multiple processes might be running.
     """
+    del deploy
     if metrics or os.getenv(constants.ENV_VAR_SERVER_METRICS_ENABLED) == 'true':
         env[constants.ENV_VAR_SERVER_METRICS_ENABLED] = 'true'
-        if deploy:
-            metrics_dir = os.path.join(tempfile.gettempdir(), 'metrics')
-            shutil.rmtree(metrics_dir, ignore_errors=True)
-            os.makedirs(metrics_dir, exist_ok=True)
-            # Refer to https://prometheus.github.io/client_python/multiprocess/
-            env['PROMETHEUS_MULTIPROC_DIR'] = metrics_dir
+        # Always set the metrics dir since we need to collect metrics from
+        # subprocesses like the executor.
+        metrics_dir = os.path.join(tempfile.gettempdir(), 'metrics')
+        shutil.rmtree(metrics_dir, ignore_errors=True)
+        os.makedirs(metrics_dir, exist_ok=True)
+        # Refer to https://prometheus.github.io/client_python/multiprocess/
+        env['PROMETHEUS_MULTIPROC_DIR'] = metrics_dir
 
 
 def check_server_healthy(