skypilot-nightly 1.0.0.dev20250607__py3-none-any.whl → 1.0.0.dev20250610__py3-none-any.whl

sky/skylet/constants.py

@@ -377,8 +377,7 @@ OVERRIDEABLE_CONFIG_KEYS_IN_TASK: List[Tuple[str, ...]] = [
 ]
 # When overriding the SkyPilot configs on the API server with the client one,
 # we skip the following keys because they are meant to be client-side configs.
-SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [('admin_policy',),
-                                                       ('api_server',),
+SKIPPED_CLIENT_OVERRIDE_KEYS: List[Tuple[str, ...]] = [('api_server',),
                                                        ('allowed_clouds',),
                                                        ('workspaces',), ('db',)]
 
@@ -419,3 +418,6 @@ ALL_CLOUDS = ('aws', 'azure', 'gcp', 'ibm', 'lambda', 'scp', 'oci',
               'kubernetes', 'runpod', 'vast', 'vsphere', 'cudo', 'fluidstack',
               'paperspace', 'do', 'nebius', 'ssh')
 # END constants used for service catalog.
+
+# The user ID of the SkyPilot system.
+SKYPILOT_SYSTEM_USER_ID = 'skypilot-system'

sky/skylet/job_lib.py

@@ -794,7 +794,8 @@ def load_job_queue(payload: str) -> List[Dict[str, Any]]:
     for job in jobs:
         job['status'] = JobStatus(job['status'])
         job['user_hash'] = job['username']
-        job['username'] = global_user_state.get_user(job['user_hash']).name
+        user = global_user_state.get_user(job['user_hash'])
+        job['username'] = user.name if user is not None else None
     return jobs
 
 

sky/skypilot_config.py

@@ -58,6 +58,11 @@ import typing
 from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
 
 import filelock
+import sqlalchemy
+from sqlalchemy import orm
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.dialects import sqlite
+from sqlalchemy.ext import declarative
 
 from sky import exceptions
 from sky import sky_logging
@@ -66,6 +71,7 @@ from sky.skylet import constants
 from sky.utils import common_utils
 from sky.utils import config_utils
 from sky.utils import context
+from sky.utils import db_utils
 from sky.utils import schemas
 from sky.utils import ux_utils
 from sky.utils.kubernetes import config_map_utils
@@ -110,6 +116,56 @@ ENV_VAR_PROJECT_CONFIG = f'{constants.SKYPILOT_ENV_VAR_PREFIX}PROJECT_CONFIG'
 _GLOBAL_CONFIG_PATH = '~/.sky/config.yaml'
 _PROJECT_CONFIG_PATH = '.sky.yaml'
 
+_SQLALCHEMY_ENGINE: Optional[sqlalchemy.engine.Engine] = None
+API_SERVER_CONFIG_KEY = 'api_server_config'
+
+Base = declarative.declarative_base()
+
+config_yaml_table = sqlalchemy.Table(
+    'config_yaml',
+    Base.metadata,
+    sqlalchemy.Column('key', sqlalchemy.Text, primary_key=True),
+    sqlalchemy.Column('value', sqlalchemy.Text),
+)
+
+
+def create_table():
+    # Create tables if they don't exist
+    Base.metadata.create_all(bind=_SQLALCHEMY_ENGINE)
+
+
+def _get_config_yaml_from_db(key: str) -> Optional[config_utils.Config]:
+    assert _SQLALCHEMY_ENGINE is not None
+    with orm.Session(_SQLALCHEMY_ENGINE) as session:
+        row = session.query(config_yaml_table).filter_by(key=key).first()
+    if row:
+        db_config = config_utils.Config(yaml.safe_load(row.value))
+        db_config.pop_nested(('db',), None)
+        return db_config
+    return None
+
+
+def _set_config_yaml_to_db(key: str, config: config_utils.Config):
+    assert _SQLALCHEMY_ENGINE is not None
+    config.pop_nested(('db',), None)
+    config_str = common_utils.dump_yaml_str(dict(config))
+    with orm.Session(_SQLALCHEMY_ENGINE) as session:
+        if (_SQLALCHEMY_ENGINE.dialect.name ==
+                db_utils.SQLAlchemyDialect.SQLITE.value):
+            insert_func = sqlite.insert
+        elif (_SQLALCHEMY_ENGINE.dialect.name ==
+              db_utils.SQLAlchemyDialect.POSTGRESQL.value):
+            insert_func = postgresql.insert
+        else:
+            raise ValueError('Unsupported database dialect')
+        insert_stmnt = insert_func(config_yaml_table).values(key=key,
+                                                             value=config_str)
+        do_update_stmt = insert_stmnt.on_conflict_do_update(
+            index_elements=[config_yaml_table.c.key],
+            set_={config_yaml_table.c.value: config_str})
+        session.execute(do_update_stmt)
+        session.commit()
+
 
 class ConfigContext:
 
@@ -257,11 +313,6 @@ def _resolve_project_config_path() -> Optional[str]:
     return None
 
 
-def _get_project_config() -> config_utils.Config:
-    """Returns the project config."""
-    return _get_config_from_path(_resolve_project_config_path())
-
-
 def _resolve_server_config_path() -> Optional[str]:
     # find the server config file
     server_config_path = _get_config_file_path(ENV_VAR_GLOBAL_CONFIG)
@@ -507,26 +558,35 @@ def _reload_config_from_internal_file(internal_config_path: str) -> None:
 
 
 def _reload_config_as_server() -> None:
+    global _SQLALCHEMY_ENGINE
     # Reset the global variables, to avoid using stale values.
     _set_loaded_config(config_utils.Config())
     _set_loaded_config_path(None)
 
-    overrides: List[config_utils.Config] = []
     server_config_path = _resolve_server_config_path()
     server_config = _get_config_from_path(server_config_path)
-    if server_config:
-        overrides.append(server_config)
 
-    # layer the configs on top of each other based on priority
-    overlaid_server_config: config_utils.Config = config_utils.Config()
-    for override in overrides:
-        overlaid_server_config = overlay_skypilot_config(
-            original_config=overlaid_server_config, override_configs=override)
     if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
-        logger.debug(
-            f'server config: \n'
-            f'{common_utils.dump_yaml_str(dict(overlaid_server_config))}')
-    _set_loaded_config(overlaid_server_config)
+        logger.debug(f'server config: \n'
+                     f'{common_utils.dump_yaml_str(dict(server_config))}')
+
+    db_url = server_config.get_nested(('db',), None)
+    if db_url and len(server_config.keys()) > 1:
+        raise ValueError(
+            'if db config is specified, no other config is allowed')
+
+    if db_url:
+        if _SQLALCHEMY_ENGINE is None:
+            _SQLALCHEMY_ENGINE = sqlalchemy.create_engine(db_url)
+            create_table()
+        db_config = _get_config_yaml_from_db(API_SERVER_CONFIG_KEY)
+        if db_config:
+            if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+                logger.debug(f'Config loaded from db:\n'
+                             f'{common_utils.dump_yaml_str(dict(db_config))}')
+            server_config = overlay_skypilot_config(server_config, db_config)
+
+    _set_loaded_config(server_config)
     _set_loaded_config_path(server_config_path)
 
 
@@ -765,17 +825,40 @@ def update_api_server_config_no_lock(config: config_utils.Config) -> None:
     Args:
         config: The config to save and sync.
     """
+
+    def is_running_pytest() -> bool:
+        return 'PYTEST_CURRENT_TEST' in os.environ
+
+    # Only allow this function to be called by the API Server in production.
+    if not is_running_pytest() and os.environ.get(
+            constants.ENV_VAR_IS_SKYPILOT_SERVER) is None:
+        raise ValueError('This function can only be called by the API Server.')
+
     global_config_path = _resolve_server_config_path()
     if global_config_path is None:
         global_config_path = get_user_config_path()
 
-    # Always save to the local file (PVC in Kubernetes, local file otherwise)
-    common_utils.dump_yaml(global_config_path, dict(config))
-
-    if config_map_utils.is_running_in_kubernetes():
-        # In Kubernetes, sync the PVC config to ConfigMap for user convenience
-        # PVC file is the source of truth, ConfigMap is just a mirror for easy
-        # access
-        config_map_utils.patch_configmap_with_config(config, global_config_path)
+    db_updated = False
+    if os.environ.get(constants.ENV_VAR_IS_SKYPILOT_SERVER) is not None:
+        existing_db_url = get_nested(('db',), None)
+        if existing_db_url:
+            new_db_url = config.get_nested(('db',), None)
+            if new_db_url and new_db_url != existing_db_url:
+                raise ValueError('Cannot change db url while server is running')
+            logger.debug('saving api_server config to db')
+            _set_config_yaml_to_db(API_SERVER_CONFIG_KEY, config)
+            db_updated = True
+
+    if not db_updated:
+        # save to the local file (PVC in Kubernetes, local file otherwise)
+        common_utils.dump_yaml(global_config_path, dict(config))
+
+        if config_map_utils.is_running_in_kubernetes():
+            # In Kubernetes, sync the PVC config to ConfigMap for user
+            # convenience.
+            # PVC file is the source of truth, ConfigMap is just a mirror for
+            # easy access.
+            config_map_utils.patch_configmap_with_config(
+                config, global_config_path)
 
     _reload_config()

sky/users/model.conf

@@ -12,4 +12,4 @@ g = _, _
 e = some(where (p.eft == allow))
 
 [matchers]
-m = g(r.sub, p.sub) && r.obj == p.obj && r.act == p.act
+m = (g(r.sub, p.sub)|| p.sub == '*') && r.obj == p.obj && r.act == p.act

sky/users/permission.py

@@ -3,7 +3,7 @@ import contextlib
 import logging
 import os
 import threading
-from typing import List
+from typing import Generator, List
 
 import casbin
 import filelock
@@ -11,8 +11,11 @@ import sqlalchemy_adapter
 
 from sky import global_user_state
 from sky import sky_logging
+from sky.skylet import constants
 from sky.users import rbac
 
+logging.getLogger('casbin.policy').setLevel(sky_logging.ERROR)
+logging.getLogger('casbin.role').setLevel(sky_logging.ERROR)
 logger = sky_logging.init_logger(__name__)
 
 # Filelocks for the policy update.
@@ -33,22 +36,24 @@ class PermissionService:
             with _lock:
                 if _enforcer_instance is None:
                     _enforcer_instance = self
-                    engine = global_user_state.SQLALCHEMY_ENGINE
+                    engine = global_user_state.initialize_and_get_db()
                     adapter = sqlalchemy_adapter.Adapter(engine)
                     model_path = os.path.join(os.path.dirname(__file__),
                                               'model.conf')
                     enforcer = casbin.Enforcer(model_path, adapter)
-                    logging.getLogger('casbin.policy').setLevel(
-                        sky_logging.ERROR)
-                    logging.getLogger('casbin.role').setLevel(sky_logging.ERROR)
                     self.enforcer = enforcer
         else:
             self.enforcer = _enforcer_instance.enforcer
-        self._maybe_initialize_policies()
+        with _policy_lock():
+            self._maybe_initialize_policies()
 
-    def _maybe_initialize_policies(self):
+    def _maybe_initialize_policies(self) -> None:
         """Initialize policies if they don't already exist."""
+        # TODO(zhwu): we should avoid running this on client side.
         logger.debug(f'Initializing policies in process: {os.getpid()}')
+        self._load_policy_no_lock()
+
+        policy_updated = False
 
         # Check if policies are already initialized by looking for existing
         # permission policies in the enforcer
@@ -66,6 +71,17 @@ class PermissionService:
                     expected_policies.append(
                         [role, item['path'], item['method']])
 
+        # Add workspace policy
+        workspace_policy_permissions = rbac.get_workspace_policy_permissions()
+        logger.debug(f'Workspace policy permissions from config: '
+                     f'{workspace_policy_permissions}')
+
+        for workspace_name, users in workspace_policy_permissions.items():
+            for user in users:
+                expected_policies.append([user, workspace_name, '*'])
+                logger.debug(f'Expected workspace policy: user={user}, '
+                             f'workspace={workspace_name}')
+
         # Check if all expected policies already exist
         policies_exist = all(
             any(policy == expected
@@ -86,48 +102,71 @@ class PermissionService:
                     for item in blocklist:
                         path = item['path']
                         method = item['method']
+                        logger.debug(f'Adding role policy: role={role}, '
+                                     f'path={path}, method={method}')
                         self.enforcer.add_policy(role, path, method)
-            self.enforcer.save_policy()
+                        policy_updated = True
+
+            for workspace_name, users in workspace_policy_permissions.items():
+                for user in users:
+                    logger.debug(f'Initializing workspace policy: user={user}, '
+                                 f'workspace={workspace_name}')
+                    self.enforcer.add_policy(user, workspace_name, '*')
+                    policy_updated = True
+            logger.debug('Policies initialized successfully')
         else:
             logger.debug('Policies already exist, skipping initialization')
 
         # Always ensure users have default roles (this is idempotent)
         all_users = global_user_state.get_all_users()
-        for user in all_users:
-            self.add_user_if_not_exists(user.id)
+        for existing_user in all_users:
+            user_added = self._add_user_if_not_exists_no_lock(existing_user.id)
+            policy_updated = policy_updated or user_added
+
+        if policy_updated:
+            self.enforcer.save_policy()
 
-    def add_user_if_not_exists(self, user: str) -> None:
+    def add_user_if_not_exists(self, user_id: str) -> None:
         """Add user role relationship."""
         with _policy_lock():
-            user_roles = self.enforcer.get_roles_for_user(user)
-            if not user_roles:
-                logger.info(f'User {user} has no roles, adding'
-                            f' default role {rbac.get_default_role()}')
-                self.enforcer.add_grouping_policy(user, rbac.get_default_role())
-                self.enforcer.save_policy()
-
-    def update_role(self, user: str, new_role: str):
+            self._add_user_if_not_exists_no_lock(user_id)
+
+    def _add_user_if_not_exists_no_lock(self, user_id: str) -> bool:
+        """Add user role relationship without lock.
+
+        Returns:
+            True if the user was added, False otherwise.
+        """
+        user_roles = self.enforcer.get_roles_for_user(user_id)
+        if not user_roles:
+            logger.info(f'User {user_id} has no roles, adding'
+                        f' default role {rbac.get_default_role()}')
+            self.enforcer.add_grouping_policy(user_id, rbac.get_default_role())
+            return True
+        return False
+
+    def update_role(self, user_id: str, new_role: str) -> None:
         """Update user role relationship."""
         with _policy_lock():
             # Get current roles
             self._load_policy_no_lock()
             # Avoid calling get_user_roles, as it will require the lock.
-            current_roles = self.enforcer.get_roles_for_user(user)
+            current_roles = self.enforcer.get_roles_for_user(user_id)
             if not current_roles:
-                logger.warning(f'User {user} has no roles')
+                logger.warning(f'User {user_id} has no roles')
             else:
                 # TODO(hailong): how to handle multiple roles?
                 current_role = current_roles[0]
                 if current_role == new_role:
-                    logger.info(f'User {user} already has role {new_role}')
+                    logger.info(f'User {user_id} already has role {new_role}')
                     return
-                self.enforcer.remove_grouping_policy(user, current_role)
+                self.enforcer.remove_grouping_policy(user_id, current_role)
 
             # Update user role
-            self.enforcer.add_grouping_policy(user, new_role)
+            self.enforcer.add_grouping_policy(user_id, new_role)
             self.enforcer.save_policy()
 
-    def get_user_roles(self, user: str) -> List[str]:
+    def get_user_roles(self, user_id: str) -> List[str]:
         """Get all roles for a user.
 
         This method returns all roles that the user has, including inherited
@@ -140,10 +179,11 @@ class PermissionService:
         Returns:
             A list of role names that the user has.
         """
-        self._load_policy()
-        return self.enforcer.get_roles_for_user(user)
+        self._load_policy_no_lock()
+        return self.enforcer.get_roles_for_user(user_id)
 
-    def check_permission(self, user: str, path: str, method: str) -> bool:
+    def check_endpoint_permission(self, user_id: str, path: str,
+                                  method: str) -> bool:
         """Check permission."""
         # We intentionally don't load the policy here, as it is a hot path, and
         # we don't support updating the policy.
@@ -151,28 +191,105 @@ class PermissionService:
         # it is a hot path in every request. It is ok to have a stale policy,
         # as long as it is eventually consistent.
         # self._load_policy_no_lock()
-        return self.enforcer.enforce(user, path, method)
+        return self.enforcer.enforce(user_id, path, method)
 
     def _load_policy_no_lock(self):
         """Load policy from storage."""
         self.enforcer.load_policy()
 
-    def _load_policy(self):
+    def load_policy(self):
         """Load policy from storage with lock."""
         with _policy_lock():
             self._load_policy_no_lock()
 
+    def check_workspace_permission(self, user_id: str,
+                                   workspace_name: str) -> bool:
+        """Check workspace permission.
+
+        This method checks if a user has permission to access a specific
+        workspace.
+
+        For private workspaces, the user must have explicit permission.
+
+        For public workspaces, the permission is granted via a wildcard policy
+        ('*').
+        """
+        if os.getenv(constants.ENV_VAR_IS_SKYPILOT_SERVER) is None:
+            # When it is not on API server, we allow all users to access all
+            # workspaces, as the workspace check has been done on API server.
+            return True
+        role = self.get_user_roles(user_id)
+        if rbac.RoleName.ADMIN.value in role:
+            return True
+        # The Casbin model matcher already handles the wildcard '*' case:
+        # m = (g(r.sub, p.sub)|| p.sub == '*') && r.obj == p.obj &&
+        # r.act == p.act
+        # This means if there's a policy ('*', workspace_name, '*'), it will
+        # match any user
+        result = self.enforcer.enforce(user_id, workspace_name, '*')
+        logger.debug(f'Workspace permission check: user={user_id}, '
+                     f'workspace={workspace_name}, result={result}')
+        return result
+
+    def add_workspace_policy(self, workspace_name: str,
+                             users: List[str]) -> None:
+        """Add workspace policy.
+
+        Args:
+            workspace_name: Name of the workspace
+            users: List of user IDs that should have access.
+                   For public workspaces, this should be ['*'].
+                   For private workspaces, this should be specific user IDs.
+        """
+        with _policy_lock():
+            for user in users:
+                logger.debug(f'Adding workspace policy: user={user}, '
+                             f'workspace={workspace_name}')
+                self.enforcer.add_policy(user, workspace_name, '*')
+            self.enforcer.save_policy()
+
+    def update_workspace_policy(self, workspace_name: str,
+                                users: List[str]) -> None:
+        """Update workspace policy.
+
+        Args:
+            workspace_name: Name of the workspace
+            users: List of user IDs that should have access.
+                   For public workspaces, this should be ['*'].
+                   For private workspaces, this should be specific user IDs.
+        """
+        with _policy_lock():
+            self._load_policy_no_lock()
+            # Remove all existing policies for this workspace
+            self.enforcer.remove_filtered_policy(1, workspace_name)
+            # Add new policies
+            for user in users:
+                logger.debug(f'Updating workspace policy: user={user}, '
+                             f'workspace={workspace_name}')
+                self.enforcer.add_policy(user, workspace_name, '*')
+            self.enforcer.save_policy()
+
+    def remove_workspace_policy(self, workspace_name: str) -> None:
+        """Remove workspace policy."""
+        with _policy_lock():
+            self.enforcer.remove_filtered_policy(1, workspace_name)
+            self.enforcer.save_policy()
+
 
 @contextlib.contextmanager
-def _policy_lock():
+def _policy_lock() -> Generator[None, None, None]:
     """Context manager for policy update lock."""
     try:
         with filelock.FileLock(POLICY_UPDATE_LOCK_PATH,
                                POLICY_UPDATE_LOCK_TIMEOUT_SECONDS):
             yield
     except filelock.Timeout as e:
-        raise RuntimeError(f'Failed to load policy due to a timeout '
+        raise RuntimeError(f'Failed to reload policy due to a timeout '
                            f'when trying to acquire the lock at '
                            f'{POLICY_UPDATE_LOCK_PATH}. '
                            'Please try again or manually remove the lock '
                            f'file if you believe it is stale.') from e
+
+
+# Singleton instance of PermissionService for other modules to use.
+permission_service = PermissionService()

sky/users/rbac.py

@@ -5,6 +5,8 @@ from typing import Dict, List
 
 from sky import sky_logging
 from sky import skypilot_config
+from sky.skylet import constants
+from sky.workspaces import utils as workspaces_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -84,3 +86,27 @@ def get_role_permissions(
             }
         }
     return config_permissions
+
+
+def get_workspace_policy_permissions() -> Dict[str, List[str]]:
+    """Get workspace policy permissions from config.
+
+    Returns:
+        A dictionary of workspace policy permissions.
+        Example:
+        {
+            'workspace1': ['user1-id', 'user2-id'],
+            'workspace2': ['user3-id', 'user4-id']
+            'default': ['*']
+        }
+    """
+    current_workspaces = skypilot_config.get_nested(('workspaces',),
+                                                    default_value={})
+    if constants.SKYPILOT_DEFAULT_WORKSPACE not in current_workspaces:
+        current_workspaces[constants.SKYPILOT_DEFAULT_WORKSPACE] = {}
+    workspaces_to_policy = {}
+    for workspace_name, workspace_config in current_workspaces.items():
+        users = workspaces_utils.get_workspace_users(workspace_config)
+        workspaces_to_policy[workspace_name] = users
+    logger.debug(f'Workspace policy permissions: {workspaces_to_policy}')
+    return workspaces_to_policy

sky/users/server.py

@@ -1,6 +1,5 @@
 """REST API for workspace management."""
 
-import hashlib
 from typing import Any, Dict, List
 
 import fastapi
@@ -8,16 +7,15 @@ import fastapi
 from sky import global_user_state
 from sky import sky_logging
 from sky.server.requests import payloads
+from sky.skylet import constants
 from sky.users import permission
 from sky.users import rbac
-from sky.utils import common_utils
+from sky.utils import common
 
 logger = sky_logging.init_logger(__name__)
 
 router = fastapi.APIRouter()
 
-permission_service = permission.PermissionService()
-
 
 @router.get('')
 async def users() -> List[Dict[str, Any]]:
@@ -25,7 +23,7 @@ async def users() -> List[Dict[str, Any]]:
     all_users = []
     user_list = global_user_state.get_all_users()
     for user in user_list:
-        user_roles = permission_service.get_user_roles(user.id)
+        user_roles = permission.permission_service.get_user_roles(user.id)
         all_users.append({
             'id': user.id,
             'name': user.name,
@@ -39,13 +37,11 @@ async def get_current_user_role(request: fastapi.Request):
     """Get current user's role."""
     # TODO(hailong): is there a reliable way to get the user
     # hash for the request without 'X-Auth-Request-Email' header?
-    if 'X-Auth-Request-Email' not in request.headers:
+    auth_user = request.state.auth_user
+    if auth_user is None:
         return {'name': '', 'role': rbac.RoleName.ADMIN.value}
-    user_name = request.headers['X-Auth-Request-Email']
-    user_hash = hashlib.md5(
-        user_name.encode()).hexdigest()[:common_utils.USER_HASH_LENGTH]
-    user_roles = permission_service.get_user_roles(user_hash)
-    return {'name': user_name, 'role': user_roles[0] if user_roles else ''}
+    user_roles = permission.permission_service.get_user_roles(auth_user.id)
+    return {'name': auth_user.name, 'role': user_roles[0] if user_roles else ''}
 
 
 @router.post('/update')
@@ -58,9 +54,14 @@ async def user_update(user_update_body: payloads.UserUpdateBody) -> None:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'Invalid role: {role}')
     user_info = global_user_state.get_user(user_id)
-    if not user_info.name:
+    if user_info is None:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'User {user_id} does not exist')
+    # Disallow updating roles for the internal users.
+    if user_info.id in [common.SERVER_ID, constants.SKYPILOT_SYSTEM_USER_ID]:
+        raise fastapi.HTTPException(status_code=400,
+                                    detail=f'Cannot update role for internal '
+                                    f'API server user {user_info.name}')
 
     # Update user role in casbin policy
-    permission_service.update_role(user_id, role)
+    permission.permission_service.update_role(user_info.id, role)

sky/utils/admin_policy_utils.py

@@ -55,6 +55,7 @@ def _get_policy_cls(
 def apply_and_use_config_in_current_request(
     entrypoint: Union['dag_lib.Dag', 'task_lib.Task'],
     request_options: Optional[admin_policy.RequestOptions] = None,
+    at_client_side: bool = False,
 ) -> Iterator['dag_lib.Dag']:
     """Applies an admin policy and override SkyPilot config for current request
 
@@ -66,7 +67,7 @@ def apply_and_use_config_in_current_request(
     Refer to `apply()` for more details.
     """
     original_config = skypilot_config.to_dict()
-    dag, mutated_config = apply(entrypoint, request_options)
+    dag, mutated_config = apply(entrypoint, request_options, at_client_side)
     if mutated_config != original_config:
         with skypilot_config.replace_skypilot_config(mutated_config):
             yield dag
@@ -77,6 +78,7 @@ def apply_and_use_config_in_current_request(
 def apply(
     entrypoint: Union['dag_lib.Dag', 'task_lib.Task'],
     request_options: Optional[admin_policy.RequestOptions] = None,
+    at_client_side: bool = False,
 ) -> Tuple['dag_lib.Dag', config_utils.Config]:
     """Applies an admin policy (if registered) to a DAG or a task.
 
@@ -105,14 +107,18 @@ def apply(
     if policy_cls is None:
         return dag, skypilot_config.to_dict()
 
-    logger.info(f'Applying policy: {policy}')
+    if at_client_side:
+        logger.info(f'Applying client admin policy: {policy}')
+    else:
+        logger.info(f'Applying server admin policy: {policy}')
     config = copy.deepcopy(skypilot_config.to_dict())
     mutated_dag = dag_lib.Dag()
     mutated_dag.name = dag.name
 
     mutated_config = None
     for task in dag.tasks:
-        user_request = admin_policy.UserRequest(task, config, request_options)
+        user_request = admin_policy.UserRequest(task, config, request_options,
+                                                at_client_side)
         try:
             mutated_user_request = policy_cls.validate_and_mutate(user_request)
         except Exception as e:  # pylint: disable=broad-except