skypilot-nightly 1.0.0.dev20250616__py3-none-any.whl → 1.0.0.dev20250618__py3-none-any.whl

sky/users/server.py

@@ -1,19 +1,30 @@
 """REST API for workspace management."""
 
-from typing import Any, Dict, List
+import contextlib
+import hashlib
+import os
+from typing import Any, Dict, Generator, List
 
 import fastapi
+import filelock
+from passlib.hash import apr_md5_crypt
 
 from sky import global_user_state
+from sky import models
 from sky import sky_logging
 from sky.server.requests import payloads
 from sky.skylet import constants
 from sky.users import permission
 from sky.users import rbac
 from sky.utils import common
+from sky.utils import common_utils
 
 logger = sky_logging.init_logger(__name__)
 
+# Filelocks for the user management.
+USER_LOCK_PATH = os.path.expanduser('~/.sky/.{user_id}.lock')
+USER_LOCK_TIMEOUT_SECONDS = 20
+
 router = fastapi.APIRouter()
 
 
@@ -39,29 +50,283 @@ async def get_current_user_role(request: fastapi.Request):
     # hash for the request without 'X-Auth-Request-Email' header?
     auth_user = request.state.auth_user
     if auth_user is None:
-        return {'name': '', 'role': rbac.RoleName.ADMIN.value}
+        return {'id': '', 'name': '', 'role': rbac.RoleName.ADMIN.value}
     user_roles = permission.permission_service.get_user_roles(auth_user.id)
-    return {'name': auth_user.name, 'role': user_roles[0] if user_roles else ''}
+    return {
+        'id': auth_user.id,
+        'name': auth_user.name,
+        'role': user_roles[0] if user_roles else ''
+    }
+
+
+@router.post('/create')
+async def user_create(user_create_body: payloads.UserCreateBody) -> None:
+    username = user_create_body.username
+    password = user_create_body.password
+    role = user_create_body.role
+
+    if not username or not password:
+        raise fastapi.HTTPException(status_code=400,
+                                    detail='Username and password are required')
+    if role and role not in rbac.get_supported_roles():
+        raise fastapi.HTTPException(status_code=400,
+                                    detail=f'Invalid role: {role}')
+
+    if not role:
+        role = rbac.get_default_role()
+
+    # Create user
+    password_hash = apr_md5_crypt.hash(password)
+    user_hash = hashlib.md5(
+        username.encode()).hexdigest()[:common_utils.USER_HASH_LENGTH]
+    with _user_lock(user_hash):
+        # Check if user already exists
+        if global_user_state.get_user_by_name(username):
+            raise fastapi.HTTPException(
+                status_code=400, detail=f'User {username!r} already exists')
+        global_user_state.add_or_update_user(
+            models.User(id=user_hash, name=username, password=password_hash))
+        permission.permission_service.update_role(user_hash, role)
 
 
 @router.post('/update')
-async def user_update(user_update_body: payloads.UserUpdateBody) -> None:
+async def user_update(request: fastapi.Request,
+                      user_update_body: payloads.UserUpdateBody) -> None:
     """Updates the user role."""
     user_id = user_update_body.user_id
     role = user_update_body.role
+    password = user_update_body.password
     supported_roles = rbac.get_supported_roles()
-    if role not in supported_roles:
+    if role and role not in supported_roles:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'Invalid role: {role}')
+    target_user_roles = permission.permission_service.get_user_roles(user_id)
+    need_update_role = role and (not target_user_roles or
+                                 (role != target_user_roles[0]))
+    current_user = request.state.auth_user
+    if current_user is not None:
+        current_user_roles = permission.permission_service.get_user_roles(
+            current_user.id)
+        if not current_user_roles:
+            raise fastapi.HTTPException(status_code=403, detail='Invalid user')
+        if current_user_roles[0] != rbac.RoleName.ADMIN.value:
+            if need_update_role:
+                raise fastapi.HTTPException(
+                    status_code=403, detail='Only admin can update user role')
+            if password and user_id != current_user.id:
+                raise fastapi.HTTPException(
+                    status_code=403,
+                    detail='Only admin can update password for other users')
     user_info = global_user_state.get_user(user_id)
     if user_info is None:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'User {user_id} does not exist')
-    # Disallow updating roles for the internal users.
-    if user_info.id in [common.SERVER_ID, constants.SKYPILOT_SYSTEM_USER_ID]:
+    # Disallow updating the internal users.
+    if need_update_role and user_info.id in [
+            common.SERVER_ID, constants.SKYPILOT_SYSTEM_USER_ID
+    ]:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'Cannot update role for internal '
                                     f'API server user {user_info.name}')
+    if password and user_info.id == constants.SKYPILOT_SYSTEM_USER_ID:
+        raise fastapi.HTTPException(
+            status_code=400,
+            detail=f'Cannot update password for internal '
+            f'API server user {user_info.name}')
+
+    with _user_lock(user_info.id):
+        if password:
+            password_hash = apr_md5_crypt.hash(password)
+            global_user_state.add_or_update_user(
+                models.User(id=user_info.id,
+                            name=user_info.name,
+                            password=password_hash))
+        if role and need_update_role:
+            # Update user role in casbin policy
+            permission.permission_service.update_role(user_info.id, role)
+
+
+@router.post('/delete')
+async def user_delete(user_delete_body: payloads.UserDeleteBody) -> None:
+    user_id = user_delete_body.user_id
+
+    user_info = global_user_state.get_user(user_id)
+    if user_info is None:
+        raise fastapi.HTTPException(status_code=400,
+                                    detail=f'User {user_id} does not exist')
+    # Disallow deleting the internal users.
+    if user_info.id in [common.SERVER_ID, constants.SKYPILOT_SYSTEM_USER_ID]:
+        raise fastapi.HTTPException(status_code=400,
+                                    detail=f'Cannot delete internal '
+                                    f'API server user {user_info.name}')
+    with _user_lock(user_id):
+        global_user_state.delete_user(user_id)
+        permission.permission_service.delete_user(user_id)
+
+
+@router.post('/import')
+async def user_import(
+        user_import_body: payloads.UserImportBody) -> Dict[str, Any]:
+    """Import users from CSV content."""
+    csv_content = user_import_body.csv_content
+
+    if not csv_content:
+        raise fastapi.HTTPException(status_code=400,
+                                    detail='CSV content is required')
+
+    # Parse CSV content
+    lines = csv_content.strip().split('\n')
+    if len(lines) < 2:
+        raise fastapi.HTTPException(
+            status_code=400,
+            detail='CSV must have at least a header row and one data row')
+
+    # Parse headers
+    headers = [h.strip().lower() for h in lines[0].split(',')]
+    required_headers = ['username', 'password', 'role']
+
+    # Check if all required headers are present
+    missing_headers = [
+        header for header in required_headers if header not in headers
+    ]
+    if missing_headers:
+        raise fastapi.HTTPException(
+            status_code=400,
+            detail=f'Missing required columns: {", ".join(missing_headers)}')
+
+    # Parse user data
+    users_to_create = []
+    parse_errors = []
+
+    for i, line in enumerate(lines[1:], start=2):
+        if not line.strip():
+            continue  # Skip empty lines
+
+        values = [v.strip() for v in line.split(',')]
+        if len(values) != len(headers):
+            parse_errors.append(f'Line {i}: Invalid number of columns')
+            continue
+
+        user_data = dict(zip(headers, values))
+
+        # Validate required fields
+        if not user_data.get('username') or not user_data.get('password'):
+            parse_errors.append(f'Line {i}: Username and password are required')
+            continue
+
+        # Validate role
+        role = user_data.get('role', '').lower()
+        if role and role not in rbac.get_supported_roles():
+            role = rbac.get_default_role()  # Default to default role if invalid
+        elif not role:
+            role = rbac.get_default_role()
+
+        users_to_create.append({
+            'username': user_data['username'],
+            'password': user_data['password'],
+            'role': role
+        })
+
+    if not users_to_create and parse_errors:
+        raise fastapi.HTTPException(
+            status_code=400,
+            detail=f'No valid users found. Errors: {"; ".join(parse_errors)}')
+
+    # Create users
+    success_count = 0
+    error_count = 0
+    creation_errors = []
+
+    for user_data in users_to_create:
+        try:
+            username = user_data['username']
+            password = user_data['password']
+            role = user_data['role']
+
+            # Check if user already exists
+            if global_user_state.get_user_by_name(username):
+                error_count += 1
+                creation_errors.append(f'{username}: User already exists')
+                continue
+
+            # Check if password is already hashed (APR1 hash)
+            if password.startswith('$apr1$'):
+                # Password is already hashed, use it directly
+                password_hash = password
+            else:
+                # Password is plain text, hash it
+                password_hash = apr_md5_crypt.hash(password)
+
+            user_hash = hashlib.md5(
+                username.encode()).hexdigest()[:common_utils.USER_HASH_LENGTH]
+
+            with _user_lock(user_hash):
+                global_user_state.add_or_update_user(
+                    models.User(id=user_hash,
+                                name=username,
+                                password=password_hash))
+                permission.permission_service.update_role(user_hash, role)
+
+            success_count += 1
+
+        except Exception as e:  # pylint: disable=broad-except
+            error_count += 1
+            creation_errors.append(f'{user_data["username"]}: {str(e)}')
+
+    return {
+        'success_count': success_count,
+        'error_count': error_count,
+        'total_processed': len(users_to_create),
+        'parse_errors': parse_errors,
+        'creation_errors': creation_errors
+    }
+
+
+@router.get('/export')
+async def user_export() -> Dict[str, Any]:
+    """Export all users as CSV content."""
+    try:
+        # Get all users
+        user_list = global_user_state.get_all_users()
+
+        # Create CSV content
+        csv_lines = ['username,password,role']  # Header
+
+        for user in user_list:
+            # Get user role
+            user_roles = permission.permission_service.get_user_roles(user.id)
+            role = user_roles[0] if user_roles else rbac.get_default_role()
+            # Avoid exporting `None` values
+            line = ''
+            if user.name:
+                line += user.name
+            line += ','
+            if user.password:
+                line += user.password
+            line += ','
+            if role:
+                line += role
+            csv_lines.append(line)
+
+        csv_content = '\n'.join(csv_lines)
+
+        return {'csv_content': csv_content, 'user_count': len(user_list)}
+
+    except Exception as e:
+        raise fastapi.HTTPException(status_code=500,
+                                    detail=f'Failed to export users: {str(e)}')
+
 
-    # Update user role in casbin policy
-    permission.permission_service.update_role(user_info.id, role)
+@contextlib.contextmanager
+def _user_lock(user_id: str) -> Generator[None, None, None]:
+    """Context manager for user lock."""
+    try:
+        with filelock.FileLock(USER_LOCK_PATH.format(user_id=user_id),
+                               USER_LOCK_TIMEOUT_SECONDS):
+            yield
+    except filelock.Timeout as e:
+        raise RuntimeError(f'Failed to update user due to a timeout '
+                           f'when trying to acquire the lock at '
+                           f'{USER_LOCK_PATH.format(user_id=user_id)}. '
+                           'Please try again or manually remove the lock '
+                           f'file if you believe it is stale.') from e

sky/utils/command_runner.py

@@ -561,7 +561,7 @@ class SSHCommandRunner(CommandRunner):
         if self.ssh_control_name is not None:
             control_path = _ssh_control_path(self.ssh_control_name)
             if control_path is not None:
-                # Suppress the `Exit request sent.` output for this comamnd
+                # Suppress the `Exit request sent.` output for this command
                 # which would interrupt the CLI spinner.
                 cmd = (f'ssh -O exit -S {control_path}/%C '
                        f'{self.ssh_user}@{self.ip} > /dev/null 2>&1')

sky/utils/common_utils.py

@@ -343,30 +343,31 @@ def get_pretty_entrypoint_cmd() -> str:
         # things like 'examples/app.py'.
         argv[0] = basename
 
-    # Redact sensitive environment variable values
-    argv = _redact_env_values(argv)
+    # Redact sensitive values from secrets arguments
+    argv = _redact_secrets_values(argv)
 
     return ' '.join(argv)
 
 
-def _redact_env_values(argv: List[str]) -> List[str]:
-    """Redact sensitive values from --env arguments.
+def _redact_secrets_values(argv: List[str]) -> List[str]:
+    """Redact sensitive values from --secret arguments.
 
     Args:
         argv: Command line arguments
 
     Returns:
-        Modified argv with redacted --env values, or original argv if any error
+        Modified argv with redacted --secret values, or original argv if any
+        error
 
     Examples:
-        ['sky', 'launch', '--env', 'HF_TOKEN=secret'] ->
-        ['sky', 'launch', '--env', 'HF_TOKEN=<redacted>']
+        ['sky', 'launch', '--secret', 'HF_TOKEN=secret'] ->
+        ['sky', 'launch', '--secret', 'HF_TOKEN=<redacted>']
 
-        ['sky', 'launch', '--env=HF_TOKEN=secret'] ->
-        ['sky', 'launch', '--env=HF_TOKEN=<redacted>']
+        ['sky', 'launch', '--secret=HF_TOKEN=secret'] ->
+        ['sky', 'launch', '--secret=HF_TOKEN=<redacted>']
 
-        ['sky', 'launch', '--env', 'HF_TOKEN'] ->
-        ['sky', 'launch', '--env', 'HF_TOKEN'] (no change)
+        ['sky', 'launch', '--secret', 'HF_TOKEN'] ->
+        ['sky', 'launch', '--secret', 'HF_TOKEN'] (no change)
     """
     try:
         if not argv:
@@ -384,7 +385,7 @@ def _redact_env_values(argv: List[str]) -> List[str]:
                 i += 1
                 continue
 
-            if arg == '--env' and i + 1 < len(argv):
+            if arg == '--secret' and i + 1 < len(argv):
                 result.append(arg)
                 next_arg = argv[i + 1]
                 # Ensure next_arg is a string and handle redaction safely
@@ -395,9 +396,10 @@ def _redact_env_values(argv: List[str]) -> List[str]:
                 else:
                     result.append(next_arg)
                 i += 2
-            elif arg.startswith('--env='):
+            elif arg.startswith('--secret='):
                 # Redact only if there's a value after the key
-                redacted = re.sub(r'^(--env=[^=]+)=.*', r'\1=<redacted>', arg)
+                redacted = re.sub(r'^(--secret=[^=]+)=.*', r'\1=<redacted>',
+                                  arg)
                 result.append(redacted)
                 i += 1
             else:

sky/utils/context.py

@@ -262,7 +262,7 @@ F = TypeVar('F', bound=Callable[..., Any])
 
 
 def contextual(func: F) -> F:
-    """Decorator to intiailize a context before executing the function.
+    """Decorator to initialize a context before executing the function.
 
     If a context is already initialized, this decorator will reset the context,
     i.e. all contextual variables set previously will be cleared.

sky/utils/controller_utils.py

@@ -254,6 +254,13 @@ def _get_cloud_dependencies_installation_commands(
         sky_check.get_cached_enabled_clouds_or_refresh(
             sky_cloud.CloudCapability.STORAGE))
     enabled_clouds = enabled_compute_clouds.union(enabled_storage_clouds)
+    enabled_k8s_and_ssh = [
+        repr(cloud)
+        for cloud in enabled_clouds
+        if isinstance(cloud, clouds.Kubernetes)
+    ]
+    k8s_and_ssh_label = ' and '.join(sorted(enabled_k8s_and_ssh))
+    k8s_dependencies_installed = False
 
     for cloud in enabled_clouds:
         cloud_python_dependencies: List[str] = copy.deepcopy(
@@ -295,10 +302,11 @@ def _get_cloud_dependencies_installation_commands(
                 '--endpoint api.nebius.cloud '
                 '--service-account-file $HOME/.nebius/credentials.json '
                 '&> /dev/null || echo "Unable to create Nebius profile."')
-        elif isinstance(cloud, clouds.Kubernetes):
+        elif (isinstance(cloud, clouds.Kubernetes) and
+              not k8s_dependencies_installed):
             step_prefix = prefix_str.replace('<step>', str(len(commands) + 1))
             commands.append(
-                f'echo -en "\\r{step_prefix}Kubernetes{empty_str}" && '
+                f'echo -en "\\r{step_prefix}{k8s_and_ssh_label}{empty_str}" && '
                 # Install k8s + skypilot dependencies
                 'sudo bash -c "if '
                 '! command -v curl &> /dev/null || '
@@ -321,6 +329,7 @@ def _get_cloud_dependencies_installation_commands(
                 'kubectl /usr/local/bin/kubectl)) && '
                 f'echo -e \'#!/bin/bash\\nexport PATH="{kubernetes_constants.SKY_K8S_EXEC_AUTH_PATH}"\\nexec "$@"\' | sudo tee /usr/local/bin/{kubernetes_constants.SKY_K8S_EXEC_AUTH_WRAPPER} > /dev/null && '  # pylint: disable=line-too-long
                 f'sudo chmod +x /usr/local/bin/{kubernetes_constants.SKY_K8S_EXEC_AUTH_WRAPPER}')  # pylint: disable=line-too-long
+            k8s_dependencies_installed = True
         elif isinstance(cloud, clouds.Cudo):
             step_prefix = prefix_str.replace('<step>', str(len(commands) + 1))
             commands.append(
@@ -422,7 +431,7 @@ def download_and_stream_latest_job_log(
         return None
 
     log_dir = list(log_dirs.values())[0]
-    log_file = os.path.join(log_dir, 'run.log')
+    log_file = os.path.expanduser(os.path.join(log_dir, 'run.log'))
 
     # Print the logs to the console.
     # TODO(zhwu): refactor this into log_utils, along with the refactoring for

sky/utils/dag_utils.py

@@ -66,7 +66,9 @@ def convert_entrypoint_to_dag(entrypoint: Any) -> 'dag_lib.Dag':
 
 def _load_chain_dag(
         configs: List[Dict[str, Any]],
-        env_overrides: Optional[List[Tuple[str, str]]] = None) -> dag_lib.Dag:
+        env_overrides: Optional[List[Tuple[str, str]]] = None,
+        secrets_overrides: Optional[List[Tuple[str,
+                                               str]]] = None) -> dag_lib.Dag:
     """Loads a chain DAG from a list of YAML configs."""
     dag_name = None
     if set(configs[0].keys()) == {'name'}:
@@ -84,7 +86,8 @@ def _load_chain_dag(
         for task_config in configs:
             if task_config is None:
                 continue
-            task = task_lib.Task.from_yaml_config(task_config, env_overrides)
+            task = task_lib.Task.from_yaml_config(task_config, env_overrides,
+                                                  secrets_overrides)
             if current_task is not None:
                 current_task >> task  # pylint: disable=pointless-statement
             current_task = task
@@ -95,6 +98,7 @@ def _load_chain_dag(
 def load_chain_dag_from_yaml(
     path: str,
     env_overrides: Optional[List[Tuple[str, str]]] = None,
+    secret_overrides: Optional[List[Tuple[str, str]]] = None,
 ) -> dag_lib.Dag:
     """Loads a chain DAG from a YAML file.
 
@@ -105,17 +109,22 @@ def load_chain_dag_from_yaml(
     the task's 'envs' section. If it is a chain dag, the envs will be updated
     for all tasks in the chain.
 
+    'secrets_overrides' is a list of (key, value) pairs that will be used to
+    update the task's 'secrets' section. If it is a chain dag, the secrets will
+    be updated for all tasks in the chain.
+
     Returns:
       A chain Dag with 1 or more tasks (an empty entrypoint would create a
       trivial task).
     """
     configs = common_utils.read_yaml_all(path)
-    return _load_chain_dag(configs, env_overrides)
+    return _load_chain_dag(configs, env_overrides, secret_overrides)
 
 
 def load_chain_dag_from_yaml_str(
     yaml_str: str,
     env_overrides: Optional[List[Tuple[str, str]]] = None,
+    secrets_overrides: Optional[List[Tuple[str, str]]] = None,
 ) -> dag_lib.Dag:
     """Loads a chain DAG from a YAML string.
 
@@ -126,12 +135,16 @@ def load_chain_dag_from_yaml_str(
     the task's 'envs' section. If it is a chain dag, the envs will be updated
     for all tasks in the chain.
 
+    'secrets_overrides' is a list of (key, value) pairs that will be used to
+    update the task's 'secrets' section. If it is a chain dag, the secrets will
+    be updated for all tasks in the chain.
+
     Returns:
       A chain Dag with 1 or more tasks (an empty entrypoint would create a
       trivial task).
     """
     configs = common_utils.read_yaml_all_str(yaml_str)
-    return _load_chain_dag(configs, env_overrides)
+    return _load_chain_dag(configs, env_overrides, secrets_overrides)
 
 
 def dump_chain_dag_to_yaml_str(dag: dag_lib.Dag) -> str:

sky/utils/kubernetes/deploy_remote_cluster.py

@@ -207,9 +207,16 @@ def prepare_hosts_info(cluster_name: str,
 
     # Get cluster-level defaults
     cluster_user = cluster_config.get('user', '')
-    cluster_identity_file = cluster_config.get('identity_file', '')
+    cluster_identity_file = os.path.expanduser(
+        cluster_config.get('identity_file', ''))
     cluster_password = cluster_config.get('password', '')
 
+    # Check if cluster identity file exists
+    if cluster_identity_file and not os.path.isfile(cluster_identity_file):
+        with ux_utils.print_exception_no_traceback():
+            raise ValueError(
+                f'SSH Identity File Missing: {cluster_identity_file}')
+
     hosts_info = []
     for host in cluster_config['hosts']:
         # Host can be a string (IP or SSH config hostname) or a dict
@@ -239,10 +246,16 @@ def prepare_hosts_info(cluster_name: str,
             # Use host-specific values or fall back to cluster defaults
             host_user = '' if is_ssh_config_host else host.get(
                 'user', cluster_user)
-            host_identity_file = '' if is_ssh_config_host else host.get(
-                'identity_file', cluster_identity_file)
+            host_identity_file = os.path.expanduser(
+                '' if is_ssh_config_host else host.
+                get('identity_file', cluster_identity_file))
             host_password = host.get('password', cluster_password)
 
+            if host_identity_file and not os.path.isfile(host_identity_file):
+                with ux_utils.print_exception_no_traceback():
+                    raise ValueError(
+                        f'SSH Identity File Missing: {host_identity_file}')
+
             hosts_info.append({
                 'ip': host['ip'],
                 'user': host_user,
@@ -836,10 +849,6 @@ def deploy_cluster(head_node,
 
     Returns: List of unsuccessful worker nodes.
     """
-    # Ensure SSH key is expanded for paths with ~ (home directory)
-    if ssh_key:
-        ssh_key = os.path.expanduser(ssh_key)
-
     history_yaml_file = os.path.join(NODE_POOLS_INFO_DIR,
                                      f'{context_name}-history.yaml')
     cert_file_path = os.path.join(NODE_POOLS_INFO_DIR,
@@ -1091,7 +1100,7 @@ def deploy_cluster(head_node,
                     f'Skipping...{NC}')
                 return node, True, False
             worker_user = worker_hosts[i]['user']
-            worker_key = os.path.expanduser(worker_hosts[i]['identity_file'])
+            worker_key = worker_hosts[i]['identity_file']
             worker_password = worker_hosts[i]['password']
             worker_askpass = create_askpass_script(worker_password)
             worker_config = worker_use_ssh_config[i]