skypilot-nightly 1.0.0.dev20250607__py3-none-any.whl → 1.0.0.dev20250610__py3-none-any.whl

sky/jobs/client/sdk.py

@@ -14,7 +14,9 @@ from sky.server import common as server_common
 from sky.server.requests import payloads
 from sky.skylet import constants
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
 from sky.utils import common_utils
+from sky.utils import context
 from sky.utils import dag_utils
 
 if typing.TYPE_CHECKING:
@@ -29,6 +31,7 @@ else:
 logger = sky_logging.init_logger(__name__)
 
 
+@context.contextual
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 def launch(
@@ -65,27 +68,32 @@ def launch(
     """
 
     dag = dag_utils.convert_entrypoint_to_dag(task)
-    sdk.validate(dag)
-    if _need_confirmation:
-        request_id = sdk.optimize(dag)
-        sdk.stream_and_get(request_id)
-        prompt = f'Launching a managed job {dag.name!r}. Proceed?'
-        if prompt is not None:
-            click.confirm(prompt, default=True, abort=True, show_default=True)
-
-    dag = client_common.upload_mounts_to_api_server(dag)
-    dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
-    body = payloads.JobsLaunchBody(
-        task=dag_str,
-        name=name,
-    )
-    response = requests.post(
-        f'{server_common.get_server_url()}/jobs/launch',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        cookies=server_common.get_api_cookie_jar(),
-    )
-    return server_common.get_request_id(response)
+    with admin_policy_utils.apply_and_use_config_in_current_request(
+            dag, at_client_side=True) as dag:
+        sdk.validate(dag)
+        if _need_confirmation:
+            request_id = sdk.optimize(dag)
+            sdk.stream_and_get(request_id)
+            prompt = f'Launching a managed job {dag.name!r}. Proceed?'
+            if prompt is not None:
+                click.confirm(prompt,
+                              default=True,
+                              abort=True,
+                              show_default=True)
+
+        dag = client_common.upload_mounts_to_api_server(dag)
+        dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
+        body = payloads.JobsLaunchBody(
+            task=dag_str,
+            name=name,
+        )
+        response = requests.post(
+            f'{server_common.get_server_url()}/jobs/launch',
+            json=json.loads(body.model_dump_json()),
+            timeout=(5, None),
+            cookies=server_common.get_api_cookie_jar(),
+        )
+        return server_common.get_request_id(response)
 
 
 @usage_lib.entrypoint

sky/jobs/server/core.py

@@ -37,6 +37,7 @@ from sky.utils import status_lib
 from sky.utils import subprocess_utils
 from sky.utils import timeline
 from sky.utils import ux_utils
+from sky.workspaces import core as workspaces_core
 
 if typing.TYPE_CHECKING:
     import sky
@@ -244,7 +245,7 @@ def launch(
 
         # Launch with the api server's user hash, so that sky status does not
         # show the owner of the controller as whatever user launched it first.
-        with common.with_server_user_hash():
+        with common.with_server_user():
             # Always launch the controller in the default workspace.
             with skypilot_config.local_active_workspace_ctx(
                     skylet_constants.SKYPILOT_DEFAULT_WORKSPACE):
@@ -455,6 +456,13 @@ def queue(refresh: bool,
 
         jobs = list(filter(user_hash_matches_or_missing, jobs))
 
+    accessible_workspaces = workspaces_core.get_workspaces()
+    jobs = list(
+        filter(
+            lambda job: job.get('workspace', skylet_constants.
+                                SKYPILOT_DEFAULT_WORKSPACE) in
+            accessible_workspaces, jobs))
+
     if skip_finished:
         # Filter out the finished jobs. If a multi-task job is partially
         # finished, we will include all its tasks.

sky/jobs/server/server.py

@@ -1,12 +1,9 @@
 """REST API for managed jobs."""
-import os
 
 import fastapi
-import httpx
 
 from sky import sky_logging
 from sky.jobs.server import core
-from sky.jobs.server import dashboard_utils
 from sky.server import common as server_common
 from sky.server import stream_utils
 from sky.server.requests import executor
@@ -14,7 +11,6 @@ from sky.server.requests import payloads
 from sky.server.requests import requests as api_requests
 from sky.skylet import constants
 from sky.utils import common
-from sky.utils import common_utils
 
 logger = sky_logging.init_logger(__name__)
 
@@ -110,94 +106,3 @@ async def download_logs(
         if jobs_download_logs_body.refresh else api_requests.ScheduleType.SHORT,
         request_cluster_name=common.JOB_CONTROLLER_NAME,
     )
-
-
-@router.get('/dashboard')
-async def dashboard(request: fastapi.Request,
-                    user_hash: str) -> fastapi.Response:
-    # TODO(cooperc): Support showing only jobs for a specific user.
-
-    # FIX(zhwu/cooperc/eric): Fix log downloading (assumes global
-    # /download_log/xx route)
-
-    # Note: before #4717, each user had their own controller, and thus their own
-    # dashboard. Now, all users share the same controller, so this isn't really
-    # necessary. TODO(cooperc): clean up.
-
-    # TODO: Put this in an executor to avoid blocking the main server thread.
-    # It can take a long time if it needs to check the controller status.
-
-    # Find the port for the dashboard of the user
-    os.environ[constants.USER_ID_ENV_VAR] = user_hash
-    server_common.reload_for_new_request(client_entrypoint=None,
-                                         client_command=None,
-                                         using_remote_api_server=False)
-    logger.info(f'Starting dashboard for user hash: {user_hash}')
-
-    with dashboard_utils.get_dashboard_lock_for_user(user_hash):
-        max_retries = 3
-        for attempt in range(max_retries):
-            port, pid = dashboard_utils.get_dashboard_session(user_hash)
-            if port == 0 or attempt > 0:
-                # Let the client know that we are waiting for starting the
-                # dashboard.
-                try:
-                    port, pid = core.start_dashboard_forwarding()
-                except Exception as e:  # pylint: disable=broad-except
-                    # We catch all exceptions to gracefully handle unknown
-                    # errors and raise an HTTPException to the client.
-                    msg = (
-                        'Dashboard failed to start: '
-                        f'{common_utils.format_exception(e, use_bracket=True)}')
-                    logger.error(msg)
-                    raise fastapi.HTTPException(status_code=503, detail=msg)
-                dashboard_utils.add_dashboard_session(user_hash, port, pid)
-
-            # Assuming the dashboard is forwarded to localhost on the API server
-            dashboard_url = f'http://localhost:{port}'
-            try:
-                # Ping the dashboard to check if it's still running
-                async with httpx.AsyncClient() as client:
-                    response = await client.request('GET',
-                                                    dashboard_url,
-                                                    timeout=5)
-                if response.is_success:
-                    break  # Connection successful, proceed with the request
-                # Raise an HTTPException here which will be caught by the
-                # following except block to retry with new connection
-                response.raise_for_status()
-            except Exception as e:  # pylint: disable=broad-except
-                # We catch all exceptions to gracefully handle unknown
-                # errors and retry or raise an HTTPException to the client.
-                # Assume an exception indicates that the dashboard connection
-                # is stale - remove it so that a new one is created.
-                dashboard_utils.remove_dashboard_session(user_hash)
-                msg = (
-                    f'Dashboard connection attempt {attempt + 1} failed with '
-                    f'{common_utils.format_exception(e, use_bracket=True)}')
-                logger.info(msg)
-                if attempt == max_retries - 1:
-                    raise fastapi.HTTPException(status_code=503, detail=msg)
-
-    # Create a client session to forward the request
-    try:
-        async with httpx.AsyncClient() as client:
-            # Make the request and get the response
-            response = await client.request(
-                method='GET',
-                url=f'{dashboard_url}',
-                headers=request.headers.raw,
-            )
-
-            # Create a new response with the content already read
-            content = await response.aread()
-            return fastapi.Response(
-                content=content,
-                status_code=response.status_code,
-                headers=dict(response.headers),
-                media_type=response.headers.get('content-type'))
-    except Exception as e:
-        msg = (f'Failed to forward request to dashboard: '
-               f'{common_utils.format_exception(e, use_bracket=True)}')
-        logger.error(msg)
-        raise fastapi.HTTPException(status_code=502, detail=msg)

sky/jobs/utils.py

@@ -1025,7 +1025,8 @@ def load_managed_job_queue(payload: str) -> List[Dict[str, Any]]:
         if 'user_hash' in job and job['user_hash'] is not None:
             # Skip jobs that do not have user_hash info.
             # TODO(cooperc): Remove check before 0.12.0.
-            job['user_name'] = global_user_state.get_user(job['user_hash']).name
+            user = global_user_state.get_user(job['user_hash'])
+            job['user_name'] = user.name if user is not None else None
     return jobs
 
 

sky/models.py

@@ -2,8 +2,13 @@
 
 import collections
 import dataclasses
+import getpass
+import os
 from typing import Any, Dict, Optional
 
+from sky.skylet import constants
+from sky.utils import common_utils
+
 
 @dataclasses.dataclass
 class User:
@@ -16,6 +21,19 @@ class User:
     def to_dict(self) -> Dict[str, Any]:
         return {'id': self.id, 'name': self.name}
 
+    def to_env_vars(self) -> Dict[str, Any]:
+        return {
+            constants.USER_ID_ENV_VAR: self.id,
+            constants.USER_ENV_VAR: self.name,
+        }
+
+    @classmethod
+    def get_current_user(cls) -> 'User':
+        """Returns the current user."""
+        user_name = os.getenv(constants.USER_ENV_VAR, getpass.getuser())
+        user_hash = common_utils.get_user_hash()
+        return User(id=user_hash, name=user_name)
+
 
 RealtimeGpuAvailability = collections.namedtuple(
     'RealtimeGpuAvailability', ['gpu', 'counts', 'capacity', 'available'])

sky/provision/kubernetes/constants.py

@@ -6,3 +6,12 @@ NO_GPU_HELP_MESSAGE = ('If your cluster contains GPUs, make sure '
                        '(e.g., skypilot.co/accelerator) are setup correctly. ')
 
 KUBERNETES_IN_CLUSTER_NAMESPACE_ENV_VAR = 'SKYPILOT_IN_CLUSTER_NAMESPACE'
+
+# Name of kubernetes exec auth wrapper script
+SKY_K8S_EXEC_AUTH_WRAPPER = 'sky-kube-exec-wrapper'
+
+# PATH envvar for kubectl exec auth execve
+SKY_K8S_EXEC_AUTH_PATH = '$HOME/skypilot-runtime/bin:$HOME/google-cloud-sdk/bin:$PATH'  # pylint: disable=line-too-long
+
+# cache directory for kubeconfig with modified exec auth
+SKY_K8S_EXEC_AUTH_KUBECONFIG_CACHE = '~/.sky/generated/kubeconfigs'

sky/provision/kubernetes/utils.py

@@ -1,6 +1,7 @@
 """Kubernetes utilities for SkyPilot."""
 import dataclasses
 import functools
+import hashlib
 import json
 import math
 import os
@@ -1555,11 +1556,11 @@ def is_kubeconfig_exec_auth(
             == schemas.RemoteIdentityOptions.LOCAL_CREDENTIALS.value):
         ctx_name = context_obj['name']
         exec_msg = ('exec-based authentication is used for '
-                    f'Kubernetes context {ctx_name!r}.'
-                    ' This may cause issues with autodown or when running '
-                    'Managed Jobs or SkyServe controller on Kubernetes. '
-                    'To fix, configure SkyPilot to create a service account '
-                    'for running pods by setting the following in '
+                    f'Kubernetes context {ctx_name!r}. '
+                    'Make sure that the corresponding cloud provider is '
+                    'also enabled through `sky check` (e.g.: GCP for GKE). '
+                    'Alternatively, configure SkyPilot to create a service '
+                    'account for running pods by setting the following in '
                     '~/.sky/config.yaml:\n'
                     '    kubernetes:\n'
                     '      remote_identity: SERVICE_ACCOUNT\n'
@@ -2877,8 +2878,8 @@ def get_context_from_config(provider_config: Dict[str, Any]) -> Optional[str]:
     context = provider_config.get('context',
                                   get_current_kube_config_context_name())
     if context == kubernetes.in_cluster_context_name():
-        # If the context (also used as the region) is in-cluster, we need to
-        # we need to use in-cluster auth by setting the context to None.
+        # If the context (also used as the region) is in-cluster, we need
+        # to use in-cluster auth by setting the context to None.
         context = None
     return context
 
@@ -3135,3 +3136,101 @@ def get_kubeconfig_paths() -> List[str]:
     for path in paths.split(kubernetes.ENV_KUBECONFIG_PATH_SEPARATOR):
         expanded.append(os.path.expanduser(path))
     return expanded
+
+
+def format_kubeconfig_exec_auth(config: Any,
+                                output_path: str,
+                                inject_wrapper: bool = True) -> bool:
+    """Reformat the kubeconfig so that exec-based authentication can be used
+    with SkyPilot. Will create a new kubeconfig file under <output_path>
+    regardless of whether a change has been made.
+
+    kubectl internally strips all environment variables except for system
+    defaults. If `inject_wrapper` is true, a wrapper executable is applied
+    to inject the relevant PATH information before exec-auth is executed.
+
+    Contents of sky-kube-exec-wrapper:
+
+    #!/bin/bash
+    export PATH="$HOME/skypilot-runtime/bin:$HOME/google-cloud-sdk:$PATH"
+    exec "$@"
+
+    refer to `skylet/constants.py` for more information.
+
+    Args:
+        config (dict): kubeconfig parsed by yaml.safe_load
+        output_path (str): Path where the potentially modified kubeconfig file
+          will be saved
+        inject_wrapper (bool): Whether to inject the wrapper script
+    Returns: whether config was updated, for logging purposes
+    """
+    updated = False
+    for user in config.get('users', []):
+        exec_info = user.get('user', {}).get('exec', {})
+        current_command = exec_info.get('command', '')
+
+        if current_command:
+            # Strip the path and keep only the executable name
+            executable = os.path.basename(current_command)
+            if executable == kubernetes_constants.SKY_K8S_EXEC_AUTH_WRAPPER:
+                # we don't want this happening recursively.
+                continue
+
+            if inject_wrapper:
+                exec_info[
+                    'command'] = kubernetes_constants.SKY_K8S_EXEC_AUTH_WRAPPER
+                if exec_info.get('args') is None:
+                    exec_info['args'] = []
+                exec_info['args'].insert(0, executable)
+                updated = True
+            elif executable != current_command:
+                exec_info['command'] = executable
+                updated = True
+
+            # Handle Nebius kubeconfigs: change --profile to 'sky'
+            if executable == 'nebius':
+                args = exec_info.get('args', [])
+                if args and '--profile' in args:
+                    try:
+                        profile_index = args.index('--profile')
+                        if profile_index + 1 < len(args):
+                            old_profile = args[profile_index + 1]
+                            if old_profile != 'sky':
+                                args[profile_index + 1] = 'sky'
+                                updated = True
+                    except ValueError:
+                        pass
+
+    os.makedirs(os.path.dirname(os.path.expanduser(output_path)), exist_ok=True)
+    with open(output_path, 'w', encoding='utf-8') as file:
+        yaml.safe_dump(config, file)
+
+    return updated
+
+
+def format_kubeconfig_exec_auth_with_cache(kubeconfig_path: str) -> str:
+    """Reformat the kubeconfig file or retrieve it from cache if it has already
+    been formatted before. Store it in the cache directory if necessary.
+
+    Having a cache for this is good if users spawn an extreme number of jobs
+    concurrently.
+
+    Args:
+        kubeconfig_path (str): kubeconfig path
+    Returns: updated kubeconfig path
+    """
+    # TODO(kyuds): GC cache files
+    with open(kubeconfig_path, 'r', encoding='utf-8') as file:
+        config = yaml.safe_load(file)
+    normalized = yaml.dump(config, sort_keys=True)
+    hashed = hashlib.sha1(normalized.encode('utf-8')).hexdigest()
+    path = os.path.expanduser(
+        f'{kubernetes_constants.SKY_K8S_EXEC_AUTH_KUBECONFIG_CACHE}/{hashed}.yaml'
+    )
+
+    # If we have already converted the same kubeconfig before, just return.
+    if os.path.isfile(path):
+        return path
+
+    format_kubeconfig_exec_auth(config, path)
+    return path

sky/serve/client/sdk.py

@@ -10,6 +10,8 @@ from sky.client import common as client_common
 from sky.server import common as server_common
 from sky.server.requests import payloads
 from sky.usage import usage_lib
+from sky.utils import admin_policy_utils
+from sky.utils import context
 from sky.utils import dag_utils
 
 if typing.TYPE_CHECKING:
@@ -23,6 +25,7 @@ else:
     requests = adaptors_common.LazyImport('requests')
 
 
+@context.contextual
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 def up(
@@ -55,30 +58,36 @@ def up(
     from sky.client import sdk  # pylint: disable=import-outside-toplevel
 
     dag = dag_utils.convert_entrypoint_to_dag(task)
-    sdk.validate(dag)
-    request_id = sdk.optimize(dag)
-    sdk.stream_and_get(request_id)
-    if _need_confirmation:
-        prompt = f'Launching a new service {service_name!r}. Proceed?'
-        if prompt is not None:
-            click.confirm(prompt, default=True, abort=True, show_default=True)
-
-    dag = client_common.upload_mounts_to_api_server(dag)
-    dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
-
-    body = payloads.ServeUpBody(
-        task=dag_str,
-        service_name=service_name,
-    )
-    response = requests.post(
-        f'{server_common.get_server_url()}/serve/up',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        cookies=server_common.get_api_cookie_jar(),
-    )
-    return server_common.get_request_id(response)
+    with admin_policy_utils.apply_and_use_config_in_current_request(
+            dag, at_client_side=True) as dag:
+        sdk.validate(dag)
+        request_id = sdk.optimize(dag)
+        sdk.stream_and_get(request_id)
+        if _need_confirmation:
+            prompt = f'Launching a new service {service_name!r}. Proceed?'
+            if prompt is not None:
+                click.confirm(prompt,
+                              default=True,
+                              abort=True,
+                              show_default=True)
+
+        dag = client_common.upload_mounts_to_api_server(dag)
+        dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
+
+        body = payloads.ServeUpBody(
+            task=dag_str,
+            service_name=service_name,
+        )
+        response = requests.post(
+            f'{server_common.get_server_url()}/serve/up',
+            json=json.loads(body.model_dump_json()),
+            timeout=(5, None),
+            cookies=server_common.get_api_cookie_jar(),
+        )
+        return server_common.get_request_id(response)
 
 
+@context.contextual
 @usage_lib.entrypoint
 @server_common.check_server_healthy_or_start
 def update(
@@ -112,30 +121,32 @@ def update(
     from sky.client import sdk  # pylint: disable=import-outside-toplevel
 
     dag = dag_utils.convert_entrypoint_to_dag(task)
-    sdk.validate(dag)
-    request_id = sdk.optimize(dag)
-    sdk.stream_and_get(request_id)
-    if _need_confirmation:
-        click.confirm(f'Updating service {service_name!r}. Proceed?',
-                      default=True,
-                      abort=True,
-                      show_default=True)
-
-    dag = client_common.upload_mounts_to_api_server(dag)
-    dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
-    body = payloads.ServeUpdateBody(
-        task=dag_str,
-        service_name=service_name,
-        mode=mode,
-    )
+    with admin_policy_utils.apply_and_use_config_in_current_request(
+            dag, at_client_side=True) as dag:
+        sdk.validate(dag)
+        request_id = sdk.optimize(dag)
+        sdk.stream_and_get(request_id)
+        if _need_confirmation:
+            click.confirm(f'Updating service {service_name!r}. Proceed?',
+                          default=True,
+                          abort=True,
+                          show_default=True)
+
+        dag = client_common.upload_mounts_to_api_server(dag)
+        dag_str = dag_utils.dump_chain_dag_to_yaml_str(dag)
+        body = payloads.ServeUpdateBody(
+            task=dag_str,
+            service_name=service_name,
+            mode=mode,
+        )
 
-    response = requests.post(
-        f'{server_common.get_server_url()}/serve/update',
-        json=json.loads(body.model_dump_json()),
-        timeout=(5, None),
-        cookies=server_common.get_api_cookie_jar(),
-    )
-    return server_common.get_request_id(response)
+        response = requests.post(
+            f'{server_common.get_server_url()}/serve/update',
+            json=json.loads(body.model_dump_json()),
+            timeout=(5, None),
+            cookies=server_common.get_api_cookie_jar(),
+        )
+        return server_common.get_request_id(response)
 
 
 @usage_lib.entrypoint

sky/serve/server/core.py

@@ -221,7 +221,7 @@ def up(
         # for the first time; otherwise it is a name conflict.
         # Since the controller may be shared among multiple users, launch the
         # controller with the API server's user hash.
-        with common.with_server_user_hash():
+        with common.with_server_user():
             with skypilot_config.local_active_workspace_ctx(
                     constants.SKYPILOT_DEFAULT_WORKSPACE):
                 controller_job_id, controller_handle = execution.launch(

sky/server/common.py

@@ -39,6 +39,7 @@ if typing.TYPE_CHECKING:
     import requests
 
     from sky import dag as dag_lib
+    from sky import models
 else:
     pydantic = adaptors_common.LazyImport('pydantic')
     requests = adaptors_common.LazyImport('requests')
@@ -419,11 +420,7 @@ def _start_api_server(deploy: bool = False,
                 dashboard_msg += (
                     'Dashboard may be stale when installed from source, '
                     'to rebuild: npm --prefix sky/dashboard install '
-                    '&& npm --prefix sky/dashboard run build\n')
-            dashboard_msg += (
-                f'{ux_utils.INDENT_LAST_SYMBOL}{colorama.Fore.GREEN}'
-                f'Dashboard: {get_dashboard_url(server_url)}')
-            dashboard_msg += f'{colorama.Style.RESET_ALL}'
+                    '&& npm --prefix sky/dashboard run build')
         logger.info(
             ux_utils.finishing_message(
                 f'SkyPilot API server started. {dashboard_msg}'))
@@ -710,7 +707,7 @@ def request_body_to_params(body: 'pydantic.BaseModel') -> Dict[str, Any]:
 
 def reload_for_new_request(client_entrypoint: Optional[str],
                            client_command: Optional[str],
-                           using_remote_api_server: bool):
+                           using_remote_api_server: bool, user: 'models.User'):
     """Reload modules, global variables, and usage message for a new request."""
     # This should be called first to make sure the logger is up-to-date.
     sky_logging.reload_logger()
@@ -719,10 +716,11 @@ def reload_for_new_request(client_entrypoint: Optional[str],
     skypilot_config.safe_reload_config()
 
     # Reset the client entrypoint and command for the usage message.
-    common_utils.set_client_status(
+    common_utils.set_request_context(
         client_entrypoint=client_entrypoint,
         client_command=client_command,
         using_remote_api_server=using_remote_api_server,
+        user=user,
     )
 
     # Clear cache should be called before reload_logger and usage reset,

sky/server/constants.py

@@ -11,8 +11,6 @@ API_VERSION = '9'
 
 # Prefix for API request names.
 REQUEST_NAME_PREFIX = 'sky.'
-# The user ID of the SkyPilot system.
-SKYPILOT_SYSTEM_USER_ID = 'skypilot-system'
 # The memory (GB) that SkyPilot tries to not use to prevent OOM.
 MIN_AVAIL_MEM_GB = 2
 # Default encoder/decoder handler name.