skypilot-nightly 1.0.0.dev20250417__py3-none-any.whl → 1.0.0.dev20250422__py3-none-any.whl

sky/server/requests/payloads.py

@@ -6,7 +6,6 @@ with the backend functions. The benefit of having the default values in the
 payloads is that a user can find the default values in the Restful API docs.
 """
 import getpass
-import json
 import os
 import typing
 from typing import Any, Dict, List, Optional, Tuple, Union
@@ -32,6 +31,19 @@ else:
 
 logger = sky_logging.init_logger(__name__)
 
+# These non-skypilot environment variables will be updated from the local
+# environment on each request when running a local API server.
+# We should avoid adding variables here, but we should include credential-
+# related variables.
+EXTERNAL_LOCAL_ENV_VARS = [
+    # Allow overriding the AWS authentication.
+    'AWS_PROFILE',
+    'AWS_ACCESS_KEY_ID',
+    'AWS_SECRET_ACCESS_KEY',
+    # Allow overriding the GCP authentication.
+    'GOOGLE_APPLICATION_CREDENTIALS',
+]
+
 
 @annotations.lru_cache(scope='global')
 def request_body_env_vars() -> dict:
@@ -39,6 +51,8 @@ def request_body_env_vars() -> dict:
     for env_var in os.environ:
         if env_var.startswith(constants.SKYPILOT_ENV_VAR_PREFIX):
             env_vars[env_var] = os.environ[env_var]
+        if common.is_api_server_local() and env_var in EXTERNAL_LOCAL_ENV_VARS:
+            env_vars[env_var] = os.environ[env_var]
     env_vars[constants.USER_ID_ENV_VAR] = common_utils.get_user_hash()
     env_vars[constants.USER_ENV_VAR] = os.getenv(constants.USER_ENV_VAR,
                                                  getpass.getuser())
@@ -47,7 +61,7 @@ def request_body_env_vars() -> dict:
     # Remove the path to config file, as the config content is included in the
     # request body and will be merged with the config on the server side.
     env_vars.pop(skypilot_config.ENV_VAR_SKYPILOT_CONFIG, None)
-    env_vars.pop(skypilot_config.ENV_VAR_USER_CONFIG, None)
+    env_vars.pop(skypilot_config.ENV_VAR_GLOBAL_CONFIG, None)
     env_vars.pop(skypilot_config.ENV_VAR_PROJECT_CONFIG, None)
     return env_vars
 
@@ -56,20 +70,9 @@ def get_override_skypilot_config_from_client() -> Dict[str, Any]:
     """Returns the override configs from the client."""
     config = skypilot_config.to_dict()
     # Remove the API server config, as we should not specify the SkyPilot
-    # server endpoint on the server side. This avoids the warning below.
+    # server endpoint on the server side. This avoids the warning at
+    # server-side.
     config.pop_nested(('api_server',), default_value=None)
-    ignored_key_values = {}
-    for nested_key in constants.SKIPPED_CLIENT_OVERRIDE_KEYS:
-        value = config.pop_nested(nested_key, default_value=None)
-        if value is not None:
-            ignored_key_values['.'.join(nested_key)] = value
-    if ignored_key_values:
-        logger.debug(f'The following keys ({json.dumps(ignored_key_values)}) '
-                     'are specified in the client SkyPilot config at '
-                     f'{skypilot_config.loaded_config_path()!r}. '
-                     'This will be ignored. If you want to specify it, '
-                     'please modify it on server side or contact your '
-                     'administrator.')
     return config
 
 
@@ -420,6 +423,15 @@ class ServeLogsBody(RequestBody):
     follow: bool = True
 
 
+class ServeDownloadLogsBody(RequestBody):
+    """The request body for the serve download logs endpoint."""
+    service_name: str
+    local_dir: str
+    targets: Optional[Union[str, serve.ServiceComponent,
+                            List[Union[str, serve.ServiceComponent]]]]
+    replica_ids: Optional[List[int]] = None
+
+
 class ServeStatusBody(RequestBody):
     """The request body for the serve status endpoint."""
     service_names: Optional[Union[str, List[str]]]

sky/server/server.py

@@ -150,7 +150,21 @@ async def lifespan(app: fastapi.FastAPI):  # pylint: disable=redefined-outer-nam
     # Shutdown: Add any cleanup code here if needed
 
 
+# Add a new middleware class to handle /internal/dashboard prefix
+class InternalDashboardPrefixMiddleware(
+        starlette.middleware.base.BaseHTTPMiddleware):
+    """Middleware to handle /internal/dashboard prefix in requests."""
+
+    async def dispatch(self, request: fastapi.Request, call_next):
+        path = request.url.path
+        if path.startswith('/internal/dashboard/'):
+            # Remove /internal/dashboard prefix and update request scope
+            request.scope['path'] = path.replace('/internal/dashboard/', '/', 1)
+        return await call_next(request)
+
+
 app = fastapi.FastAPI(prefix='/api/v1', debug=True, lifespan=lifespan)
+app.add_middleware(InternalDashboardPrefixMiddleware)
 app.add_middleware(
     cors.CORSMiddleware,
     # TODO(zhwu): in production deployment, we should restrict the allowed
@@ -1101,6 +1115,35 @@ async def complete_storage_name(incomplete: str,) -> List[str]:
     return global_user_state.get_storage_names_start_with(incomplete)
 
 
+# Add a route to serve static files
+@app.get('/{full_path:path}')
+async def serve_static_or_dashboard(full_path: str):
+    """Serves static files for any unmatched routes.
+
+    Handles the /dashboard prefix from Next.js configuration.
+    """
+    # Check if the path starts with 'dashboard/' and remove it if it does
+    if full_path.startswith('dashboard/'):
+        full_path = full_path[len('dashboard/'):]
+
+    # Try to serve the file directly from the out directory first
+    file_path = os.path.join(server_constants.DASHBOARD_DIR, full_path)
+    if os.path.isfile(file_path):
+        return fastapi.responses.FileResponse(file_path)
+
+    # If file not found, serve the index.html for client-side routing.
+    # For example, the non-matched arbitrary route (/ or /test) from
+    # client will be redirected to the index.html.
+    index_path = os.path.join(server_constants.DASHBOARD_DIR, 'index.html')
+    try:
+        with open(index_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        return fastapi.responses.HTMLResponse(content=content)
+    except Exception as e:
+        logger.error(f'Error serving dashboard: {e}')
+        raise fastapi.HTTPException(status_code=500, detail=str(e))
+
+
 if __name__ == '__main__':
     import uvicorn
 

sky/setup_files/MANIFEST.in

@@ -15,3 +15,4 @@ include sky/jobs/dashboard/static/*
 include sky/templates/*
 include sky/utils/kubernetes/*
 include sky/server/html/*
+recursive-include sky/dashboard/out *

sky/sky_logging.py

@@ -18,6 +18,12 @@ _FORMAT = '%(levelname).1s %(asctime)s %(filename)s:%(lineno)d] %(message)s'
 _DATE_FORMAT = '%m-%d %H:%M:%S'
 _SENSITIVE_LOGGER = ['sky.provisioner', 'sky.optimizer']
 
+DEBUG = logging.DEBUG
+INFO = logging.INFO
+WARNING = logging.WARNING
+ERROR = logging.ERROR
+CRITICAL = logging.CRITICAL
+
 
 def _show_logging_prefix():
     return env_options.Options.SHOW_DEBUG_INFO.get(
@@ -127,6 +133,10 @@ def set_logging_level(logger: str, level: int):
         logger.setLevel(original_level)
 
 
+def logging_enabled(logger: logging.Logger, level: int) -> bool:
+    return logger.level <= level
+
+
 @contextlib.contextmanager
 def silent():
     """Make all sky_logging.print() and logger.{info, warning...} silent.

sky/skypilot_config.py

@@ -1,6 +1,6 @@
 """Immutable user configurations (EXPERIMENTAL).
 
-On module import, we attempt to parse the config located at _USER_CONFIG_PATH
+On module import, we attempt to parse the config located at _GLOBAL_CONFIG_PATH
 (default: ~/.sky/config.yaml). Caller can then use
 
   >> skypilot_config.loaded()
@@ -50,8 +50,8 @@ then:
 """
 import contextlib
 import copy
+import json
 import os
-import pprint
 import threading
 import typing
 from typing import Any, Dict, Iterator, List, Optional, Tuple
@@ -80,8 +80,8 @@ logger = sky_logging.init_logger(__name__)
 #     path as the config file. Do not use any other config files.
 #     This behavior is subject to change and should not be relied on by users.
 # Else,
-# (1) If env var {ENV_VAR_USER_CONFIG} exists, use its path as the user
-#     config file. Else, use the default path {_USER_CONFIG_PATH}.
+# (1) If env var {ENV_VAR_GLOBAL_CONFIG} exists, use its path as the user
+#     config file. Else, use the default path {_GLOBAL_CONFIG_PATH}.
 # (2) If env var {ENV_VAR_PROJECT_CONFIG} exists, use its path as the project
 #     config file. Else, use the default path {_PROJECT_CONFIG_PATH}.
 # (3) Override any config keys in (1) with the ones in (2).
@@ -97,21 +97,16 @@ logger = sky_logging.init_logger(__name__)
 # use the same config file.
 ENV_VAR_SKYPILOT_CONFIG = f'{constants.SKYPILOT_ENV_VAR_PREFIX}CONFIG'
 
-# (Used by users) Environment variables for setting non-default user and
-# project config files on clients.
-ENV_VAR_USER_CONFIG = f'{constants.SKYPILOT_ENV_VAR_PREFIX}USER_CONFIG'
+# Environment variables for setting non-default server and user
+# config files.
+ENV_VAR_GLOBAL_CONFIG = f'{constants.SKYPILOT_ENV_VAR_PREFIX}GLOBAL_CONFIG'
+# Environment variables for setting non-default project config files.
 ENV_VAR_PROJECT_CONFIG = f'{constants.SKYPILOT_ENV_VAR_PREFIX}PROJECT_CONFIG'
 
-# (Used by server) Environment variable for setting the server config file.
-ENV_VAR_SERVER_CONFIG = f'{constants.SKYPILOT_ENV_VAR_PREFIX}SERVER_CONFIG'
-
 # Path to the client config files.
-_USER_CONFIG_PATH = '~/.sky/config.yaml'
+_GLOBAL_CONFIG_PATH = '~/.sky/config.yaml'
 _PROJECT_CONFIG_PATH = '.sky.yaml'
 
-# Path to the server config file.
-_SERVER_CONFIG_PATH = _USER_CONFIG_PATH
-
 # The loaded config.
 _dict = config_utils.Config()
 _loaded_config_path: Optional[str] = None
@@ -121,24 +116,24 @@ _reload_config_lock = threading.Lock()
 
 def get_user_config_path() -> str:
     """Returns the path to the user config file."""
-    return _USER_CONFIG_PATH
+    return _GLOBAL_CONFIG_PATH
 
 
 def get_user_config() -> config_utils.Config:
     """Returns the user config."""
     # find the user config file
-    user_config_path = _get_config_file_path(ENV_VAR_USER_CONFIG)
+    user_config_path = _get_config_file_path(ENV_VAR_GLOBAL_CONFIG)
     if user_config_path:
         logger.debug('using user config file specified by '
-                     f'{ENV_VAR_USER_CONFIG}: {user_config_path}')
+                     f'{ENV_VAR_GLOBAL_CONFIG}: {user_config_path}')
         user_config_path = os.path.expanduser(user_config_path)
         if not os.path.exists(user_config_path):
             with ux_utils.print_exception_no_traceback():
                 raise FileNotFoundError(
                     'Config file specified by env var '
-                    f'{ENV_VAR_USER_CONFIG} ({user_config_path!r}) '
+                    f'{ENV_VAR_GLOBAL_CONFIG} ({user_config_path!r}) '
                     'does not exist. Please double check the path or unset the '
-                    f'env var: unset {ENV_VAR_USER_CONFIG}')
+                    f'env var: unset {ENV_VAR_GLOBAL_CONFIG}')
     else:
         user_config_path = get_user_config_path()
         logger.debug(f'using default user config file: {user_config_path}')
@@ -185,20 +180,20 @@ def _get_project_config() -> config_utils.Config:
 def get_server_config() -> config_utils.Config:
     """Returns the server config."""
     # find the server config file
-    server_config_path = _get_config_file_path(ENV_VAR_SERVER_CONFIG)
+    server_config_path = _get_config_file_path(ENV_VAR_GLOBAL_CONFIG)
     if server_config_path:
         logger.debug('using server config file specified by '
-                     f'{ENV_VAR_SERVER_CONFIG}: {server_config_path}')
+                     f'{ENV_VAR_GLOBAL_CONFIG}: {server_config_path}')
         server_config_path = os.path.expanduser(server_config_path)
         if not os.path.exists(server_config_path):
             with ux_utils.print_exception_no_traceback():
                 raise FileNotFoundError(
                     'Config file specified by env var '
-                    f'{ENV_VAR_SERVER_CONFIG} ({server_config_path!r}) '
+                    f'{ENV_VAR_GLOBAL_CONFIG} ({server_config_path!r}) '
                     'does not exist. Please double check the path or unset the '
-                    f'env var: unset {ENV_VAR_SERVER_CONFIG}')
+                    f'env var: unset {ENV_VAR_GLOBAL_CONFIG}')
     else:
-        server_config_path = _SERVER_CONFIG_PATH
+        server_config_path = _GLOBAL_CONFIG_PATH
         logger.debug(f'using default server config file: {server_config_path}')
         server_config_path = os.path.expanduser(server_config_path)
 
@@ -314,8 +309,9 @@ def _parse_config_file(config_path: str) -> config_utils.Config:
     try:
         config_dict = common_utils.read_yaml(config_path)
         config = config_utils.Config.from_dict(config_dict)
-        logger.debug(
-            f'Config loaded from {config_path}:\n{pprint.pformat(config)}')
+        if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+            logger.debug(f'Config loaded from {config_path}:\n'
+                         f'{common_utils.dump_yaml_str(dict(config))}')
     except yaml.YAMLError as e:
         logger.error(f'Error in loading config file ({config_path}):', e)
     if config:
@@ -359,7 +355,10 @@ def _reload_config_as_server() -> None:
     for override in overrides:
         overlaid_server_config = overlay_skypilot_config(
             original_config=overlaid_server_config, override_configs=override)
-    logger.debug(f'final server config: {overlaid_server_config}')
+    if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+        logger.debug(
+            f'server config: \n'
+            f'{common_utils.dump_yaml_str(dict(overlaid_server_config))}')
     _dict = overlaid_server_config
 
 
@@ -381,7 +380,10 @@ def _reload_config_as_client() -> None:
     for override in overrides:
         overlaid_client_config = overlay_skypilot_config(
             original_config=overlaid_client_config, override_configs=override)
-    logger.debug(f'final client config: {overlaid_client_config}')
+    if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+        logger.debug(
+            f'client config (before task and CLI overrides): \n'
+            f'{common_utils.dump_yaml_str(dict(overlaid_client_config))}')
     _dict = overlaid_client_config
 
 
@@ -414,10 +416,26 @@ def override_skypilot_config(
         yield
         return
     original_config = _dict
+    override_configs = config_utils.Config(override_configs)
+    disallowed_diff_keys = []
+    for key in constants.SKIPPED_CLIENT_OVERRIDE_KEYS:
+        value = override_configs.pop_nested(key, default_value=None)
+        if (value is not None and
+                value != original_config.get_nested(key, default_value=None)):
+            disallowed_diff_keys.append('.'.join(key))
+    # Only warn if there is a diff in disallowed override keys, as the client
+    # use the same config file when connecting to a local server.
+    if disallowed_diff_keys:
+        logger.warning(
+            f'The following keys ({json.dumps(disallowed_diff_keys)}) have '
+            'different values in the client SkyPilot config with the server '
+            'and will be ignored. Remove these keys to disable this warning. '
+            'If you want to specify it, please modify it on server side or '
+            'contact your administrator.')
     config = _dict.get_nested(
         keys=tuple(),
         default_value=None,
-        override_configs=override_configs,
+        override_configs=dict(override_configs),
         allowed_override_keys=None,
         disallowed_override_keys=constants.SKIPPED_CLIENT_OVERRIDE_KEYS)
     try:
@@ -439,14 +457,14 @@ def override_skypilot_config(
                 '=== SkyPilot config on API server ===\n'
                 f'{common_utils.dump_yaml_str(dict(original_config))}\n'
                 '=== Your local SkyPilot config ===\n'
-                f'{common_utils.dump_yaml_str(override_configs)}\n'
+                f'{common_utils.dump_yaml_str(dict(override_configs))}\n'
                 f'Details: {e}') from e
     finally:
         _dict = original_config
         _config_overridden = False
 
 
-def _compose_cli_config(cli_config: Optional[str],) -> config_utils.Config:
+def _compose_cli_config(cli_config: Optional[List[str]]) -> config_utils.Config:
     """Composes the skypilot CLI config.
     CLI config can either be:
     - A path to a config file
@@ -457,18 +475,19 @@ def _compose_cli_config(cli_config: Optional[str],) -> config_utils.Config:
         return config_utils.Config()
 
     config_source = 'CLI'
-    maybe_config_path = os.path.expanduser(cli_config)
     try:
+        maybe_config_path = os.path.expanduser(cli_config[0])
         if os.path.isfile(maybe_config_path):
+            if len(cli_config) != 1:
+                raise ValueError(
+                    'Cannot use multiple --config flags with a config file.')
             config_source = maybe_config_path
             # cli_config is a path to a config file
             parsed_config = OmegaConf.to_object(
                 OmegaConf.load(maybe_config_path))
         else:  # cli_config is a comma-separated list of key-value pairs
-            variables: List[str] = []
-            variables = cli_config.split(',')
             parsed_config = OmegaConf.to_object(
-                OmegaConf.from_dotlist(variables))
+                OmegaConf.from_dotlist(cli_config))
         _validate_config(parsed_config, config_source)
     except ValueError as e:
         raise ValueError(f'Invalid config override: {cli_config}. '
@@ -479,7 +498,7 @@ def _compose_cli_config(cli_config: Optional[str],) -> config_utils.Config:
     return parsed_config
 
 
-def apply_cli_config(cli_config: Optional[str]) -> Dict[str, Any]:
+def apply_cli_config(cli_config: Optional[List[str]]) -> Dict[str, Any]:
     """Applies the CLI provided config.
     SAFETY:
     This function directly modifies the global _dict variable.
@@ -491,7 +510,9 @@ def apply_cli_config(cli_config: Optional[str]) -> Dict[str, Any]:
     """
     global _dict
     parsed_config = _compose_cli_config(cli_config)
-    logger.debug(f'applying following CLI overrides: {parsed_config}')
+    if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
+        logger.debug(f'applying following CLI overrides: \n'
+                     f'{common_utils.dump_yaml_str(dict(parsed_config))}')
     _dict = overlay_skypilot_config(original_config=_dict,
                                     override_configs=parsed_config)
     return parsed_config

sky/templates/kubernetes-ray.yml.j2

@@ -384,8 +384,12 @@ available_node_types:
                 set +e
                 
                 if [ ! -z "$MISSING_PACKAGES" ]; then
-                  echo "Installing missing packages: $MISSING_PACKAGES";
-                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $MISSING_PACKAGES;
+                  # Install missing packages individually to avoid failure installation breaks the whole install process,
+                  # e.g. fuse3 is not available on some distributions.
+                  echo "Installing missing packages individually: $MISSING_PACKAGES";
+                  for pkg in $MISSING_PACKAGES; do
+                    DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $pkg || echo "Optional package $pkg installation failed, skip.";
+                  done
                 fi;
 
                 {% if k8s_fuse_device_required %}

sky/utils/config_utils.py

@@ -171,7 +171,6 @@ def _get_nested(configs: Optional[Dict[str, Any]],
             curr = value
         else:
             return default_value
-    logger.debug(f'Config: {".".join(keys)} -> {curr}')
     return curr
 
 

sky/utils/controller_utils.py

@@ -393,7 +393,6 @@ def download_and_stream_latest_job_log(
             f'Failed to stream the logs for the user program at '
             f'{log_file}: {common_utils.format_exception(e)}',
             exc_info=True)
-        # Return the log_file anyway.
 
     return log_file
 

{skypilot_nightly-1.0.0.dev20250417.dist-info → skypilot_nightly-1.0.0.dev20250422.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250417
+Version: 1.0.0.dev20250422
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0