skypilot-nightly 1.0.0.dev20250828__py3-none-any.whl → 1.0.0.dev20250831__py3-none-any.whl

sky/global_user_state.py

@@ -206,6 +206,7 @@ cluster_event_table = sqlalchemy.Table(
     sqlalchemy.Column('reason', sqlalchemy.Text, primary_key=True),
     sqlalchemy.Column('transitioned_at', sqlalchemy.Integer, primary_key=True),
     sqlalchemy.Column('type', sqlalchemy.Text),
+    sqlalchemy.Column('request_id', sqlalchemy.Text, server_default=None),
 )
 
 ssh_key_table = sqlalchemy.Table(
@@ -745,6 +746,7 @@ def add_cluster_event(cluster_name: str,
             elif last_event == reason:
                 return
         try:
+            request_id = common_utils.get_current_request_id()
             session.execute(
                 insert_func(cluster_event_table).values(
                     cluster_hash=cluster_hash,
@@ -754,6 +756,7 @@ def add_cluster_event(cluster_name: str,
                     reason=reason,
                     transitioned_at=transitioned_at,
                     type=event_type.value,
+                    request_id=request_id,
                 ))
             session.commit()
         except sqlalchemy.exc.IntegrityError as e:
@@ -2082,19 +2085,51 @@ def get_cluster_yaml_str(cluster_yaml_path: Optional[str]) -> Optional[str]:
         row = session.query(cluster_yaml_table).filter_by(
             cluster_name=cluster_name).first()
     if row is None:
-        # If the cluster yaml is not in the database, check if it exists
-        # on the local file system and migrate it to the database.
-        # TODO(syang): remove this check once we have a way to migrate the
-        # cluster from file to database. Remove on v0.12.0.
-        if cluster_yaml_path is not None and os.path.exists(cluster_yaml_path):
-            with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
-                yaml_str = f.read()
-            set_cluster_yaml(cluster_name, yaml_str)
-            return yaml_str
-        return None
+        return _set_cluster_yaml_from_file(cluster_yaml_path, cluster_name)
     return row.yaml
 
 
+def get_cluster_yaml_str_multiple(cluster_yaml_paths: List[str]) -> List[str]:
+    """Get the cluster yaml from the database or the local file system.
+    """
+    assert _SQLALCHEMY_ENGINE is not None
+    cluster_names_to_yaml_paths = {}
+    for cluster_yaml_path in cluster_yaml_paths:
+        cluster_name, _ = os.path.splitext(os.path.basename(cluster_yaml_path))
+        cluster_names_to_yaml_paths[cluster_name] = cluster_yaml_path
+
+    cluster_names = list(cluster_names_to_yaml_paths.keys())
+    with orm.Session(_SQLALCHEMY_ENGINE) as session:
+        rows = session.query(cluster_yaml_table).filter(
+            cluster_yaml_table.c.cluster_name.in_(cluster_names)).all()
+    row_cluster_names_to_yaml = {row.cluster_name: row.yaml for row in rows}
+
+    yaml_strs = []
+    for cluster_name in cluster_names:
+        if cluster_name in row_cluster_names_to_yaml:
+            yaml_strs.append(row_cluster_names_to_yaml[cluster_name])
+        else:
+            yaml_str = _set_cluster_yaml_from_file(
+                cluster_names_to_yaml_paths[cluster_name], cluster_name)
+            yaml_strs.append(yaml_str)
+    return yaml_strs
+
+
+def _set_cluster_yaml_from_file(cluster_yaml_path: str,
+                                cluster_name: str) -> Optional[str]:
+    """Set the cluster yaml in the database from a file."""
+    # If the cluster yaml is not in the database, check if it exists
+    # on the local file system and migrate it to the database.
+    # TODO(syang): remove this check once we have a way to migrate the
+    # cluster from file to database. Remove on v0.12.0.
+    if cluster_yaml_path is not None and os.path.exists(cluster_yaml_path):
+        with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
+            yaml_str = f.read()
+        set_cluster_yaml(cluster_name, yaml_str)
+        return yaml_str
+    return None
+
+
 def get_cluster_yaml_dict(cluster_yaml_path: Optional[str]) -> Dict[str, Any]:
     """Get the cluster yaml as a dictionary from the database.
 
@@ -2106,6 +2141,19 @@ def get_cluster_yaml_dict(cluster_yaml_path: Optional[str]) -> Dict[str, Any]:
     return yaml_utils.safe_load(yaml_str)
 
 
+def get_cluster_yaml_dict_multiple(
+        cluster_yaml_paths: List[str]) -> List[Dict[str, Any]]:
+    """Get the cluster yaml as a dictionary from the database."""
+    yaml_strs = get_cluster_yaml_str_multiple(cluster_yaml_paths)
+    yaml_dicts = []
+    for idx, yaml_str in enumerate(yaml_strs):
+        if yaml_str is None:
+            raise ValueError(
+                f'Cluster yaml {cluster_yaml_paths[idx]} not found.')
+        yaml_dicts.append(yaml_utils.safe_load(yaml_str))
+    return yaml_dicts
+
+
 @_init_db
 def set_cluster_yaml(cluster_name: str, yaml_str: str) -> None:
     """Set the cluster yaml in the database."""

sky/provision/aws/config.py

@@ -87,6 +87,9 @@ def bootstrap_instances(
         use_internal_ips=config.provider_config.get('use_internal_ips', False),
         vpc_name=config.provider_config.get('vpc_name'))
 
+    max_efa_interfaces = config.provider_config.get('max_efa_interfaces', 0)
+    enable_efa = max_efa_interfaces > 0
+
     # Cluster workers should be in a security group that permits traffic within
     # the group, and also SSH access from outside.
     if security_group_ids is None:
@@ -103,7 +106,8 @@ def bootstrap_instances(
             extended_ip_rules = []
         security_group_ids = _configure_security_group(ec2, vpc_id,
                                                        expected_sg_name,
-                                                       extended_ip_rules)
+                                                       extended_ip_rules,
+                                                       enable_efa)
         if expected_sg_name != aws_cloud.DEFAULT_SECURITY_GROUP_NAME:
             logger.debug('Attempting to create the default security group.')
             # Attempt to create the default security group. This is needed
@@ -114,7 +118,7 @@ def bootstrap_instances(
             try:
                 _configure_security_group(ec2, vpc_id,
                                           aws_cloud.DEFAULT_SECURITY_GROUP_NAME,
-                                          [])
+                                          [], enable_efa)
                 logger.debug('Default security group created.')
             except exceptions.NoClusterLaunchedError as e:
                 if 'not authorized to perform: ec2:CreateSecurityGroup' in str(
@@ -148,6 +152,37 @@ def bootstrap_instances(
     return config
 
 
+def _configure_placement_group(ec2: 'mypy_boto3_ec2.ServiceResource',
+                               placement_group_name: str):
+    """Configure placement group for the cluster."""
+    # Create the placement group
+    logger.info(f'Creating placement group {placement_group_name}.')
+    try:
+        ec2.meta.client.create_placement_group(GroupName=placement_group_name,
+                                               Strategy='cluster')
+    except aws.botocore_exceptions().ClientError as exc:
+        if exc.response.get(
+                'Error', {}).get('Code') == 'InvalidPlacementGroup.Duplicate':
+            logger.debug(
+                f'Placement group {placement_group_name} already exists.')
+        else:
+            raise exc
+
+
+def delete_placement_group(ec2: 'mypy_boto3_ec2.ServiceResource',
+                           placement_group_name: str):
+    """Delete the placement group."""
+    try:
+        ec2.meta.client.delete_placement_group(GroupName=placement_group_name)
+    except aws.botocore_exceptions().ClientError as exc:
+        if exc.response.get('Error',
+                            {}).get('Code') == 'InvalidPlacementGroup.Unknown':
+            logger.debug(
+                f'Placement group {placement_group_name} does not exist.')
+        else:
+            raise exc
+
+
 def _configure_iam_role(iam) -> Dict[str, Any]:
 
     def _get_instance_profile(profile_name: str):
@@ -557,7 +592,8 @@ def _get_subnet_and_vpc_id(ec2: 'mypy_boto3_ec2.ServiceResource',
 
 def _configure_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
                               vpc_id: str, expected_sg_name: str,
-                              extended_ip_rules: List) -> List[str]:
+                              extended_ip_rules: List,
+                              enable_efa: bool) -> List[str]:
     security_group = _get_or_create_vpc_security_group(ec2, vpc_id,
                                                        expected_sg_name)
     sg_ids = [security_group.id]
@@ -583,16 +619,55 @@ def _configure_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
         },
         *extended_ip_rules,
     ]
+    outbound_rules = []
+    if enable_efa:
+        # EFA requires that outbound rules permit the same security group to
+        # communicate with each other
+        # Refer to https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start-nccl.html#nccl-start-base-setup # pylint: disable=line-too-long
+        outbound_rules.append({
+            'FromPort': -1,
+            'ToPort': -1,
+            'IpProtocol': '-1',
+            'UserIdGroupPairs': [{
+                'GroupId': i
+            } for i in sg_ids],
+        })
     # upsert the default security group
     if not security_group.ip_permissions:
         # If users specify security groups, we should not change the rules
         # of these security groups. Here we change it because it is the default
         # security group for SkyPilot.
         security_group.authorize_ingress(IpPermissions=inbound_rules)
+    if _need_to_update_outbound_rules(security_group, outbound_rules):
+        security_group.authorize_egress(IpPermissions=outbound_rules)
 
     return sg_ids
 
 
+def _need_to_update_outbound_rules(
+    security_group: Any,
+    outbound_rules: List[Dict[str, Any]],
+) -> bool:
+    """Check if we need to update the outbound rules of the security group."""
+    if not security_group.ip_permissions_egress:
+        return True  # No outbound rules, we need to add them
+    existing_group_ids = []
+    for rule in security_group.ip_permissions_egress:
+        if 'UserIdGroupPairs' in rule:
+            group_pairs = rule['UserIdGroupPairs']
+            for pair in group_pairs:
+                existing_group_ids.append(pair['GroupId'])
+    logger.debug(f'Existing group ids: {existing_group_ids}')
+    for rule in outbound_rules:
+        if 'UserIdGroupPairs' in rule:
+            group_pairs = rule['UserIdGroupPairs']
+            for pair in group_pairs:
+                if pair['GroupId'] not in existing_group_ids:
+                    logger.debug(f'New group id: {pair["GroupId"]}')
+                    return True  # New group id, we need to add it
+    return False  # No need to update
+
+
 def _get_or_create_vpc_security_group(ec2: 'mypy_boto3_ec2.ServiceResource',
                                       vpc_id: str,
                                       expected_sg_name: str) -> Any:

sky/provision/aws/instance.py

@@ -184,9 +184,15 @@ def _merge_tag_specs(tag_specs: List[Dict[str, Any]],
             tag_specs += [user_tag_spec]
 
 
-def _create_instances(ec2_fail_fast, cluster_name: str,
-                      node_config: Dict[str, Any], tags: Dict[str, str],
-                      count: int, associate_public_ip_address: bool) -> List:
+def _create_instances(
+    ec2_fail_fast,
+    cluster_name: str,
+    node_config: Dict[str, Any],
+    tags: Dict[str, str],
+    count: int,
+    associate_public_ip_address: bool,
+    max_efa_interfaces: int,
+) -> List:
     tags = {
         'Name': cluster_name,
         constants.TAG_RAY_CLUSTER_NAME: cluster_name,
@@ -239,7 +245,36 @@ def _create_instances(ec2_fail_fast, cluster_name: str,
                 # Whether the VM(s) should have a public IP.
                 'AssociatePublicIpAddress': associate_public_ip_address,
                 'Groups': security_group_ids,
+                'InterfaceType': 'efa'
+                                 if max_efa_interfaces > 0 else 'interface',
             }]
+            # Due to AWS limitation, if an instance type supports multiple
+            # network cards, we cannot assign public IP addresses to the
+            # instance during creation, which will raise the following error:
+            #   (InvalidParameterCombination) when calling the RunInstances
+            #   operation: The associatePublicIPAddress parameter cannot be
+            #   specified when launching with multiple network interfaces.
+            # So we only attach multiple network interfaces if public IP is
+            # not required.
+            # TODO(hailong): support attaching/detaching elastic IP to expose
+            # public IP in this case.
+            if max_efa_interfaces > 1 and not associate_public_ip_address:
+                instance_type = conf['InstanceType']
+                for i in range(1, max_efa_interfaces):
+                    interface_type = 'efa-only'
+                    # Special handling for P5 instances
+                    # Refer to https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-acc-inst-types.html#efa-for-p5 for more details. # pylint: disable=line-too-long
+                    if (instance_type == 'p5.48xlarge' or
+                            instance_type == 'p5e.48xlarge'):
+                        interface_type = 'efa' if i % 4 == 0 else 'efa-only'
+                    network_interfaces.append({
+                        'SubnetId': subnet_id,
+                        'DeviceIndex': 1,
+                        'NetworkCardIndex': i,
+                        'AssociatePublicIpAddress': False,
+                        'Groups': security_group_ids,
+                        'InterfaceType': interface_type,
+                    })
             conf['NetworkInterfaces'] = network_interfaces
 
             instances = _ec2_call_with_retry_on_server_error(
@@ -289,6 +324,7 @@ def run_instances(region: str, cluster_name_on_cloud: str,
     zone = None
     resumed_instance_ids: List[str] = []
     created_instance_ids: List[str] = []
+    max_efa_interfaces = config.provider_config.get('max_efa_interfaces', 0)
 
     # sort tags by key to support deterministic unit test stubbing
     tags = dict(sorted(copy.deepcopy(config.tags).items()))
@@ -504,7 +540,8 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                     tags,
                     reservation_count,
                     associate_public_ip_address=(
-                        not config.provider_config['use_internal_ips']))
+                        not config.provider_config['use_internal_ips']),
+                    max_efa_interfaces=max_efa_interfaces)
                 created_instances.extend(created_reserved_instances)
                 to_start_count -= reservation_count
                 if to_start_count <= 0:
@@ -527,7 +564,8 @@ def run_instances(region: str, cluster_name_on_cloud: str,
                 tags,
                 to_start_count,
                 associate_public_ip_address=(
-                    not config.provider_config['use_internal_ips']))
+                    not config.provider_config['use_internal_ips']),
+                max_efa_interfaces=max_efa_interfaces)
 
             created_instances.extend(created_remaining_instances)
         created_instances.sort(key=lambda x: x.id)
@@ -686,6 +724,7 @@ def terminate_instances(
                                   filters,
                                   included_instances=None,
                                   excluded_instances=None)
+    instance_list = list(instances)
     default_sg = aws_config.get_security_group_from_vpc_id(
         ec2, _get_vpc_id(provider_config),
         aws_cloud.DEFAULT_SECURITY_GROUP_NAME)
@@ -719,7 +758,7 @@ def terminate_instances(
         # exist. We must block on instance termination so that we can
         # delete the security group.
         instances.terminate()
-        for instance in instances:
+        for instance in instance_list:
             instance.wait_until_terminated()
 
     # TODO(suquark): Currently, the implementation of GCP and Azure will

sky/provision/docker_utils.py

@@ -371,7 +371,7 @@ class DockerInitializer:
             'mkdir -p ~/.ssh;'
             'cat /tmp/host_ssh_authorized_keys >> ~/.ssh/authorized_keys;'
             'sudo service ssh start;'
-            'sudo sed -i "s/mesg n/tty -s \&\& mesg n/" ~/.profile;'
+            'sudo sed -i "s/mesg n/tty -s \\&\\& mesg n/" ~/.profile;'
             f'{SETUP_ENV_VARS_CMD}',
             run_env='docker')
 

sky/provision/kubernetes/utils.py

@@ -1,4 +1,5 @@
 """Kubernetes utilities for SkyPilot."""
+import copy
 import dataclasses
 import datetime
 import enum
@@ -1082,6 +1083,14 @@ class KarpenterAutoscaler(Autoscaler):
     can_query_backend: bool = False
 
 
+class CoreweaveAutoscaler(Autoscaler):
+    """CoreWeave autoscaler
+    """
+
+    label_formatter: Any = CoreWeaveLabelFormatter
+    can_query_backend: bool = False
+
+
 class GenericAutoscaler(Autoscaler):
     """Generic autoscaler
     """
@@ -1094,6 +1103,7 @@ class GenericAutoscaler(Autoscaler):
 AUTOSCALER_TYPE_TO_AUTOSCALER = {
     kubernetes_enums.KubernetesAutoscalerType.GKE: GKEAutoscaler,
     kubernetes_enums.KubernetesAutoscalerType.KARPENTER: KarpenterAutoscaler,
+    kubernetes_enums.KubernetesAutoscalerType.COREWEAVE: CoreweaveAutoscaler,
     kubernetes_enums.KubernetesAutoscalerType.GENERIC: GenericAutoscaler,
 }
 
@@ -2706,11 +2716,11 @@ def get_endpoint_debug_message(context: Optional[str] = None) -> str:
 
 
 def combine_pod_config_fields(
-    cluster_yaml_path: str,
+    cluster_yaml_obj: Dict[str, Any],
     cluster_config_overrides: Dict[str, Any],
     cloud: Optional[clouds.Cloud] = None,
     context: Optional[str] = None,
-) -> None:
+) -> Dict[str, Any]:
     """Adds or updates fields in the YAML with fields from the
     ~/.sky/config.yaml's kubernetes.pod_spec dict.
     This can be used to add fields to the YAML that are not supported by
@@ -2749,9 +2759,7 @@ def combine_pod_config_fields(
                     - name: my-secret
         ```
     """
-    with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
-        yaml_content = f.read()
-    yaml_obj = yaml_utils.safe_load(yaml_content)
+    merged_cluster_yaml_obj = copy.deepcopy(cluster_yaml_obj)
     # We don't use override_configs in `get_effective_region_config`, as merging
     # the pod config requires special handling.
     if isinstance(cloud, clouds.SSH):
@@ -2778,26 +2786,20 @@ def combine_pod_config_fields(
 
     # Merge the kubernetes config into the YAML for both head and worker nodes.
     config_utils.merge_k8s_configs(
-        yaml_obj['available_node_types']['ray_head_default']['node_config'],
-        kubernetes_config)
-
-    # Write the updated YAML back to the file
-    yaml_utils.dump_yaml(cluster_yaml_path, yaml_obj)
+        merged_cluster_yaml_obj['available_node_types']['ray_head_default']
+        ['node_config'], kubernetes_config)
+    return merged_cluster_yaml_obj
 
 
-def combine_metadata_fields(cluster_yaml_path: str,
+def combine_metadata_fields(cluster_yaml_obj: Dict[str, Any],
                             cluster_config_overrides: Dict[str, Any],
-                            context: Optional[str] = None) -> None:
+                            context: Optional[str] = None) -> Dict[str, Any]:
     """Updates the metadata for all Kubernetes objects created by SkyPilot with
     fields from the ~/.sky/config.yaml's kubernetes.custom_metadata dict.
 
     Obeys the same add or update semantics as combine_pod_config_fields().
     """
-
-    with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
-        yaml_content = f.read()
-    yaml_obj = yaml_utils.safe_load(yaml_content)
-
+    merged_cluster_yaml_obj = copy.deepcopy(cluster_yaml_obj)
     # Get custom_metadata from global config
     custom_metadata = skypilot_config.get_effective_region_config(
         cloud='kubernetes',
@@ -2819,22 +2821,42 @@ def combine_metadata_fields(cluster_yaml_path: str,
     # List of objects in the cluster YAML to be updated
     combination_destinations = [
         # Service accounts
-        yaml_obj['provider']['autoscaler_service_account']['metadata'],
-        yaml_obj['provider']['autoscaler_role']['metadata'],
-        yaml_obj['provider']['autoscaler_role_binding']['metadata'],
-        yaml_obj['provider']['autoscaler_service_account']['metadata'],
-        # Pod spec
-        yaml_obj['available_node_types']['ray_head_default']['node_config']
+        merged_cluster_yaml_obj['provider']['autoscaler_service_account']
+        ['metadata'],
+        merged_cluster_yaml_obj['provider']['autoscaler_role']['metadata'],
+        merged_cluster_yaml_obj['provider']['autoscaler_role_binding']
         ['metadata'],
+        merged_cluster_yaml_obj['provider']['autoscaler_service_account']
+        ['metadata'],
+        # Pod spec
+        merged_cluster_yaml_obj['available_node_types']['ray_head_default']
+        ['node_config']['metadata'],
         # Services for pods
-        *[svc['metadata'] for svc in yaml_obj['provider']['services']]
+        *[
+            svc['metadata']
+            for svc in merged_cluster_yaml_obj['provider']['services']
+        ]
     ]
 
     for destination in combination_destinations:
         config_utils.merge_k8s_configs(destination, custom_metadata)
 
-    # Write the updated YAML back to the file
-    yaml_utils.dump_yaml(cluster_yaml_path, yaml_obj)
+    return merged_cluster_yaml_obj
+
+
+def combine_pod_config_fields_and_metadata(
+        cluster_yaml_obj: Dict[str, Any],
+        cluster_config_overrides: Dict[str, Any],
+        cloud: Optional[clouds.Cloud] = None,
+        context: Optional[str] = None) -> Dict[str, Any]:
+    """Combines pod config fields and metadata fields"""
+    combined_yaml_obj = combine_pod_config_fields(cluster_yaml_obj,
+                                                  cluster_config_overrides,
+                                                  cloud, context)
+    combined_yaml_obj = combine_metadata_fields(combined_yaml_obj,
+                                                cluster_config_overrides,
+                                                context)
+    return combined_yaml_obj
 
 
 def merge_custom_metadata(

sky/schemas/db/global_user_state/007_cluster_event_request_id.py

@@ -0,0 +1,34 @@
+"""Add request_id to cluster_events.
+
+Revision ID: 007
+Revises: 006
+Create Date: 2025-08-28
+
+"""
+# pylint: disable=invalid-name
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sky.utils.db import db_utils
+
+# revision identifiers, used by Alembic.
+revision: str = '007'
+down_revision: Union[str, Sequence[str], None] = '006'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade():
+    """Add request_id column to cluster_events."""
+    with op.get_context().autocommit_block():
+        db_utils.add_column_to_table_alembic('cluster_events',
+                                             'request_id',
+                                             sa.Text(),
+                                             server_default=None)
+
+
+def downgrade():
+    """No-op for backward compatibility."""
+    pass

sky/server/common.py

@@ -911,8 +911,7 @@ def reload_for_new_request(client_entrypoint: Optional[str],
 
     # Clear cache should be called before reload_logger and usage reset,
     # otherwise, the latest env var will not be used.
-    for func in annotations.FUNCTIONS_NEED_RELOAD_CACHE:
-        func.cache_clear()
+    annotations.clear_request_level_cache()
 
     # We need to reset usage message, so that the message is up-to-date with the
     # latest information in the context, e.g. client entrypoint and run id.

sky/server/daemons.py

@@ -7,8 +7,10 @@ from typing import Callable
 from sky import sky_logging
 from sky import skypilot_config
 from sky.server import constants as server_constants
+from sky.utils import annotations
 from sky.utils import common
 from sky.utils import env_options
+from sky.utils import timeline
 from sky.utils import ux_utils
 
 logger = sky_logging.init_logger(__name__)
@@ -67,6 +69,10 @@ class InternalRequestDaemon:
                     sky_logging.reload_logger()
                     level = self.refresh_log_level()
                     self.event_fn()
+                # Clear request level cache after each run to avoid
+                # using too much memory.
+                annotations.clear_request_level_cache()
+                timeline.save_timeline()
             except Exception:  # pylint: disable=broad-except
                 # It is OK to fail to run the event, as the event is not
                 # critical, but we should log the error.

sky/server/requests/executor.py

@@ -383,7 +383,8 @@ def _request_execution_wrapper(request_id: str,
         # config, as there can be some logs during override that needs to be
         # captured in the log file.
         try:
-            with override_request_env_and_config(request_body, request_id), \
+            with sky_logging.add_debug_log_handler(request_id), \
+                override_request_env_and_config(request_body, request_id), \
                 tempstore.tempdir():
                 if sky_logging.logging_enabled(logger, sky_logging.DEBUG):
                     config = skypilot_config.to_dict()

sky/server/requests/payloads.py

@@ -71,7 +71,9 @@ EXTERNAL_LOCAL_ENV_VARS = [
 def request_body_env_vars() -> dict:
     env_vars = {}
     for env_var in os.environ:
-        if env_var.startswith(constants.SKYPILOT_ENV_VAR_PREFIX):
+        if (env_var.startswith(constants.SKYPILOT_ENV_VAR_PREFIX) and
+                not env_var.startswith(
+                    constants.SKYPILOT_SERVER_ENV_VAR_PREFIX)):
             env_vars[env_var] = os.environ[env_var]
         if common.is_api_server_local() and env_var in EXTERNAL_LOCAL_ENV_VARS:
             env_vars[env_var] = os.environ[env_var]
@@ -307,6 +309,7 @@ class StatusBody(RequestBody):
     cluster_names: Optional[List[str]] = None
     refresh: common_lib.StatusRefreshMode = common_lib.StatusRefreshMode.NONE
     all_users: bool = True
+    include_credentials: bool = False
 
 
 class StartBody(RequestBody):