skypilot-nightly 1.0.0.dev20250408__py3-none-any.whl → 1.0.0.dev20250411__py3-none-any.whl

sky/provision/do/utils.py

@@ -15,6 +15,7 @@ from sky.adaptors import do
 from sky.provision import common
 from sky.provision import constants as provision_constants
 from sky.provision.do import constants
+from sky.utils import annotations
 from sky.utils import common_utils
 
 logger = sky_logging.init_logger(__name__)
@@ -31,7 +32,6 @@ MAX_BACKOFF_FACTOR = 10
 MAX_ATTEMPTS = 6
 SSH_KEY_NAME_ON_DO = f'sky-key-{common_utils.get_user_hash()}'
 
-CREDENTIALS_PATH = '~/.config/doctl/config.yaml'
 _client = None
 _ssh_key_id = None
 
@@ -40,31 +40,34 @@ class DigitalOceanError(Exception):
     pass
 
 
-def _init_client():
-    global _client, CREDENTIALS_PATH
-    assert _client is None
-    CREDENTIALS_PATH = None
+@annotations.lru_cache(scope='request')
+def get_credentials_path():
+    credentials_path = None
     credentials_found = 0
     for path in POSSIBLE_CREDENTIALS_PATHS:
         if os.path.exists(path):
-            CREDENTIALS_PATH = path
-            credentials_found += 1
             logger.debug(f'Digital Ocean credential path found at {path}')
-    if not credentials_found > 1:
-        logger.debug('more than 1 credential file found')
-    if CREDENTIALS_PATH is None:
-        raise DigitalOceanError(
-            'no credentials file found from '
-            f'the following paths {POSSIBLE_CREDENTIALS_PATHS}')
+            credentials_path = path
+            credentials_found += 1
+    if credentials_found > 1:
+        logger.debug('More than 1 credential file found')
+    return credentials_path
 
+
+def _init_client():
+    global _client
+    assert _client is None
     # attempt default context
-    credentials = common_utils.read_yaml(CREDENTIALS_PATH)
+    if get_credentials_path() is None:
+        raise DigitalOceanError(
+            'No credentials found, please run `doctl auth init`')
+    credentials = common_utils.read_yaml(get_credentials_path())
     default_token = credentials.get('access-token', None)
     if default_token is not None:
         try:
             test_client = do.pydo.Client(token=default_token)
             test_client.droplets.list()
-            logger.debug('trying `default` context')
+            logger.debug('Trying `default` context')
             _client = test_client
             return _client
         except do.exceptions().HttpResponseError:
@@ -76,7 +79,7 @@ def _init_client():
             try:
                 test_client = do.pydo.Client(token=api_token)
                 test_client.droplets.list()
-                logger.debug(f'using {context} context')
+                logger.debug(f'Using "{context}" context')
                 _client = test_client
                 break
             except do.exceptions().HttpResponseError:

sky/provision/gcp/config.py

@@ -571,35 +571,45 @@ def get_usable_vpc_and_subnet(
 
     specific_vpc_to_use = config.provider_config.get('vpc_name', None)
     if specific_vpc_to_use is not None:
+        if '/' in specific_vpc_to_use:
+            # VPC can also be specified in the format PROJECT_ID/VPC_NAME.
+            # This enables use of shared VPCs.
+            split_vpc_value = specific_vpc_to_use.split('/')
+            if len(split_vpc_value) != 2:
+                raise ValueError(f'Invalid VPC name: {specific_vpc_to_use}. '
+                                 'Please specify the VPC name in the format '
+                                 'PROJECT_ID/VPC_NAME.')
+            project_id = split_vpc_value[0]
+            specific_vpc_to_use = split_vpc_value[1]
+
         vpcnets_all = _list_vpcnets(project_id,
                                     compute,
                                     filter=f'name={specific_vpc_to_use}')
-        # On GCP, VPC names are unique, so it'd be 0 or 1 VPC found.
-        assert (len(vpcnets_all) <=
-                1), (f'{len(vpcnets_all)} VPCs found with the same name '
-                     f'{specific_vpc_to_use}')
-        if len(vpcnets_all) == 1:
-            # Skip checking any firewall rules if the user has specified a VPC.
-            logger.info(f'Using user-specified VPC {specific_vpc_to_use!r}.')
-            subnets = _list_subnets(project_id,
-                                    region,
-                                    compute,
-                                    network=specific_vpc_to_use)
-            if not subnets:
-                _skypilot_log_error_and_exit_for_failover(
-                    'SUBNET_NOT_FOUND_FOR_VPC',
-                    f'No subnet for region {region} found for specified VPC '
-                    f'{specific_vpc_to_use!r}. '
-                    f'Check the subnets of VPC {specific_vpc_to_use!r} at '
-                    'https://console.cloud.google.com/networking/networks')
-            return specific_vpc_to_use, subnets[0]
-        else:
+        if not vpcnets_all:
             # VPC with this name not found. Error out and let SkyPilot failover.
             _skypilot_log_error_and_exit_for_failover(
                 'VPC_NOT_FOUND',
                 f'No VPC with name {specific_vpc_to_use!r} is found. '
                 'To fix: specify a correct VPC name.')
             # Should not reach here.
+            assert False
+
+        # On GCP, VPC names are unique within a project.
+        assert len(vpcnets_all) == 1, (vpcnets_all, specific_vpc_to_use)
+        # Skip checking any firewall rules if the user has specified a VPC.
+        logger.info(f'Using user-specified VPC {specific_vpc_to_use!r}.')
+        subnets = _list_subnets(project_id,
+                                region,
+                                compute,
+                                network=specific_vpc_to_use)
+        if not subnets:
+            _skypilot_log_error_and_exit_for_failover(
+                'SUBNET_NOT_FOUND_FOR_VPC',
+                f'No subnet for region {region} found for specified VPC '
+                f'{specific_vpc_to_use!r}. '
+                f'Check the subnets of VPC {specific_vpc_to_use!r} at '
+                'https://console.cloud.google.com/networking/networks')
+        return specific_vpc_to_use, subnets[0]
 
     subnets_all = _list_subnets(project_id, region, compute)
 

sky/serve/client/sdk.py

@@ -74,6 +74,7 @@ def up(
         f'{server_common.get_server_url()}/serve/up',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response)
 
@@ -132,6 +133,7 @@ def update(
         f'{server_common.get_server_url()}/serve/update',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response)
 
@@ -173,6 +175,7 @@ def down(
         f'{server_common.get_server_url()}/serve/down',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response)
 
@@ -207,6 +210,7 @@ def terminate_replica(service_name: str, replica_id: int,
         f'{server_common.get_server_url()}/serve/terminate-replica',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response)
 
@@ -279,6 +283,7 @@ def status(
         f'{server_common.get_server_url()}/serve/status',
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
+        cookies=server_common.get_api_cookie_jar(),
     )
     return server_common.get_request_id(response)
 
@@ -365,6 +370,7 @@ def tail_logs(service_name: str,
         json=json.loads(body.model_dump_json()),
         timeout=(5, None),
         stream=True,
+        cookies=server_common.get_api_cookie_jar(),
     )
     request_id = server_common.get_request_id(response)
     sdk.stream_response(request_id, response, output_stream)

sky/server/common.py

@@ -3,6 +3,7 @@
 import dataclasses
 import enum
 import functools
+from http.cookiejar import MozillaCookieJar
 import json
 import os
 import pathlib
@@ -80,6 +81,18 @@ class ApiServerInfo:
     api_version: ApiVersion
 
 
+def get_api_cookie_jar() -> requests.cookies.RequestsCookieJar:
+    """Returns the cookie jar used by the client to access the API server."""
+    cookie_file = os.environ.get(server_constants.API_COOKIE_FILE_ENV_VAR)
+    cookie_jar = requests.cookies.RequestsCookieJar()
+    if cookie_file and os.path.exists(cookie_file):
+        cookie_path = pathlib.Path(cookie_file).expanduser().resolve()
+        file_cookie_jar = MozillaCookieJar(cookie_path)
+        file_cookie_jar.load()
+        cookie_jar.update(file_cookie_jar)
+    return cookie_jar
+
+
 @annotations.lru_cache(scope='global')
 def get_server_url(host: Optional[str] = None) -> str:
     endpoint = DEFAULT_SERVER_URL
@@ -117,7 +130,9 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
     server_url = endpoint if endpoint is not None else get_server_url()
     while time_out_try_count <= RETRY_COUNT_ON_TIMEOUT:
         try:
-            response = requests.get(f'{server_url}/api/health', timeout=2.5)
+            response = requests.get(f'{server_url}/api/health',
+                                    timeout=2.5,
+                                    cookies=get_api_cookie_jar())
             if response.status_code == 200:
                 try:
                     result = response.json()

sky/server/constants.py

@@ -1,5 +1,7 @@
 """Constants for the API servers."""
 
+from sky.skylet import constants
+
 # API server version, whenever there is a change in API server that requires a
 # restart of the local API server or error out when the client does not match
 # the server version.
@@ -19,3 +21,6 @@ API_SERVER_REQUEST_DB_PATH = '~/.sky/api_server/requests.db'
 # The interval (seconds) for the cluster status to be refreshed in the
 # background.
 CLUSTER_REFRESH_DAEMON_INTERVAL_SECONDS = 60
+
+# Environment variable for a file path to the API cookie file.
+API_COOKIE_FILE_ENV_VAR = f'{constants.SKYPILOT_ENV_VAR_PREFIX}API_COOKIE_FILE'

sky/setup_files/dependencies.py

@@ -9,7 +9,7 @@ import sys
 from typing import Dict, List
 
 install_requires = [
-    'wheel',
+    'wheel<0.46.0',  # https://github.com/skypilot-org/skypilot/issues/5153
     'cachetools',
     # NOTE: ray requires click>=7.0.
     'click >= 7.0',

sky/skylet/log_lib.py

@@ -149,6 +149,7 @@ def run_with_log(
     process_stream: bool = True,
     line_processor: Optional[log_utils.LineProcessor] = None,
     streaming_prefix: Optional[str] = None,
+    log_cmd: bool = False,
     **kwargs,
 ) -> Union[int, Tuple[int, str, str]]:
     """Runs a command and logs its output to a file.
@@ -182,6 +183,9 @@ def run_with_log(
     # the terminal output when typing in the terminal that starts the API
     # server.
     stdin = kwargs.pop('stdin', subprocess.DEVNULL)
+    if log_cmd:
+        with open(log_path, 'a', encoding='utf-8') as f:
+            print(f'Running command: {cmd}', file=f)
     with subprocess.Popen(cmd,
                           stdout=stdout_arg,
                           stderr=stderr_arg,

sky/skypilot_config.py

@@ -52,7 +52,6 @@ import contextlib
 import copy
 import os
 import pprint
-import tempfile
 import typing
 from typing import Any, Dict, Iterator, Optional, Tuple
 
@@ -92,6 +91,7 @@ CONFIG_PATH = '~/.sky/config.yaml'
 # The loaded config.
 _dict = config_utils.Config()
 _loaded_config_path: Optional[str] = None
+_config_overridden: bool = False
 
 
 def get_nested(keys: Tuple[str, ...],
@@ -178,7 +178,10 @@ def _reload_config() -> None:
 
 
 def loaded_config_path() -> Optional[str]:
-    """Returns the path to the loaded config file."""
+    """Returns the path to the loaded config file, or
+    '<overridden>' if the config is overridden."""
+    if _config_overridden:
+        return '<overridden>'
     return _loaded_config_path
 
 
@@ -195,31 +198,30 @@ def loaded() -> bool:
 def override_skypilot_config(
         override_configs: Optional[Dict[str, Any]]) -> Iterator[None]:
     """Overrides the user configurations."""
+    global _dict, _config_overridden
     # TODO(SKY-1215): allow admin user to extend the disallowed keys or specify
     # allowed keys.
     if not override_configs:
         # If no override configs (None or empty dict), do nothing.
         yield
         return
-    original_env_config_path = _loaded_config_path
-    original_config = dict(_dict)
+    original_config = _dict
     config = _dict.get_nested(
         keys=tuple(),
         default_value=None,
         override_configs=override_configs,
         allowed_override_keys=None,
         disallowed_override_keys=constants.SKIPPED_CLIENT_OVERRIDE_KEYS)
-    with tempfile.NamedTemporaryFile(
-            mode='w',
-            prefix='skypilot_config',
-            # Have to avoid deleting the file as the underlying function needs
-            # to read the config file, and we need to close the file mode='w'
-            # to enable reading.
-            delete=False) as f:
-        common_utils.dump_yaml(f.name, dict(config))
-        os.environ[ENV_VAR_SKYPILOT_CONFIG] = f.name
     try:
-        _reload_config()
+        common_utils.validate_schema(
+            config,
+            schemas.get_config_schema(),
+            f'Invalid config {config}. See: '
+            'https://docs.skypilot.co/en/latest/reference/config.html. '  # pylint: disable=line-too-long
+            'Error: ',
+            skip_none=False)
+        _config_overridden = True
+        _dict = config
         yield
     except exceptions.InvalidSkyPilotConfigError as e:
         with ux_utils.print_exception_no_traceback():
@@ -227,23 +229,10 @@ def override_skypilot_config(
                 'Failed to override the SkyPilot config on API '
                 'server with your local SkyPilot config:\n'
                 '=== SkyPilot config on API server ===\n'
-                f'{common_utils.dump_yaml_str(original_config)}\n'
+                f'{common_utils.dump_yaml_str(dict(original_config))}\n'
                 '=== Your local SkyPilot config ===\n'
                 f'{common_utils.dump_yaml_str(override_configs)}\n'
                 f'Details: {e}') from e
-
     finally:
-        if original_env_config_path is not None:
-            os.environ[ENV_VAR_SKYPILOT_CONFIG] = original_env_config_path
-        else:
-            os.environ.pop(ENV_VAR_SKYPILOT_CONFIG, None)
-        # Reload the config to restore the original config to avoid the next
-        # request reusing the same process to use the config for the current
-        # request.
-        _reload_config()
-
-        try:
-            os.remove(f.name)
-        except Exception:  # pylint: disable=broad-except
-            # Failing to delete the file is not critical.
-            pass
+        _dict = original_config
+        _config_overridden = False

sky/task.py

@@ -552,15 +552,35 @@ class Task:
                              estimated_size_gigabytes=estimated_size_gigabytes)
 
         # Experimental configs.
-        experimnetal_configs = config.pop('experimental', None)
-        cluster_config_override = None
-        if experimnetal_configs is not None:
-            cluster_config_override = experimnetal_configs.pop(
+        experimental_configs = config.pop('experimental', None)
+
+        # Handle the top-level config field
+        config_override = config.pop('config', None)
+
+        # Handle backward compatibility with experimental.config_overrides
+        # TODO: Remove experimental.config_overrides in 0.11.0.
+        if experimental_configs is not None:
+            exp_config_override = experimental_configs.pop(
                 'config_overrides', None)
+            if exp_config_override is not None:
+                logger.warning(
+                    f'{colorama.Fore.YELLOW}`experimental.config_overrides` '
+                    'field is deprecated in the task YAML. Use the `config` '
+                    f'field to set config overrides.{colorama.Style.RESET_ALL}')
+                if config_override is not None:
+                    logger.warning(
+                        f'{colorama.Fore.YELLOW}Both top-level `config` and '
+                        f'`experimental.config_overrides` are specified. '
+                        f'Using top-level `config`.{colorama.Style.RESET_ALL}')
+                else:
+                    config_override = exp_config_override
             logger.debug('Overriding skypilot config with task-level config: '
-                         f'{cluster_config_override}')
-        assert not experimnetal_configs, ('Invalid task args: '
-                                          f'{experimnetal_configs.keys()}')
+                         f'{config_override}')
+            assert not experimental_configs, ('Invalid task args: '
+                                              f'{experimental_configs.keys()}')
+
+        # Store the final config override for use in resource setup
+        cluster_config_override = config_override
 
         # Parse resources field.
         resources_config = config.pop('resources', {})

sky/utils/schemas.py

@@ -473,6 +473,8 @@ def _filter_schema(schema: dict, keys_to_keep: List[Tuple[str, ...]]) -> dict:
 
 
 def _experimental_task_schema() -> dict:
+    # TODO: experimental.config_overrides has been deprecated in favor of the
+    # top-level `config` field. Remove in v0.11.0.
     config_override_schema = _filter_schema(
         get_config_schema(), constants.OVERRIDEABLE_CONFIG_KEYS_IN_TASK)
     return {
@@ -555,6 +557,9 @@ def get_task_schema():
             'file_mounts_mapping': {
                 'type': 'object',
             },
+            'config': _filter_schema(
+                get_config_schema(),
+                constants.OVERRIDEABLE_CONFIG_KEYS_IN_TASK),
             **_experimental_task_schema(),
         }
     }
@@ -604,13 +609,6 @@ def get_cluster_schema():
 
 
 _NETWORK_CONFIG_SCHEMA = {
-    'vpc_name': {
-        'oneOf': [{
-            'type': 'string',
-        }, {
-            'type': 'null',
-        }],
-    },
     'use_internal_ips': {
         'type': 'boolean',
     },
@@ -767,6 +765,13 @@ def get_config_schema():
                 },
                 'security_group_name':
                     (_PRORPERTY_NAME_OR_CLUSTER_NAME_TO_PROPERTY),
+                'vpc_name': {
+                    'oneOf': [{
+                        'type': 'string',
+                    }, {
+                        'type': 'null',
+                    }],
+                },
                 **_LABELS_SCHEMA,
                 **_NETWORK_CONFIG_SCHEMA,
             },
@@ -805,6 +810,19 @@ def get_config_schema():
                 'enable_gvnic': {
                     'type': 'boolean'
                 },
+                'vpc_name': {
+                    'oneOf': [
+                        {
+                            'type': 'string',
+                            # vpc-name or project-id/vpc-name
+                            # VPC name and Project ID have -, a-z, and 0-9.
+                            'pattern': '^(?:[-a-z0-9]+/)?[-a-z0-9]+$'
+                        },
+                        {
+                            'type': 'null',
+                        }
+                    ],
+                },
                 **_LABELS_SCHEMA,
                 **_NETWORK_CONFIG_SCHEMA,
             },

{skypilot_nightly-1.0.0.dev20250408.dist-info → skypilot_nightly-1.0.0.dev20250411.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250408
+Version: 1.0.0.dev20250411
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -19,7 +19,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Distributed Computing
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: wheel
+Requires-Dist: wheel<0.46.0
 Requires-Dist: cachetools
 Requires-Dist: click>=7.0
 Requires-Dist: colorama