skypilot-nightly 1.0.0.dev20250617__py3-none-any.whl → 1.0.0.dev20250619__py3-none-any.whl

sky/jobs/state.py

@@ -113,7 +113,7 @@ def create_table(cursor, conn):
 
     # `job_info` contains the mapping from job_id to the job_name, as well as
     # information used by the scheduler.
-    cursor.execute("""\
+    cursor.execute(f"""\
         CREATE TABLE IF NOT EXISTS job_info (
         spot_job_id INTEGER PRIMARY KEY AUTOINCREMENT,
         name TEXT,
@@ -123,7 +123,7 @@ def create_table(cursor, conn):
         env_file_path TEXT,
         user_hash TEXT,
         workspace TEXT DEFAULT NULL,
-        priority INTEGER DEFAULT 500,
+        priority INTEGER DEFAULT {constants.DEFAULT_PRIORITY},
         entrypoint TEXT DEFAULT NULL,
         original_user_yaml_path TEXT DEFAULT NULL)""")
 
@@ -148,12 +148,13 @@ def create_table(cursor, conn):
                                  'TEXT DEFAULT NULL',
                                  value_to_replace_existing_entries='default')
 
-    db_utils.add_column_to_table(cursor,
-                                 conn,
-                                 'job_info',
-                                 'priority',
-                                 'INTEGER',
-                                 value_to_replace_existing_entries=500)
+    db_utils.add_column_to_table(
+        cursor,
+        conn,
+        'job_info',
+        'priority',
+        'INTEGER',
+        value_to_replace_existing_entries=constants.DEFAULT_PRIORITY)
 
     db_utils.add_column_to_table(cursor, conn, 'job_info', 'entrypoint', 'TEXT')
     db_utils.add_column_to_table(cursor, conn, 'job_info',

sky/jobs/utils.py

@@ -945,7 +945,7 @@ def dump_managed_job_queue() -> str:
     # Figure out what the highest priority blocking job is. We need to know in
     # order to determine if other jobs are blocked by a higher priority job, or
     # just by the limited controller resources.
-    highest_blocking_priority = 0
+    highest_blocking_priority = constants.MIN_PRIORITY
     for job in jobs:
         if job['schedule_state'] not in (
                 # LAUNCHING and ALIVE_BACKOFF jobs will block other jobs with

sky/logs/__init__.py

@@ -0,0 +1,17 @@
+"""Sky logging agents."""
+from typing import Optional
+
+from sky import exceptions
+from sky import skypilot_config
+from sky.logs.agent import LoggingAgent
+from sky.logs.gcp import GCPLoggingAgent
+
+
+def get_logging_agent() -> Optional[LoggingAgent]:
+    store = skypilot_config.get_nested(('logs', 'store'), None)
+    if store is None:
+        return None
+    if store == 'gcp':
+        return GCPLoggingAgent(skypilot_config.get_nested(('logs', 'gcp'), {}))
+    raise exceptions.InvalidSkyPilotConfigError(
+        f'Invalid logging store: {store}')

sky/logs/agent.py

@@ -0,0 +1,73 @@
+"""Base class for all logging agents."""
+import abc
+import os
+import shlex
+from typing import Any, Dict
+
+from sky.skylet import constants
+from sky.utils import common_utils
+from sky.utils import resources_utils
+
+
+class LoggingAgent(abc.ABC):
+    """Base class for all logging agents.
+
+    Each agent should implement the `get_setup_command` and
+    `get_credential_file_mounts` methods to return the setup command and
+    credential file mounts for the agent for provisioner to setup the agent
+    on each node.
+    """
+
+    @abc.abstractmethod
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        pass
+
+    @abc.abstractmethod
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        pass
+
+
+class FluentbitAgent(LoggingAgent):
+    """Base class for logging store that use fluentbit as the agent."""
+
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        install_cmd = (
+            'if ! command -v fluent-bit >/dev/null 2>&1; then '
+            'sudo apt-get install -y gnupg; '
+            # pylint: disable=line-too-long
+            'curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh; '
+            'fi')
+        cfg = self.fluentbit_config(cluster_name)
+        cfg_path = os.path.join(constants.LOGGING_CONFIG_DIR, 'fluentbit.yaml')
+        config_cmd = (f'mkdir -p {constants.LOGGING_CONFIG_DIR} && '
+                      f'echo {shlex.quote(cfg)} > {cfg_path}')
+        start_cmd = ('nohup $(command -v fluent-bit || '
+                     'echo "/opt/fluent-bit/bin/fluent-bit") '
+                     f'-c {cfg_path} > /tmp/fluentbit.log 2>&1 &')
+        return f'set -e; {install_cmd}; {config_cmd}; {start_cmd}'
+
+    def fluentbit_config(self,
+                         cluster_name: resources_utils.ClusterName) -> str:
+        cfg_dict = {
+            'pipeline': {
+                'inputs': [{
+                    'name': 'tail',
+                    'path': f'{constants.SKY_LOGS_DIRECTORY}/*/*.log',
+                    'path_key': 'log_path',
+                    # Shorten the refresh interval from 60s to 1s since every
+                    # job creates a new log file and we must be responsive
+                    # for this: the VM might be autodown within a minute
+                    # right after the job completion.
+                    'refresh_interval': 1,
+                }],
+                'outputs': [self.fluentbit_output_config(cluster_name)],
+            }
+        }
+        return common_utils.dump_yaml_str(cfg_dict)
+
+    @abc.abstractmethod
+    def fluentbit_output_config(
+            self, cluster_name: resources_utils.ClusterName) -> Dict[str, Any]:
+        pass

sky/logs/gcp.py

@@ -0,0 +1,91 @@
+"""GCP logging agent."""
+
+from typing import Any, Dict, Optional
+
+import pydantic
+
+from sky.clouds import gcp
+from sky.logs.agent import FluentbitAgent
+from sky.utils import resources_utils
+
+
+class _GCPLoggingConfig(pydantic.BaseModel):
+    """Configuration for GCP logging agent."""
+    project_id: Optional[str] = None
+    credentials_file: Optional[str] = None
+    additional_labels: Optional[Dict[str, str]] = None
+
+
+class _StackdriverOutputConfig(pydantic.BaseModel):
+    """Auxiliary model for building stackdriver output config in YAML.
+
+    Ref: https://docs.fluentbit.io/manual/1.7/pipeline/outputs/stackdriver
+    """
+    name: str = 'stackdriver'
+    match: str = '*'
+    export_to_project_id: Optional[str] = None
+    labels: Optional[Dict[str, str]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        config = self.model_dump(exclude_none=True)
+        if self.labels:
+            # Replace the label format from `{k: v}` to `k=v`
+            label_str = ','.join([f'{k}={v}' for k, v in self.labels.items()])
+            config['labels'] = label_str
+        return config
+
+
+class GCPLoggingAgent(FluentbitAgent):
+    """GCP logging agent."""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.config = _GCPLoggingConfig(**config)
+
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        credential_path = gcp.DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH
+        if self.config.credentials_file:
+            credential_path = self.config.credentials_file
+        # Set GOOGLE_APPLICATION_CREDENTIALS and check whether credentials
+        # is valid.
+        # Stackdriver only support service account credentials or credentials
+        # from metadata server (only available on GCE or GKE). If the default
+        # credentials uploaded by API server is NOT a service account key and
+        # there is NO metadata server available, the logging agent will fail to
+        # authenticate and we require the user to upload a service account key
+        # via logs.gcp.credentials_file in this case.
+        # Also note that we use env var instead of YAML config to specify the
+        # service account key file path in order to resolve the home directory
+        # more reliably.
+        # Ref: https://github.com/fluent/fluent-bit/issues/8804
+        # TODO(aylei): check whether the credentials config is valid before
+        # provision.
+        pre_cmd = (f'export GOOGLE_APPLICATION_CREDENTIALS={credential_path}; '
+                   f'cat {credential_path} | grep "service_account" || '
+                   f'(echo "Credentials file {credential_path} is not a '
+                   'service account key, check metadata server" && '
+                   'curl -s http://metadata.google.internal >/dev/null || '
+                   f'(echo "Neither service account key nor metadata server is '
+                   'available. Set logs.gcp.credentials_file to a service '
+                   'account key in server config and retry." && '
+                   'exit 1;))')
+        return pre_cmd + ' && ' + super().get_setup_command(cluster_name)
+
+    def fluentbit_output_config(
+            self, cluster_name: resources_utils.ClusterName) -> Dict[str, Any]:
+        display_name = cluster_name.display_name
+        unique_name = cluster_name.name_on_cloud
+
+        return _StackdriverOutputConfig(
+            export_to_project_id=self.config.project_id,
+            labels={
+                'skypilot_cluster_name': display_name,
+                'skypilot_cluster_id': unique_name,
+                **(self.config.additional_labels or {})
+            },
+        ).to_dict()
+
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        if self.config.credentials_file:
+            return {self.config.credentials_file: self.config.credentials_file}
+        return {}

sky/models.py

@@ -17,6 +17,7 @@ class User:
     id: str
     # Display name of the user
     name: Optional[str] = None
+    password: Optional[str] = None
 
     def to_dict(self) -> Dict[str, Any]:
         return {'id': self.id, 'name': self.name}

sky/provision/common.py

@@ -6,6 +6,7 @@ import os
 from typing import Any, Dict, List, Optional, Tuple
 
 from sky import sky_logging
+from sky.utils import env_options
 from sky.utils import resources_utils
 
 # NOTE: we can use pydantic instead of dataclasses or namedtuples, because
@@ -244,6 +245,15 @@ class SocketEndpoint(Endpoint):
 
     def url(self, override_ip: Optional[str] = None) -> str:
         host = override_ip if override_ip else self.host
+        if env_options.Options.RUNNING_IN_BUILDKITE.get(
+        ) and 'localhost' in host:
+            # In Buildkite CI, we run a kind (Kubernetes in Docker) cluster.
+            # The controller pod runs inside this kind cluster, which itself
+            # runs in a container. When the pod tries to access 'localhost',
+            # it can't reach the host machine's localhost. Using
+            # 'host.docker.internal' allows the pod to properly communicate
+            # with services running on the host machine's localhost.
+            host = 'host.docker.internal'
         return f'{host}{":" + str(self.port) if self.port else ""}'
 
 

sky/provision/instance_setup.py

@@ -8,6 +8,7 @@ import time
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
 from sky import exceptions
+from sky import logs
 from sky import provision
 from sky import sky_logging
 from sky.provision import common
@@ -21,6 +22,7 @@ from sky.utils import accelerator_registry
 from sky.utils import command_runner
 from sky.utils import common_utils
 from sky.utils import env_options
+from sky.utils import resources_utils
 from sky.utils import subprocess_utils
 from sky.utils import timeline
 from sky.utils import ux_utils
@@ -557,3 +559,36 @@ def internal_file_mounts(cluster_name: str, common_file_mounts: Dict[str, str],
         ssh_credentials=ssh_credentials,
         max_workers=subprocess_utils.get_max_workers_for_file_mounts(
             common_file_mounts, cluster_info.provider_name))
+
+
+@common.log_function_start_end
+@timeline.event
+def setup_logging_on_cluster(logging_agent: logs.LoggingAgent,
+                             cluster_name: resources_utils.ClusterName,
+                             cluster_info: common.ClusterInfo,
+                             ssh_credentials: Dict[str, Any]) -> None:
+    """Setup logging agent (fluentbit) on all nodes after provisioning."""
+    _hint_worker_log_path(cluster_name.name_on_cloud, cluster_info,
+                          'logging_setup')
+
+    @_auto_retry()
+    def _setup_node(runner: command_runner.CommandRunner, log_path: str):
+        cmd = logging_agent.get_setup_command(cluster_name)
+        logger.info(f'Running command on node: {cmd}')
+        returncode, stdout, stderr = runner.run(cmd,
+                                                stream_logs=False,
+                                                require_outputs=True,
+                                                log_path=log_path,
+                                                source_bashrc=True)
+        if returncode:
+            raise RuntimeError(f'Failed to setup logging agent\n{cmd}\n'
+                               f'(exit code {returncode}). Error: '
+                               f'===== stdout ===== \n{stdout}\n'
+                               f'===== stderr ====={stderr}')
+
+    _parallel_ssh_with_cache(_setup_node,
+                             cluster_name.name_on_cloud,
+                             stage_name='logging_setup',
+                             digest=None,
+                             cluster_info=cluster_info,
+                             ssh_credentials=ssh_credentials)

sky/provision/provisioner.py

@@ -16,6 +16,7 @@ import sky
 from sky import clouds
 from sky import exceptions
 from sky import global_user_state
+from sky import logs
 from sky import provision
 from sky import sky_logging
 from sky import skypilot_config
@@ -648,6 +649,15 @@ def _post_provision_setup(
             logger.debug('Ray cluster is ready. Skip starting ray cluster on '
                          'worker nodes.')
 
+        logging_agent = logs.get_logging_agent()
+        if logging_agent:
+            status.update(
+                ux_utils.spinner_message('Setting up logging agent',
+                                         provision_logging.config.log_path))
+            instance_setup.setup_logging_on_cluster(logging_agent, cluster_name,
+                                                    cluster_info,
+                                                    ssh_credentials)
+
         instance_setup.start_skylet_on_head_node(cluster_name.name_on_cloud,
                                                  cluster_info, ssh_credentials)
 
@@ -672,6 +682,7 @@ def post_provision_runtime_setup(
        and other necessary files to the VM.
     3. Run setup commands to install dependencies.
     4. Start ray cluster and skylet.
+    5. (Optional) Setup logging agent.
 
     Raises:
         RuntimeError: If the setup process encounters any error.

sky/resources.py

@@ -225,7 +225,7 @@ class Resources:
           autostop: the autostop configuration to use. For launched resources,
             may or may not correspond to the actual current autostop config.
           priority: the priority for this resource configuration. Must be an
-            integer from 0 to 1000, where higher values indicate higher priority.
+            integer from -1000 to 1000, where higher values indicate higher priority.
             If None, no priority is set.
           volumes: the volumes to mount on the instance.
           _docker_login_config: the docker configuration to use. This includes
@@ -631,7 +631,7 @@ class Resources:
     def priority(self) -> Optional[int]:
         """The priority for this resource configuration.
 
-        Higher values indicate higher priority. Valid range is 0-1000.
+        Higher values indicate higher priority. Valid range is -1000 to 1000.
         """
         return self._priority
 
@@ -824,14 +824,15 @@ class Resources:
         """Sets the priority for this resource configuration.
 
         Args:
-            priority: Priority value from 0 to 1000, where higher values
+            priority: Priority value from -1000 to 1000, where higher values
                 indicate higher priority. If None, no priority is set.
         """
         if priority is not None:
-            if not 0 <= priority <= 1000:
+            if not constants.MIN_PRIORITY <= priority <= constants.MAX_PRIORITY:
                 with ux_utils.print_exception_no_traceback():
-                    raise ValueError(f'Priority must be between 0 and 1000. '
-                                     f'Found: {priority}')
+                    raise ValueError(
+                        f'Priority must be between {constants.MIN_PRIORITY} and'
+                        f' {constants.MAX_PRIORITY}. Found: {priority}')
         self._priority = priority
 
     def _set_volumes(

sky/serve/server/core.py

@@ -297,6 +297,8 @@ def up(
             assert task.service is not None
             protocol = ('http'
                         if task.service.tls_credential is None else 'https')
+            socket_endpoint = socket_endpoint.replace('https://', '').replace(
+                'http://', '')
             endpoint = f'{protocol}://{socket_endpoint}'
 
         logger.info(
@@ -716,6 +718,9 @@ def status(
             else:
                 protocol = ('https'
                             if service_record['tls_encrypted'] else 'http')
+                if endpoint is not None:
+                    endpoint = endpoint.replace('https://',
+                                                '').replace('http://', '')
                 service_record['endpoint'] = f'{protocol}://{endpoint}'
 
     return service_records

sky/server/common.py

@@ -13,7 +13,7 @@ import subprocess
 import sys
 import time
 import typing
-from typing import Any, Dict, Literal, Optional
+from typing import Any, Dict, Literal, Optional, Tuple
 from urllib import parse
 import uuid
 
@@ -128,6 +128,8 @@ class ApiServerInfo:
     version: Optional[str] = None
     version_on_disk: Optional[str] = None
     commit: Optional[str] = None
+    user: Optional[Dict[str, Any]] = None
+    basic_auth_enabled: bool = False
 
 
 def get_api_cookie_jar_path() -> pathlib.Path:
@@ -261,11 +263,15 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
             version = result.get('version')
             version_on_disk = result.get('version_on_disk')
             commit = result.get('commit')
+            user = result.get('user')
+            basic_auth_enabled = result.get('basic_auth_enabled')
             server_info = ApiServerInfo(status=ApiServerStatus.HEALTHY,
                                         api_version=api_version,
                                         version=version,
                                         version_on_disk=version_on_disk,
-                                        commit=commit)
+                                        commit=commit,
+                                        user=user,
+                                        basic_auth_enabled=basic_auth_enabled)
             if api_version is None or version is None or commit is None:
                 logger.warning(f'API server response missing '
                                f'version info. {server_url} may '
@@ -320,7 +326,8 @@ def get_request_id(response: 'requests.Response') -> RequestId:
 
 def _start_api_server(deploy: bool = False,
                       host: str = '127.0.0.1',
-                      foreground: bool = False):
+                      foreground: bool = False,
+                      enable_basic_auth: bool = False):
     """Starts a SkyPilot API server locally."""
     server_url = get_server_url(host)
     assert server_url in AVAILABLE_LOCAL_API_SERVER_URLS, (
@@ -354,6 +361,8 @@ def _start_api_server(deploy: bool = False,
         if foreground:
             # Replaces the current process with the API server
             os.environ[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
+            if enable_basic_auth:
+                os.environ[constants.ENV_VAR_ENABLE_BASIC_AUTH] = 'true'
             os.execvp(args[0], args)
 
         log_path = os.path.expanduser(constants.API_SERVER_LOGS)
@@ -365,6 +374,8 @@ def _start_api_server(deploy: bool = False,
         # the API server.
         server_env = os.environ.copy()
         server_env[constants.ENV_VAR_IS_SKYPILOT_SERVER] = 'true'
+        if enable_basic_auth:
+            server_env[constants.ENV_VAR_ENABLE_BASIC_AUTH] = 'true'
         with open(log_path, 'w', encoding='utf-8') as log_file:
             # Because the log file is opened using a with statement, it may seem
             # that the file will be closed when the with statement is exited
@@ -428,10 +439,10 @@ def _start_api_server(deploy: bool = False,
 
 def check_server_healthy(
     endpoint: Optional[str] = None
-) -> Literal[
+) -> Tuple[Literal[
         # Use an incomplete list of Literals here to enforce raising for other
         # enum values.
-        ApiServerStatus.HEALTHY, ApiServerStatus.NEEDS_AUTH]:
+        ApiServerStatus.HEALTHY, ApiServerStatus.NEEDS_AUTH], ApiServerInfo]:
     """Check if the API server is healthy.
 
     Args:
@@ -508,7 +519,7 @@ def check_server_healthy(
 
         hinted_for_server_install_version_mismatch = True
 
-    return api_server_status
+    return api_server_status, api_server_info
 
 
 def _get_version_info_hint(server_info: ApiServerInfo) -> str:
@@ -559,10 +570,11 @@ def get_skypilot_version_on_disk() -> str:
 
 def check_server_healthy_or_start_fn(deploy: bool = False,
                                      host: str = '127.0.0.1',
-                                     foreground: bool = False):
+                                     foreground: bool = False,
+                                     enable_basic_auth: bool = False):
     api_server_status = None
     try:
-        api_server_status = check_server_healthy()
+        api_server_status, _ = check_server_healthy()
         if api_server_status == ApiServerStatus.NEEDS_AUTH:
             endpoint = get_server_url()
             with ux_utils.print_exception_no_traceback():
@@ -580,7 +592,7 @@ def check_server_healthy_or_start_fn(deploy: bool = False,
             # have started the server while we were waiting for the lock.
             api_server_info = get_api_server_status(endpoint)
             if api_server_info.status == ApiServerStatus.UNHEALTHY:
-                _start_api_server(deploy, host, foreground)
+                _start_api_server(deploy, host, foreground, enable_basic_auth)
 
 
 def check_server_healthy_or_start(func):

sky/server/requests/payloads.py

@@ -336,10 +336,28 @@ class ClusterJobsDownloadLogsBody(RequestBody):
     local_dir: str = constants.SKY_LOGS_DIRECTORY
 
 
+class UserCreateBody(RequestBody):
+    """The request body for the user create endpoint."""
+    username: str
+    password: str
+    role: Optional[str] = None
+
+
+class UserDeleteBody(RequestBody):
+    """The request body for the user delete endpoint."""
+    user_id: str
+
+
 class UserUpdateBody(RequestBody):
     """The request body for the user update endpoint."""
     user_id: str
-    role: str
+    role: Optional[str] = None
+    password: Optional[str] = None
+
+
+class UserImportBody(RequestBody):
+    """The request body for the user import endpoint."""
+    csv_content: str
 
 
 class DownloadBody(RequestBody):