skypilot-nightly 1.0.0.dev20250502__py3-none-any.whl → 1.0.0.dev20251203__py3-none-any.whl

sky/logs/__init__.py

@@ -0,0 +1,21 @@
+"""Sky logging agents."""
+from typing import Optional
+
+from sky import exceptions
+from sky import skypilot_config
+from sky.logs.agent import LoggingAgent
+from sky.logs.aws import CloudwatchLoggingAgent
+from sky.logs.gcp import GCPLoggingAgent
+
+
+def get_logging_agent() -> Optional[LoggingAgent]:
+    store = skypilot_config.get_nested(('logs', 'store'), None)
+    if store is None:
+        return None
+    if store == 'gcp':
+        return GCPLoggingAgent(skypilot_config.get_nested(('logs', 'gcp'), {}))
+    elif store == 'aws':
+        return CloudwatchLoggingAgent(
+            skypilot_config.get_nested(('logs', 'aws'), {}))
+    raise exceptions.InvalidSkyPilotConfigError(
+        f'Invalid logging store: {store}')

sky/logs/agent.py

@@ -0,0 +1,108 @@
+"""Base class for all logging agents."""
+import abc
+import os
+import shlex
+from typing import Any, Dict
+
+from sky.skylet import constants
+from sky.utils import resources_utils
+from sky.utils import yaml_utils
+
+
+class LoggingAgent(abc.ABC):
+    """Base class for all logging agents.
+
+    Each agent should implement the `get_setup_command` and
+    `get_credential_file_mounts` methods to return the setup command and
+    credential file mounts for the agent for provisioner to setup the agent
+    on each node.
+    """
+
+    @abc.abstractmethod
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        pass
+
+    @abc.abstractmethod
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        pass
+
+
+class FluentbitAgent(LoggingAgent):
+    """Base class for logging store that use fluentbit as the agent."""
+
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        install_cmd = (
+            # pylint: disable=line-too-long
+            'if ! command -v fluent-bit >/dev/null 2>&1 && [ ! -f /opt/fluent-bit/bin/fluent-bit ]; then '
+            'sudo apt-get update; sudo apt-get install -y gnupg; '
+            # pylint: disable=line-too-long
+            'sudo sh -c \'curl -L https://packages.fluentbit.io/fluentbit.key | gpg --dearmor > /usr/share/keyrings/fluentbit-keyring.gpg\'; '
+            # pylint: disable=line-too-long
+            'os_id=$(grep -oP \'(?<=^ID=).*\' /etc/os-release 2>/dev/null || lsb_release -is 2>/dev/null | tr \'[:upper:]\' \'[:lower:]\'); '
+            # pylint: disable=line-too-long
+            'codename=$(grep -oP \'(?<=VERSION_CODENAME=).*\' /etc/os-release 2>/dev/null || lsb_release -cs 2>/dev/null); '
+            # pylint: disable=line-too-long
+            'echo "deb [signed-by=/usr/share/keyrings/fluentbit-keyring.gpg] https://packages.fluentbit.io/$os_id/$codename $codename main" | sudo tee /etc/apt/sources.list.d/fluent-bit.list; '
+            'sudo apt-get update; '
+            'sudo apt-get install -y fluent-bit; '
+            'fi')
+        cfg = self.fluentbit_config(cluster_name)
+        cfg_path = os.path.join(constants.LOGGING_CONFIG_DIR, 'fluentbit.yaml')
+        config_cmd = (f'mkdir -p {constants.LOGGING_CONFIG_DIR} && '
+                      f'echo {shlex.quote(cfg)} > {cfg_path}')
+        kill_prior_cmd = (
+            'if [ -f "/tmp/fluentbit.pid" ]; then '
+            # pylint: disable=line-too-long
+            'echo "Killing prior fluent-bit process $(cat /tmp/fluentbit.pid)"; '
+            'kill "$(cat /tmp/fluentbit.pid)" || true; '
+            'fi')
+        start_cmd = ('nohup $(command -v fluent-bit || '
+                     'echo "/opt/fluent-bit/bin/fluent-bit") '
+                     f'-c {cfg_path} > /tmp/fluentbit.log 2>&1 & '
+                     'echo $! > /tmp/fluentbit.pid')
+        return ('set -e; '
+                f'{install_cmd}; '
+                f'{config_cmd}; '
+                f'{kill_prior_cmd}; '
+                f'{start_cmd}')
+
+    def fluentbit_config(self,
+                         cluster_name: resources_utils.ClusterName) -> str:
+        cfg_dict = {
+            'parsers': [{
+                'name': 'sky-ray-parser',
+                'format': 'regex',
+                # pylint: disable=line-too-long
+                'regex': r'(?:\x1b\[[\d;]+m)?\((?<worker_name>[^,]+)(?:,\s*rank=(?<rank>\d+))?(?:,\s*pid=(?<pid>\d+))(?:,\s*ip=(?<ip>[\d.]+))?\)(?:\x1b\[[\d;]+m)?\s*(?<log_line>.*)',
+                'types': 'rank:integer pid:integer',
+            }],
+            'pipeline': {
+                'inputs': [{
+                    'name': 'tail',
+                    'path': f'{constants.SKY_LOGS_DIRECTORY}/*/*.log',
+                    'path_key': 'log_path',
+                    # Shorten the refresh interval from 60s to 1s since every
+                    # job creates a new log file and we must be responsive
+                    # for this: the VM might be autodown within a minute
+                    # right after the job completion.
+                    'refresh_interval': 1,
+                }],
+                'filters': [{
+                    'name': 'parser',
+                    'match': '*',
+                    'key_name': 'log',
+                    'parser': 'sky-ray-parser',
+                    'preserve_key': 'on',  # preserve field for backwards compat
+                    'reserve_data': 'on',
+                }],
+                'outputs': [self.fluentbit_output_config(cluster_name)],
+            }
+        }
+        return yaml_utils.dump_yaml_str(cfg_dict)
+
+    @abc.abstractmethod
+    def fluentbit_output_config(
+            self, cluster_name: resources_utils.ClusterName) -> Dict[str, Any]:
+        pass

sky/logs/aws.py

@@ -0,0 +1,243 @@
+"""AWS CloudWatch logging agent."""
+
+from typing import Any, Dict, Optional
+
+import pydantic
+
+from sky.logs.agent import FluentbitAgent
+from sky.utils import resources_utils
+from sky.utils import yaml_utils
+
+EC2_MD_URL = '"${AWS_EC2_METADATA_SERVICE_ENDPOINT:-http://169.254.169.254/}"'
+
+
+class _CloudwatchLoggingConfig(pydantic.BaseModel):
+    """Configuration for AWS CloudWatch logging agent."""
+    region: Optional[str] = None
+    credentials_file: Optional[str] = None
+    log_group_name: str = 'skypilot-logs'
+    log_stream_prefix: str = 'skypilot-'
+    auto_create_group: bool = True
+    additional_tags: Optional[Dict[str, str]] = None
+
+
+class _CloudWatchOutputConfig(pydantic.BaseModel):
+    """Auxiliary model for building CloudWatch output config in YAML.
+
+    Ref: https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch
+    """
+    name: str = 'cloudwatch_logs'
+    match: str = '*'
+    region: Optional[str] = None
+    log_group_name: Optional[str] = None
+    log_stream_prefix: Optional[str] = None
+    auto_create_group: bool = True
+    additional_tags: Optional[Dict[str, str]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        config = self.model_dump(exclude_none=True)
+        if 'auto_create_group' in config:
+            config['auto_create_group'] = 'true' if config[
+                'auto_create_group'] else 'false'
+        return config
+
+
+class CloudwatchLoggingAgent(FluentbitAgent):
+    """AWS CloudWatch logging agent.
+
+    This agent forwards logs from SkyPilot clusters to AWS CloudWatch using
+    Fluent Bit. It supports authentication via IAM roles (preferred), AWS
+    credentials file, or environment variables.
+
+    Example configuration:
+    ```yaml
+    logs:
+      store: aws
+      aws:
+        region: us-west-2
+        log_group_name: skypilot-logs
+        log_stream_prefix: my-cluster-
+        auto_create_group: true
+    ```
+    """
+
+    def __init__(self, config: Dict[str, Any]):
+        """Initialize the CloudWatch logging agent.
+
+        Args:
+            config: The configuration for the CloudWatch logging agent.
+                   See the class docstring for the expected format.
+        """
+        self.config = _CloudwatchLoggingConfig(**config)
+        super().__init__()
+
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        """Get the command to set up the CloudWatch logging agent.
+
+        Args:
+            cluster_name: The name of the cluster.
+
+        Returns:
+            The command to set up the CloudWatch logging agent.
+        """
+
+        if self.config.credentials_file:
+            credential_path = self.config.credentials_file
+
+        # Set AWS credentials and check whether credentials are valid.
+        # CloudWatch plugin supports IAM roles, credentials file, and
+        # environment variables. We prefer IAM roles when available
+        # (on EC2 instances). If credentials file is provided, we use
+        # it. Otherwise, we check if credentials are available in
+        # the environment.
+        pre_cmd = ''
+        if self.config.credentials_file:
+            pre_cmd = (
+                f'export AWS_SHARED_CREDENTIALS_FILE={credential_path}; '
+                f'if [ ! -f {credential_path} ]; then '
+                f'echo "ERROR: AWS credentials file {credential_path} '
+                f'not found. Please check if the file exists and is '
+                f'accessible." && exit 1; '
+                f'fi; '
+                f'if ! grep -q "\\[.*\\]" {credential_path} || '
+                f'! grep -q "aws_access_key_id" {credential_path}; then '
+                f'echo "ERROR: AWS credentials file {credential_path} is '
+                f'invalid. It should contain a profile section '
+                f'[profile_name] and aws_access_key_id." && exit 1; '
+                f'fi;')
+        else:
+            # Check if we're running on EC2 with an IAM role or if
+            # AWS credentials are available in the environment
+            pre_cmd = (
+                f'if ! curl -s -m 1 {EC2_MD_URL}'
+                'latest/meta-data/iam/security-credentials/ > /dev/null; '
+                'then '
+                # failed EC2 check, look for env vars
+                'if [ -z "$AWS_ACCESS_KEY_ID" ] || '
+                '[ -z "$AWS_SECRET_ACCESS_KEY" ]; then '
+                'echo "ERROR: AWS CloudWatch logging configuration error. '
+                'Not running on EC2 with IAM role and AWS credentials not '
+                'found in environment. Please do one of the following: '
+                '1. Run on an EC2 instance with an IAM role that has '
+                'CloudWatch permissions, 2. Set AWS_ACCESS_KEY_ID and '
+                'AWS_SECRET_ACCESS_KEY environment variables, or '
+                '3. Provide a credentials file via logs.aws.credentials_file '
+                'in SkyPilot config." && exit 1; '
+                'fi; '
+                'fi;')
+
+        # If region is specified, set it in the environment
+        if self.config.region:
+            pre_cmd += (f' export AWS_REGION={self.config.region}'
+                        f' AWS_DEFAULT_REGION={self.config.region};'
+                        ' command -v aws &>/dev/null && '
+                        f'aws configure set region {self.config.region};')
+        else:
+            # If region is not specified, check if it's available in
+            # the environment or credentials file
+            pre_cmd += (
+                ' if [ -z "$AWS_REGION" ] && '
+                '[ -z "$AWS_DEFAULT_REGION" ]; then '
+                'echo "WARNING: AWS region not specified in configuration or '
+                'environment. CloudWatch logging may fail if the region '
+                'cannot be determined. Consider setting logs.aws.region in '
+                'SkyPilot config."; '
+                'fi; ')
+
+        # Add a test command to verify AWS credentials work with CloudWatch
+        pre_cmd += (
+            ' echo "Verifying AWS CloudWatch access..."; '
+            'if command -v aws > /dev/null; then '
+            'aws cloudwatch list-metrics --namespace AWS/Logs --max-items 1 '
+            '> /dev/null 2>&1 || '
+            '{ echo "ERROR: Failed to access AWS CloudWatch. Please check '
+            'your credentials and permissions."; '
+            'echo "The IAM role or user must have cloudwatch:ListMetrics '
+            'and logs:* permissions."; '
+            'exit 1; }; '
+            'else echo "AWS CLI not installed, skipping CloudWatch access '
+            'verification."; '
+            'fi; ')
+
+        return pre_cmd + ' ' + super().get_setup_command(cluster_name)
+
+    def fluentbit_config(self,
+                         cluster_name: resources_utils.ClusterName) -> str:
+        """Get the Fluent Bit configuration for CloudWatch.
+
+        This overrides the base method to add a fallback output for local file
+        logging in case CloudWatch logging fails.
+
+        Args:
+            cluster_name: The name of the cluster.
+
+        Returns:
+            The Fluent Bit configuration as a YAML string.
+        """
+        cfg_dict = yaml_utils.read_yaml_str(
+            super().fluentbit_config(cluster_name))
+        display_name = cluster_name.display_name
+        unique_name = cluster_name.name_on_cloud
+        # Build tags for the log stream
+        tags = {
+            'skypilot.cluster_name': display_name,
+            'skypilot.cluster_id': unique_name,
+        }
+
+        # Add additional tags if provided
+        if self.config.additional_tags:
+            tags.update(self.config.additional_tags)
+
+        log_processors = []
+        for key, value in tags.items():
+            log_processors.append({
+                'name': 'content_modifier',
+                'action': 'upsert',
+                'key': key,
+                'value': value
+            })
+
+        # Add log processors to config
+        processors_config = cfg_dict['pipeline']['inputs'][0].get(
+            'processors', {})
+        processors_logs_config = processors_config.get('logs', [])
+        processors_logs_config.extend(log_processors)
+        processors_config['logs'] = processors_logs_config
+        cfg_dict['pipeline']['inputs'][0]['processors'] = processors_config
+
+        return yaml_utils.dump_yaml_str(cfg_dict)
+
+    def fluentbit_output_config(
+            self, cluster_name: resources_utils.ClusterName) -> Dict[str, Any]:
+        """Get the Fluent Bit output configuration for CloudWatch.
+
+        Args:
+            cluster_name: The name of the cluster.
+
+        Returns:
+            The Fluent Bit output configuration for CloudWatch.
+        """
+        unique_name = cluster_name.name_on_cloud
+
+        # Format the log stream name to include cluster information
+        # This helps with identifying logs in CloudWatch
+        log_stream_prefix = f'{self.config.log_stream_prefix}{unique_name}-'
+
+        # Create the CloudWatch output configuration with error handling options
+        return _CloudWatchOutputConfig(
+            region=self.config.region,
+            log_group_name=self.config.log_group_name,
+            log_stream_prefix=log_stream_prefix,
+            auto_create_group=self.config.auto_create_group,
+        ).to_dict()
+
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        """Get the credential file mounts for the CloudWatch logging agent.
+
+        Returns:
+            A dictionary mapping local credential file paths to remote paths.
+        """
+        if self.config.credentials_file:
+            return {self.config.credentials_file: self.config.credentials_file}
+        return {}

sky/logs/gcp.py

@@ -0,0 +1,91 @@
+"""GCP logging agent."""
+
+from typing import Any, Dict, Optional
+
+import pydantic
+
+from sky.clouds import gcp
+from sky.logs.agent import FluentbitAgent
+from sky.utils import resources_utils
+
+
+class _GCPLoggingConfig(pydantic.BaseModel):
+    """Configuration for GCP logging agent."""
+    project_id: Optional[str] = None
+    credentials_file: Optional[str] = None
+    additional_labels: Optional[Dict[str, str]] = None
+
+
+class _StackdriverOutputConfig(pydantic.BaseModel):
+    """Auxiliary model for building stackdriver output config in YAML.
+
+    Ref: https://docs.fluentbit.io/manual/1.7/pipeline/outputs/stackdriver
+    """
+    name: str = 'stackdriver'
+    match: str = '*'
+    export_to_project_id: Optional[str] = None
+    labels: Optional[Dict[str, str]] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        config = self.model_dump(exclude_none=True)
+        if self.labels:
+            # Replace the label format from `{k: v}` to `k=v`
+            label_str = ','.join([f'{k}={v}' for k, v in self.labels.items()])
+            config['labels'] = label_str
+        return config
+
+
+class GCPLoggingAgent(FluentbitAgent):
+    """GCP logging agent."""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.config = _GCPLoggingConfig(**config)
+
+    def get_setup_command(self,
+                          cluster_name: resources_utils.ClusterName) -> str:
+        credential_path = gcp.DEFAULT_GCP_APPLICATION_CREDENTIAL_PATH
+        if self.config.credentials_file:
+            credential_path = self.config.credentials_file
+        # Set GOOGLE_APPLICATION_CREDENTIALS and check whether credentials
+        # is valid.
+        # Stackdriver only support service account credentials or credentials
+        # from metadata server (only available on GCE or GKE). If the default
+        # credentials uploaded by API server is NOT a service account key and
+        # there is NO metadata server available, the logging agent will fail to
+        # authenticate and we require the user to upload a service account key
+        # via logs.gcp.credentials_file in this case.
+        # Also note that we use env var instead of YAML config to specify the
+        # service account key file path in order to resolve the home directory
+        # more reliably.
+        # Ref: https://github.com/fluent/fluent-bit/issues/8804
+        # TODO(aylei): check whether the credentials config is valid before
+        # provision.
+        pre_cmd = (f'export GOOGLE_APPLICATION_CREDENTIALS={credential_path}; '
+                   f'cat {credential_path} | grep "service_account" || '
+                   f'(echo "Credentials file {credential_path} is not a '
+                   'service account key, check metadata server" && '
+                   'curl -s http://metadata.google.internal >/dev/null || '
+                   f'(echo "Neither service account key nor metadata server is '
+                   'available. Set logs.gcp.credentials_file to a service '
+                   'account key in server config and retry." && '
+                   'exit 1;))')
+        return pre_cmd + ' && ' + super().get_setup_command(cluster_name)
+
+    def fluentbit_output_config(
+            self, cluster_name: resources_utils.ClusterName) -> Dict[str, Any]:
+        display_name = cluster_name.display_name
+        unique_name = cluster_name.name_on_cloud
+
+        return _StackdriverOutputConfig(
+            export_to_project_id=self.config.project_id,
+            labels={
+                'skypilot_cluster_name': display_name,
+                'skypilot_cluster_id': unique_name,
+                **(self.config.additional_labels or {})
+            },
+        ).to_dict()
+
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        if self.config.credentials_file:
+            return {self.config.credentials_file: self.config.credentials_file}
+        return {}