skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

sky/utils/auth_utils.py

@@ -0,0 +1,153 @@
+"""Utils for managing SkyPilot SSH key pairs."""
+
+import functools
+import os
+from typing import Tuple
+
+import filelock
+
+from sky import global_user_state
+from sky import sky_logging
+from sky.utils import common_utils
+
+logger = sky_logging.init_logger(__name__)
+
+MAX_TRIALS = 64
+# TODO(zhwu): Support user specified key pair.
+# We intentionally not have the ssh key pair to be stored in
+# ~/.sky/api_server/clients, i.e. sky.server.common.API_SERVER_CLIENT_DIR,
+# because ssh key pair need to persist across API server restarts, while
+# the former dir is ephemeral.
+_SSH_KEY_PATH_PREFIX = '~/.sky/clients/{user_hash}/ssh'
+
+
+def get_ssh_key_and_lock_path(user_hash: str) -> Tuple[str, str, str]:
+    user_ssh_key_prefix = _SSH_KEY_PATH_PREFIX.format(user_hash=user_hash)
+
+    os.makedirs(os.path.expanduser(user_ssh_key_prefix),
+                exist_ok=True,
+                mode=0o700)
+    private_key_path = os.path.join(user_ssh_key_prefix, 'sky-key')
+    public_key_path = os.path.join(user_ssh_key_prefix, 'sky-key.pub')
+    lock_path = os.path.join(user_ssh_key_prefix, '.__internal-sky-key.lock')
+    return private_key_path, public_key_path, lock_path
+
+
+def _generate_rsa_key_pair() -> Tuple[str, str]:
+    # Keep the import of the cryptography local to avoid expensive
+    # third-party imports when not needed.
+    # pylint: disable=import-outside-toplevel
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives import serialization
+    from cryptography.hazmat.primitives.asymmetric import rsa
+
+    key = rsa.generate_private_key(backend=default_backend(),
+                                   public_exponent=65537,
+                                   key_size=2048)
+
+    private_key = key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption()).decode(
+            'utf-8').strip()
+
+    public_key = key.public_key().public_bytes(
+        serialization.Encoding.OpenSSH,
+        serialization.PublicFormat.OpenSSH).decode('utf-8').strip()
+
+    return public_key, private_key
+
+
+def _save_key_pair(private_key_path: str, public_key_path: str,
+                   private_key: str, public_key: str) -> None:
+    key_dir = os.path.dirname(private_key_path)
+    os.makedirs(key_dir, exist_ok=True, mode=0o700)
+
+    with open(
+            private_key_path,
+            'w',
+            encoding='utf-8',
+            opener=functools.partial(os.open, mode=0o600),
+    ) as f:
+        f.write(private_key)
+
+    with open(public_key_path,
+              'w',
+              encoding='utf-8',
+              opener=functools.partial(os.open, mode=0o644)) as f:
+        f.write(public_key)
+
+
+def get_or_generate_keys() -> Tuple[str, str]:
+    """Returns the absolute private and public key paths."""
+    user_hash = common_utils.get_user_hash()
+    private_key_path, public_key_path, lock_path = get_ssh_key_and_lock_path(
+        user_hash)
+    private_key_path = os.path.expanduser(private_key_path)
+    public_key_path = os.path.expanduser(public_key_path)
+    lock_path = os.path.expanduser(lock_path)
+
+    lock_dir = os.path.dirname(lock_path)
+    # We should have the folder ~/.sky/generated/ssh to have 0o700 permission,
+    # as the ssh configs will be written to this folder as well in
+    # backend_utils.SSHConfigHelper
+    os.makedirs(lock_dir, exist_ok=True, mode=0o700)
+    with filelock.FileLock(lock_path, timeout=10):
+        if not os.path.exists(private_key_path):
+            ssh_public_key, ssh_private_key, exists = (
+                global_user_state.get_ssh_keys(user_hash))
+            if not exists:
+                ssh_public_key, ssh_private_key = _generate_rsa_key_pair()
+                global_user_state.set_ssh_keys(user_hash, ssh_public_key,
+                                               ssh_private_key)
+            _save_key_pair(private_key_path, public_key_path, ssh_private_key,
+                           ssh_public_key)
+    assert os.path.exists(public_key_path), (
+        'Private key found, but associated public key '
+        f'{public_key_path} does not exist.')
+    return private_key_path, public_key_path
+
+
+def create_ssh_key_files_from_db(private_key_path: str) -> bool:
+    """Creates the ssh key files from the database.
+
+    Returns:
+        True if the ssh key files are created successfully, False otherwise.
+    """
+    # Assume private key path is in the format of
+    # ~/.sky/clients/<user_hash>/ssh/sky-key
+    separated_path = os.path.normpath(private_key_path).split(os.path.sep)
+    assert separated_path[-1] == 'sky-key'
+    assert separated_path[-2] == 'ssh'
+    user_hash = separated_path[-3]
+
+    private_key_path_generated, public_key_path, lock_path = (
+        get_ssh_key_and_lock_path(user_hash))
+    assert private_key_path == os.path.expanduser(private_key_path_generated), (
+        f'Private key path {private_key_path} does not '
+        'match the generated path '
+        f'{os.path.expanduser(private_key_path_generated)}')
+    private_key_path = os.path.expanduser(private_key_path)
+    public_key_path = os.path.expanduser(public_key_path)
+    lock_path = os.path.expanduser(lock_path)
+    lock_dir = os.path.dirname(lock_path)
+
+    if os.path.exists(private_key_path) and os.path.exists(public_key_path):
+        return True
+    # We should have the folder ~/.sky/generated/ssh to have 0o700 permission,
+    # as the ssh configs will be written to this folder as well in
+    # backend_utils.SSHConfigHelper
+    os.makedirs(lock_dir, exist_ok=True, mode=0o700)
+    with filelock.FileLock(lock_path, timeout=10):
+        if not os.path.exists(private_key_path):
+            ssh_public_key, ssh_private_key, exists = (
+                global_user_state.get_ssh_keys(user_hash))
+            if not exists:
+                logger.debug(f'SSH keys not found for user {user_hash}')
+                return False
+            _save_key_pair(private_key_path, public_key_path, ssh_private_key,
+                           ssh_public_key)
+    assert os.path.exists(public_key_path), (
+        'Private key found, but associated public key '
+        f'{public_key_path} does not exist.')
+    return True

sky/utils/benchmark_utils.py

@@ -0,0 +1,60 @@
+"""Utility functions for benchmarking."""
+
+import functools
+import logging
+import time
+from typing import Callable, Optional
+
+from sky import sky_logging
+
+logger = sky_logging.init_logger(__name__)
+
+
+def log_execution_time(func: Optional[Callable] = None,
+                       *,
+                       name: Optional[str] = None,
+                       level: int = logging.DEBUG,
+                       precision: int = 4) -> Callable:
+    """Mark a function and log its execution time.
+
+    Args:
+        func: Function to decorate.
+        name: Name of the function.
+        level: Logging level.
+        precision: Number of decimal places (default: 4).
+
+    Usage:
+        from sky.utils import benchmark_utils
+
+        @benchmark_utils.log_execution_time
+        def my_function():
+            pass
+
+        @benchmark_utils.log_execution_time(name='my_module.my_function2')
+        def my_function2():
+            pass
+    """
+
+    def decorator(f: Callable) -> Callable:
+
+        @functools.wraps(f)
+        def wrapper(*args, **kwargs):
+            nonlocal name
+            name = name or f.__name__
+            start_time = time.perf_counter()
+            try:
+                result = f(*args, **kwargs)
+                return result
+            finally:
+                end_time = time.perf_counter()
+                execution_time = end_time - start_time
+                log = (f'Method {name} executed in '
+                       f'{execution_time:.{precision}f}')
+                logger.log(level, log)
+
+        return wrapper
+
+    if func is None:
+        return decorator
+    else:
+        return decorator(func)

sky/utils/cli_utils/status_utils.py

@@ -6,11 +6,12 @@ import click
 import colorama
 
 from sky import backends
+from sky.schemas.api import responses
 from sky.utils import common_utils
-from sky.utils import controller_utils
 from sky.utils import log_utils
 from sky.utils import resources_utils
 from sky.utils import status_lib
+from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
     from sky.provision.kubernetes import utils as kubernetes_utils
@@ -33,24 +34,23 @@ class StatusColumn:
     def __init__(self,
                  name: str,
                  calc_func: Callable,
-                 trunc_length: int = 0,
+                 truncate: bool = True,
                  show_by_default: bool = True):
         self.name = name
         self.calc_func = calc_func
-        self.trunc_length = trunc_length
+        self.truncate: bool = truncate
         self.show_by_default = show_by_default
 
     def calc(self, record):
-        val = self.calc_func(record)
-        if self.trunc_length != 0:
-            val = common_utils.truncate_long_string(str(val), self.trunc_length)
+        val = self.calc_func(record, self.truncate)
         return val
 
 
-def show_status_table(cluster_records: List[_ClusterRecord],
+def show_status_table(cluster_records: List[responses.StatusResponse],
                       show_all: bool,
                       show_user: bool,
-                      query_clusters: Optional[List[str]] = None) -> int:
+                      query_clusters: Optional[List[str]] = None,
+                      show_workspaces: bool = False) -> int:
     """Compute cluster table values and display.
 
     Returns:
@@ -58,7 +58,6 @@ def show_status_table(cluster_records: List[_ClusterRecord],
         STOPPED.
     """
     # TODO(zhwu): Update the information for autostop clusters.
-
     status_columns = [
         StatusColumn('NAME', _get_name),
     ]
@@ -68,19 +67,24 @@ def show_status_table(cluster_records: List[_ClusterRecord],
             StatusColumn('USER_ID', _get_user_hash, show_by_default=False))
 
     status_columns += [
-        StatusColumn('LAUNCHED', _get_launched),
-        StatusColumn('RESOURCES',
-                     _get_resources,
-                     trunc_length=70 if not show_all else 0),
-        StatusColumn('REGION', _get_region, show_by_default=False),
-        StatusColumn('ZONE', _get_zone, show_by_default=False),
+        StatusColumn('WORKSPACE',
+                     _get_workspace,
+                     show_by_default=show_workspaces),
+        StatusColumn('INFRA', _get_infra, truncate=not show_all),
+        StatusColumn('RESOURCES', _get_resources, truncate=not show_all),
         StatusColumn('STATUS', _get_status_colored),
         StatusColumn('AUTOSTOP', _get_autostop),
-        StatusColumn('HEAD_IP', _get_head_ip, show_by_default=False),
-        StatusColumn('COMMAND',
-                     _get_command,
-                     trunc_length=COMMAND_TRUNC_LENGTH if not show_all else 0),
+        StatusColumn('LAUNCHED', _get_launched),
     ]
+    if show_all:
+        status_columns += [
+            StatusColumn('HEAD_IP', _get_head_ip, show_by_default=False),
+            StatusColumn('COMMAND',
+                         _get_command,
+                         truncate=not show_all,
+                         show_by_default=False),
+            StatusColumn('LAST_EVENT', _get_last_event, show_by_default=False),
+        ]
 
     columns = []
     for status_column in status_columns:
@@ -102,17 +106,16 @@ def show_status_table(cluster_records: List[_ClusterRecord],
 
     if query_clusters:
         cluster_names = {record['name'] for record in cluster_records}
-        not_found_clusters = [
-            repr(cluster)
-            for cluster in query_clusters
-            if cluster not in cluster_names
-        ]
-        cluster_str = 'Cluster'
-        if len(not_found_clusters) > 1:
-            cluster_str += 's'
-        cluster_str += ' '
-        cluster_str += ', '.join(not_found_clusters)
-        click.echo(f'{cluster_str} not found.')
+        not_found_clusters = ux_utils.get_non_matched_query(
+            query_clusters, cluster_names)
+        not_found_clusters = [repr(cluster) for cluster in not_found_clusters]
+        if not_found_clusters:
+            cluster_str = 'Cluster'
+            if len(not_found_clusters) > 1:
+                cluster_str += 's'
+            cluster_str += ' '
+            cluster_str += ', '.join(not_found_clusters)
+            click.echo(f'{cluster_str} not found.')
     elif not cluster_records:
         click.echo('No existing clusters.')
     return num_pending_autostop
@@ -134,7 +137,8 @@ def get_total_cost_of_displayed_records(
 
 def show_cost_report_table(cluster_records: List[_ClusterCostReportRecord],
                            show_all: bool,
-                           controller_name: Optional[str] = None):
+                           controller_name: Optional[str] = None,
+                           days: Optional[int] = None):
     """Compute cluster table values and display for cost report.
 
     For each cluster, this shows: cluster name, resources, launched time,
@@ -160,10 +164,10 @@ def show_cost_report_table(cluster_records: List[_ClusterCostReportRecord],
     status_columns = [
         StatusColumn('NAME', _get_name),
         StatusColumn('LAUNCHED', _get_launched),
-        StatusColumn('DURATION', _get_duration, trunc_length=20),
+        StatusColumn('DURATION', _get_duration, truncate=False),
         StatusColumn('RESOURCES',
                      _get_resources_for_cost_report,
-                     trunc_length=70 if not show_all else 0),
+                     truncate=False),
         StatusColumn('STATUS',
                      _get_status_for_cost_report,
                      show_by_default=True),
@@ -197,22 +201,21 @@ def show_cost_report_table(cluster_records: List[_ClusterCostReportRecord],
         cluster_table.add_row(row)
 
     if cluster_records:
+        controller_record = cluster_records[0]
         if controller_name is not None:
-            controller = controller_utils.Controllers.from_name(controller_name)
-            if controller is None:
-                raise ValueError(f'Controller {controller_name} not found.')
-            autostop_minutes, _ = (
-                controller_utils.get_controller_autostop_config(
-                    controller=controller))
-            if autostop_minutes is not None:
+            autostop = controller_record.get('autostop', None)
+            autostop_str = ''
+            if autostop is not None:
                 autostop_str = (f'{colorama.Style.DIM} (will be autostopped if '
-                                f'idle for {autostop_minutes}min)'
+                                f'idle for {autostop}min)'
                                 f'{colorama.Style.RESET_ALL}')
             click.echo(f'\n{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
                        f'{controller_name}{colorama.Style.RESET_ALL}'
                        f'{autostop_str}')
         else:
-            click.echo(f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}Clusters'
+            days_str = '' if days is None else f' (last {days} days)'
+            click.echo(f'{colorama.Fore.CYAN}{colorama.Style.BRIGHT}'
+                       f'Clusters{days_str}'
                        f'{colorama.Style.RESET_ALL}')
         click.echo(cluster_table)
 
@@ -220,47 +223,80 @@ def show_cost_report_table(cluster_records: List[_ClusterCostReportRecord],
 # Some of these lambdas are invoked on both _ClusterRecord and
 # _ClusterCostReportRecord, which is okay as we guarantee the queried fields
 # exist in those cases.
-_get_name = (lambda cluster_record: cluster_record['name'])
-_get_user_hash = (lambda cluster_record: cluster_record['user_hash'])
-_get_user_name = (lambda cluster_record: cluster_record.get('user_name', '-'))
-_get_launched = (lambda cluster_record: log_utils.readable_time_duration(
+_get_name = (lambda cluster_record, _: cluster_record['name'])
+_get_user_hash = (lambda cluster_record, _: cluster_record['user_hash'])
+_get_user_name = (
+    lambda cluster_record, _: cluster_record.get('user_name', '-'))
+_get_launched = (lambda cluster_record, _: log_utils.readable_time_duration(
     cluster_record['launched_at']))
-_get_region = (
-    lambda clusters_status: clusters_status['handle'].launched_resources.region)
-_get_command = (lambda cluster_record: cluster_record['last_use'])
-_get_duration = (lambda cluster_record: log_utils.readable_time_duration(
+_get_duration = (lambda cluster_record, _: log_utils.readable_time_duration(
     0, cluster_record['duration'], absolute=True))
 
 
-def _get_status(cluster_record: _ClusterRecord) -> status_lib.ClusterStatus:
-    return cluster_record['status']
+def _get_command(cluster_record: _ClusterRecord, truncate: bool = True) -> str:
+    command = cluster_record.get('last_use', '-')
+    if truncate:
+        return common_utils.truncate_long_string(command, COMMAND_TRUNC_LENGTH)
+    return command
 
 
-def _get_status_colored(cluster_record: _ClusterRecord) -> str:
-    return _get_status(cluster_record).colored_str()
+def _get_status(cluster_record: _ClusterRecord,
+                truncate: bool = True) -> status_lib.ClusterStatus:
+    del truncate
+    return cluster_record['status']
 
 
-def _get_resources(cluster_record: _ClusterRecord) -> str:
-    if 'resources_str' in cluster_record:
-        return cluster_record['resources_str']
-    handle = cluster_record['handle']
-    if isinstance(handle, backends.LocalDockerResourceHandle):
-        resources_str = 'docker'
-    elif isinstance(handle, backends.CloudVmRayResourceHandle):
-        resources_str = resources_utils.get_readable_resources_repr(handle)
-    else:
-        raise ValueError(f'Unknown handle type {type(handle)} encountered.')
-    return resources_str
+def _get_workspace(cluster_record: _ClusterRecord,
+                   truncate: bool = True) -> str:
+    del truncate
+    return cluster_record['workspace']
 
 
-def _get_zone(cluster_record: _ClusterRecord) -> str:
-    zone_str = cluster_record['handle'].launched_resources.zone
-    if zone_str is None:
-        zone_str = '-'
-    return zone_str
+def _get_status_colored(cluster_record: _ClusterRecord,
+                        truncate: bool = True) -> str:
+    del truncate
+    return _get_status(cluster_record).colored_str()
 
 
-def _get_autostop(cluster_record: _ClusterRecord) -> str:
+def _get_resources(cluster_record: _ClusterRecord,
+                   truncate: bool = True) -> str:
+    """Get the resources information for a cluster.
+
+    Returns:
+        A string in one of the following formats:
+        - For cloud VMs: "Nx instance_type" (e.g., "1x m6i.2xlarge")
+        - For K8S/SSH: "Nx (...)"
+        - "-" if no resource information is available
+    """
+    handle = cluster_record['handle']
+    if isinstance(handle, backends.CloudVmRayResourceHandle):
+        launched_resources = handle.launched_resources
+        if launched_resources is None:
+            return '-'
+
+        # For cloud VMs, show instance type directly
+        # For K8S/SSH, show (...) as the resource type
+        resources_str = cluster_record.get('resources_str', None)
+        if not truncate:
+            resources_str_full = cluster_record.get('resources_str_full', None)
+            if resources_str_full is not None:
+                resources_str = resources_str_full
+        if resources_str is None:
+            resources_str_simple, resources_str_full = (
+                resources_utils.get_readable_resources_repr(
+                    handle, simplified_only=truncate))
+            if truncate:
+                resources_str = resources_str_simple
+            else:
+                assert resources_str_full is not None
+                resources_str = resources_str_full
+
+        return resources_str
+    return '-'
+
+
+def _get_autostop(cluster_record: _ClusterRecord, truncate: bool = True) -> str:
+    del truncate
     autostop_str = ''
     separation = ''
     if cluster_record['autostop'] >= 0:
@@ -275,7 +311,8 @@ def _get_autostop(cluster_record: _ClusterRecord) -> str:
     return autostop_str
 
 
-def _get_head_ip(cluster_record: _ClusterRecord) -> str:
+def _get_head_ip(cluster_record: _ClusterRecord, truncate: bool = True) -> str:
+    del truncate  # Unused
     handle = cluster_record['handle']
     if not isinstance(handle, backends.CloudVmRayResourceHandle):
         return '-'
@@ -284,17 +321,46 @@ def _get_head_ip(cluster_record: _ClusterRecord) -> str:
     return handle.head_ip
 
 
+def _get_last_event(cluster_record: _ClusterRecord,
+                    truncate: bool = True) -> str:
+    del truncate
+    if cluster_record.get('last_event', None) is None:
+        return 'No recorded events.'
+    return cluster_record['last_event']
+
+
 def _is_pending_autostop(cluster_record: _ClusterRecord) -> bool:
     # autostop < 0 means nothing scheduled.
     return cluster_record['autostop'] >= 0 and _get_status(
         cluster_record) != status_lib.ClusterStatus.STOPPED
 
 
+def _get_infra(cluster_record: _ClusterRecord, truncate: bool = True) -> str:
+    """Get the infrastructure information for a cluster.
+
+    Returns:
+        A string in one of the following formats:
+        - AWS/region (e.g., "AWS/us-east-1")
+        - K8S/context (e.g., "K8S/my-ctx")
+        - SSH/hostname (e.g., "SSH/my-tobi-box")
+        - "-" if no infrastructure information is available
+    """
+    handle = cluster_record['handle']
+    if isinstance(handle, backends.CloudVmRayResourceHandle):
+        if handle.launched_resources is None:
+            # If launched_resources is None, try to get infra from the record
+            return cluster_record.get('infra', '-')
+        return handle.launched_resources.infra.formatted_str(truncate)
+    return '-'
+
+
 # ---- 'sky cost-report' helper functions below ----
 
 
 def _get_status_value_for_cost_report(
-        cluster_cost_report_record: _ClusterCostReportRecord) -> int:
+        cluster_cost_report_record: _ClusterCostReportRecord,
+        truncate: bool = True) -> int:
+    del truncate
     status = cluster_cost_report_record['status']
     if status is None:
         return -1
@@ -302,7 +368,9 @@ def _get_status_value_for_cost_report(
 
 
 def _get_status_for_cost_report(
-        cluster_cost_report_record: _ClusterCostReportRecord) -> str:
+        cluster_cost_report_record: _ClusterCostReportRecord,
+        truncate: bool = True) -> str:
+    del truncate
     status = cluster_cost_report_record['status']
     if status is None:
         return f'{colorama.Style.DIM}TERMINATED{colorama.Style.RESET_ALL}'
@@ -310,7 +378,9 @@ def _get_status_for_cost_report(
 
 
 def _get_resources_for_cost_report(
-        cluster_cost_report_record: _ClusterCostReportRecord) -> str:
+        cluster_cost_report_record: _ClusterCostReportRecord,
+        truncate: bool = True) -> str:
+    del truncate
     launched_nodes = cluster_cost_report_record['num_nodes']
     launched_resources = cluster_cost_report_record['resources']
 
@@ -322,7 +392,9 @@ def _get_resources_for_cost_report(
 
 
 def _get_price_for_cost_report(
-        cluster_cost_report_record: _ClusterCostReportRecord) -> str:
+        cluster_cost_report_record: _ClusterCostReportRecord,
+        truncate: bool = True) -> str:
+    del truncate
     launched_nodes = cluster_cost_report_record['num_nodes']
     launched_resources = cluster_cost_report_record['resources']
 
@@ -332,7 +404,9 @@ def _get_price_for_cost_report(
 
 
 def _get_estimated_cost_for_cost_report(
-        cluster_cost_report_record: _ClusterCostReportRecord) -> str:
+        cluster_cost_report_record: _ClusterCostReportRecord,
+        truncate: bool = True) -> str:
+    del truncate
     cost = cluster_cost_report_record['total_cost']
 
     if not cost:
@@ -342,18 +416,17 @@ def _get_estimated_cost_for_cost_report(
 
 
 def show_kubernetes_cluster_status_table(
-        clusters: List['kubernetes_utils.KubernetesSkyPilotClusterInfo'],
+        clusters: List['kubernetes_utils.KubernetesSkyPilotClusterInfoPayload'],
         show_all: bool) -> None:
     """Compute cluster table values and display for Kubernetes clusters."""
     status_columns = [
-        StatusColumn('USER', lambda c: c.user),
-        StatusColumn('NAME', lambda c: c.cluster_name),
-        StatusColumn('LAUNCHED',
-                     lambda c: log_utils.readable_time_duration(c.launched_at)),
-        StatusColumn('RESOURCES',
-                     lambda c: c.resources_str,
-                     trunc_length=70 if not show_all else 0),
-        StatusColumn('STATUS', lambda c: c.status.colored_str()),
+        StatusColumn('USER', lambda c, _: c.user),
+        StatusColumn('NAME', lambda c, _: c.cluster_name),
+        StatusColumn('RESOURCES', lambda c, _: c.resources_str, truncate=False),
+        StatusColumn('STATUS', lambda c, _: c.status.colored_str()),
+        StatusColumn(
+            'LAUNCHED',
+            lambda c, _: log_utils.readable_time_duration(c.launched_at)),
         # TODO(romilb): We should consider adding POD_NAME field here when --all
         #  is passed to help users fetch pod name programmatically.
     ]