skypilot-nightly 1.0.0.dev20250616__py3-none-any.whl → 1.0.0.dev20250618__py3-none-any.whl

sky/client/cli.py

@@ -25,7 +25,6 @@ each other.
 """
 import collections
 import copy
-import datetime
 import fnmatch
 import functools
 import os
@@ -34,7 +33,6 @@ import shlex
 import shutil
 import subprocess
 import sys
-import textwrap
 import traceback
 import typing
 from typing import (Any, Callable, Dict, Generator, List, Optional, Set, Tuple,
@@ -52,15 +50,12 @@ from sky import backends
 from sky import catalog
 from sky import clouds
 from sky import exceptions
-from sky import global_user_state
 from sky import jobs as managed_jobs
 from sky import models
 from sky import serve as serve_lib
 from sky import sky_logging
 from sky import skypilot_config
 from sky.adaptors import common as adaptors_common
-from sky.benchmark import benchmark_state
-from sky.benchmark import benchmark_utils
 from sky.client import sdk
 from sky.data import storage_utils
 from sky.provision.kubernetes import constants as kubernetes_constants
@@ -236,6 +231,22 @@ def _parse_env_var(env_var: str) -> Tuple[str, str]:
     return ret[0], ret[1]
 
 
+def _parse_secret_var(secret_var: str) -> Tuple[str, str]:
+    """Parse secret vars into a (KEY, VAL) pair."""
+    if '=' not in secret_var:
+        value = os.environ.get(secret_var)
+        if value is None:
+            raise click.UsageError(
+                f'{secret_var} is not set in local environment.')
+        return (secret_var, value)
+    ret = tuple(secret_var.split('=', 1))
+    if len(ret) != 2:
+        raise click.UsageError(
+            f'Invalid secret var: {secret_var}. Must be in the form of KEY=VAL '
+            'or KEY.')
+    return ret[0], ret[1]
+
+
 def _async_call_or_wait(request_id: str, async_call: bool,
                         request_name: str) -> Any:
     short_request_id = request_id[:8]
@@ -461,6 +472,23 @@ _TASK_OPTIONS = [
 
         3. ``--env MY_ENV3``: set ``$MY_ENV3`` on the cluster to be the
         same value of ``$MY_ENV3`` in the local environment.""",
+    ),
+    click.option(
+        '--secret',
+        required=False,
+        type=_parse_secret_var,
+        multiple=True,
+        help="""\
+        Secret variable to set on the remote node. These variables will be
+        redacted in logs and YAML outputs for security. It can be specified
+        multiple times. Examples:
+
+        \b
+        1. ``--secret API_KEY=secret123``: set ``$API_KEY`` on the cluster to
+        be secret123.
+
+        2. ``--secret JWT_SECRET``: set ``$JWT_SECRET`` on the cluster to be
+        the same value of ``$JWT_SECRET`` in the local environment.""",
     )
 ]
 _TASK_OPTIONS_WITH_NAME = [
@@ -873,6 +901,7 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
     network_tier: Optional[str] = None,
     ports: Optional[Tuple[str, ...]] = None,
     env: Optional[List[Tuple[str, str]]] = None,
+    secret: Optional[List[Tuple[str, str]]] = None,
     field_to_ignore: Optional[List[str]] = None,
     # job launch specific
     job_recovery: Optional[str] = None,
@@ -921,7 +950,9 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
     if is_yaml:
         assert entrypoint is not None
         usage_lib.messages.usage.update_user_task_yaml(entrypoint)
-        dag = dag_utils.load_chain_dag_from_yaml(entrypoint, env_overrides=env)
+        dag = dag_utils.load_chain_dag_from_yaml(entrypoint,
+                                                 env_overrides=env,
+                                                 secret_overrides=secret)
         if len(dag.tasks) > 1:
             # When the dag has more than 1 task. It is unclear how to
             # override the params for the dag. So we just ignore the
@@ -940,6 +971,7 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
         task.set_resources({sky.Resources()})
         # env update has been done for DAG in load_chain_dag_from_yaml for YAML.
         task.update_envs(env)
+        task.update_secrets(secret)
 
     # Override.
     if workdir is not None:
@@ -1248,6 +1280,7 @@ def launch(
         image_id: Optional[str],
         env_file: Optional[Dict[str, str]],
         env: List[Tuple[str, str]],
+        secret: List[Tuple[str, str]],
         disk_size: Optional[int],
         disk_tier: Optional[str],
         network_tier: Optional[str],
@@ -1302,6 +1335,7 @@ def launch(
         use_spot=use_spot,
         image_id=image_id,
         env=env,
+        secret=secret,
         disk_size=disk_size,
         disk_tier=disk_tier,
         network_tier=network_tier,
@@ -1418,6 +1452,7 @@ def exec(cluster: Optional[str],
          image_id: Optional[str],
          env_file: Optional[Dict[str, str]],
          env: List[Tuple[str, str]],
+         secret: List[Tuple[str, str]],
          cpus: Optional[str],
          memory: Optional[str],
          disk_size: Optional[int],
@@ -1516,6 +1551,7 @@ def exec(cluster: Optional[str],
         image_id=image_id,
         num_nodes=num_nodes,
         env=env,
+        secret=secret,
         disk_size=disk_size,
         disk_tier=disk_tier,
         network_tier=network_tier,
@@ -4163,12 +4199,6 @@ def storage_delete(names: List[str], all: bool, yes: bool, async_call: bool):  #
                          f'{colorama.Style.RESET_ALL}')
 
 
-@cli.group(cls=_NaturalOrderGroup, hidden=True)
-def bench():
-    """SkyPilot Benchmark CLI."""
-    raise click.UsageError('The benchmark CLI is currently disabled.')
-
-
 @cli.group(cls=_NaturalOrderGroup)
 def jobs():
     """Managed Jobs CLI (jobs with auto-recovery)."""
@@ -4235,6 +4265,7 @@ def jobs_launch(
     job_recovery: Optional[str],
     env_file: Optional[Dict[str, str]],
     env: List[Tuple[str, str]],
+    secret: List[Tuple[str, str]],
     disk_size: Optional[int],
     disk_tier: Optional[str],
     network_tier: Optional[str],
@@ -4282,6 +4313,7 @@ def jobs_launch(
         use_spot=use_spot,
         image_id=image_id,
         env=env,
+        secret=secret,
         disk_size=disk_size,
         disk_tier=disk_tier,
         network_tier=network_tier,
@@ -4615,6 +4647,7 @@ def _generate_task_with_service(
     image_id: Optional[str],
     env_file: Optional[Dict[str, str]],
     env: List[Tuple[str, str]],
+    secret: Optional[List[Tuple[str, str]]],
     gpus: Optional[str],
     instance_type: Optional[str],
     ports: Optional[Tuple[str]],
@@ -4647,6 +4680,7 @@ def _generate_task_with_service(
         use_spot=use_spot,
         image_id=image_id,
         env=env,
+        secret=secret,
         disk_size=disk_size,
         disk_tier=disk_tier,
         network_tier=network_tier,
@@ -4756,6 +4790,7 @@ def serve_up(
     image_id: Optional[str],
     env_file: Optional[Dict[str, str]],
     env: List[Tuple[str, str]],
+    secret: List[Tuple[str, str]],
     gpus: Optional[str],
     instance_type: Optional[str],
     ports: Tuple[str],
@@ -4816,6 +4851,7 @@ def serve_up(
         image_id=image_id,
         env_file=env_file,
         env=env,
+        secret=secret,
         disk_size=disk_size,
         disk_tier=disk_tier,
         network_tier=network_tier,
@@ -4864,11 +4900,12 @@ def serve_up(
 @timeline.event
 @usage_lib.entrypoint
 def serve_update(
-        service_name: str, service_yaml: Tuple[str, ...],
-        workdir: Optional[str], infra: Optional[str], cloud: Optional[str],
-        region: Optional[str], zone: Optional[str], num_nodes: Optional[int],
-        use_spot: Optional[bool], image_id: Optional[str],
-        env_file: Optional[Dict[str, str]], env: List[Tuple[str, str]],
+        service_name: str, service_yaml: Tuple[str,
+                                               ...], workdir: Optional[str],
+        infra: Optional[str], cloud: Optional[str], region: Optional[str],
+        zone: Optional[str], num_nodes: Optional[int], use_spot: Optional[bool],
+        image_id: Optional[str], env_file: Optional[Dict[str, str]],
+        env: List[Tuple[str, str]], secret: List[Tuple[str, str]],
         gpus: Optional[str], instance_type: Optional[str], ports: Tuple[str],
         cpus: Optional[str], memory: Optional[str], disk_size: Optional[int],
         disk_tier: Optional[str], network_tier: Optional[str], mode: str,
@@ -4920,6 +4957,7 @@ def serve_update(
         image_id=image_id,
         env_file=env_file,
         env=env,
+        secret=secret,
         disk_size=disk_size,
         disk_tier=disk_tier,
         network_tier=network_tier,
@@ -5325,626 +5363,6 @@ def serve_logs(
             raise
 
 
-# ==============================
-# Sky Benchmark CLIs
-# ==============================
-
-
-@ux_utils.print_exception_no_traceback()
-def _get_candidate_configs(
-        entrypoint_yaml_path: str) -> Optional[List[Dict[str, str]]]:
-    """Gets benchmark candidate configs from a YAML file.
-
-    Benchmark candidates are configured in the YAML file as a list of
-    dictionaries. Each dictionary defines a candidate config
-    by overriding resources. For example:
-
-    resources:
-        cloud: aws
-        candidates:
-        - {accelerators: K80}
-        - {instance_type: g4dn.2xlarge}
-        - {cloud: gcp, accelerators: V100} # overrides cloud
-    """
-    config = common_utils.read_yaml(os.path.expanduser(entrypoint_yaml_path))
-    if not isinstance(config, dict):
-        raise ValueError(f'Invalid YAML file: {entrypoint_yaml_path}. '
-                         'The YAML file should be parsed into a dictionary.')
-    if config.get('resources') is None:
-        return None
-
-    resources = config['resources']
-    if not isinstance(resources, dict):
-        raise ValueError(
-            f'Invalid resources configuration in {entrypoint_yaml_path}. '
-            'Resources must be a dictionary.')
-    if resources.get('candidates') is None:
-        return None
-
-    candidates = resources['candidates']
-    if not isinstance(candidates, list):
-        raise ValueError('Resource candidates must be a list of dictionaries.')
-    for candidate in candidates:
-        if not isinstance(candidate, dict):
-            raise ValueError('Each resource candidate must be a dictionary.')
-    return candidates
-
-
-@bench.command('launch', cls=_DocumentedCodeCommand)
-@config_option(expose_value=True)
-@click.argument('entrypoint',
-                required=True,
-                type=str,
-                nargs=-1,
-                **_get_shell_complete_args(_complete_file_name))
-@click.option('--benchmark',
-              '-b',
-              required=True,
-              type=str,
-              help='Benchmark name.')
-@_add_click_options(_TASK_OPTIONS_WITH_NAME + _COMMON_OPTIONS)
-@click.option('--gpus',
-              required=False,
-              type=str,
-              help=('Comma-separated list of GPUs to run benchmark on. '
-                    'Example values: "T4:4,V100:8" (without blank spaces).'))
-@click.option(
-    '--ports',
-    required=False,
-    type=str,
-    multiple=True,
-    help=('Ports to open on the cluster. '
-          'If specified, overrides the "ports" config in the YAML. '),
-)
-@click.option(
-    '--idle-minutes-to-autostop',
-    '-i',
-    default=None,
-    type=int,
-    required=False,
-    help=('Automatically stop the cluster after this many minutes '
-          'of idleness after setup/file_mounts. This is equivalent to '
-          'running `sky launch -d ...` and then `sky autostop -i <minutes>`. '
-          'If not set, the cluster will not be autostopped.'))
-# Disabling quote check here, as there seems to be a bug in pylint,
-# which incorrectly recognizes the help string as a docstring.
-# pylint: disable=bad-docstring-quotes
-@click.option('--yes',
-              '-y',
-              is_flag=True,
-              default=False,
-              required=False,
-              help='Skip confirmation prompt.')
-@usage_lib.entrypoint
-def benchmark_launch(
-    entrypoint: str,
-    benchmark: str,
-    name: Optional[str],
-    workdir: Optional[str],
-    infra: Optional[str],
-    cloud: Optional[str],
-    region: Optional[str],
-    zone: Optional[str],
-    gpus: Optional[str],
-    num_nodes: Optional[int],
-    use_spot: Optional[bool],
-    image_id: Optional[str],
-    env_file: Optional[Dict[str, str]],
-    env: List[Tuple[str, str]],
-    cpus: Optional[str],
-    memory: Optional[str],
-    disk_size: Optional[int],
-    disk_tier: Optional[str],
-    ports: Tuple[str],
-    idle_minutes_to_autostop: Optional[int],
-    yes: bool,
-    async_call: bool,  # pylint: disable=unused-argument
-    config_override: Optional[Dict[str, Any]] = None,
-) -> None:
-    """Benchmark a task on different resources.
-
-    Example usage: `sky bench launch mytask.yaml -b mytask --gpus V100,T4`
-    will benchmark your task on a V100 cluster and a T4 cluster simultaneously.
-    Alternatively, specify the benchmarking resources in your YAML (see doc),
-    which allows benchmarking on many more resource fields.
-    """
-    # TODO(zhwu): move benchmark to SkyPilot API server
-    env = _merge_env_vars(env_file, env)
-    record = benchmark_state.get_benchmark_from_name(benchmark)
-    if record is not None:
-        raise click.BadParameter(f'Benchmark {benchmark} already exists. '
-                                 'To delete the previous benchmark result, '
-                                 f'run `sky bench delete {benchmark}`.')
-    entrypoint = ' '.join(entrypoint)
-    if not entrypoint:
-        raise click.BadParameter('Please specify a task yaml to benchmark.')
-
-    is_yaml, config = _check_yaml(entrypoint)
-    if not is_yaml:
-        raise click.BadParameter(
-            'Sky Benchmark does not support command line tasks. '
-            'Please provide a YAML file.')
-    assert config is not None, (is_yaml, config)
-    cloud, region, zone = _handle_infra_cloud_region_zone_options(
-        infra, cloud, region, zone)
-
-    click.secho('Benchmarking a task from YAML: ', fg='cyan', nl=False)
-    click.secho(entrypoint, bold=True)
-
-    candidates = _get_candidate_configs(entrypoint)
-    # Check if the candidate configs are specified in both CLI and YAML.
-    if candidates is not None:
-        message = ('is specified in both CLI and resources.candidates '
-                   'in the YAML. Please specify only one of them.')
-        if cloud is not None:
-            if any('cloud' in candidate for candidate in candidates):
-                raise click.BadParameter(f'cloud {message}')
-        if region is not None:
-            if any('region' in candidate for candidate in candidates):
-                raise click.BadParameter(f'region {message}')
-        if zone is not None:
-            if any('zone' in candidate for candidate in candidates):
-                raise click.BadParameter(f'zone {message}')
-        if gpus is not None:
-            if any('accelerators' in candidate for candidate in candidates):
-                raise click.BadParameter(f'gpus (accelerators) {message}')
-        if use_spot is not None:
-            if any('use_spot' in candidate for candidate in candidates):
-                raise click.BadParameter(f'use_spot {message}')
-        if image_id is not None:
-            if any('image_id' in candidate for candidate in candidates):
-                raise click.BadParameter(f'image_id {message}')
-        if disk_size is not None:
-            if any('disk_size' in candidate for candidate in candidates):
-                raise click.BadParameter(f'disk_size {message}')
-        if disk_tier is not None:
-            if any('disk_tier' in candidate for candidate in candidates):
-                raise click.BadParameter(f'disk_tier {message}')
-        if ports:
-            if any('ports' in candidate for candidate in candidates):
-                raise click.BadParameter(f'ports {message}')
-
-    # The user can specify the benchmark candidates in either of the two ways:
-    # 1. By specifying resources.candidates in the YAML.
-    # 2. By specifying gpu types as a command line argument (--gpus).
-    override_gpu = None
-    if gpus is not None:
-        gpu_list = gpus.split(',')
-        gpu_list = [gpu.strip() for gpu in gpu_list]
-        if ' ' in gpus:
-            raise click.BadParameter('Remove blanks in --gpus.')
-
-        if len(gpu_list) == 1:
-            override_gpu = gpu_list[0]
-        else:
-            # If len(gpu_list) > 1, gpus is interpreted
-            # as a list of benchmark candidates.
-            if candidates is None:
-                candidates = [{'accelerators': gpu} for gpu in gpu_list]
-                override_gpu = None
-            else:
-                raise ValueError('Provide benchmark candidates in either '
-                                 '--gpus or resources.candidates in the YAML.')
-    if candidates is None:
-        candidates = [{}]
-
-    if 'resources' not in config:
-        config['resources'] = {}
-    resources_config = config['resources']
-
-    # Override the yaml config with the command line arguments.
-    if name is not None:
-        config['name'] = name
-    if workdir is not None:
-        config['workdir'] = workdir
-    if num_nodes is not None:
-        config['num_nodes'] = num_nodes
-    override_params = _parse_override_params(cloud=cloud,
-                                             region=region,
-                                             zone=zone,
-                                             gpus=override_gpu,
-                                             cpus=cpus,
-                                             memory=memory,
-                                             use_spot=use_spot,
-                                             image_id=image_id,
-                                             disk_size=disk_size,
-                                             disk_tier=disk_tier,
-                                             ports=ports,
-                                             config_override=config_override)
-    _pop_and_ignore_fields_in_override_params(
-        override_params, field_to_ignore=['cpus', 'memory'])
-    resources_config.update(override_params)
-    if 'cloud' in resources_config:
-        cloud = resources_config.pop('cloud')
-        if cloud is not None:
-            resources_config['cloud'] = str(cloud)
-    if 'region' in resources_config:
-        if resources_config['region'] is None:
-            resources_config.pop('region')
-    if 'zone' in resources_config:
-        if resources_config['zone'] is None:
-            resources_config.pop('zone')
-    if 'accelerators' in resources_config:
-        if resources_config['accelerators'] is None:
-            resources_config.pop('accelerators')
-    if 'image_id' in resources_config:
-        if resources_config['image_id'] is None:
-            resources_config.pop('image_id')
-
-    # Fully generate the benchmark candidate configs.
-    clusters, candidate_configs = benchmark_utils.generate_benchmark_configs(
-        benchmark, config, candidates)
-    # Show the benchmarking VM instances selected by the optimizer.
-    # This also detects the case where the user requested infeasible resources.
-    benchmark_utils.print_benchmark_clusters(benchmark, clusters, config,
-                                             candidate_configs)
-    if not yes:
-        plural = 's' if len(candidates) > 1 else ''
-        prompt = f'Launching {len(candidates)} cluster{plural}. Proceed?'
-        click.confirm(prompt, default=True, abort=True, show_default=True)
-
-    # Configs that are only accepted by the CLI.
-    commandline_args: Dict[str, Any] = {}
-    # Set the default idle minutes to autostop as 5, mimicking
-    # the serverless execution.
-    if idle_minutes_to_autostop is None:
-        idle_minutes_to_autostop = 5
-    commandline_args['idle-minutes-to-autostop'] = idle_minutes_to_autostop
-    if env:
-        commandline_args['env'] = [f'{k}={v}' for k, v in env]
-
-    # Launch the benchmarking clusters in detach mode in parallel.
-    benchmark_created = benchmark_utils.launch_benchmark_clusters(
-        benchmark, clusters, candidate_configs, commandline_args)
-
-    # If at least one cluster is created, print the following messages.
-    if benchmark_created:
-        logger.info(
-            f'\n{colorama.Fore.CYAN}Benchmark name: '
-            f'{colorama.Style.BRIGHT}{benchmark}{colorama.Style.RESET_ALL}'
-            '\nTo see the benchmark results: '
-            f'{ux_utils.BOLD}sky bench show '
-            f'{benchmark}{ux_utils.RESET_BOLD}'
-            '\nTo teardown the clusters: '
-            f'{ux_utils.BOLD}sky bench down '
-            f'{benchmark}{ux_utils.RESET_BOLD}')
-        subprocess_utils.run('sky bench ls')
-    else:
-        logger.error('No benchmarking clusters are created.')
-        subprocess_utils.run('sky status')
-
-
-@bench.command('ls', cls=_DocumentedCodeCommand)
-@config_option(expose_value=False)
-@usage_lib.entrypoint
-def benchmark_ls() -> None:
-    """List the benchmark history."""
-    benchmarks = benchmark_state.get_benchmarks()
-    columns = [
-        'BENCHMARK',
-        'TASK',
-        'LAUNCHED',
-    ]
-
-    max_num_candidates = 1
-    for benchmark in benchmarks:
-        benchmark_results = benchmark_state.get_benchmark_results(
-            benchmark['name'])
-        num_candidates = len(benchmark_results)
-        if num_candidates > max_num_candidates:
-            max_num_candidates = num_candidates
-
-    if max_num_candidates == 1:
-        columns += ['CANDIDATE']
-    else:
-        columns += [f'CANDIDATE {i}' for i in range(1, max_num_candidates + 1)]
-    benchmark_table = log_utils.create_table(columns)
-
-    for benchmark in benchmarks:
-        if benchmark['task'] is not None:
-            task = benchmark['task']
-        else:
-            task = '-'
-        row = [
-            # BENCHMARK
-            benchmark['name'],
-            # TASK
-            task,
-            # LAUNCHED
-            datetime.datetime.fromtimestamp(benchmark['launched_at']),
-        ]
-
-        benchmark_results = benchmark_state.get_benchmark_results(
-            benchmark['name'])
-        # RESOURCES
-        for b in benchmark_results:
-            num_nodes = b['num_nodes']
-            resources = b['resources']
-            postfix_spot = '[Spot]' if resources.use_spot else ''
-            instance_type = resources.instance_type + postfix_spot
-            if resources.accelerators is None:
-                accelerators = ''
-            else:
-                accelerator, count = list(resources.accelerators.items())[0]
-                accelerators = f' ({accelerator}:{count})'
-            # For brevity, skip the cloud names.
-            resources_str = f'{num_nodes}x {instance_type}{accelerators}'
-            row.append(resources_str)
-        row += [''] * (max_num_candidates - len(benchmark_results))
-        benchmark_table.add_row(row)
-    if benchmarks:
-        click.echo(benchmark_table)
-    else:
-        click.echo('No benchmark history found.')
-
-
-@bench.command('show', cls=_DocumentedCodeCommand)
-@config_option(expose_value=False)
-@click.argument('benchmark', required=True, type=str)
-# TODO(woosuk): Add --all option to show all the collected information
-# (e.g., setup time, warmup steps, total steps, etc.).
-@usage_lib.entrypoint
-def benchmark_show(benchmark: str) -> None:
-    """Show a benchmark report."""
-    record = benchmark_state.get_benchmark_from_name(benchmark)
-    if record is None:
-        raise click.BadParameter(f'Benchmark {benchmark} does not exist.')
-    benchmark_utils.update_benchmark_state(benchmark)
-
-    click.echo(
-        textwrap.dedent("""\
-        Legend:
-        - #STEPS: Number of steps taken.
-        - SEC/STEP, $/STEP: Average time (cost) per step.
-        - EST(hr), EST($): Estimated total time (cost) to complete the benchmark.
-    """))
-    columns = [
-        'CLUSTER',
-        'RESOURCES',
-        'STATUS',
-        'DURATION',
-        'SPENT($)',
-        '#STEPS',
-        'SEC/STEP',
-        '$/STEP',
-        'EST(hr)',
-        'EST($)',
-    ]
-
-    cluster_table = log_utils.create_table(columns)
-    rows = []
-    benchmark_results = benchmark_state.get_benchmark_results(benchmark)
-    for result in benchmark_results:
-        num_nodes = result['num_nodes']
-        resources = result['resources']
-        row = [
-            # CLUSTER
-            result['cluster'],
-            # RESOURCES
-            f'{num_nodes}x {resources}',
-            # STATUS
-            result['status'].value,
-        ]
-
-        record = result['record']
-        if (record is None or record.start_time is None or
-                record.last_time is None):
-            row += ['-'] * (len(columns) - len(row))
-            rows.append(row)
-            continue
-
-        duration_str = log_utils.readable_time_duration(record.start_time,
-                                                        record.last_time,
-                                                        absolute=True)
-        duration = record.last_time - record.start_time
-        spent = num_nodes * resources.get_cost(duration)
-        spent_str = f'{spent:.4f}'
-
-        num_steps = record.num_steps_so_far
-        if num_steps is None:
-            num_steps = '-'
-
-        seconds_per_step = record.seconds_per_step
-        if seconds_per_step is None:
-            seconds_per_step_str = '-'
-            cost_per_step_str = '-'
-        else:
-            seconds_per_step_str = f'{seconds_per_step:.4f}'
-            cost_per_step = num_nodes * resources.get_cost(seconds_per_step)
-            cost_per_step_str = f'{cost_per_step:.6f}'
-
-        total_time = record.estimated_total_seconds
-        if total_time is None:
-            total_time_str = '-'
-            total_cost_str = '-'
-        else:
-            total_time_str = f'{total_time / 3600:.2f}'
-            total_cost = num_nodes * resources.get_cost(total_time)
-            total_cost_str = f'{total_cost:.2f}'
-
-        row += [
-            # DURATION
-            duration_str,
-            # SPENT($)
-            spent_str,
-            # STEPS
-            num_steps,
-            # SEC/STEP
-            seconds_per_step_str,
-            # $/STEP
-            cost_per_step_str,
-            # EST(hr)
-            total_time_str,
-            # EST($)
-            total_cost_str,
-        ]
-        rows.append(row)
-
-    cluster_table.add_rows(rows)
-    click.echo(cluster_table)
-
-    finished = [
-        row for row in rows
-        if row[2] == benchmark_state.BenchmarkStatus.FINISHED.value
-    ]
-    if any(row[5] == '-' for row in finished):
-        # No #STEPS. SkyCallback was unused.
-        click.secho(
-            'SkyCallback logs are not found in this benchmark. '
-            'Consider using SkyCallback to get more detailed information '
-            'in real time.',
-            fg='yellow')
-    elif any(row[6] != '-' and row[-1] == '-' for row in rows):
-        # No EST($). total_steps is not specified and cannot be inferred.
-        click.secho(
-            'Cannot estimate total time and cost because '
-            'the total number of steps cannot be inferred by SkyCallback. '
-            'To get the estimation, specify the total number of steps in '
-            'either `sky_callback.init` or `Sky*Callback`.',
-            fg='yellow')
-
-
-@bench.command('down', cls=_DocumentedCodeCommand)
-@config_option(expose_value=False)
-@click.argument('benchmark', required=True, type=str)
-@click.option(
-    '--exclude',
-    '-e',
-    'clusters_to_exclude',
-    required=False,
-    type=str,
-    multiple=True,
-    help=('Cluster name(s) to exclude from termination. '
-          'Typically, you might want to see the benchmark results in '
-          '`sky bench show` and exclude a "winner" cluster from termination '
-          'to finish the running task.'))
-@click.option('--yes',
-              '-y',
-              is_flag=True,
-              default=False,
-              required=False,
-              help='Skip confirmation prompt.')
-@usage_lib.entrypoint
-def benchmark_down(
-    benchmark: str,
-    clusters_to_exclude: List[str],
-    yes: bool,
-) -> None:
-    """Tear down all clusters belonging to a benchmark."""
-    record = benchmark_state.get_benchmark_from_name(benchmark)
-    if record is None:
-        raise click.BadParameter(f'Benchmark {benchmark} does not exist.')
-
-    clusters = benchmark_state.get_benchmark_clusters(benchmark)
-    to_stop: List[str] = []
-    for cluster in clusters:
-        if cluster in clusters_to_exclude:
-            continue
-        if global_user_state.get_cluster_from_name(cluster) is None:
-            continue
-        to_stop.append(cluster)
-
-    _down_or_stop_clusters(to_stop, down=True, no_confirm=yes)
-
-
-@bench.command('delete', cls=_DocumentedCodeCommand)
-@config_option(expose_value=False)
-@click.argument('benchmarks', required=False, type=str, nargs=-1)
-@click.option('--all',
-              '-a',
-              default=None,
-              is_flag=True,
-              help='Delete all benchmark reports from the history.')
-@click.option('--yes',
-              '-y',
-              is_flag=True,
-              default=False,
-              required=False,
-              help='Skip confirmation prompt.')
-@usage_lib.entrypoint
-# pylint: disable=redefined-builtin
-def benchmark_delete(benchmarks: Tuple[str], all: Optional[bool],
-                     yes: bool) -> None:
-    """Delete benchmark reports from the history."""
-    if not benchmarks and all is None:
-        raise click.BadParameter(
-            'Either specify benchmarks or use --all to delete all benchmarks.')
-    to_delete = []
-    if benchmarks:
-        for benchmark in benchmarks:
-            record = benchmark_state.get_benchmark_from_name(benchmark)
-            if record is None:
-                print(f'Benchmark {benchmark} not found.')
-            else:
-                to_delete.append(record)
-    if all:
-        to_delete = benchmark_state.get_benchmarks()
-        if benchmarks:
-            print('Both --all and benchmark(s) specified '
-                  'for sky bench delete. Letting --all take effect.')
-
-    to_delete = [r['name'] for r in to_delete]
-    if not to_delete:
-        return
-
-    benchmark_list = ', '.join(to_delete)
-    plural = 's' if len(to_delete) > 1 else ''
-    if not yes:
-        click.confirm(
-            f'Deleting the benchmark{plural}: {benchmark_list}. Proceed?',
-            default=True,
-            abort=True,
-            show_default=True)
-
-    progress = rich_progress.Progress(transient=True,
-                                      redirect_stdout=False,
-                                      redirect_stderr=False)
-    task = progress.add_task(
-        f'[bold cyan]Deleting {len(to_delete)} benchmark{plural}: ',
-        total=len(to_delete))
-
-    def _delete_benchmark(benchmark: str) -> None:
-        clusters = benchmark_state.get_benchmark_clusters(benchmark)
-        records = []
-        for cluster in clusters:
-            record = global_user_state.get_cluster_from_name(cluster)
-            records.append(record)
-        num_clusters = len([r for r in records if r is not None])
-
-        if num_clusters > 0:
-            plural = 's' if num_clusters > 1 else ''
-            message = (f'{colorama.Fore.YELLOW}Benchmark {benchmark} '
-                       f'has {num_clusters} un-terminated cluster{plural}. '
-                       f'Terminate the cluster{plural} with '
-                       f'{ux_utils.BOLD} sky bench down {benchmark} '
-                       f'{ux_utils.RESET_BOLD} '
-                       'before deleting the benchmark report.')
-            success = False
-        else:
-            bucket_name = benchmark_state.get_benchmark_from_name(
-                benchmark)['bucket']
-            handle = global_user_state.get_handle_from_storage_name(bucket_name)
-            assert handle is not None, bucket_name
-            bucket_type = list(handle.sky_stores.keys())[0]
-            benchmark_utils.remove_benchmark_logs(benchmark, bucket_name,
-                                                  bucket_type)
-            benchmark_state.delete_benchmark(benchmark)
-            message = (f'{colorama.Fore.GREEN}Benchmark report for '
-                       f'{benchmark} deleted.{colorama.Style.RESET_ALL}')
-            success = True
-
-        progress.stop()
-        click.secho(message)
-        if success:
-            progress.update(task, advance=1)
-        progress.start()
-
-    with progress:
-        subprocess_utils.run_in_parallel(_delete_benchmark, to_delete)
-        progress.live.transient = False
-        progress.refresh()
-
-
 @cli.group(cls=_NaturalOrderGroup, hidden=True)
 def local():
     """SkyPilot local tools CLI."""
@@ -6080,10 +5498,19 @@ def api():
               'to manage the process lifecycle and collect logs directly. '
               'This is useful when the API server is managed by systems '
               'like systemd and Kubernetes.')
+@click.option('--enable-basic-auth',
+              is_flag=True,
+              default=False,
+              required=False,
+              help='Enable basic authentication in the SkyPilot API server.')
 @usage_lib.entrypoint
-def api_start(deploy: bool, host: Optional[str], foreground: bool):
+def api_start(deploy: bool, host: Optional[str], foreground: bool,
+              enable_basic_auth: bool):
     """Starts the SkyPilot API server locally."""
-    sdk.api_start(deploy=deploy, host=host, foreground=foreground)
+    sdk.api_start(deploy=deploy,
+                  host=host,
+                  foreground=foreground,
+                  enable_basic_auth=enable_basic_auth)
 
 
 @api.command('stop', cls=_DocumentedCodeCommand)