skypilot-nightly 1.0.0.dev20250211__py3-none-any.whl → 1.0.0.dev20250213__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '1064e8a074909c5d13f862357ed0e3186079a8e8'
+_SKYPILOT_COMMIT_SHA = 'c49961417a83b049b3f3435a252c8ec5ea0fb5e6'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250211'
+__version__ = '1.0.0.dev20250213'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/cli.py

@@ -623,7 +623,8 @@ def _launch_with_confirm(
             click.confirm(prompt, default=True, abort=True, show_default=True)
 
     if not confirm_shown:
-        click.secho(f'Running task on cluster {cluster}...', fg='yellow')
+        click.secho('Running on cluster: ', fg='cyan', nl=False)
+        click.secho(cluster)
 
     sky.launch(
         dag,
@@ -722,7 +723,6 @@ def _pop_and_ignore_fields_in_override_params(
 def _make_task_or_dag_from_entrypoint_with_overrides(
     entrypoint: Tuple[str, ...],
     *,
-    entrypoint_name: str = 'Task',
     name: Optional[str] = None,
     workdir: Optional[str] = None,
     cloud: Optional[str] = None,
@@ -754,19 +754,15 @@ def _make_task_or_dag_from_entrypoint_with_overrides(
     entrypoint: Optional[str]
     if is_yaml:
         # Treat entrypoint as a yaml.
-        click.secho(f'{entrypoint_name} from YAML spec: ',
-                    fg='yellow',
-                    nl=False)
-        click.secho(entrypoint, bold=True)
+        click.secho('YAML to run: ', fg='cyan', nl=False)
+        click.secho(entrypoint)
     else:
         if not entrypoint:
             entrypoint = None
         else:
             # Treat entrypoint as a bash command.
-            click.secho(f'{entrypoint_name} from command: ',
-                        fg='yellow',
-                        nl=False)
-            click.secho(entrypoint, bold=True)
+            click.secho('Command to run: ', fg='cyan', nl=False)
+            click.secho(entrypoint)
 
     override_params = _parse_override_params(cloud=cloud,
                                              region=region,
@@ -1333,7 +1329,8 @@ def exec(
                                'supports a single task only.')
     task = task_or_dag
 
-    click.secho(f'Executing task on cluster {cluster}...', fg='yellow')
+    click.secho('Submitting job to cluster: ', fg='cyan', nl=False)
+    click.secho(cluster)
     sky.exec(task, backend=backend, cluster_name=cluster, detach_run=detach_run)
 
 
@@ -1982,7 +1979,7 @@ def cost_report(all: bool):  # pylint: disable=redefined-builtin
 def queue(clusters: List[str], skip_finished: bool, all_users: bool):
     # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
     """Show the job queue for cluster(s)."""
-    click.secho('Fetching and parsing job queue...', fg='yellow')
+    click.secho('Fetching and parsing job queue...', fg='cyan')
     if clusters:
         clusters = _get_glob_clusters(clusters)
     else:
@@ -3785,7 +3782,7 @@ def jobs_queue(all: bool, refresh: bool, skip_finished: bool):
       watch -n60 sky jobs queue
 
     """
-    click.secho('Fetching managed job statuses...', fg='yellow')
+    click.secho('Fetching managed jobs...', fg='cyan')
     with rich_utils.safe_status(
             ux_utils.spinner_message('Checking managed jobs')):
         _, msg = _get_managed_jobs(refresh=refresh,
@@ -3938,7 +3935,7 @@ def jobs_dashboard(port: Optional[int]):
     # see if the controller is UP first, which is slow; (2) not have to run SSH
     # port forwarding first (we'd just launch a local dashboard which would make
     # REST API calls to the controller dashboard server).
-    click.secho('Checking if jobs controller is up...', fg='yellow')
+    click.secho('Checking if jobs controller is up...', fg='cyan')
     hint = ('Dashboard is not available if jobs controller is not up. Run a '
             'managed job first.')
     backend_utils.is_controller_accessible(
@@ -4032,7 +4029,6 @@ def _generate_task_with_service(
         disk_size=disk_size,
         disk_tier=disk_tier,
         ports=ports,
-        entrypoint_name='Service',
     )
     if isinstance(task, sky.Dag):
         raise click.UsageError(
@@ -4197,7 +4193,7 @@ def serve_up(
         ports=ports,
         not_supported_cmd='sky serve up',
     )
-    click.secho('Service Spec:', fg='cyan')
+    click.secho('Service spec:', fg='cyan')
     click.echo(task.service)
 
     click.secho('Each replica will use the following resources (estimated):',
@@ -4315,7 +4311,7 @@ def serve_update(
         ports=ports,
         not_supported_cmd='sky serve update',
     )
-    click.secho('Service Spec:', fg='cyan')
+    click.secho('Service spec:', fg='cyan')
     click.echo(task.service)
 
     click.secho('New replica will use the following resources (estimated):',
@@ -4767,7 +4763,7 @@ def benchmark_launch(
             'Please provide a YAML file.')
     assert config is not None, (is_yaml, config)
 
-    click.secho('Benchmarking a task from YAML spec: ', fg='yellow', nl=False)
+    click.secho('Benchmarking a task from YAML: ', fg='cyan', nl=False)
     click.secho(entrypoint, bold=True)
 
     candidates = _get_candidate_configs(entrypoint)

sky/clouds/kubernetes.py

@@ -464,7 +464,9 @@ class Kubernetes(clouds.Cloud):
         # CPU resources on the node instead within the pod.
         custom_ray_options = {
             'object-store-memory': 500000000,
-            'num-cpus': str(int(cpus)),
+            # 'num-cpus' must be an integer, but we should not set it to 0 if
+            # cpus is <1.
+            'num-cpus': str(max(int(cpus), 1)),
         }
         deploy_vars = {
             'instance_type': resources.instance_type,

sky/clouds/runpod.py

@@ -177,6 +177,11 @@ class RunPod(clouds.Cloud):
         hourly_cost = self.instance_type_to_hourly_cost(
             instance_type=instance_type, use_spot=use_spot)
 
+        # default to root
+        docker_username_for_runpod = (resources.docker_username_for_runpod
+                                      if resources.docker_username_for_runpod
+                                      is not None else 'root')
+
         return {
             'instance_type': instance_type,
             'custom_resources': custom_resources,
@@ -184,6 +189,7 @@ class RunPod(clouds.Cloud):
             'image_id': image_id,
             'use_spot': use_spot,
             'bid_per_gpu': str(hourly_cost),
+            'docker_username_for_runpod': docker_username_for_runpod,
         }
 
     def _get_feasible_launchable_resources(

sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py

@@ -28,6 +28,7 @@ REGIONS = [
     'asia-northeast-2',
     'us-east-1',
     'us-east-2',
+    'us-east-3',
     'us-west-2',
     'us-west-1',
     'us-south-1',

sky/execution.py

@@ -259,8 +259,8 @@ def _execute(
             bold = colorama.Style.BRIGHT
             reset = colorama.Style.RESET_ALL
             logger.info(
-                f'{yellow}Launching an unmanaged spot task, which does not '
-                f'automatically recover from preemptions.{reset}\n{yellow}To '
+                f'{yellow}Launching a spot job that does not '
+                f'automatically recover from preemptions. To '
                 'get automatic recovery, use managed job instead: '
                 f'{reset}{bold}sky jobs launch{reset} {yellow}or{reset} '
                 f'{bold}sky.jobs.launch(){reset}.')

sky/optimizer.py

@@ -884,10 +884,8 @@ class Optimizer:
                 # Add a new line for better readability, when there are multiple
                 # tasks.
                 logger.info('')
-            logger.info(
-                f'{colorama.Style.BRIGHT}Considered resources {task_str}'
-                f'({task.num_nodes} node{plural}):'
-                f'{colorama.Style.RESET_ALL}')
+            logger.info(f'Considered resources {task_str}'
+                        f'({task.num_nodes} node{plural}):')
 
             # Only print 1 row per cloud.
             # The following code is to generate the table

sky/provision/instance_setup.py

@@ -15,9 +15,12 @@ from sky.provision import docker_utils
 from sky.provision import logging as provision_logging
 from sky.provision import metadata_utils
 from sky.skylet import constants
+from sky.usage import constants as usage_constants
+from sky.usage import usage_lib
 from sky.utils import accelerator_registry
 from sky.utils import command_runner
 from sky.utils import common_utils
+from sky.utils import env_options
 from sky.utils import subprocess_utils
 from sky.utils import timeline
 from sky.utils import ux_utils
@@ -67,6 +70,30 @@ MAYBE_SKYLET_RESTART_CMD = (f'{constants.ACTIVATE_SKY_REMOTE_PYTHON_ENV}; '
                             'sky.skylet.attempt_skylet;')
 
 
+def _set_usage_run_id_cmd() -> str:
+    """Gets the command to set the usage run id.
+
+    The command saves the current usage run id to the file, so that the skylet
+    can use it to report the heartbeat.
+
+    We use a function instead of a constant so that the usage run id is the
+    latest one when the function is called.
+    """
+    return (
+        f'cat {usage_constants.USAGE_RUN_ID_FILE} || '
+        # The run id is retrieved locally for the current run, so that the
+        # remote cluster will be set with the same run id as the initial
+        # launch operation.
+        f'echo "{usage_lib.messages.usage.run_id}" > '
+        f'{usage_constants.USAGE_RUN_ID_FILE}')
+
+
+def _set_skypilot_env_var_cmd() -> str:
+    """Sets the skypilot environment variables on the remote machine."""
+    env_vars = env_options.Options.all_options()
+    return '; '.join([f'export {k}={v}' for k, v in env_vars.items()])
+
+
 def _auto_retry(should_retry: Callable[[Exception], bool] = lambda _: True):
     """Decorator that retries the function if it fails.
 
@@ -450,11 +477,17 @@ def start_skylet_on_head_node(cluster_name: str,
     logger.info(f'Running command on head node: {MAYBE_SKYLET_RESTART_CMD}')
     # We need to source bashrc for skylet to make sure the autostop event can
     # access the path to the cloud CLIs.
-    returncode, stdout, stderr = head_runner.run(MAYBE_SKYLET_RESTART_CMD,
-                                                 stream_logs=False,
-                                                 require_outputs=True,
-                                                 log_path=log_path_abs,
-                                                 source_bashrc=True)
+    set_usage_run_id_cmd = _set_usage_run_id_cmd()
+    # Set the skypilot environment variables, including the usage type, debug
+    # info, and other options.
+    set_skypilot_env_var_cmd = _set_skypilot_env_var_cmd()
+    returncode, stdout, stderr = head_runner.run(
+        f'{set_usage_run_id_cmd}; {set_skypilot_env_var_cmd}; '
+        f'{MAYBE_SKYLET_RESTART_CMD}',
+        stream_logs=False,
+        require_outputs=True,
+        log_path=log_path_abs,
+        source_bashrc=True)
     if returncode:
         raise RuntimeError('Failed to start skylet on the head node '
                            f'(exit code {returncode}). Error: '

sky/provision/kubernetes/utils.py

@@ -2178,52 +2178,54 @@ def get_kubernetes_node_info(
 
     lf, _ = detect_gpu_label_formatter(context)
     if not lf:
-        label_key = None
+        label_keys = []
     else:
         label_keys = lf.get_label_keys()
 
     node_info_dict: Dict[str, KubernetesNodeInfo] = {}
 
-    for label_key in label_keys:
-        for node in nodes:
-            allocated_qty = 0
+    for node in nodes:
+        accelerator_name = None
+        # Determine the accelerator name from the node labels and pick the
+        # first one found. We assume that the node has only one accelerator type
+        # (e.g., either GPU or TPU).
+        for label_key in label_keys:
             if lf is not None and label_key in node.metadata.labels:
                 accelerator_name = lf.get_accelerator_from_label_value(
                     node.metadata.labels.get(label_key))
-            else:
-                accelerator_name = None
+                break
 
-            accelerator_count = get_node_accelerator_count(
-                node.status.allocatable)
+        allocated_qty = 0
+        accelerator_count = get_node_accelerator_count(node.status.allocatable)
 
-            if pods is None:
-                accelerators_available = -1
+        if pods is None:
+            accelerators_available = -1
 
-            else:
-                for pod in pods:
-                    # Get all the pods running on the node
-                    if (pod.spec.node_name == node.metadata.name and
-                            pod.status.phase in ['Running', 'Pending']):
-                        # Iterate over all the containers in the pod and sum the
-                        # GPU requests
-                        for container in pod.spec.containers:
-                            if container.resources.requests:
-                                allocated_qty += get_node_accelerator_count(
-                                    container.resources.requests)
-
-                accelerators_available = accelerator_count - allocated_qty
-
-            # Exclude multi-host TPUs from being processed.
-            # TODO(Doyoung): Remove the logic when adding support for
-            # multi-host TPUs.
-            if is_multi_host_tpu(node.metadata.labels):
-                continue
+        else:
+            for pod in pods:
+                # Get all the pods running on the node
+                if (pod.spec.node_name == node.metadata.name and
+                        pod.status.phase in ['Running', 'Pending']):
+                    # Iterate over all the containers in the pod and sum the
+                    # GPU requests
+                    for container in pod.spec.containers:
+                        if container.resources.requests:
+                            allocated_qty += get_node_accelerator_count(
+                                container.resources.requests)
+
+            accelerators_available = accelerator_count - allocated_qty
+
+        # Exclude multi-host TPUs from being processed.
+        # TODO(Doyoung): Remove the logic when adding support for
+        # multi-host TPUs.
+        if is_multi_host_tpu(node.metadata.labels):
+            continue
 
-            node_info_dict[node.metadata.name] = KubernetesNodeInfo(
-                name=node.metadata.name,
-                accelerator_type=accelerator_name,
-                total={'accelerator_count': int(accelerator_count)},
-                free={'accelerators_available': int(accelerators_available)})
+        node_info_dict[node.metadata.name] = KubernetesNodeInfo(
+            name=node.metadata.name,
+            accelerator_type=accelerator_name,
+            total={'accelerator_count': int(accelerator_count)},
+            free={'accelerators_available': int(accelerators_available)})
 
     return node_info_dict
 

sky/provision/provisioner.py

@@ -450,6 +450,13 @@ def _post_provision_setup(
         logger.info(f'{indent_str}{colorama.Style.DIM}{vm_str}{plural} {verb} '
                     f'up.{colorama.Style.RESET_ALL}')
 
+        # It's promised by the cluster config that docker_config does not
+        # exist for docker-native clouds, i.e. they provide docker containers
+        # instead of full VMs, like Kubernetes and RunPod, as it requires some
+        # special handlings to run docker inside their docker virtualization.
+        # For their Docker image settings, we do them when provisioning the
+        # cluster. See provision/{cloud}/instance.py:get_cluster_info for more
+        # details.
         if docker_config:
             status.update(
                 ux_utils.spinner_message(

sky/provision/runpod/utils.py

@@ -186,7 +186,7 @@ def delete_pod_template(template_name: str) -> None:
         runpod.runpod.api.graphql.run_graphql_query(
             f'mutation {{deleteTemplate(templateName: "{template_name}")}}')
     except runpod.runpod.error.QueryError as e:
-        logger.warning(f'Failed to delete template {template_name}: {e}'
+        logger.warning(f'Failed to delete template {template_name}: {e} '
                        'Please delete it manually.')
 
 
@@ -195,8 +195,9 @@ def delete_register_auth(registry_auth_id: str) -> None:
     try:
         runpod.runpod.delete_container_registry_auth(registry_auth_id)
     except runpod.runpod.error.QueryError as e:
-        logger.warning(f'Failed to delete registry auth {registry_auth_id}: {e}'
-                       'Please delete it manually.')
+        logger.warning(
+            f'Failed to delete registry auth {registry_auth_id}: {e} '
+            'Please delete it manually.')
 
 
 def _create_template_for_docker_login(

sky/resources.py

@@ -67,6 +67,7 @@ class Resources:
         # Internal use only.
         # pylint: disable=invalid-name
         _docker_login_config: Optional[docker_utils.DockerLoginConfig] = None,
+        _docker_username_for_runpod: Optional[str] = None,
         _is_image_managed: Optional[bool] = None,
         _requires_fuse: Optional[bool] = None,
         _cluster_config_overrides: Optional[Dict[str, Any]] = None,
@@ -148,6 +149,9 @@ class Resources:
           _docker_login_config: the docker configuration to use. This includes
             the docker username, password, and registry server. If None, skip
             docker login.
+          _docker_username_for_runpod: the login username for the docker
+            containers. This is used by RunPod to set the ssh user for the
+            docker containers.
           _requires_fuse: whether the task requires FUSE mounting support. This
             is used internally by certain cloud implementations to do additional
             setup for FUSE mounting. This flag also safeguards against using
@@ -234,6 +238,12 @@ class Resources:
 
         self._docker_login_config = _docker_login_config
 
+        # TODO(andyl): This ctor param seems to be unused.
+        # We always use `Task.set_resources` and `Resources.copy` to set the
+        # `docker_username_for_runpod`. But to keep the consistency with
+        # `_docker_login_config`, we keep it here.
+        self._docker_username_for_runpod = _docker_username_for_runpod
+
         self._requires_fuse = _requires_fuse
 
         self._cluster_config_overrides = _cluster_config_overrides
@@ -479,6 +489,10 @@ class Resources:
     def requires_fuse(self, value: Optional[bool]) -> None:
         self._requires_fuse = value
 
+    @property
+    def docker_username_for_runpod(self) -> Optional[str]:
+        return self._docker_username_for_runpod
+
     def _set_cpus(
         self,
         cpus: Union[None, int, float, str],
@@ -1065,6 +1079,10 @@ class Resources:
         cloud_specific_variables = self.cloud.make_deploy_resources_variables(
             self, cluster_name, region, zones, num_nodes, dryrun)
 
+        # TODO(andyl): Should we print some warnings if users' envs share
+        # same names with the cloud specific variables, but not enabled
+        # since it's not on the particular cloud?
+
         # Docker run options
         docker_run_options = skypilot_config.get_nested(
             ('docker', 'run_options'),
@@ -1277,6 +1295,9 @@ class Resources:
             labels=override.pop('labels', self.labels),
             _docker_login_config=override.pop('_docker_login_config',
                                               self._docker_login_config),
+            _docker_username_for_runpod=override.pop(
+                '_docker_username_for_runpod',
+                self._docker_username_for_runpod),
             _is_image_managed=override.pop('_is_image_managed',
                                            self._is_image_managed),
             _requires_fuse=override.pop('_requires_fuse', self._requires_fuse),
@@ -1438,6 +1459,8 @@ class Resources:
         resources_fields['labels'] = config.pop('labels', None)
         resources_fields['_docker_login_config'] = config.pop(
             '_docker_login_config', None)
+        resources_fields['_docker_username_for_runpod'] = config.pop(
+            '_docker_username_for_runpod', None)
         resources_fields['_is_image_managed'] = config.pop(
             '_is_image_managed', None)
         resources_fields['_requires_fuse'] = config.pop('_requires_fuse', None)
@@ -1486,6 +1509,9 @@ class Resources:
         if self._docker_login_config is not None:
             config['_docker_login_config'] = dataclasses.asdict(
                 self._docker_login_config)
+        if self._docker_username_for_runpod is not None:
+            config['_docker_username_for_runpod'] = (
+                self._docker_username_for_runpod)
         add_if_not_none('_cluster_config_overrides',
                         self._cluster_config_overrides)
         if self._is_image_managed is not None:

sky/skylet/constants.py

@@ -110,6 +110,8 @@ DOCKER_LOGIN_ENV_VARS = {
     DOCKER_SERVER_ENV_VAR,
 }
 
+RUNPOD_DOCKER_USERNAME_ENV_VAR = 'SKYPILOT_RUNPOD_DOCKER_USERNAME'
+
 # Commands for disable GPU ECC, which can improve the performance of the GPU
 # for some workloads by 30%. This will only be applied when a user specify
 # `nvidia_gpus.disable_ecc: true` in ~/.sky/config.yaml.

sky/skylet/events.py

@@ -20,6 +20,7 @@ from sky.serve import serve_utils
 from sky.skylet import autostop_lib
 from sky.skylet import constants
 from sky.skylet import job_lib
+from sky.usage import usage_lib
 from sky.utils import cluster_yaml_utils
 from sky.utils import common_utils
 from sky.utils import ux_utils
@@ -90,6 +91,14 @@ class ServiceUpdateEvent(SkyletEvent):
         serve_utils.update_service_status()
 
 
+class UsageHeartbeatReportEvent(SkyletEvent):
+    """Skylet event for reporting usage."""
+    EVENT_INTERVAL_SECONDS = 600
+
+    def _run(self):
+        usage_lib.send_heartbeat(interval_seconds=self.EVENT_INTERVAL_SECONDS)
+
+
 class AutostopEvent(SkyletEvent):
     """Skylet event for autostop.
 

sky/skylet/skylet.py

@@ -25,6 +25,8 @@ EVENTS = [
     # unhealthy, this event will correctly update the controller
     # status to CONTROLLER_FAILED.
     events.ServiceUpdateEvent(),
+    # Report usage heartbeat every 10 minutes.
+    events.UsageHeartbeatReportEvent(),
 ]
 
 while True:

sky/task.py

@@ -121,6 +121,9 @@ def _check_docker_login_config(task_envs: Dict[str, str]) -> bool:
 
     If any of the docker login env vars is set, all of them must be set.
 
+    Returns:
+        True if there is a valid docker login config in task_envs.
+        False otherwise.
     Raises:
         ValueError: if any of the docker login env vars is set, but not all of
             them are set.
@@ -168,6 +171,23 @@ def _with_docker_login_config(
     return type(resources)(new_resources)
 
 
+def _with_docker_username_for_runpod(
+    resources: Union[Set['resources_lib.Resources'],
+                     List['resources_lib.Resources']],
+    task_envs: Dict[str, str],
+) -> Union[Set['resources_lib.Resources'], List['resources_lib.Resources']]:
+    docker_username_for_runpod = task_envs.get(
+        constants.RUNPOD_DOCKER_USERNAME_ENV_VAR)
+
+    # We should not call r.copy() if docker_username_for_runpod is None,
+    # to prevent `DummyResources` instance becoming a `Resources` instance.
+    if docker_username_for_runpod is None:
+        return resources
+    return (type(resources)(
+        r.copy(_docker_username_for_runpod=docker_username_for_runpod)
+        for r in resources))
+
+
 class Task:
     """Task: a computation to be run on the cloud."""
 
@@ -582,6 +602,8 @@ class Task:
         if _check_docker_login_config(self._envs):
             self.resources = _with_docker_login_config(self.resources,
                                                        self._envs)
+        self.resources = _with_docker_username_for_runpod(
+            self.resources, self._envs)
         return self
 
     @property
@@ -647,6 +669,9 @@ class Task:
             resources = {resources}
         # TODO(woosuk): Check if the resources are None.
         self.resources = _with_docker_login_config(resources, self.envs)
+        # Only have effect on RunPod.
+        self.resources = _with_docker_username_for_runpod(
+            self.resources, self.envs)
 
         # Evaluate if the task requires FUSE and set the requires_fuse flag
         for _, storage_obj in self.storage_mounts.items():

sky/templates/kubernetes-ray.yml.j2

@@ -373,15 +373,16 @@ available_node_types:
                 done;
                 if [ ! -z "$INSTALL_FIRST" ]; then
                   echo "Installing core packages: $INSTALL_FIRST";
-                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $INSTALL_FIRST;
+                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $INSTALL_FIRST;
                 fi;
                 # SSH and other packages are not necessary, so we disable set -e
                 set +e
                 
                 if [ ! -z "$MISSING_PACKAGES" ]; then
                   echo "Installing missing packages: $MISSING_PACKAGES";
-                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y $MISSING_PACKAGES;
+                  DEBIAN_FRONTEND=noninteractive $(prefix_cmd) apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $MISSING_PACKAGES;
                 fi;
+              
                 $(prefix_cmd) mkdir -p /var/run/sshd;
                 $(prefix_cmd) sed -i "s/PermitRootLogin prohibit-password/PermitRootLogin yes/" /etc/ssh/sshd_config;
                 $(prefix_cmd) sed "s@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g" -i /etc/pam.d/sshd;

sky/templates/runpod-ray.yml.j2

@@ -25,7 +25,7 @@ provider:
   {%- endif %}
 
 auth:
-  ssh_user: root
+  ssh_user: {{docker_username_for_runpod}}
   ssh_private_key: {{ssh_private_key}}
 
 available_node_types:

sky/usage/constants.py

@@ -3,7 +3,6 @@
 LOG_URL = 'http://usage.skypilot.co:9090/loki/api/v1/push'  # pylint: disable=line-too-long
 
 USAGE_MESSAGE_SCHEMA_VERSION = 1
-
 PRIVACY_POLICY_PATH = '~/.sky/privacy_policy'
 
 USAGE_POLICY_MESSAGE = (
@@ -15,3 +14,5 @@ USAGE_POLICY_MESSAGE = (
 
 USAGE_MESSAGE_REDACT_KEYS = ['setup', 'run', 'envs']
 USAGE_MESSAGE_REDACT_TYPES = {str, dict}
+
+USAGE_RUN_ID_FILE = '~/.sky/usage_run_id'

sky/usage/usage_lib.py

@@ -44,6 +44,7 @@ def _get_current_timestamp_ns() -> int:
 class MessageType(enum.Enum):
     """Types for messages to be sent to Loki."""
     USAGE = 'usage'
+    HEARTBEAT = 'heartbeat'
     # TODO(zhwu): Add more types, e.g., cluster_lifecycle.
 
 
@@ -67,8 +68,9 @@ class MessageToReport:
         properties = self.__dict__.copy()
         return {k: v for k, v in properties.items() if not k.startswith('_')}
 
-    def __repr__(self):
-        raise NotImplementedError
+    def __repr__(self) -> str:
+        d = self.get_properties()
+        return json.dumps(d)
 
 
 class UsageMessageToReport(MessageToReport):
@@ -160,10 +162,6 @@ class UsageMessageToReport(MessageToReport):
         self.exception: Optional[str] = None  # entrypoint_context
         self.stacktrace: Optional[str] = None  # entrypoint_context
 
-    def __repr__(self) -> str:
-        d = self.get_properties()
-        return json.dumps(d)
-
     def update_entrypoint(self, msg: str):
         self.entrypoint = msg
 
@@ -275,16 +273,43 @@ class UsageMessageToReport(MessageToReport):
                                            name_or_fn)
 
 
+class HeartbeatMessageToReport(MessageToReport):
+    """Message to be reported to Grafana Loki for heartbeat on a cluster."""
+
+    def __init__(self, interval_seconds: int = 600):
+        super().__init__(constants.USAGE_MESSAGE_SCHEMA_VERSION)
+        # This interval_seconds is mainly for recording the heartbeat interval
+        # in the heartbeat message, so that the collector can use it.
+        self.interval_seconds = interval_seconds
+
+    def get_properties(self) -> Dict[str, Any]:
+        properties = super().get_properties()
+        # The run id is set by the skylet, which will always be the same for
+        # the entire lifetime of the run.
+        with open(os.path.expanduser(constants.USAGE_RUN_ID_FILE),
+                  'r',
+                  encoding='utf-8') as f:
+            properties['run_id'] = f.read().strip()
+        return properties
+
+
 class MessageCollection:
     """A collection of messages."""
 
     def __init__(self):
-        self._messages = {MessageType.USAGE: UsageMessageToReport()}
+        self._messages = {
+            MessageType.USAGE: UsageMessageToReport(),
+            MessageType.HEARTBEAT: HeartbeatMessageToReport()
+        }
 
     @property
-    def usage(self):
+    def usage(self) -> UsageMessageToReport:
         return self._messages[MessageType.USAGE]
 
+    @property
+    def heartbeat(self) -> HeartbeatMessageToReport:
+        return self._messages[MessageType.HEARTBEAT]
+
     def reset(self, message_type: MessageType):
         self._messages[message_type] = self._messages[message_type].__class__()
 
@@ -308,13 +333,25 @@ def _send_to_loki(message_type: MessageType):
 
     message = messages[message_type]
 
+    # In case the message has no start time, set it to the current time.
+    message.start()
     message.send_time = _get_current_timestamp_ns()
-    log_timestamp = message.start_time
+    # Use send time instead of start time to avoid the message being dropped
+    # by Loki, due to the timestamp being too old. We still have the start time
+    # in the message for dashboard.
+    log_timestamp = message.send_time
 
     environment = 'prod'
     if env_options.Options.IS_DEVELOPER.get():
         environment = 'dev'
-    prom_labels = {'type': message_type.value, 'environment': environment}
+    prom_labels = {
+        'type': message_type.value,
+        'environment': environment,
+        'schema_version': message.schema_version,
+    }
+    if message_type == MessageType.USAGE:
+        prom_labels['new_cluster'] = (message.original_cluster_status != 'UP'
+                                      and message.final_cluster_status == 'UP')
 
     headers = {'Content-type': 'application/json'}
     payload = {
@@ -392,7 +429,7 @@ def prepare_json_from_yaml_config(
 def _send_local_messages():
     """Send all messages not been uploaded to Loki."""
     for msg_type, message in messages.items():
-        if not message.message_sent:
+        if not message.message_sent and msg_type != MessageType.HEARTBEAT:
             # Avoid the fallback entrypoint to send the message again
             # in normal case.
             try:
@@ -402,6 +439,11 @@ def _send_local_messages():
                              f'exception caught: {type(e)}({e})')
 
 
+def send_heartbeat(interval_seconds: int = 600):
+    messages.heartbeat.interval_seconds = interval_seconds
+    _send_to_loki(MessageType.HEARTBEAT)
+
+
 @contextlib.contextmanager
 def entrypoint_context(name: str, fallback: bool = False):
     """Context manager for entrypoint.

sky/utils/env_options.py

@@ -1,6 +1,7 @@
 """Global environment options for sky."""
 import enum
 import os
+from typing import Dict
 
 
 class Options(enum.Enum):
@@ -35,3 +36,8 @@ class Options(enum.Enum):
     def env_key(self) -> str:
         """The environment variable key name."""
         return self.value[0]
+
+    @classmethod
+    def all_options(cls) -> Dict[str, bool]:
+        """Returns all options as a dictionary."""
+        return {option.env_key: option.get() for option in list(Options)}

{skypilot_nightly-1.0.0.dev20250211.dist-info → skypilot_nightly-1.0.0.dev20250213.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250211
+Version: 1.0.0.dev20250213
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20250211.dist-info → skypilot_nightly-1.0.0.dev20250213.dist-info}/RECORD

@@ -1,20 +1,20 @@
-sky/__init__.py,sha256=1ezOy54DC-pEqSUADXHnHCcwQ1lU5dYJ6c14AjKNI1g,5560
+sky/__init__.py,sha256=ulWt-DtbuSpxFnOQtMOqMTPqAYFd6WgWd_T-bxS7_QM,5560
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=MNc9uHnvQ1EsEl8SsrYcYCGbxcnDbR6gaRCXVNd5RZE,22338
 sky/check.py,sha256=xzLlxUkBCrzpOho8lw65EvKLPl_b9lA2nteF5MSYbDQ,10885
-sky/cli.py,sha256=B-YWYiKnfSGdSOXtAY8SRGOGhneUeNPBjXFZ0FuLZ8w,214131
+sky/cli.py,sha256=_Q-XlsLN73e8BJilClajL7VOG8vINVJ_xRjENOpJdDA,213928
 sky/cloud_stores.py,sha256=PcLT57_8SZy7o6paAluElfBynaLkbaOq3l-8dNg1AVM,23672
 sky/core.py,sha256=fE1rn4Ku94S0XmWTO5-6t6eT6aaJImNczRqEnTe8v7Q,38742
 sky/dag.py,sha256=f3sJlkH4bE6Uuz3ozNtsMhcBpRx7KmC9Sa4seDKt4hU,3104
 sky/exceptions.py,sha256=SEhRubPlk-crkflPC5P_Z085iLrSd3UScYwc790QwYw,9378
-sky/execution.py,sha256=dpbk1kGRkGHT0FCJKGvjqeV3qIGEN2K20NDZbVrcAvI,28483
+sky/execution.py,sha256=vNUE9Z8hCSQeil7h3kdote2r6nkbrGXSqqmK6ru594Q,28453
 sky/global_user_state.py,sha256=cTwltMCDIIBaapuGgARxFwpDJDCiKKyVW-PP_qtWuCA,30241
-sky/optimizer.py,sha256=d5BPAEZVrS3a2oBclSwo8MWkHQKQ3u4tcyawOANN0_0,59836
-sky/resources.py,sha256=D3jteQxKOUydoNm7VDl90p02dwP3RpbO3gqNcl4dpOI,70327
+sky/optimizer.py,sha256=H5cpKELOQmnFpox0QXMB4P7jGhJxzXog4Ht_TYJaGuA,59758
+sky/resources.py,sha256=W7VO5nTizr-KIhOamOs7oSwmBGLjQZhQM6DoYbiAOsg,71648
 sky/sky_logging.py,sha256=7Zk9mL1TDxFkGsy3INMBKYlqsbognVGSMzAsHZdZlhw,5891
 sky/skypilot_config.py,sha256=FN93hSG-heQCHBnemlIK2TwrJngKbpx4vMXNUzPIzV8,9087
 sky/status_lib.py,sha256=J7Jb4_Dz0v2T64ttOdyUgpokvl4S0sBJrMfH7Fvo51A,1457
-sky/task.py,sha256=zri5_Ghh5-fjDf2rl997ZmL4IlXetW9u9XXJIRUJ3Qg,51353
+sky/task.py,sha256=Z74bBkOx1bFmGMoQRD3qbYxIZ5qi2AC5htY1KIsmvT0,52394
 sky/adaptors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/adaptors/aws.py,sha256=FNNC8B-iqSSiCLPiWGK_PLm1R-Kt4yI5JPIpdE0QJxQ,7565
 sky/adaptors/azure.py,sha256=yjM8nAPW-mlSXfmA8OmJNnSIrZ9lQx2-GxiI-TIVrwE,21910
@@ -51,11 +51,11 @@ sky/clouds/do.py,sha256=od4gMTrs2W5IkaDDr_oexOSdIOqn94vKq2U_QZcrpRk,11311
 sky/clouds/fluidstack.py,sha256=u2I6jXEtTqgqRWi2EafMsKqc8VkUq1cR6CSDUvk72_U,12407
 sky/clouds/gcp.py,sha256=6QOnefFsYiLCcnajjduLHsayqJ641bBu42jPTpvy7Mc,55007
 sky/clouds/ibm.py,sha256=0ArRTQx1_DpTNGByFhukzFedEDzmVjBsGiiques1bQ0,21447
-sky/clouds/kubernetes.py,sha256=oZg4Lpn2ZBikyc5NTJIziUPEY0xs2mtz546ButhkZ7g,31541
+sky/clouds/kubernetes.py,sha256=ocf8ZUlMbOcPb-n8JrSFix9rH75g089sC1JAd84JUXQ,31653
 sky/clouds/lambda_cloud.py,sha256=42AmcN2X_wdBMuAw606nR_pQCBAy5QFiAo711_WRqDE,12672
 sky/clouds/oci.py,sha256=VpPxpMJv52QePVdwdK9EuiMyqjp70dk8_rgUVv5Y-2w,27028
 sky/clouds/paperspace.py,sha256=F0Sj1RcqRb5fPjrr8qbdeY-JdfAHcRPc902pZOi4erw,10889
-sky/clouds/runpod.py,sha256=Wtaarp27_LTu5_E2agC7tTr2vhN1D4sblr2vZTT4vBI,11580
+sky/clouds/runpod.py,sha256=4gZTbUO4I8bzio5x9Km42ZIujZzCx2eszdkN0Mz5fqE,11893
 sky/clouds/scp.py,sha256=JHyMqkAAqr9lJq79IVjj3rU1g-ZCCGLZTJEzIhYsw7c,15845
 sky/clouds/vast.py,sha256=vQV489qkZMfDtt_SnXParPY49gkgKx5LZAEOsk65kIo,11231
 sky/clouds/vsphere.py,sha256=rrNf6_uHy4ukjHwaN35XVh2-Xj9k43-QGQkiEXyHYJk,12273
@@ -85,7 +85,7 @@ sky/clouds/service_catalog/data_fetchers/fetch_azure.py,sha256=7YVnoGDGGZI2TK02b
 sky/clouds/service_catalog/data_fetchers/fetch_cudo.py,sha256=52P48lvWN0s1ArjeLPeLemPRpxjSRcHincRle0nqdm4,3440
 sky/clouds/service_catalog/data_fetchers/fetch_fluidstack.py,sha256=yKuAFbjBRNz_e2RNNDT_aHHAuKQ86Ac7GKgIie5O6Pg,7273
 sky/clouds/service_catalog/data_fetchers/fetch_gcp.py,sha256=HLxdxA9DMSi19mgpVM_cERV4o-xh_tJ9vmkGm1wOaIE,30868
-sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=MN54h0CAGPHQAeF2eTmuESq3b0-d1kDARRUM6OkivCk,4962
+sky/clouds/service_catalog/data_fetchers/fetch_lambda_cloud.py,sha256=Bi5ta91p4SkFCoaEJUPKPjDB0FZ24DMR4NcKFpKCkxU,4979
 sky/clouds/service_catalog/data_fetchers/fetch_vast.py,sha256=zR9icM3ty5C8tGw13pQbsBtQQMgG4kl1j_jSGqqrgOA,4741
 sky/clouds/service_catalog/data_fetchers/fetch_vsphere.py,sha256=Opp2r3KSzXPtwk3lKNbO8IX9QzjoRSwy1kW3jPjtS1c,21453
 sky/clouds/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -115,10 +115,10 @@ sky/provision/__init__.py,sha256=hb_z69_7-FH1I8aDpFKNj2x_a8spzceWcovklutNgP8,637
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
 sky/provision/docker_utils.py,sha256=ENm0LkyrYWic3Ikyacho8X5uDMvGsbkZQsb6kNH1DuI,19629
-sky/provision/instance_setup.py,sha256=8Pudbpke6ah0xufr2UwtsDnNZ64-aAYkz8M44ZA0huI,23218
+sky/provision/instance_setup.py,sha256=YBFOwZQLBzpUjYoVQcX0KItej1rCBRWM23Dw9lg_q24,24386
 sky/provision/logging.py,sha256=yZWgejrFBhhRjAtvFu5N5bRXIMK5TuwNjp1vKQqz2pw,2103
 sky/provision/metadata_utils.py,sha256=LrxeV4wD2QPzNdXV_npj8q-pr35FatxBBjF_jSbpOT0,4013
-sky/provision/provisioner.py,sha256=ZOgFOO0NB4QZVPwd4qikRqi615Bq67n0Vcl3cTDVxNE,29153
+sky/provision/provisioner.py,sha256=UuMO2wp98Nk1nue25TtIaMU79Aj14v9IoIAFO_CUy9w,29636
 sky/provision/aws/__init__.py,sha256=mxq8PeWJqUtalDozTNpbtENErRZ1ktEs8uf2aG9UUgU,731
 sky/provision/aws/config.py,sha256=-4mr5uxgsl_8eLm_4DfP8JurZGSysGuY0iDeBTHnX5Q,25943
 sky/provision/aws/instance.py,sha256=3-R8ohuN8ooNh2Fqqb7-c4vNFy1xsw2GQF4PHg3APhE,40843
@@ -153,7 +153,7 @@ sky/provision/kubernetes/config.py,sha256=bXwOGdSAnXCkDreew0KsSUqSv3ZrptNeevqat7
 sky/provision/kubernetes/instance.py,sha256=AQikdRgNklpeMgiEd4w2Hh7kGssVABsy0aCh9xsKi5Y,50313
 sky/provision/kubernetes/network.py,sha256=EpNjRQ131CXepqbdkoRKFu4szVrm0oKEpv1l8EgOkjU,12364
 sky/provision/kubernetes/network_utils.py,sha256=52BZY_5ynCH6IXlivKObYyAHDgQCJyAJIjmM7J4MpFo,11393
-sky/provision/kubernetes/utils.py,sha256=4kSEx6NZB3MAsDqCxLO-elo7EO6Coh-9wypwVqs3jgk,109895
+sky/provision/kubernetes/utils.py,sha256=swOe6ozgSoucDtoJCExs0HLLWYuoi5HkIGMMSp7fEzc,109962
 sky/provision/kubernetes/manifests/smarter-device-manager-configmap.yaml,sha256=AMzYzlY0JIlfBWj5eX054Rc1XDW2thUcLSOGMJVhIdA,229
 sky/provision/kubernetes/manifests/smarter-device-manager-daemonset.yaml,sha256=RtTq4F1QUmR2Uunb6zuuRaPhV7hpesz4saHjn3Ncsb4,2010
 sky/provision/lambda_cloud/__init__.py,sha256=6EEvSgtUeEiup9ivIFevHmgv0GqleroO2X0K7TRa2nE,612
@@ -172,7 +172,7 @@ sky/provision/paperspace/utils.py,sha256=uOmxbDKjV6skFizC4gYXSxRuEqso5ck2kF7MbtN
 sky/provision/runpod/__init__.py,sha256=6HYvHI27EaLrX1SS0vWVhdLu5HDBeZCdvAeDJuwM5pk,556
 sky/provision/runpod/config.py,sha256=9ulZJVL7nHuxhTdoj8D7lNn7SdicJ5zc6FIcHIG9tcg,321
 sky/provision/runpod/instance.py,sha256=FNalpTQMvnmACdtFsGvVPVhEkzdqrlmd_pExn33bIQ8,10358
-sky/provision/runpod/utils.py,sha256=2L4MUrFi4rQOP26IeAkPNQ9oWGPJhH5fvz7NAbL3nNQ,12421
+sky/provision/runpod/utils.py,sha256=F3zsxPV3IY5C73J-zNZEfcXL8bsZBgNWzlmtSO6qOqI,12425
 sky/provision/runpod/api/__init__.py,sha256=eJwjPeQZ5B7chf4-Bl4YeI2Uo9aLX4M1rr2NmPk89_E,112
 sky/provision/runpod/api/commands.py,sha256=oh77PS0H0wZudHV8II9ceRuaFQ8FN4NJ4S3-6_PeqPM,4238
 sky/provision/runpod/api/pods.py,sha256=GMwxgNr9NnHPfyh2Y9b8S_vLhrLY4h7LybFBBQNAyfw,4948
@@ -215,12 +215,12 @@ sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/attempt_skylet.py,sha256=GZ6ITjjA0m-da3IxXXfoHR6n4pjp3X3TOXUqVvSrV0k,2136
 sky/skylet/autostop_lib.py,sha256=JPDHmByuhoNYXSUHl-OnyeJUkOFWn7gDM1FrS7Kr3E8,4478
 sky/skylet/configs.py,sha256=UtnpmEL0F9hH6PSjhsps7xgjGZ6qzPOfW1p2yj9tSng,1887
-sky/skylet/constants.py,sha256=EUSW4yH59eqBDLMIdmQWIYd3nAJBFoUeo5v9MGiginI,16057
-sky/skylet/events.py,sha256=0bOjUYpphuAficD9wDB5NOan2vwJDaRqdnm4sl0RK0U,12535
+sky/skylet/constants.py,sha256=bTvojyYHLhUxMI-xD8d6Fj6snQiK4-PLw-baRFKCcfQ,16125
+sky/skylet/events.py,sha256=__7bt6Z8q2W1vwTQv4yug-oAXDwSf8zBeRxb8HFM36U,12792
 sky/skylet/job_lib.py,sha256=Rk-C069cusJIRXsks8xqCb016JSt7GlpU7LrpX0qFJk,42785
 sky/skylet/log_lib.py,sha256=oFEBd85vDYFrIyyZKekH30yc4rRYILC0F0o-COQ64oE,20445
 sky/skylet/log_lib.pyi,sha256=rRk4eUX0RHGs1QL9CXsJq6RE7FqqxZlfuPJOLXTvg7I,4453
-sky/skylet/skylet.py,sha256=Tpv4yYR3jwxZsYeFPexB1gS1bCL5_AAfPzGKLsknPhA,1147
+sky/skylet/skylet.py,sha256=mWmqCvxSlfdVU_L8NL6P52jmCt3smd8K0HdyNBfMPeI,1234
 sky/skylet/subprocess_daemon.py,sha256=gcL-_Hea7-SrBUyZfAbo40RBFbaeuBmPCW0dm4YYkPo,3537
 sky/skylet/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/skylet/providers/command_runner.py,sha256=DdBKP0QX325_N3zAVYwnmXmfbfXNqkzWQZpF9DSR7Go,16259
@@ -250,20 +250,20 @@ sky/templates/jobs-controller.yaml.j2,sha256=FfagMkhXZdUWR6HtJHJ3JEZzJy4eov5CQZH
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
 sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8wyVEbRNFHRsBuLsw,626
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=iw7mypHszg6Ggq9MbyiYMFOkSlXaQZulaxqC5IWYGCc,3381
-sky/templates/kubernetes-ray.yml.j2,sha256=EHUDvALvhaPB44U7cdgXStV6v8Qh8yn5J4T6XFnmZoM,28856
+sky/templates/kubernetes-ray.yml.j2,sha256=x3Eq1ejG577E6eAZtJvpTlzXRCW5beMhqApV3J8BEZY,29019
 sky/templates/kubernetes-ssh-jump.yml.j2,sha256=k5W5sOIMppU7dDkJMwPlqsUcb92y7L5_TVG3hkgMy8M,2747
 sky/templates/lambda-ray.yml.j2,sha256=HyvO_tX2vxwSsc4IFVSqGuIbjLMk0bevP9bcxb8ZQII,4498
 sky/templates/local-ray.yml.j2,sha256=FNHeyHF6nW9nU9QLIZceUWfvrFTTcO51KqhTnYCEFaA,1185
 sky/templates/oci-ray.yml.j2,sha256=92dvXGaUd2Kwep9fgTjOsAPJiBLr8GQTjy7pFvuPAyE,4562
 sky/templates/paperspace-ray.yml.j2,sha256=HQjZNamrB_a4fOMCxQXSVdV5JIHtbGtAE0JzEO8uuVQ,4021
-sky/templates/runpod-ray.yml.j2,sha256=bUiF4Y_EkCA_GKLtTzPXbajdL-NOUiJ38Pe4dZd2dys,4284
+sky/templates/runpod-ray.yml.j2,sha256=95392Jvk-PwrEHIF2C5i-EnaqC31nTq4UTYZuQxUe1k,4310
 sky/templates/scp-ray.yml.j2,sha256=I9u8Ax-lit-d6UrCC9BVU8avst8w1cwK6TrzZBcz_JM,5608
 sky/templates/sky-serve-controller.yaml.j2,sha256=W4i1-OGRU2WDvauLC4EDXcYrNxj7mzRFSvSqzAKfehc,2020
 sky/templates/vast-ray.yml.j2,sha256=KaZLBJfI6FzAVRVq0NNM0_SN0RQUrDIehnJJ_LnvwnY,2990
 sky/templates/vsphere-ray.yml.j2,sha256=cOQ-qdpxGA2FHajMMhTJI-SmlYzdPterX4Gsiq-nkb0,3587
 sky/usage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sky/usage/constants.py,sha256=8xpg9vhDU9A3eObtpkNFjwa42oCazqGEv4yw_vJSO7U,590
-sky/usage/usage_lib.py,sha256=rjsekywo8IB_lJwRKBaWfQZ_znUJ-mIu1b9iWkCog88,18211
+sky/usage/constants.py,sha256=k7PQ-QP1p3tDgnzvy7QoxJjuTXWDUyVkbtPcIEvDsYM,632
+sky/usage/usage_lib.py,sha256=jpRt-24WVxYyd-XJz3_lSHboUKmWy8x8lRvvO-JO68g,20026
 sky/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/utils/accelerator_registry.py,sha256=cpPS9_MahQPt0ev4qPT-qyGpe12YD78UNj_gAvt720Q,4052
 sky/utils/admin_policy_utils.py,sha256=_Vt_jTTYCXmMdryj0vrrumFPewa93qHnzUqBDXjAhRU,5981
@@ -275,7 +275,7 @@ sky/utils/control_master_utils.py,sha256=90hnxiAUP20gbJ9e3MERh7rb04ZO_I3LsljNjR2
 sky/utils/controller_utils.py,sha256=SUrhK46ouBH2rm7azfFLIWr-T9-voYAdiXl2z5fG4Qw,45948
 sky/utils/dag_utils.py,sha256=l_0O3RUfe9OdQ9mtbhdlHpJVD4VAF_HQ3A75dgsYIjM,6099
 sky/utils/db_utils.py,sha256=K2-OHPg0FeHCarevMdWe0IWzm6wWumViEeYeJuGoFUE,3747
-sky/utils/env_options.py,sha256=E5iwRFBUY2Iq6e0y0c1Mv5OSQ4MRNdk0-p38xUyVerc,1366
+sky/utils/env_options.py,sha256=aaD6GoYK0LaZIqjOEZ-R7eccQuiRriW3EuLWtOI5En8,1578
 sky/utils/kubernetes_enums.py,sha256=imGqHSa8O07zD_6xH1SDMM7dBU5lF5fzFFlQuQy00QM,1384
 sky/utils/log_utils.py,sha256=AjkgSrk0GVOUbnnCEC2f4lsf2HOIXkZETCxR0BJw2-U,14152
 sky/utils/resources_utils.py,sha256=06Kx6AfbBdwBYGmIYFEY_qm6OBc2a5esZMPvIX7gCvc,7787
@@ -298,9 +298,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7Z
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20250211.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20250211.dist-info/METADATA,sha256=4PYhrvC32in1nSpeg1_y7Sx5Qx2mu5vqzLyUyRsAVbc,21397
-skypilot_nightly-1.0.0.dev20250211.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-skypilot_nightly-1.0.0.dev20250211.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20250211.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20250211.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250213.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250213.dist-info/METADATA,sha256=hLWjPTEc6CnhdlKg2uvdj1VZgjmf3XNt0fJV3R61GTc,21397
+skypilot_nightly-1.0.0.dev20250213.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+skypilot_nightly-1.0.0.dev20250213.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250213.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250213.dist-info/RECORD,,