skypilot-nightly 1.0.0.dev20250204__py3-none-any.whl → 1.0.0.dev20250205__py3-none-any.whl

sky/clouds/vast.py

@@ -0,0 +1,279 @@
+""" Vast Cloud. """
+
+import typing
+from typing import Dict, Iterator, List, Optional, Tuple, Union
+
+from sky import clouds
+from sky.clouds import service_catalog
+from sky.utils import resources_utils
+
+if typing.TYPE_CHECKING:
+    from sky import resources as resources_lib
+
+
+@clouds.CLOUD_REGISTRY.register
+class Vast(clouds.Cloud):
+    """ Vast GPU Cloud
+
+    _REPR | The string representation for the Vast GPU cloud object.
+    """
+    _REPR = 'Vast'
+    _CLOUD_UNSUPPORTED_FEATURES = {
+        clouds.CloudImplementationFeatures.MULTI_NODE:
+            ('Multi-node not supported yet, as the interconnection among nodes '
+             'are non-trivial on Vast.'),
+        clouds.CloudImplementationFeatures.CUSTOM_DISK_TIER:
+            ('Customizing disk tier is not supported yet on Vast.'),
+        clouds.CloudImplementationFeatures.OPEN_PORTS:
+            ('Opening ports is currently not supported on Vast.'),
+        clouds.CloudImplementationFeatures.STORAGE_MOUNTING:
+            ('Mounting object stores is not supported on Vast.'),
+    }
+    #
+    # Vast doesn't have a max cluster name limit. This number
+    # is reasonably large and exists to play nicely with the
+    # other providers
+    #
+    _MAX_CLUSTER_NAME_LEN_LIMIT = 120
+    _regions: List[clouds.Region] = []
+
+    PROVISIONER_VERSION = clouds.ProvisionerVersion.SKYPILOT
+    STATUS_VERSION = clouds.StatusVersion.SKYPILOT
+
+    @classmethod
+    def _unsupported_features_for_resources(
+        cls, resources: 'resources_lib.Resources'
+    ) -> Dict[clouds.CloudImplementationFeatures, str]:
+        """The features not supported based on the resources provided.
+
+        This method is used by check_features_are_supported() to check if the
+        cloud implementation supports all the requested features.
+
+        Returns:
+            A dict of {feature: reason} for the features not supported by the
+            cloud implementation.
+        """
+        del resources  # unused
+        return cls._CLOUD_UNSUPPORTED_FEATURES
+
+    @classmethod
+    def _max_cluster_name_length(cls) -> Optional[int]:
+        return cls._MAX_CLUSTER_NAME_LEN_LIMIT
+
+    @classmethod
+    def regions_with_offering(cls, instance_type: str,
+                              accelerators: Optional[Dict[str, int]],
+                              use_spot: bool, region: Optional[str],
+                              zone: Optional[str]) -> List[clouds.Region]:
+        assert zone is None, 'Vast does not support zones.'
+        del accelerators, zone  # unused
+        regions = service_catalog.get_region_zones_for_instance_type(
+            instance_type, use_spot, 'vast')
+
+        if region is not None:
+            regions = [r for r in regions if r.name == region]
+        return regions
+
+    @classmethod
+    def get_vcpus_mem_from_instance_type(
+        cls,
+        instance_type: str,
+    ) -> Tuple[Optional[float], Optional[float]]:
+        return service_catalog.get_vcpus_mem_from_instance_type(instance_type,
+                                                                clouds='vast')
+
+    @classmethod
+    def zones_provision_loop(
+        cls,
+        *,
+        region: str,
+        num_nodes: int,
+        instance_type: str,
+        accelerators: Optional[Dict[str, int]] = None,
+        use_spot: bool = False,
+    ) -> Iterator[None]:
+        del num_nodes  # unused
+        regions = cls.regions_with_offering(instance_type,
+                                            accelerators,
+                                            use_spot,
+                                            region=region,
+                                            zone=None)
+        for r in regions:
+            assert r.zones is None, r
+            yield r.zones
+
+    def instance_type_to_hourly_cost(self,
+                                     instance_type: str,
+                                     use_spot: bool,
+                                     region: Optional[str] = None,
+                                     zone: Optional[str] = None) -> float:
+        return service_catalog.get_hourly_cost(instance_type,
+                                               use_spot=use_spot,
+                                               region=region,
+                                               zone=zone,
+                                               clouds='vast')
+
+    def accelerators_to_hourly_cost(self,
+                                    accelerators: Dict[str, int],
+                                    use_spot: bool,
+                                    region: Optional[str] = None,
+                                    zone: Optional[str] = None) -> float:
+        """Returns the hourly cost of the accelerators, in dollars/hour."""
+        del accelerators, use_spot, region, zone  # unused
+        return 0.0  # Vast includes accelerators in the hourly cost.
+
+    def get_egress_cost(self, num_gigabytes: float) -> float:
+        return 0.0
+
+    @classmethod
+    def get_default_instance_type(
+            cls,
+            cpus: Optional[str] = None,
+            memory: Optional[str] = None,
+            disk_tier: Optional[resources_utils.DiskTier] = None
+    ) -> Optional[str]:
+        """Returns the default instance type for Vast."""
+        return service_catalog.get_default_instance_type(cpus=cpus,
+                                                         memory=memory,
+                                                         disk_tier=disk_tier,
+                                                         clouds='vast')
+
+    @classmethod
+    def get_accelerators_from_instance_type(
+            cls, instance_type: str) -> Optional[Dict[str, Union[int, float]]]:
+        return service_catalog.get_accelerators_from_instance_type(
+            instance_type, clouds='vast')
+
+    @classmethod
+    def get_zone_shell_cmd(cls) -> Optional[str]:
+        return None
+
+    def make_deploy_resources_variables(
+            self,
+            resources: 'resources_lib.Resources',
+            cluster_name: resources_utils.ClusterName,
+            region: 'clouds.Region',
+            zones: Optional[List['clouds.Zone']],
+            num_nodes: int,
+            dryrun: bool = False) -> Dict[str, Optional[str]]:
+        del zones, dryrun, cluster_name, num_nodes  # unused
+
+        r = resources
+        acc_dict = self.get_accelerators_from_instance_type(r.instance_type)
+        custom_resources = resources_utils.make_ray_custom_resources_str(
+            acc_dict)
+
+        if r.image_id is None:
+            image_id = 'vastai/base:0.0.2'
+        elif r.extract_docker_image() is not None:
+            image_id = r.extract_docker_image()
+        else:
+            image_id = r.image_id[r.region]
+
+        return {
+            'instance_type': resources.instance_type,
+            'custom_resources': custom_resources,
+            'region': region.name,
+            'image_id': image_id,
+        }
+
+    def _get_feasible_launchable_resources(
+        self, resources: 'resources_lib.Resources'
+    ) -> 'resources_utils.FeasibleResources':
+        """Returns a list of feasible resources for the given resources."""
+        if resources.instance_type is not None:
+            assert resources.is_launchable(), resources
+            resources = resources.copy(accelerators=None)
+            return resources_utils.FeasibleResources([resources], [], None)
+
+        def _make(instance_list):
+            resource_list = []
+            for instance_type in instance_list:
+                r = resources.copy(
+                    cloud=Vast(),
+                    instance_type=instance_type,
+                    accelerators=None,
+                    cpus=None,
+                )
+                resource_list.append(r)
+            return resource_list
+
+        # Currently, handle a filter on accelerators only.
+        accelerators = resources.accelerators
+        if accelerators is None:
+            # Return a default instance type
+            default_instance_type = Vast.get_default_instance_type(
+                cpus=resources.cpus,
+                memory=resources.memory,
+                disk_tier=resources.disk_tier)
+            if default_instance_type is None:
+                # TODO: Add hints to all return values in this method to help
+                #  users understand why the resources are not launchable.
+                return resources_utils.FeasibleResources([], [], None)
+            else:
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
+
+        assert len(accelerators) == 1, resources
+        acc, acc_count = list(accelerators.items())[0]
+        (instance_list, fuzzy_candidate_list
+        ) = service_catalog.get_instance_type_for_accelerator(
+            acc,
+            acc_count,
+            use_spot=resources.use_spot,
+            cpus=resources.cpus,
+            region=resources.region,
+            zone=resources.zone,
+            memory=resources.memory,
+            clouds='vast')
+        if instance_list is None:
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
+
+    @classmethod
+    def check_credentials(cls) -> Tuple[bool, Optional[str]]:
+        """ Verify that the user has valid credentials for Vast. """
+        try:
+            import vastai_sdk as _vast  # pylint: disable=import-outside-toplevel
+            vast = _vast.VastAI()
+
+            # We only support file pased credential passing
+            if vast.creds_source != 'FILE':
+                return False, (
+                    'error \n'  # First line is indented by 4 spaces
+                    '    Credentials can be set up by running: \n'
+                    '        $ pip install vastai\n'
+                    '        $ echo [key] > ~/.vast_api_key\n'
+                    '    For more information, see https://skypilot.readthedocs.io/en/latest/getting-started/installation.html#vast'  # pylint: disable=line-too-long
+                )
+
+            return True, None
+
+        except ImportError:
+            return False, ('Failed to import vast. '
+                           'To install, run: pip install skypilot[vast]')
+
+    def get_credential_file_mounts(self) -> Dict[str, str]:
+        return {
+            '~/.config/vastai/vast_api_key': '~/.config/vastai/vast_api_key'
+        }
+
+    @classmethod
+    def get_user_identities(cls) -> Optional[List[List[str]]]:
+        # NOTE: used for very advanced SkyPilot functionality
+        # Can implement later if desired
+        return None
+
+    def instance_type_exists(self, instance_type: str) -> bool:
+        return service_catalog.instance_type_exists(instance_type, 'vast')
+
+    def validate_region_zone(self, region: Optional[str], zone: Optional[str]):
+        return service_catalog.validate_region_zone(region, zone, clouds='vast')
+
+    @classmethod
+    def get_image_size(cls, image_id: str, region: Optional[str]) -> float:
+        # TODO: use 0.0 for now to allow all images. We should change this to
+        # return the docker image size.
+        return 0.0

sky/jobs/dashboard/templates/index.html

@@ -10,7 +10,6 @@
         integrity="sha384-rbsA2VBKQhggwzxH7pPCaAqO46MgnOM80zW1RWuH61DGLwZJEdK2Kadq2F9CUG65" crossorigin="anonymous">
     <style>
         :root {
-            --primary-color: #0d6efd;
             --secondary-color: #6c757d;
             --success-color: #198754;
             --warning-color: #ffc107;
@@ -98,10 +97,38 @@
 
         /* Allow Details column to wrap */
         .table th:nth-child(12),  /* Details column */
+
         .table td:nth-child(12) {
-            white-space: normal;  /* Allow text wrapping */
-            max-width: 250px;     /* Limit width to prevent excessive stretching */
-            word-wrap: break-word; /* Break long words if needed */
+            max-width: 250px;     /* Limit width */
+            overflow: hidden;     /* Hide overflow */
+            text-overflow: ellipsis; /* Show ellipsis for overflow */
+            position: relative;   /* For tooltip positioning */
+            cursor: pointer !important;  /* Force show pointer cursor */
+            padding-right: 24px;  /* Make room for the arrow */
+        }
+
+        .table td:nth-child(12)::after {
+            content: '▼';
+            position: absolute;
+            right: 8px;
+            top: 12px;
+            font-size: 0.6em;
+            opacity: 0.5;
+            display: none;  /* Hide by default */
+        }
+
+        .table td:nth-child(12).expandable::after {
+            display: block;  /* Only show for expandable content */
+        }
+
+        .table td:nth-child(12).expanded::after {
+            transform: rotate(180deg);
+        }
+
+        .table td:nth-child(12).expanded {
+            max-width: none;
+            white-space: normal;
+            word-wrap: break-word;
         }
 
         .badge {
@@ -190,11 +217,11 @@
         }
 
         .status-container:hover::after {
-            content: attr(title);
+            content: attr(data-tooltip);
             position: absolute;
-            left: 0;          /* Changed from 50% */
-            bottom: 100%;
-            transform: translateY(-8px);  /* Removed translateX */
+            left: 0;
+            top: 100%;
+            transform: translateY(8px);
             z-index: 1001;
             background-color: rgba(33, 37, 41, 0.9);
             color: white;
@@ -214,11 +241,11 @@
         .status-container:hover::before {
             content: '';
             position: absolute;
-            left: 20px;       /* Changed from 50% */
-            bottom: 100%;
-            transform: none;   /* Removed transform */
+            left: 20px;
+            top: 100%;          /* Changed from bottom: 100% */
+            transform: none;
             border: 8px solid transparent;
-            border-top-color: rgba(33, 37, 41, 0.9);
+            border-bottom-color: rgba(33, 37, 41, 0.9);  /* Changed from border-top-color */
             z-index: 1001;
             pointer-events: none;
             opacity: 0;
@@ -236,7 +263,7 @@
         .table {
             overflow: visible !important;
         }
-        
+
         .fixed-header-table {
             overflow: visible !important;
         }
@@ -262,9 +289,9 @@
         }
 
         #last-updated:hover::after {
-            content: attr(title);
+            content: attr(data-tooltip);
             position: absolute;
-            bottom: 100%;
+            top: 100%;           /* Changed from bottom: 100% to top: 100% */
             left: 50%;
             transform: translateX(-50%);
             padding: 0.5rem 1rem;
@@ -274,34 +301,33 @@
             font-size: 0.875rem;
             white-space: nowrap;
             z-index: 1000;
-            margin-bottom: 8px;
+            margin-top: 8px;     /* Changed from margin-bottom to margin-top */
         }
 
         #last-updated:hover::before {
             content: '';
             position: absolute;
-            bottom: 100%;
+            top: 100%;           /* Changed from bottom: 100% to top: 100% */
             left: 50%;
             transform: translateX(-50%);
             border: 8px solid transparent;
-            border-top-color: rgba(33, 37, 41, 0.9);
-            margin-bottom: -8px;
+            border-bottom-color: rgba(33, 37, 41, 0.9);  /* Changed from border-top-color to border-bottom-color */
+            margin-top: -8px;    /* Changed from margin-bottom to margin-top */
             z-index: 1000;
         }
 
         .clickable-badge {
             cursor: pointer;
             transition: transform 0.2s, opacity 0.2s;
-            opacity: 0.4; 
+            opacity: 0.4;
         }
 
         .clickable-badge:hover {
-            transform: scale(1.1);
+            transform: scale(1.05);
             opacity: 1;
         }
 
         .clickable-badge.selected-filter {
-            transform: scale(1.1);
             opacity: 1;
             box-shadow: 0 0 0 2px #fff, 0 0 0 4px currentColor;
         }
@@ -313,6 +339,39 @@
         #last-updated:hover::before {
             z-index: 1001;
         }
+
+        /* Add tooltip styles for refresh label */
+        .refresh-label {
+            position: relative;
+            cursor: help;
+        }
+
+        .refresh-label:hover::after {
+            content: attr(data-tooltip);
+            position: absolute;
+            left: 50%;
+            top: 100%;
+            transform: translateX(-50%) translateY(8px);
+            z-index: 1001;
+            background-color: rgba(33, 37, 41, 0.9);
+            color: white;
+            padding: 0.5rem 1rem;
+            border-radius: 6px;
+            font-size: 0.875rem;
+            white-space: nowrap;
+            box-shadow: 0 2px 12px rgba(0, 0, 0, 0.15);
+        }
+
+        .refresh-label:hover::before {
+            content: '';
+            position: absolute;
+            left: 50%;
+            top: 100%;
+            transform: translateX(-50%);
+            border: 8px solid transparent;
+            border-bottom-color: rgba(33, 37, 41, 0.9);
+            z-index: 1001;
+        }
     </style>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.1/moment.min.js"></script>
     <script
@@ -328,14 +387,14 @@
                 <div class="d-flex align-items-center">
                     <div class="form-check form-switch me-3">
                         <input class="form-check-input" type="checkbox" id="refresh-toggle" checked>
-                        <label class="form-check-label" for="refresh-toggle">
+                        <label class="form-check-label refresh-label" for="refresh-toggle" data-tooltip="Refreshes every 30 seconds">
                             Auto-refresh
                             <span class="refresh-indicator">
                                 <span class="refresh-spinner" id="refresh-spinner"></span>
                             </span>
                         </label>
                     </div>
-                    <p class="text-muted mb-0" id="last-updated"></p>
+                    <p class="text-muted mb-0" id="last-updated" data-tooltip="{{ utcTimestamp }}"></p>
                 </div>
             </div>
         </header>
@@ -349,7 +408,7 @@
         </select>
 
         {% if rows %}
-        <p>Filter By : 
+        <p>Filter by status:
             <span class="badge bg-secondary clickable-badge selected-filter me-2" data-status="ALL">All</span>
             {% set status_dict = {} %}
             {% for row in rows %}
@@ -362,7 +421,7 @@
             {% endfor %}
             {% for status, count in status_dict|dictsort %}
                 <span class="me-2">
-                    <span class="me-1">; {{ count }}</span>
+                    <span class="me-1">| {{ count }}</span>
                     {% if status.startswith('RUNNING') %}
                     <span class="badge bg-primary clickable-badge" data-status="{{ status }}">{{ status }}</span>
                     {% elif status.startswith('PENDING') or status.startswith('SUBMITTED') %}
@@ -414,7 +473,7 @@
                     <td>{{ row[7] }}</td>
                     <td>
                         <!-- Status column with tooltip -->
-                        <div class="status-container" style="position: relative;" title="{{ row[14] }}">
+                        <div class="status-container" style="position: relative;" data-tooltip="{{ row[14] }}">
                             {% if row[9].startswith('RUNNING') %}
                             <span class="badge bg-primary">{{ row[9].split()[0] }}</span>{{ row[9][row[9].split()[0]|length:] }}
                             {% elif row[9].startswith('PENDING') or row[9].startswith('SUBMITTED') %}
@@ -436,12 +495,12 @@
                     <td>{{ row[11] }}</td>  {# Cluster #}
                     <td>{{ row[12] }}</td>  {# Region #}
                     <td>{{ row[8] }}</td>   {# Recoveries #}
-                    <td>{{ row[13] }}</td>  {# Details #}
+                    <td data-full-text="{{ row[13] }}">{{ row[13] }}</td>  {# Details #}
                     <td>
                         {% if row[1]|string|replace(' \u21B3', '') and row[1]|string|replace(' \u21B3', '') != '-' %}
-                        <a href="{{ url_for('download_log', job_id=row[1]|string|replace(' \u21B3', '')) }}" 
+                        <a href="{{ url_for('download_log', job_id=row[1]|string|replace(' \u21B3', '')) }}"
                            class="btn btn-sm btn-outline-secondary">
-                            Controller Log
+                            controller log
                         </a>
                         {% endif %}
                     </td>
@@ -515,7 +574,7 @@
             var localTimestamp = moment.utc(timestamp).tz(moment.tz.guess()).format('YYYY-MM-DD HH:mm:ss z');
             var utcTimestamp = moment.utc(timestamp).format('YYYY-MM-DD HH:mm:ss UTC');
             document.getElementById("last-updated").textContent = "Last updated: " + localTimestamp;
-            document.getElementById("last-updated").title = utcTimestamp;
+            document.getElementById("last-updated").setAttribute('data-tooltip', utcTimestamp);
         });
     </script>
     <script>
@@ -567,7 +626,7 @@
                 var statusCell = row.querySelector("td:nth-child(7)"); // Status is now in the 7th column
                 if (statusCell) {
                     var statusText = statusCell.textContent.trim().split(' ')[0]; // Get first word of status
-                    
+
                     if (status === '' || statusText === status) {
                         row.style.display = "";
                     } else {
@@ -589,18 +648,6 @@
         window.addEventListener('beforeunload', function() {
             document.getElementById('refresh-spinner').style.display = 'inline-block';
         });
-
-        // Enhance table row hover effect
-        document.querySelectorAll('tbody tr').forEach(row => {
-            row.addEventListener('mouseenter', function() {
-                this.style.transform = 'scale(1.002)';
-                this.style.transition = 'transform 0.2s';
-            });
-            
-            row.addEventListener('mouseleave', function() {
-                this.style.transform = 'scale(1)';
-            });
-        });
     </script>
     <script>
         // Update column indices for job table
@@ -615,20 +662,20 @@
         document.addEventListener("DOMContentLoaded", function() {
             const statusFilter = document.getElementById('status-filter');
             const badges = document.querySelectorAll('.clickable-badge');
-            
+
             // Set initial state
             const savedFilter = localStorage.getItem("statusFilter") || '';
             updateSelectedBadge(savedFilter);
-            
+
             badges.forEach(function(badge) {
                 badge.addEventListener('click', function() {
                     const status = this.dataset.status;
                     const currentFilter = statusFilter.value;
-                    
+
                     // If clicking the already selected filter, clear it (show all)
-                    const newStatus = (status === currentFilter || (status === 'ALL' && currentFilter === '')) ? '' : 
+                    const newStatus = (status === currentFilter || (status === 'ALL' && currentFilter === '')) ? '' :
                                     (status === 'ALL' ? '' : status);
-                    
+
                     // Update filter and UI
                     statusFilter.value = newStatus;
                     filterStatus(newStatus);
@@ -636,11 +683,11 @@
                     updateSelectedBadge(newStatus);
                 });
             });
-            
+
             function updateSelectedBadge(selectedStatus) {
                 badges.forEach(badge => {
                     badge.classList.remove('selected-filter');
-                    if ((selectedStatus === '' && badge.dataset.status === 'ALL') || 
+                    if ((selectedStatus === '' && badge.dataset.status === 'ALL') ||
                         badge.dataset.status === selectedStatus) {
                         badge.classList.add('selected-filter');
                     }
@@ -653,7 +700,7 @@
         document.addEventListener("DOMContentLoaded", function() {
             const header = document.querySelector('header');
             const container = document.querySelector('.container');
-            
+
             window.addEventListener('scroll', function() {
                 if (container.getBoundingClientRect().top < 0) {
                     header.classList.add('sticky');
@@ -663,7 +710,25 @@
             });
         });
     </script>
+<script>
+    // Add click handler for Details cells
+    document.addEventListener('DOMContentLoaded', function() {
+        const detailsCells = document.querySelectorAll('#jobs-table td:nth-child(12)');
+
+        detailsCells.forEach(cell => {
+            // Check if content is truncated
+            if (cell.scrollWidth > cell.clientWidth) {
+                cell.classList.add('expandable');
+            }
 
+            cell.addEventListener('click', function() {
+                if (this.scrollWidth > this.clientWidth || this.classList.contains('expanded')) {
+                    this.classList.toggle('expanded');
+                }
+            });
+        });
+    });
+</script>
 </body>
 
 </html>

sky/jobs/scheduler.py

@@ -60,6 +60,14 @@ logger = sky_logging.init_logger('sky.jobs.controller')
 _MANAGED_JOB_SCHEDULER_LOCK = '~/.sky/locks/managed_job_scheduler.lock'
 _ALIVE_JOB_LAUNCH_WAIT_INTERVAL = 0.5
 
+# Based on testing, assume a running job uses 350MB memory.
+JOB_MEMORY_MB = 350
+# Past 2000 simultaneous jobs, we become unstable.
+# See https://github.com/skypilot-org/skypilot/issues/4649.
+MAX_JOB_LIMIT = 2000
+# Number of ongoing launches launches allowed per CPU.
+LAUNCHES_PER_CPU = 4
+
 
 @lru_cache(maxsize=1)
 def _get_lock_path() -> str:
@@ -247,15 +255,16 @@ def _set_alive_waiting(job_id: int) -> None:
 
 
 def _get_job_parallelism() -> int:
-    # Assume a running job uses 350MB memory.
-    # We observe 230-300 in practice.
-    job_memory = 350 * 1024 * 1024
-    return max(psutil.virtual_memory().total // job_memory, 1)
+    job_memory = JOB_MEMORY_MB * 1024 * 1024
+
+    job_limit = min(psutil.virtual_memory().total // job_memory, MAX_JOB_LIMIT)
+
+    return max(job_limit, 1)
 
 
 def _get_launch_parallelism() -> int:
     cpus = os.cpu_count()
-    return cpus * 4 if cpus is not None else 1
+    return cpus * LAUNCHES_PER_CPU if cpus is not None else 1
 
 
 def _can_start_new_job() -> bool:

sky/provision/__init__.py

@@ -22,6 +22,7 @@ from sky.provision import kubernetes
 from sky.provision import lambda_cloud
 from sky.provision import oci
 from sky.provision import runpod
+from sky.provision import vast
 from sky.provision import vsphere
 from sky.utils import command_runner
 from sky.utils import timeline