skypilot-nightly 1.0.0.dev20250206__tar.gz → 1.0.0.dev20250208__tar.gz

{skypilot_nightly-1.0.0.dev20250206/skypilot_nightly.egg-info → skypilot_nightly-1.0.0.dev20250208}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: skypilot-nightly
-Version: 1.0.0.dev20250206
+Version: 1.0.0.dev20250208
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -244,7 +244,7 @@ To get the latest features and fixes, use the nightly build or [install from sou
 pip install "skypilot-nightly[kubernetes,aws,gcp,azure,oci,lambda,runpod,fluidstack,paperspace,cudo,ibm,scp]"
 ```
 
-[Current supported infra](https://docs.skypilot.co/en/latest/getting-started/installation.html) (Kubernetes; AWS, GCP, Azure, OCI, Lambda Cloud, Fluidstack, RunPod, Cudo, Paperspace, Cloudflare, Samsung, IBM, VMware vSphere):
+[Current supported infra](https://docs.skypilot.co/en/latest/getting-started/installation.html) (Kubernetes; AWS, GCP, Azure, OCI, Lambda Cloud, Fluidstack, RunPod, Cudo, Digital Ocean, Paperspace, Cloudflare, Samsung, IBM, Vast.ai, VMware vSphere):
 <p align="center">
   <img alt="SkyPilot" src="https://raw.githubusercontent.com/skypilot-org/skypilot/master/docs/source/images/cloud-logos-light.png" width=85%>
 </p>

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/README.md

@@ -86,7 +86,7 @@ To get the latest features and fixes, use the nightly build or [install from sou
 pip install "skypilot-nightly[kubernetes,aws,gcp,azure,oci,lambda,runpod,fluidstack,paperspace,cudo,ibm,scp]"
 ```
 
-[Current supported infra](https://docs.skypilot.co/en/latest/getting-started/installation.html) (Kubernetes; AWS, GCP, Azure, OCI, Lambda Cloud, Fluidstack, RunPod, Cudo, Paperspace, Cloudflare, Samsung, IBM, VMware vSphere):
+[Current supported infra](https://docs.skypilot.co/en/latest/getting-started/installation.html) (Kubernetes; AWS, GCP, Azure, OCI, Lambda Cloud, Fluidstack, RunPod, Cudo, Digital Ocean, Paperspace, Cloudflare, Samsung, IBM, Vast.ai, VMware vSphere):
 <p align="center">
   <picture>
     <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/skypilot-org/skypilot/master/docs/source/images/cloud-logos-dark.png">

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = '1e284afda6c2808e521e629a025267a12744b0db'
+_SKYPILOT_COMMIT_SHA = '54fe787d1fb31687cc78eb307db59d54c2d79076'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20250206'
+__version__ = '1.0.0.dev20250208'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/authentication.py

@@ -289,7 +289,7 @@ def setup_lambda_authentication(config: Dict[str, Any]) -> Dict[str, Any]:
     config['auth']['ssh_public_key'] = PUBLIC_SSH_KEY_PATH
 
     # TODO(zhwu): we need to avoid uploading the public ssh key to the
-    # nodes, as that will cause problem when the node is used as spot
+    # nodes, as that will cause problem when the node is used as jobs
     # controller, i.e., the public and private key on the node may
     # not match.
     file_mounts = config['file_mounts']

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/backends/backend_utils.py

@@ -844,11 +844,6 @@ def write_cluster_config(
 
     # User-supplied global instance tags from ~/.sky/config.yaml.
     labels = skypilot_config.get_nested((str(cloud).lower(), 'labels'), {})
-    # Deprecated: instance_tags have been replaced by labels. For backward
-    # compatibility, we support them and the schema allows them only if
-    # `labels` are not specified. This should be removed after 0.8.0.
-    labels = skypilot_config.get_nested((str(cloud).lower(), 'instance_tags'),
-                                        labels)
     # labels is a dict, which is guaranteed by the type check in
     # schemas.py
     assert isinstance(labels, dict), labels
@@ -997,7 +992,7 @@ def write_cluster_config(
 
     # Read the cluster name from the tmp yaml file, to take the backward
     # compatbility restortion above into account.
-    # TODO: remove this after 2 minor releases, 0.8.0.
+    # TODO: remove this after 2 minor releases, 0.10.0.
     yaml_config = common_utils.read_yaml(tmp_yaml_path)
     config_dict['cluster_name_on_cloud'] = yaml_config['cluster_name']
 

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/backends/cloud_vm_ray_backend.py

@@ -2446,7 +2446,7 @@ class CloudVmRayResourceHandle(backends.backend.ResourceHandle):
             # can be None. We need to update it here, even when force_cached is
             # set to True.
             # TODO: We can remove `self.cached_external_ips is None` after
-            # version 0.8.0.
+            # all clouds moved to new provisioner.
             if force_cached and self.cached_external_ips is None:
                 raise RuntimeError(
                     'Tried to use cached cluster info, but it\'s missing for '
@@ -3416,17 +3416,11 @@ class CloudVmRayBackend(backends.Backend['CloudVmRayResourceHandle']):
             managed_job_code = managed_job_codegen.set_pending(
                 job_id, managed_job_dag)
             # Set the managed job to PENDING state to make sure that this
-            # managed job appears in the `sky jobs queue`, when there are
-            # already 2x vCPU controller processes running on the controller VM,
-            # e.g., 16 controller processes running on a controller with 8
-            # vCPUs.
-            # The managed job should be set to PENDING state *after* the
-            # controller process job has been queued, as our skylet on spot
-            # controller will set the managed job in FAILED state if the
-            # controller process job does not exist.
-            # We cannot set the managed job to PENDING state in the codegen for
-            # the controller process job, as it will stay in the job pending
-            # table and not be executed until there is an empty slot.
+            # managed job appears in the `sky jobs queue`, even if it needs to
+            # wait to be submitted.
+            # We cannot set the managed job to PENDING state in the job template
+            # (jobs-controller.yaml.j2), as it may need to wait for the run
+            # commands to be scheduled on the job controller in high-load cases.
             job_submit_cmd = job_submit_cmd + ' && ' + managed_job_code
 
         returncode, stdout, stderr = self.run_on_head(handle,

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/clouds/do.py

@@ -257,6 +257,12 @@ class DO(clouds.Cloud):
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:
         """Verify that the user has valid credentials for DO."""
+
+        try:
+            do.exceptions()
+        except ImportError as err:
+            return False, str(err)
+
         try:
             # attempt to make a CURL request for listing instances
             do_utils.client().droplets.list()

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/clouds/service_catalog/data_fetchers/fetch_aws.py

@@ -306,6 +306,12 @@ def _get_instance_types_df(region: str) -> Union[str, 'pd.DataFrame']:
                 assert find_num_in_name is not None, row['InstanceType']
                 num_in_name = find_num_in_name.group(1)
                 acc_count = int(num_in_name) // 2
+            if row['InstanceType'] == 'p5en.48xlarge':
+                # TODO(andyl): Check if this workaround still needed after
+                # v0.10.0 released. Currently, the acc_name returned by the
+                # AWS API is 'NVIDIA', which is incorrect. See #4652.
+                acc_name = 'H200'
+                acc_count = 8
             return pd.Series({
                 'AcceleratorName': acc_name,
                 'AcceleratorCount': acc_count,

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/exceptions.py

@@ -81,6 +81,14 @@ class ManagedJobReachedMaxRetriesError(Exception):
     pass
 
 
+class ManagedJobStatusError(Exception):
+    """Raised when a managed job task status update is invalid.
+
+    For instance, a RUNNING job cannot become SUBMITTED.
+    """
+    pass
+
+
 class ResourcesMismatchError(Exception):
     """Raised when resources are mismatched."""
     pass

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/jobs/controller.py

@@ -68,7 +68,7 @@ class JobsController:
             else:
                 assert task.name is not None, task
                 task_name = task.name
-                # This is guaranteed by the spot_launch API, where we fill in
+                # This is guaranteed by the jobs.launch API, where we fill in
                 # the task.name with
                 # dag_utils.maybe_infer_and_fill_dag_and_task_names.
                 assert task_name is not None, self._dag
@@ -137,8 +137,8 @@ class JobsController:
                 1. The optimizer cannot find a feasible solution.
                 2. Precheck errors: invalid cluster name, failure in getting
                 cloud user identity, or unsupported feature.
-            exceptions.SpotJobReachedMaxRetryError: This will be raised when
-                all prechecks passed but the maximum number of retries is
+            exceptions.ManagedJobReachedMaxRetriesError: This will be raised
+                when all prechecks passed but the maximum number of retries is
                 reached for `sky.launch`. The failure of `sky.launch` can be
                 due to:
                 1. Any of the underlying failover exceptions is due to resources
@@ -482,8 +482,6 @@ def _cleanup(job_id: int, dag_yaml: str):
         when reaching here, as we currently only support chain DAGs, and only
         task is executed at a time.
     """
-    # NOTE: The code to get cluster name is same as what we did in the spot
-    # controller, we should keep it in sync with JobsController.__init__()
     dag, _ = _get_dag_and_name(dag_yaml)
     for task in dag.tasks:
         assert task.name is not None, task

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/jobs/core.py

@@ -472,24 +472,3 @@ def sync_down_logs(
                                        job_name=name,
                                        controller=controller,
                                        local_dir=local_dir)
-
-
-spot_launch = common_utils.deprecated_function(
-    launch,
-    name='sky.jobs.launch',
-    deprecated_name='spot_launch',
-    removing_version='0.8.0',
-    override_argument={'use_spot': True})
-spot_queue = common_utils.deprecated_function(queue,
-                                              name='sky.jobs.queue',
-                                              deprecated_name='spot_queue',
-                                              removing_version='0.8.0')
-spot_cancel = common_utils.deprecated_function(cancel,
-                                               name='sky.jobs.cancel',
-                                               deprecated_name='spot_cancel',
-                                               removing_version='0.8.0')
-spot_tail_logs = common_utils.deprecated_function(
-    tail_logs,
-    name='sky.jobs.tail_logs',
-    deprecated_name='spot_tail_logs',
-    removing_version='0.8.0')

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/jobs/recovery_strategy.py

@@ -263,8 +263,8 @@ class StrategyExecutor:
                 1. The optimizer cannot find a feasible solution.
                 2. Precheck errors: invalid cluster name, failure in getting
                 cloud user identity, or unsupported feature.
-            exceptions.SpotJobReachedMaxRetryError: This will be raised when
-                all prechecks passed but the maximum number of retries is
+            exceptions.ManagedJobReachedMaxRetriesError: This will be raised
+                when all prechecks passed but the maximum number of retries is
                 reached for `sky.launch`. The failure of `sky.launch` can be
                 due to:
                 1. Any of the underlying failover exceptions is due to resources

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/jobs/state.py

@@ -11,6 +11,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import colorama
 
+from sky import exceptions
 from sky import sky_logging
 from sky.utils import common_utils
 from sky.utils import db_utils
@@ -32,7 +33,7 @@ logger = sky_logging.init_logger(__name__)
 # the same content as the `task_name` column.
 # The `job_id` is now not really a job id, but a only a unique
 # identifier/primary key for all the tasks. We will use `spot_job_id`
-# to identify the spot job.
+# to identify the job.
 # TODO(zhwu): schema migration may be needed.
 def create_table(cursor, conn):
     # Enable WAL mode to avoid locking issues.
@@ -420,9 +421,16 @@ def set_submitted(job_id: int, task_id: int, run_timestamp: str,
             run_timestamp=(?),
             specs=(?)
             WHERE spot_job_id=(?) AND
-            task_id=(?)""",
+            task_id=(?) AND
+            status=(?) AND
+            end_at IS null""",
             (resources_str, submit_time, ManagedJobStatus.SUBMITTED.value,
-             run_timestamp, json.dumps(specs), job_id, task_id))
+             run_timestamp, json.dumps(specs), job_id, task_id,
+             ManagedJobStatus.PENDING.value))
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                f'Failed to set the task to submitted. '
+                f'({cursor.rowcount} rows updated)')
     callback_func('SUBMITTED')
 
 
@@ -434,7 +442,14 @@ def set_starting(job_id: int, task_id: int, callback_func: CallbackType):
             """\
             UPDATE spot SET status=(?)
             WHERE spot_job_id=(?) AND
-            task_id=(?)""", (ManagedJobStatus.STARTING.value, job_id, task_id))
+            task_id=(?) AND
+            status=(?) AND
+            end_at IS null""", (ManagedJobStatus.STARTING.value, job_id,
+                                task_id, ManagedJobStatus.SUBMITTED.value))
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                f'Failed to set the task to starting. '
+                f'({cursor.rowcount} rows updated)')
     callback_func('STARTING')
 
 
@@ -447,15 +462,25 @@ def set_started(job_id: int, task_id: int, start_time: float,
             """\
             UPDATE spot SET status=(?), start_at=(?), last_recovered_at=(?)
             WHERE spot_job_id=(?) AND
-            task_id=(?)""",
+            task_id=(?) AND
+            status IN (?, ?) AND
+            end_at IS null""",
             (
                 ManagedJobStatus.RUNNING.value,
                 start_time,
                 start_time,
                 job_id,
                 task_id,
+                ManagedJobStatus.STARTING.value,
+                # If the task is empty, we will jump straight from PENDING to
+                # RUNNING
+                ManagedJobStatus.PENDING.value,
             ),
         )
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                f'Failed to set the task to started. '
+                f'({cursor.rowcount} rows updated)')
     callback_func('STARTED')
 
 
@@ -468,8 +493,15 @@ def set_recovering(job_id: int, task_id: int, callback_func: CallbackType):
                 UPDATE spot SET
                 status=(?), job_duration=job_duration+(?)-last_recovered_at
                 WHERE spot_job_id=(?) AND
-                task_id=(?)""",
-            (ManagedJobStatus.RECOVERING.value, time.time(), job_id, task_id))
+                task_id=(?) AND
+                status=(?) AND
+                end_at IS null""",
+            (ManagedJobStatus.RECOVERING.value, time.time(), job_id, task_id,
+             ManagedJobStatus.RUNNING.value))
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                f'Failed to set the task to recovering. '
+                f'({cursor.rowcount} rows updated)')
     callback_func('RECOVERING')
 
 
@@ -482,8 +514,15 @@ def set_recovered(job_id: int, task_id: int, recovered_time: float,
             UPDATE spot SET
             status=(?), last_recovered_at=(?), recovery_count=recovery_count+1
             WHERE spot_job_id=(?) AND
-            task_id=(?)""",
-            (ManagedJobStatus.RUNNING.value, recovered_time, job_id, task_id))
+            task_id=(?) AND
+            status=(?) AND
+            end_at IS null""",
+            (ManagedJobStatus.RUNNING.value, recovered_time, job_id, task_id,
+             ManagedJobStatus.RECOVERING.value))
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                f'Failed to set the task to recovered. '
+                f'({cursor.rowcount} rows updated)')
     logger.info('==== Recovered. ====')
     callback_func('RECOVERED')
 
@@ -496,10 +535,16 @@ def set_succeeded(job_id: int, task_id: int, end_time: float,
             """\
             UPDATE spot SET
             status=(?), end_at=(?)
-            WHERE spot_job_id=(?) AND task_id=(?)
-            AND end_at IS null""",
-            (ManagedJobStatus.SUCCEEDED.value, end_time, job_id, task_id))
-
+            WHERE spot_job_id=(?) AND
+            task_id=(?) AND
+            status=(?) AND
+            end_at IS null""",
+            (ManagedJobStatus.SUCCEEDED.value, end_time, job_id, task_id,
+             ManagedJobStatus.RUNNING.value))
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                f'Failed to set the task to succeeded. '
+                f'({cursor.rowcount} rows updated)')
     callback_func('SUCCEEDED')
     logger.info('Job succeeded.')
 
@@ -571,7 +616,9 @@ def set_failed(
                 {set_str}
                 WHERE spot_job_id=(?) {task_query_str} AND end_at IS null""",
                 (end_time, *list(fields_to_set.values()), job_id, *task_value))
-    if callback_func:
+
+        updated = cursor.rowcount > 0
+    if callback_func and updated:
         callback_func('FAILED')
     logger.info(failure_reason)
 
@@ -586,12 +633,15 @@ def set_cancelling(job_id: int, callback_func: CallbackType):
         rows = cursor.execute(
             """\
             UPDATE spot SET
-            status=(?), end_at=(?)
+            status=(?)
             WHERE spot_job_id=(?) AND end_at IS null""",
-            (ManagedJobStatus.CANCELLING.value, time.time(), job_id))
-        if rows.rowcount > 0:
-            logger.info('Cancelling the job...')
-            callback_func('CANCELLING')
+            (ManagedJobStatus.CANCELLING.value, job_id))
+        updated = rows.rowcount > 0
+    if updated:
+        logger.info('Cancelling the job...')
+        callback_func('CANCELLING')
+    else:
+        logger.info('Cancellation skipped, job is already terminal')
 
 
 def set_cancelled(job_id: int, callback_func: CallbackType):
@@ -607,9 +657,12 @@ def set_cancelled(job_id: int, callback_func: CallbackType):
             WHERE spot_job_id=(?) AND status=(?)""",
             (ManagedJobStatus.CANCELLED.value, time.time(), job_id,
              ManagedJobStatus.CANCELLING.value))
-        if rows.rowcount > 0:
-            logger.info('Job cancelled.')
-            callback_func('CANCELLED')
+        updated = rows.rowcount > 0
+    if updated:
+        logger.info('Job cancelled.')
+        callback_func('CANCELLED')
+    else:
+        logger.info('Cancellation skipped, job is not CANCELLING')
 
 
 def set_local_log_file(job_id: int, task_id: Optional[int],
@@ -705,8 +758,9 @@ def get_jobs_to_check_status(job_id: Optional[int] = None) -> List[int]:
         job_id: Optional job ID to check. If None, checks all jobs.
 
     Returns a list of job_ids, including the following:
-    - For jobs with schedule state: jobs that have schedule state not DONE
-    - For legacy jobs (no schedule state): jobs that are in non-terminal status
+    - Jobs that have a schedule_state that is not DONE
+    - Jobs have schedule_state DONE but are in a non-terminal status
+    - Legacy jobs (that is, no schedule state) that are in non-terminal status
     """
     job_filter = '' if job_id is None else 'AND spot.spot_job_id=(?)'
     job_value = () if job_id is None else (job_id,)
@@ -719,7 +773,9 @@ def get_jobs_to_check_status(job_id: Optional[int] = None) -> List[int]:
 
     # Get jobs that are either:
     # 1. Have schedule state that is not DONE, or
-    # 2. Have no schedule state (legacy) AND are in non-terminal status
+    # 2. Have schedule state DONE AND are in non-terminal status (unexpected
+    #    inconsistent state), or
+    # 3. Have no schedule state (legacy) AND are in non-terminal status
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         rows = cursor.execute(
             f"""\
@@ -728,14 +784,23 @@ def get_jobs_to_check_status(job_id: Optional[int] = None) -> List[int]:
             LEFT OUTER JOIN job_info
             ON spot.spot_job_id=job_info.spot_job_id
             WHERE (
+                -- non-legacy jobs that are not DONE
                 (job_info.schedule_state IS NOT NULL AND
                  job_info.schedule_state IS NOT ?)
                 OR
-                (job_info.schedule_state IS NULL AND
+                -- legacy or that are in non-terminal status or
+                -- DONE jobs that are in non-terminal status
+                ((-- legacy jobs
+                  job_info.schedule_state IS NULL OR
+                  -- non-legacy DONE jobs
+                  job_info.schedule_state IS ?
+                 ) AND
+                 -- non-terminal
                  status NOT IN ({status_filter_str}))
             )
             {job_filter}
             ORDER BY spot.spot_job_id DESC""", [
+                ManagedJobScheduleState.DONE.value,
                 ManagedJobScheduleState.DONE.value, *terminal_status_values,
                 *job_value
             ]).fetchall()

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/jobs/utils.py

@@ -6,11 +6,9 @@ ManagedJobCodeGen.
 """
 import collections
 import enum
-import inspect
 import os
 import pathlib
 import shlex
-import shutil
 import textwrap
 import time
 import traceback
@@ -53,7 +51,6 @@ JOB_CONTROLLER_NAME: str = (
 LEGACY_JOB_CONTROLLER_NAME: str = (
     f'sky-spot-controller-{common_utils.get_user_hash()}')
 SIGNAL_FILE_PREFIX = '/tmp/sky_jobs_controller_signal_{}'
-LEGACY_SIGNAL_FILE_PREFIX = '/tmp/sky_spot_controller_signal_{}'
 # Controller checks its job's status every this many seconds.
 JOB_STATUS_CHECK_GAP_SECONDS = 20
 
@@ -248,16 +245,35 @@ def update_managed_jobs_statuses(job_id: Optional[int] = None):
         schedule_state = tasks[0]['schedule_state']
 
         # Backwards compatibility: this job was submitted when ray was still
-        # used for managing the parallelism of job controllers.
+        # used for managing the parallelism of job controllers, before #4485.
         # TODO(cooperc): Remove before 0.11.0.
         if (schedule_state is
                 managed_job_state.ManagedJobScheduleState.INVALID):
             _handle_legacy_job(job_id)
             continue
 
-        # For jobs with schedule state:
+        # Handle jobs with schedule state (non-legacy jobs):
         pid = tasks[0]['controller_pid']
-        if pid is None:
+        if schedule_state == managed_job_state.ManagedJobScheduleState.DONE:
+            # There are two cases where we could get a job that is DONE.
+            # 1. At query time (get_jobs_to_check_status), the job was not yet
+            #    DONE, but since then (before get_managed_jobs is called) it has
+            #    hit a terminal status, marked itself done, and exited. This is
+            #    fine.
+            # 2. The job is DONE, but in a non-terminal status. This is
+            #    unexpected. For instance, the task status is RUNNING, but the
+            #    job schedule_state is DONE.
+            if all(task['status'].is_terminal() for task in tasks):
+                # Turns out this job is fine, even though it got pulled by
+                # get_jobs_to_check_status. Probably case #1 above.
+                continue
+
+            logger.error(f'Job {job_id} has DONE schedule state, but some '
+                         f'tasks are not terminal. Task statuses: '
+                         f'{", ".join(task["status"].value for task in tasks)}')
+            failure_reason = ('Inconsistent internal job state. This is a bug.')
+        elif pid is None:
+            # Non-legacy job and controller process has not yet started.
             if schedule_state in (
                     managed_job_state.ManagedJobScheduleState.INACTIVE,
                     managed_job_state.ManagedJobScheduleState.WAITING):
@@ -458,17 +474,12 @@ def cancel_jobs_by_id(job_ids: Optional[List[int]]) -> str:
 
         # Send the signal to the jobs controller.
         signal_file = pathlib.Path(SIGNAL_FILE_PREFIX.format(job_id))
-        legacy_signal_file = pathlib.Path(
-            LEGACY_SIGNAL_FILE_PREFIX.format(job_id))
         # Filelock is needed to prevent race condition between signal
         # check/removal and signal writing.
         with filelock.FileLock(str(signal_file) + '.lock'):
             with signal_file.open('w', encoding='utf-8') as f:
                 f.write(UserSignal.CANCEL.value)
                 f.flush()
-            # Backward compatibility for managed jobs launched before #3419. It
-            # can be removed in the future 0.8.0 release.
-            shutil.copy(str(signal_file), str(legacy_signal_file))
         cancelled_job_ids.append(job_id)
 
     if not cancelled_job_ids:
@@ -1116,28 +1127,15 @@ class ManagedJobCodeGen:
 
       >> codegen = ManagedJobCodeGen.show_jobs(...)
     """
-    # TODO: the try..except.. block is for backward compatibility. Remove it in
-    # v0.8.0.
     _PREFIX = textwrap.dedent("""\
-        managed_job_version = 0
-        try:
-            from sky.jobs import utils
-            from sky.jobs import constants as managed_job_constants
-            from sky.jobs import state as managed_job_state
-
-            managed_job_version = managed_job_constants.MANAGED_JOBS_VERSION
-        except ImportError:
-            from sky.spot import spot_state as managed_job_state
-            from sky.spot import spot_utils as utils
+        from sky.jobs import utils
+        from sky.jobs import state as managed_job_state
         """)
 
     @classmethod
     def get_job_table(cls) -> str:
         code = textwrap.dedent("""\
-        if managed_job_version < 1:
-            job_table = utils.dump_spot_job_queue()
-        else:
-            job_table = utils.dump_managed_job_queue()
+        job_table = utils.dump_managed_job_queue()
         print(job_table, flush=True)
         """)
         return cls._build(code)
@@ -1173,29 +1171,9 @@ class ManagedJobCodeGen:
                     job_id: Optional[int],
                     follow: bool = True,
                     controller: bool = False) -> str:
-        # We inspect the source code of the function here for backward
-        # compatibility.
-        # TODO: change to utils.stream_logs(job_id, job_name, follow) in v0.8.0.
-        # Import libraries required by `stream_logs`. The try...except... block
-        # should be removed in v0.8.0.
-        code = textwrap.dedent("""\
-        import os
-        import time
-
-        from sky.skylet import job_lib, log_lib
-        from sky.skylet import constants
-        from sky.utils import ux_utils
-        try:
-            from sky.jobs.utils import stream_logs_by_id
-        except ImportError:
-            from sky.spot.spot_utils import stream_logs_by_id
-        from typing import Optional
-        """)
-        code += inspect.getsource(stream_logs)
-        code += textwrap.dedent(f"""\
-
-        msg = stream_logs({job_id!r}, {job_name!r},
-                           follow={follow}, controller={controller})
+        code = textwrap.dedent(f"""\
+        msg = utils.stream_logs({job_id!r}, {job_name!r},
+                                follow={follow}, controller={controller})
         print(msg, flush=True)
         """)
         return cls._build(code)

{skypilot_nightly-1.0.0.dev20250206 → skypilot_nightly-1.0.0.dev20250208}/sky/provision/azure/instance.py

@@ -15,7 +15,6 @@ from sky import status_lib
 from sky.adaptors import azure
 from sky.provision import common
 from sky.provision import constants
-from sky.provision.azure import config as config_lib
 from sky.utils import common_utils
 from sky.utils import subprocess_utils
 from sky.utils import ux_utils
@@ -52,10 +51,6 @@ _RESOURCE_NETWORK_INTERFACE_TYPE = 'Microsoft.Network/networkInterfaces'
 
 _RESOURCE_GROUP_NOT_FOUND_ERROR_MESSAGE = 'ResourceGroupNotFound'
 _POLL_INTERVAL = 1
-# TODO(Doyoung): _LEGACY_NSG_NAME can be remove this after 0.8.0 to ignore
-# legacy nsg names.
-_LEGACY_NSG_NAME = 'ray-{cluster_name_on_cloud}-nsg'
-_SECOND_LEGACY_NSG_NAME = 'sky-{cluster_name_on_cloud}-nsg'
 
 
 class AzureInstanceStatus(enum.Enum):
@@ -987,32 +982,6 @@ def query_instances(
     return statuses
 
 
-# TODO(Doyoung): _get_cluster_nsg can be remove this after 0.8.0 to ignore
-# legacy nsg names.
-def _get_cluster_nsg(network_client: Client, resource_group: str,
-                     cluster_name_on_cloud: str) -> NetworkSecurityGroup:
-    """Retrieve the NSG associated with the given name of the cluster."""
-    list_network_security_groups = _get_azure_sdk_function(
-        client=network_client.network_security_groups, function_name='list')
-    legacy_nsg_name = _LEGACY_NSG_NAME.format(
-        cluster_name_on_cloud=cluster_name_on_cloud)
-    second_legacy_nsg_name = _SECOND_LEGACY_NSG_NAME.format(
-        cluster_name_on_cloud=cluster_name_on_cloud)
-    _, nsg_name = config_lib.get_cluster_id_and_nsg_name(
-        resource_group=resource_group,
-        cluster_name_on_cloud=cluster_name_on_cloud)
-    possible_nsg_names = [nsg_name, legacy_nsg_name, second_legacy_nsg_name]
-    for nsg in list_network_security_groups(resource_group):
-        if nsg.name in possible_nsg_names:
-            return nsg
-
-    # Raise an error if no matching NSG is found
-    raise ValueError('Failed to find a matching NSG for cluster '
-                     f'{cluster_name_on_cloud!r} in resource group '
-                     f'{resource_group!r}. Expected NSG names were: '
-                     f'{possible_nsg_names}.')
-
-
 def open_ports(
     cluster_name_on_cloud: str,
     ports: List[str],