skypilot-nightly 1.0.0.dev20250528__py3-none-any.whl → 1.0.0.dev20250529__py3-none-any.whl

sky/jobs/state.py

@@ -120,7 +120,8 @@ def create_table(cursor, conn):
         dag_yaml_path TEXT,
         env_file_path TEXT,
         user_hash TEXT,
-        workspace TEXT DEFAULT NULL)""")
+        workspace TEXT DEFAULT NULL,
+        priority INTEGER DEFAULT 500)""")
 
     db_utils.add_column_to_table(cursor, conn, 'job_info', 'schedule_state',
                                  'TEXT')
@@ -142,6 +143,14 @@ def create_table(cursor, conn):
                                  'workspace',
                                  'TEXT DEFAULT NULL',
                                  value_to_replace_existing_entries='default')
+
+    db_utils.add_column_to_table(cursor,
+                                 conn,
+                                 'job_info',
+                                 'priority',
+                                 'INTEGER',
+                                 value_to_replace_existing_entries=500)
+
     conn.commit()
 
 
@@ -199,6 +208,7 @@ columns = [
     'env_file_path',
     'user_hash',
     'workspace',
+    'priority',
 ]
 
 
@@ -215,7 +225,7 @@ class ManagedJobStatus(enum.Enum):
     reset to INIT or SETTING_UP multiple times (depending on the preemptions).
 
     However, a managed job only has one ManagedJobStatus on the jobs controller.
-        ManagedJobStatus = [PENDING, SUBMITTED, STARTING, RUNNING, ...]
+        ManagedJobStatus = [PENDING, STARTING, RUNNING, ...]
     Mapping from JobStatus to ManagedJobStatus:
         INIT            ->  STARTING/RECOVERING
         SETTING_UP      ->  RUNNING
@@ -235,10 +245,14 @@ class ManagedJobStatus(enum.Enum):
     # PENDING: Waiting for the jobs controller to have a slot to run the
     # controller process.
     PENDING = 'PENDING'
+    # SUBMITTED: This state used to be briefly set before immediately changing
+    # to STARTING. Its use was removed in #5682. We keep it for backwards
+    # compatibility, so we can still parse old jobs databases that may have jobs
+    # in this state.
+    # TODO(cooperc): remove this in v0.12.0
+    DEPRECATED_SUBMITTED = 'SUBMITTED'
     # The submitted_at timestamp of the managed job in the 'spot' table will be
     # set to the time when the job controller begins running.
-    # SUBMITTED: The jobs controller starts the controller process.
-    SUBMITTED = 'SUBMITTED'
     # STARTING: The controller process is launching the cluster for the managed
     # job.
     STARTING = 'STARTING'
@@ -314,7 +328,6 @@ class ManagedJobStatus(enum.Enum):
 
 _SPOT_STATUS_TO_COLOR = {
     ManagedJobStatus.PENDING: colorama.Fore.BLUE,
-    ManagedJobStatus.SUBMITTED: colorama.Fore.BLUE,
     ManagedJobStatus.STARTING: colorama.Fore.BLUE,
     ManagedJobStatus.RUNNING: colorama.Fore.GREEN,
     ManagedJobStatus.RECOVERING: colorama.Fore.CYAN,
@@ -326,6 +339,8 @@ _SPOT_STATUS_TO_COLOR = {
     ManagedJobStatus.FAILED_CONTROLLER: colorama.Fore.RED,
     ManagedJobStatus.CANCELLING: colorama.Fore.YELLOW,
     ManagedJobStatus.CANCELLED: colorama.Fore.YELLOW,
+    # TODO(cooperc): backwards compatibility, remove this in v0.12.0
+    ManagedJobStatus.DEPRECATED_SUBMITTED: colorama.Fore.BLUE,
 }
 
 
@@ -342,8 +357,12 @@ class ManagedJobScheduleState(enum.Enum):
     - LAUNCHING -> ALIVE: The launch attempt was completed. It may have
       succeeded or failed. The job controller is not allowed to sky.launch again
       without transitioning to ALIVE_WAITING and then LAUNCHING.
+    - LAUNCHING -> ALIVE_BACKOFF: The launch failed to find resources, and is
+      in backoff waiting for resources.
     - ALIVE -> ALIVE_WAITING: The job controller wants to sky.launch again,
       either for recovery or to launch a subsequent task.
+    - ALIVE_BACKOFF -> ALIVE_WAITING: The backoff period has ended, and the job
+      controller wants to try to launch again.
     - ALIVE_WAITING -> LAUNCHING: The scheduler has determined that the job
       controller may launch again.
     - LAUNCHING, ALIVE, or ALIVE_WAITING -> DONE: The job controller is exiting
@@ -357,6 +376,7 @@ class ManagedJobScheduleState(enum.Enum):
     state or vice versa. (In fact, schedule state is defined on the job and
     status on the task.)
     - INACTIVE or WAITING should only be seen when a job is PENDING.
+    - ALIVE_BACKOFF should only be seen when a job is STARTING.
     - ALIVE_WAITING should only be seen when a job is RECOVERING, has multiple
       tasks, or needs to retry launching.
     - LAUNCHING and ALIVE can be seen in many different statuses.
@@ -382,6 +402,9 @@ class ManagedJobScheduleState(enum.Enum):
     # The job is running sky.launch, or soon will, using a limited number of
     # allowed launch slots.
     LAUNCHING = 'LAUNCHING'
+    # The job is alive, but is in backoff waiting for resources - a special case
+    # of ALIVE.
+    ALIVE_BACKOFF = 'ALIVE_BACKOFF'
     # The controller for the job is running, but it's not currently launching.
     ALIVE = 'ALIVE'
     # The job is in a terminal state. (Not necessarily SUCCEEDED.)
@@ -411,11 +434,11 @@ def set_pending(job_id: int, task_id: int, task_name: str, resources_str: str):
              ManagedJobStatus.PENDING.value))
 
 
-def set_submitted(job_id: int, task_id: int, run_timestamp: str,
-                  submit_time: float, resources_str: str,
-                  specs: Dict[str, Union[str,
-                                         int]], callback_func: CallbackType):
-    """Set the task to submitted.
+def set_starting(job_id: int, task_id: int, run_timestamp: str,
+                 submit_time: float, resources_str: str,
+                 specs: Dict[str, Union[str,
+                                        int]], callback_func: CallbackType):
+    """Set the task to starting state.
 
     Args:
         job_id: The managed job ID.
@@ -432,6 +455,7 @@ def set_submitted(job_id: int, task_id: int, run_timestamp: str,
     # make it easier to find them based on one of the values.
     # Also, using the earlier timestamp should be closer to the term
     # `submit_at`, which represents the time the managed task is submitted.
+    logger.info('Launching the spot cluster...')
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         cursor.execute(
             """\
@@ -445,19 +469,54 @@ def set_submitted(job_id: int, task_id: int, run_timestamp: str,
             task_id=(?) AND
             status=(?) AND
             end_at IS null""",
-            (resources_str, submit_time, ManagedJobStatus.SUBMITTED.value,
+            (resources_str, submit_time, ManagedJobStatus.STARTING.value,
              run_timestamp, json.dumps(specs), job_id, task_id,
              ManagedJobStatus.PENDING.value))
         if cursor.rowcount != 1:
             raise exceptions.ManagedJobStatusError(
-                f'Failed to set the task to submitted. '
+                'Failed to set the task to starting. '
                 f'({cursor.rowcount} rows updated)')
+    # SUBMITTED is no longer used, but we keep it for backward compatibility.
+    # TODO(cooperc): remove this in v0.12.0
     callback_func('SUBMITTED')
+    callback_func('STARTING')
 
 
-def set_starting(job_id: int, task_id: int, callback_func: CallbackType):
-    """Set the task to starting state."""
-    logger.info('Launching the spot cluster...')
+def set_backoff_pending(job_id: int, task_id: int):
+    """Set the task to PENDING state if it is in backoff.
+
+    This should only be used to transition from STARTING or RECOVERING back to
+    PENDING.
+    """
+    with db_utils.safe_cursor(_DB_PATH) as cursor:
+        cursor.execute(
+            """\
+            UPDATE spot SET status=(?)
+            WHERE spot_job_id=(?) AND
+            task_id=(?) AND
+            status IN (?, ?) AND
+            end_at IS null""", (ManagedJobStatus.PENDING.value, job_id, task_id,
+                                ManagedJobStatus.STARTING.value,
+                                ManagedJobStatus.RECOVERING.value))
+        logger.debug('back to PENDING')
+        if cursor.rowcount != 1:
+            raise exceptions.ManagedJobStatusError(
+                'Failed to set the task back to pending. '
+                f'({cursor.rowcount} rows updated)')
+    # Do not call callback_func here, as we don't use the callback for PENDING.
+
+
+def set_restarting(job_id: int, task_id: int, recovering: bool):
+    """Set the task back to STARTING or RECOVERING from PENDING.
+
+    This should not be used for the initial transition from PENDING to STARTING.
+    In that case, use set_starting instead. This function should only be used
+    after using set_backoff_pending to transition back to PENDING during
+    launch retry backoff.
+    """
+    target_status = ManagedJobStatus.STARTING.value
+    if recovering:
+        target_status = ManagedJobStatus.RECOVERING.value
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         cursor.execute(
             """\
@@ -465,13 +524,15 @@ def set_starting(job_id: int, task_id: int, callback_func: CallbackType):
             WHERE spot_job_id=(?) AND
             task_id=(?) AND
             status=(?) AND
-            end_at IS null""", (ManagedJobStatus.STARTING.value, job_id,
-                                task_id, ManagedJobStatus.SUBMITTED.value))
+            end_at IS null""",
+            (target_status, job_id, task_id, ManagedJobStatus.PENDING.value))
+        logger.debug(f'back to {target_status}')
         if cursor.rowcount != 1:
             raise exceptions.ManagedJobStatusError(
-                f'Failed to set the task to starting. '
+                f'Failed to set the task back to {target_status}. '
                 f'({cursor.rowcount} rows updated)')
-    callback_func('STARTING')
+    # Do not call callback_func here, as it should only be invoked for the
+    # initial (pre-`set_backoff_pending`) transition to STARTING or RECOVERING.
 
 
 def set_started(job_id: int, task_id: int, start_time: float,
@@ -1004,16 +1065,16 @@ def get_local_log_file(job_id: int, task_id: Optional[int]) -> Optional[str]:
 
 
 def scheduler_set_waiting(job_id: int, dag_yaml_path: str, env_file_path: str,
-                          user_hash: str) -> None:
+                          user_hash: str, priority: int) -> None:
     """Do not call without holding the scheduler lock."""
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         updated_count = cursor.execute(
             'UPDATE job_info SET '
             'schedule_state = (?), dag_yaml_path = (?), env_file_path = (?), '
-            '  user_hash = (?) '
+            '  user_hash = (?), priority = (?) '
             'WHERE spot_job_id = (?) AND schedule_state = (?)',
             (ManagedJobScheduleState.WAITING.value, dag_yaml_path,
-             env_file_path, user_hash, job_id,
+             env_file_path, user_hash, priority, job_id,
              ManagedJobScheduleState.INACTIVE.value)).rowcount
         assert updated_count == 1, (job_id, updated_count)
 
@@ -1043,15 +1104,28 @@ def scheduler_set_alive(job_id: int) -> None:
         assert updated_count == 1, (job_id, updated_count)
 
 
-def scheduler_set_alive_waiting(job_id: int) -> None:
+def scheduler_set_alive_backoff(job_id: int) -> None:
     """Do not call without holding the scheduler lock."""
     with db_utils.safe_cursor(_DB_PATH) as cursor:
         updated_count = cursor.execute(
             'UPDATE job_info SET '
             'schedule_state = (?) '
             'WHERE spot_job_id = (?) AND schedule_state = (?)',
+            (ManagedJobScheduleState.ALIVE_BACKOFF.value, job_id,
+             ManagedJobScheduleState.LAUNCHING.value)).rowcount
+        assert updated_count == 1, (job_id, updated_count)
+
+
+def scheduler_set_alive_waiting(job_id: int) -> None:
+    """Do not call without holding the scheduler lock."""
+    with db_utils.safe_cursor(_DB_PATH) as cursor:
+        updated_count = cursor.execute(
+            'UPDATE job_info SET '
+            'schedule_state = (?) '
+            'WHERE spot_job_id = (?) AND schedule_state IN (?, ?)',
             (ManagedJobScheduleState.ALIVE_WAITING.value, job_id,
-             ManagedJobScheduleState.ALIVE.value)).rowcount
+             ManagedJobScheduleState.ALIVE.value,
+             ManagedJobScheduleState.ALIVE_BACKOFF.value)).rowcount
         assert updated_count == 1, (job_id, updated_count)
 
 
@@ -1099,32 +1173,53 @@ def get_num_alive_jobs() -> int:
         return cursor.execute(
             'SELECT COUNT(*) '
             'FROM job_info '
-            'WHERE schedule_state IN (?, ?, ?)',
+            'WHERE schedule_state IN (?, ?, ?, ?)',
             (ManagedJobScheduleState.ALIVE_WAITING.value,
              ManagedJobScheduleState.LAUNCHING.value,
-             ManagedJobScheduleState.ALIVE.value)).fetchone()[0]
+             ManagedJobScheduleState.ALIVE.value,
+             ManagedJobScheduleState.ALIVE_BACKOFF.value)).fetchone()[0]
 
 
 def get_waiting_job() -> Optional[Dict[str, Any]]:
     """Get the next job that should transition to LAUNCHING.
 
+    Selects the highest-priority (lowest numerical value) WAITING or
+    ALIVE_WAITING job, provided its priority value is less than or equal to any
+    currently LAUNCHING or ALIVE_BACKOFF job.
+
     Backwards compatibility note: jobs submitted before #4485 will have no
     schedule_state and will be ignored by this SQL query.
     """
     with db_utils.safe_cursor(_DB_PATH) as cursor:
-        row = cursor.execute(
+        # Get the highest-priority (lowest numerical value) WAITING or
+        # ALIVE_WAITING job whose priority value is less than or equal to
+        # the highest priority (numerically smallest) LAUNCHING or
+        # ALIVE_BACKOFF job's priority.
+        waiting_job_row = cursor.execute(
             'SELECT spot_job_id, schedule_state, dag_yaml_path, env_file_path '
             'FROM job_info '
-            'WHERE schedule_state in (?, ?) '
-            'ORDER BY spot_job_id LIMIT 1',
+            'WHERE schedule_state IN (?, ?) '
+            'AND priority <= COALESCE('
+            '    (SELECT MIN(priority) '
+            '     FROM job_info '
+            '     WHERE schedule_state IN (?, ?)), '
+            '    1000'
+            ')'
+            'ORDER BY priority ASC, spot_job_id ASC LIMIT 1',
             (ManagedJobScheduleState.WAITING.value,
-             ManagedJobScheduleState.ALIVE_WAITING.value)).fetchone()
+             ManagedJobScheduleState.ALIVE_WAITING.value,
+             ManagedJobScheduleState.LAUNCHING.value,
+             ManagedJobScheduleState.ALIVE_BACKOFF.value)).fetchone()
+
+        if waiting_job_row is None:
+            return None
+
         return {
-            'job_id': row[0],
-            'schedule_state': ManagedJobScheduleState(row[1]),
-            'dag_yaml_path': row[2],
-            'env_file_path': row[3],
-        } if row is not None else None
+            'job_id': waiting_job_row[0],
+            'schedule_state': ManagedJobScheduleState(waiting_job_row[1]),
+            'dag_yaml_path': waiting_job_row[2],
+            'env_file_path': waiting_job_row[3],
+        }
 
 
 def get_workspace(job_id: int) -> str:

sky/jobs/utils.py

@@ -953,6 +953,22 @@ def dump_managed_job_queue() -> str:
             job['region'] = '-'
             job['zone'] = '-'
 
+        # Add details about schedule state / backoff.
+        state_details = None
+        if job['schedule_state'] == 'ALIVE_BACKOFF':
+            state_details = 'In backoff, waiting for resources'
+        elif job['schedule_state'] in ('WAITING', 'ALIVE_WAITING'):
+            state_details = 'Waiting for other jobs to launch'
+
+        if state_details and job['failure_reason']:
+            job['details'] = f'{state_details} - {job["failure_reason"]}'
+        elif state_details:
+            job['details'] = state_details
+        elif job['failure_reason']:
+            job['details'] = f'Failure: {job["failure_reason"]}'
+        else:
+            job['details'] = None
+
     return message_utils.encode_payload(jobs)
 
 
@@ -981,7 +997,7 @@ def _get_job_status_from_tasks(
         # Use the first non-succeeded status.
         if managed_task_status != managed_job_state.ManagedJobStatus.SUCCEEDED:
             # TODO(zhwu): we should not blindly use the first non-
-            # succeeded as the status could be changed to SUBMITTED
+            # succeeded as the status could be changed to PENDING
             # when going from one task to the next one, which can be
             # confusing.
             break
@@ -1063,6 +1079,7 @@ def format_job_table(
         'TASK',
         *(['WORKSPACE'] if show_workspace else []),
         'NAME',
+        'PRIORITY',
         *user_cols,
         'REQUESTED',
         'SUBMITTED',
@@ -1092,7 +1109,10 @@ def format_job_table(
         # by the task_id.
         jobs[get_hash(task)].append(task)
 
-    def generate_details(failure_reason: Optional[str]) -> str:
+    def generate_details(details: Optional[str],
+                         failure_reason: Optional[str]) -> str:
+        if details is not None:
+            return details
         if failure_reason is not None:
             return f'Failure: {failure_reason}'
         return '-'
@@ -1131,6 +1151,7 @@ def format_job_table(
             submitted_at = None
             end_at: Optional[int] = 0
             recovery_cnt = 0
+            priority = job_tasks[0].get('priority', '-')
             managed_job_status, current_task_id = _get_job_status_from_tasks(
                 job_tasks)
             for task in job_tasks:
@@ -1166,6 +1187,7 @@ def format_job_table(
                 '',
                 *([''] if show_workspace else []),
                 job_name,
+                str(priority),
                 *user_values,
                 '-',
                 submitted,
@@ -1175,13 +1197,14 @@ def format_job_table(
                 status_str,
             ]
             if show_all:
+                details = job_tasks[current_task_id].get('details')
                 failure_reason = job_tasks[current_task_id]['failure_reason']
                 job_values.extend([
                     '-',
                     '-',
                     '-',
                     job_tasks[0]['schedule_state'],
-                    generate_details(failure_reason),
+                    generate_details(details, failure_reason),
                 ])
             if tasks_have_k8s_user:
                 job_values.insert(0, job_tasks[0].get('user', '-'))
@@ -1195,11 +1218,13 @@ def format_job_table(
             submitted = log_utils.readable_time_duration(task['submitted_at'])
             user_values = get_user_column_values(task)
             task_workspace = '-' if len(job_tasks) > 1 else workspace
+            priority = task.get('priority', '-')
             values = [
                 task['job_id'] if len(job_tasks) == 1 else ' \u21B3',
                 task['task_id'] if len(job_tasks) > 1 else '-',
                 *([task_workspace] if show_workspace else []),
                 task['task_name'],
+                str(priority),
                 *user_values,
                 task['resources'],
                 # SUBMITTED
@@ -1244,7 +1269,8 @@ def format_job_table(
                     infra.formatted_str(),
                     task['cluster_resources'],
                     schedule_state,
-                    generate_details(task['failure_reason']),
+                    generate_details(task.get('details'),
+                                     task['failure_reason']),
                 ])
             if tasks_have_k8s_user:
                 values.insert(0, task.get('user', '-'))

sky/resources.py

@@ -98,7 +98,7 @@ class Resources:
     """
     # If any fields changed, increment the version. For backward compatibility,
     # modify the __setstate__ method to handle the old version.
-    _VERSION = 24
+    _VERSION = 25
 
     def __init__(
         self,
@@ -294,6 +294,8 @@ class Resources:
                 }
         else:
             self._image_id = image_id
+        if isinstance(self._cloud, clouds.Kubernetes):
+            _maybe_add_docker_prefix_to_image_id(self._image_id)
         self._is_image_managed = _is_image_managed
 
         if isinstance(disk_tier, str):
@@ -2075,6 +2077,10 @@ class Resources:
         if version < 24:
             self._volumes = None
 
+        if version < 25:
+            if isinstance(state.get('_cloud', None), clouds.Kubernetes):
+                _maybe_add_docker_prefix_to_image_id(state['_image_id'])
+
         self.__dict__.update(state)
 
 
@@ -2111,3 +2117,12 @@ class LaunchableResources(Resources):
         """
         self.assert_launchable()
         return typing.cast(LaunchableResources, super().copy(**override))
+
+
+def _maybe_add_docker_prefix_to_image_id(
+        image_id_dict: Optional[Dict[Optional[str], str]]) -> None:
+    if image_id_dict is None:
+        return
+    for k, v in image_id_dict.items():
+        if not v.startswith('docker:'):
+            image_id_dict[k] = f'docker:{v}'

sky/server/common.py

@@ -159,7 +159,8 @@ def get_server_url(host: Optional[str] = None) -> str:
 
 
 @annotations.lru_cache(scope='global')
-def get_dashboard_url(server_url: str) -> str:
+def get_dashboard_url(server_url: str,
+                      starting_page: Optional[str] = None) -> str:
     # The server_url may include username or password with the
     # format of https://username:password@example.com:8080/path
     # We need to remove the username and password and only
@@ -172,7 +173,10 @@ def get_dashboard_url(server_url: str) -> str:
     if parsed.path:
         dashboard_url = f'{dashboard_url}{parsed.path}'
     dashboard_url = dashboard_url.rstrip('/')
-    return f'{dashboard_url}/dashboard'
+    dashboard_url = f'{dashboard_url}/dashboard'
+    if starting_page:
+        dashboard_url = f'{dashboard_url}/{starting_page}'
+    return dashboard_url
 
 
 @annotations.lru_cache(scope='global')

sky/server/html/token_page.html

@@ -100,6 +100,9 @@
             color: #5f6368;
             margin-top: 30px;
         }
+        .local-port-info {
+            display: none;
+        }
     </style>
 </head>
 <body>
@@ -114,14 +117,18 @@
                 <path d="M16.632 21.3918L15.2651 27.6605L21.3357 25.6091L30.3276 16.6172L16.632 21.3918Z" fill="#39A4DD"/>
             </svg>
         </div>
-        <h1>Sign in to SkyPilot CLI</h1>
+        <h1 class="no-local-port">Sign in to SkyPilot CLI</h1>
+        <h1 class="local-port-info">Successfully signed into SkyPilot CLI</h1>
         <p class="user-identifier">USER_PLACEHOLDER</p>
-        <p>You are seeing this page because a SkyPilot command requires authentication.</p>
-        <p>Please copy the following token and paste it into your SkyPilot CLI prompt:</p>
-        <div id="token-box" class="code-block">SKYPILOT_API_SERVER_USER_TOKEN_PLACEHOLDER</div>
-        <button id="copy-btn" class="copy-button">Copy Token</button>
+        <!-- display token info by default -->
+        <p class="no-local-port">You are seeing this page because a SkyPilot command requires authentication.</p>
+        <p class="no-local-port">Please copy the following token and paste it into your SkyPilot CLI prompt:</p>
+        <div id="token-box" class="code-block no-local-port">SKYPILOT_API_SERVER_USER_TOKEN_PLACEHOLDER</div>
+        <button id="copy-btn" class="copy-button no-local-port">Copy Token</button>
+        <p class="footer-text no-local-port">You can close this tab after copying the token.</p>
 
-        <p class="footer-text">You can close this tab after copying the token.</p>
+        <!-- don't display local port info unless successful -->
+        <p class="local-port-info">You can now close this tab.</p>
     </div>
 
     <script>
@@ -154,6 +161,25 @@
                 copyBtn.textContent = 'Copy Token';
             }, 2000);
         });
+
+        function hideTokenInfo() {
+            const noLocalPortElems = document.querySelectorAll('.no-local-port');
+            noLocalPortElems.forEach(elem => {
+                elem.style.display = 'none';
+            });
+            const localPortInfoElems = document.querySelectorAll('.local-port-info');
+            localPortInfoElems.forEach(elem => {
+                elem.classList.remove('local-port-info');
+            });
+        }
+
+        if (window.location.search.includes('local_port=')) {
+            const uri = `http://localhost:${window.location.search.split('local_port=')[1]}`;
+            fetch(uri, {
+                method: 'POST',
+                body: 'SKYPILOT_API_SERVER_USER_TOKEN_PLACEHOLDER'
+            }).then(hideTokenInfo)
+        }
     </script>
 </body>
 </html>

sky/server/server.py

@@ -272,7 +272,9 @@ app.include_router(workspaces_rest.router,
 
 
 @app.get('/token')
-async def token(request: fastapi.Request) -> fastapi.responses.HTMLResponse:
+async def token(request: fastapi.Request,
+                local_port: Optional[int] = None) -> fastapi.responses.Response:
+    del local_port  # local_port is used by the served js, but ignored by server
     user = _get_auth_user_header(request)
 
     token_data = {

sky/setup_files/dependencies.py

@@ -118,7 +118,13 @@ extras_require: Dict[str, List[str]] = {
     # We need google-api-python-client>=2.69.0 to enable 'discardLocalSsd'
     # parameter for stopping instances. Reference:
     # https://github.com/googleapis/google-api-python-client/commit/f6e9d3869ed605b06f7cbf2e8cf2db25108506e6
-    'gcp': ['google-api-python-client>=2.69.0', 'google-cloud-storage'],
+    'gcp': [
+        'google-api-python-client>=2.69.0',
+        'google-cloud-storage',
+        # see https://github.com/conda/conda/issues/13619
+        # see https://github.com/googleapis/google-api-python-client/issues/2554
+        'pyopenssl >= 23.2.0, <24.3.0',
+    ],
     'ibm': [
         'ibm-cloud-sdk-core',
         'ibm-vpc',

sky/skylet/constants.py

@@ -89,7 +89,7 @@ TASK_ID_LIST_ENV_VAR = f'{SKYPILOT_ENV_VAR_PREFIX}TASK_IDS'
 # cluster yaml is updated.
 #
 # TODO(zongheng,zhanghao): make the upgrading of skylet automatic?
-SKYLET_VERSION = '12'
+SKYLET_VERSION = '13'
 # The version of the lib files that skylet/jobs use. Whenever there is an API
 # change for the job_lib or log_lib, we need to bump this version, so that the
 # user can be notified to update their SkyPilot version on the remote cluster.

sky/task.py

@@ -292,6 +292,8 @@ class Task:
         self.resources: Union[List[sky.Resources],
                               Set[sky.Resources]] = {sky.Resources()}
         self._service: Optional[service_spec.SkyServiceSpec] = None
+        # The priority of the managed job running this task.
+        self._job_priority: Optional[int] = None
         # Resources that this task cannot run on.
         self.blocked_resources = blocked_resources
 
@@ -629,6 +631,10 @@ class Task:
             service = service_spec.SkyServiceSpec.from_yaml_config(service)
         task.set_service(service)
 
+        job = config.pop('job', None)
+        if job is not None and 'priority' in job:
+            task.set_job_priority(job['priority'])
+
         assert not config, f'Invalid task args: {config.keys()}'
         return task
 
@@ -831,6 +837,23 @@ class Task:
         self._service = service
         return self
 
+    @property
+    def job_priority(self) -> Optional[int]:
+        """The priority of the managed job running this task."""
+        return self._job_priority
+
+    def set_job_priority(self, priority: int) -> 'Task':
+        """Sets the job priority for this task.
+
+        Args:
+          priority: an integer between 0 and 1000.
+
+        Returns:
+          self: The current task, with job priority set.
+        """
+        self._job_priority = priority
+        return self
+
     def set_time_estimator(self, func: Callable[['sky.Resources'],
                                                 int]) -> 'Task':
         """Sets a func mapping resources to estimated time (secs).
@@ -1274,6 +1297,9 @@ class Task:
         if self.service is not None:
             add_if_not_none('service', self.service.to_yaml_config())
 
+        if self.job_priority is not None:
+            add_if_not_none('job', {'priority': self.job_priority})
+
         add_if_not_none('num_nodes', self.num_nodes)
 
         if self.inputs is not None:

sky/templates/jobs-controller.yaml.j2

@@ -66,7 +66,8 @@ run: |
   # managed_job_codegen.set_pending() before we get here.
   python -u -m sky.jobs.scheduler {{remote_user_yaml_path}} \
     --job-id $SKYPILOT_INTERNAL_JOB_ID \
-    --env-file {{remote_env_file_path}}
+    --env-file {{remote_env_file_path}} \
+    --priority {{priority}}
 
 
 envs: