skypilot-nightly 1.0.0.dev2024053101__py3-none-any.whl → 1.0.0.dev2025022801__py3-none-any.whl

sky/jobs/dashboard/dashboard.py

@@ -6,13 +6,18 @@ https://github.com/ray-project/ray/tree/master/dashboard/client/src) and/or get
 rid of the SSH port-forwarding business (see cli.py's job_dashboard()
 comment).
 """
+import collections
 import datetime
+import enum
+import os
 import pathlib
 
 import flask
 import yaml
 
 from sky import jobs as managed_jobs
+from sky.client import sdk
+from sky.jobs import constants as managed_job_constants
 from sky.utils import common_utils
 from sky.utils import controller_utils
 
@@ -26,25 +31,112 @@ def _is_running_on_jobs_controller() -> bool:
     """
     if pathlib.Path('~/.sky/sky_ray.yml').expanduser().exists():
         config = yaml.safe_load(
-            pathlib.Path('~/.sky/sky_ray.yml').expanduser().read_text())
+            pathlib.Path('~/.sky/sky_ray.yml').expanduser().read_text(
+                encoding='utf-8'))
         cluster_name = config.get('cluster_name', '')
-        candidate_controller_names = (
-            controller_utils.Controllers.JOBS_CONTROLLER.value.
-            candidate_cluster_names)
         # We use startswith instead of exact match because the cluster name in
         # the yaml file is cluster_name_on_cloud which may have additional
         # suffices.
-        return any(
-            cluster_name.startswith(name)
-            for name in candidate_controller_names)
+        return cluster_name.startswith(
+            controller_utils.Controllers.JOBS_CONTROLLER.value.cluster_name)
     return False
 
 
+# Column indices for job table
+class JobTableColumns(enum.IntEnum):
+    """Column indices for the jobs table in the dashboard.
+
+    - DROPDOWN (0): Column for expandable dropdown arrow
+    - ID (1): Job ID column
+    - TASK (2): Task name/number column
+    - NAME (3): Job name column
+    - RESOURCES (4): Resources used by job
+    - SUBMITTED (5): Job submission timestamp
+    - TOTAL_DURATION (6): Total time since job submission
+    - JOB_DURATION (7): Actual job runtime
+    - RECOVERIES (8): Number of job recoveries
+    - STATUS (9): Current job status
+    - STARTED (10): Job start timestamp
+    - CLUSTER (11): Cluster name
+    - REGION (12): Cloud region
+    - FAILOVER (13): Job failover history
+    - DETAILS (14): Job details
+    - ACTIONS (15): Available actions column
+    """
+    DROPDOWN = 0
+    ID = 1
+    TASK = 2
+    NAME = 3
+    RESOURCES = 4
+    SUBMITTED = 5
+    TOTAL_DURATION = 6
+    JOB_DURATION = 7
+    RECOVERIES = 8
+    STATUS = 9
+    STARTED = 10
+    CLUSTER = 11
+    REGION = 12
+    DETAILS = 13
+    FAILOVER = 14
+    ACTIONS = 15
+
+
+# Column headers matching the indices above
+JOB_TABLE_COLUMNS = [
+    '', 'ID', 'Task', 'Name', 'Resources', 'Submitted', 'Total Duration',
+    'Job Duration', 'Status', 'Started', 'Cluster', 'Region', 'Failover',
+    'Recoveries', 'Details', 'Actions'
+]
+
+# This column is given by format_job_table but should be ignored.
+SCHED_STATE_COLUMN = 12
+
+
+def _extract_launch_history(log_content: str) -> str:
+    """Extract launch history from log content.
+
+    Args:
+        log_content: Content of the log file.
+    Returns:
+        A formatted string containing the launch history.
+    """
+    launches = []
+    current_launch = None
+
+    for line in log_content.splitlines():
+        if 'Launching on' in line:
+            try:
+                parts = line.split(']')
+                if len(parts) >= 2:
+                    timestamp = parts[0].split()[1:3]
+                    message = parts[1].replace('[0m⚙︎', '').strip()
+                    formatted_line = f'{" ".join(timestamp)} {message}'
+                    if current_launch:
+                        prev_time, prev_target = current_launch.rsplit(
+                            ' Launching on ', 1)
+                        launches.append(
+                            f'{prev_time} Tried to launch on {prev_target}')
+
+                    # Store the current launch
+                    current_launch = formatted_line
+            except IndexError:
+                launches.append(line.strip())
+
+    # Add the final (successful) launch at the beginning
+    if current_launch:
+        result = [current_launch]
+        result.extend(launches)
+        return '\n'.join(result)
+
+    return 'No launch history found'
+
+
 @app.route('/')
 def home():
     if not _is_running_on_jobs_controller():
         # Experimental: run on laptop (refresh is very slow).
-        all_managed_jobs = managed_jobs.queue(refresh=True, skip_finished=False)
+        request_id = managed_jobs.queue(refresh=True, skip_finished=False)
+        all_managed_jobs = sdk.get(request_id)
     else:
         job_table = managed_jobs.dump_managed_job_queue()
         all_managed_jobs = managed_jobs.load_managed_job_queue(job_table)
@@ -52,36 +144,90 @@ def home():
     timestamp = datetime.datetime.now(datetime.timezone.utc)
     rows = managed_jobs.format_job_table(all_managed_jobs,
                                          show_all=True,
+                                         show_user=False,
                                          return_rows=True)
-    # Add an empty column for the dropdown button. This will be added in the
-    # jobs/templates/index.html file.
-    rows = [[''] + row for row in rows]
-
-    # FIXME(zongheng): make the job table/queue funcs return structured info so
-    # that we don't have to do things like row[-5] below.
-    columns = [
-        '', 'ID', 'Task', 'Name', 'Resources', 'Submitted', 'Total Duration',
-        'Job Duration', 'Recoveries', 'Status', 'Started', 'Cluster', 'Region',
-        'Failure'
+
+    status_counts = collections.defaultdict(int)
+    for task in all_managed_jobs:
+        if not task['status'].is_terminal():
+            status_counts[task['status'].value] += 1
+
+    # Add an empty column for the dropdown button and actions column
+    # Exclude SCHED. STATE column
+    rows = [
+        [''] + row[:SCHED_STATE_COLUMN] + row[SCHED_STATE_COLUMN + 1:] +
+        # Add empty cell for failover and actions column
+        [''] + [''] for row in rows
     ]
-    if rows and len(rows[0]) != len(columns):
+
+    # Add log content as failover history for each job
+    for row in rows:
+        job_id = str(row[JobTableColumns.ID]).strip().replace(' ⤳', '')
+        if job_id and job_id != '-':
+            try:
+                log_path = os.path.join(
+                    os.path.expanduser(
+                        managed_job_constants.JOBS_CONTROLLER_LOGS_DIR),
+                    f'{job_id}.log')
+                if os.path.exists(log_path):
+                    with open(log_path, 'r', encoding='utf-8') as f:
+                        log_content = f.read()
+                        row[JobTableColumns.FAILOVER] = _extract_launch_history(
+                            log_content)
+                else:
+                    row[JobTableColumns.FAILOVER] = 'Log file not found'
+            except (IOError, OSError) as e:
+                row[JobTableColumns.FAILOVER] = f'Error reading log: {str(e)}'
+    app.logger.error('All managed jobs:')
+
+    # Validate column count
+    if rows and len(rows[0]) != len(JOB_TABLE_COLUMNS):
         raise RuntimeError(
-            'Dashboard code and managed job queue code are out of sync.')
+            f'Dashboard code and managed job queue code are out of sync. '
+            f'Expected {(JOB_TABLE_COLUMNS)} columns, got {(rows[0])}')
 
-    # Fix STATUS color codes: '\x1b[33mCANCELLED\x1b[0m' -> 'CANCELLED'.
+    # Fix STATUS color codes: '\x1b[33mCANCELLED\x1b[0m' -> 'CANCELLED'
     for row in rows:
-        row[-5] = common_utils.remove_color(row[-5])
-    # Remove filler rows ([''], ..., ['-']).
-    rows = [row for row in rows if ''.join(map(str, row)) != '']
+        row[JobTableColumns.STATUS] = common_utils.remove_color(
+            row[JobTableColumns.STATUS])
+
+    # Remove filler rows ([''], ..., ['-'])
+    rows = [
+        row for row in rows
+        if ''.join(map(str, row[:JobTableColumns.ACTIONS])) != ''
+    ]
+
+    # Get all unique status values
+    status_values = sorted(
+        list(set(row[JobTableColumns.STATUS] for row in rows)))
 
     rendered_html = flask.render_template(
         'index.html',
-        columns=columns,
+        columns=JOB_TABLE_COLUMNS,
         rows=rows,
         last_updated_timestamp=timestamp,
+        status_values=status_values,
+        status_counts=status_counts,
     )
     return rendered_html
 
 
+@app.route('/download_log/<job_id>')
+def download_log(job_id):
+    try:
+        log_path = os.path.join(
+            os.path.expanduser(managed_job_constants.JOBS_CONTROLLER_LOGS_DIR),
+            f'{job_id}.log')
+        if not os.path.exists(log_path):
+            flask.abort(404)
+        return flask.send_file(log_path,
+                               mimetype='text/plain',
+                               as_attachment=True,
+                               download_name=f'job_{job_id}.log')
+    except (IOError, OSError) as e:
+        app.logger.error(f'Error downloading log for job {job_id}: {str(e)}')
+        flask.abort(500)
+
+
 if __name__ == '__main__':
     app.run()