skypilot-nightly 1.0.0.dev20251203__py3-none-any.whl → 1.0.0.dev20260112__py3-none-any.whl

sky/backends/task_codegen.py

@@ -4,6 +4,7 @@ import copy
 import inspect
 import json
 import math
+import os
 import textwrap
 from typing import Dict, List, Optional, Tuple
 
@@ -146,6 +147,7 @@ class TaskCodeGen:
         if [ $(findmnt -t fuse.rclone --noheading | wc -l) -gt 0 ] && \
            [ -d {constants.RCLONE_MOUNT_CACHED_LOG_DIR} ] && \
            [ "$(ls -A {constants.RCLONE_MOUNT_CACHED_LOG_DIR})" ]; then
+            FLUSH_START_TIME=$(date +%s)
             flushed=0
             # extra second on top of --vfs-cache-poll-interval to
             # avoid race condition between rclone log line creation and this check.
@@ -158,13 +160,32 @@ class TaskCodeGen:
                     exitcode=0
                     tac $file | grep "vfs cache: cleaned:" -m 1 | grep "in use 0, to upload 0, uploading 0" -q || exitcode=$?
                     if [ $exitcode -ne 0 ]; then
-                        echo "skypilot: cached mount is still uploading to remote"
+                        ELAPSED=$(($(date +%s) - FLUSH_START_TIME))
+                        # Extract the last vfs cache status line to show what we're waiting for
+                        CACHE_STATUS=$(tac $file | grep "vfs cache: cleaned:" -m 1 | sed 's/.*vfs cache: cleaned: //' 2>/dev/null)
+                        # Extract currently uploading files from recent log lines (show up to 2 files)
+                        UPLOADING_FILES=$(tac $file | head -30 | grep -E "queuing for upload" | head -2 | sed 's/.*INFO  : //' | sed 's/: vfs cache:.*//' | tr '\\n' ',' | sed 's/,$//' | sed 's/,/, /g' 2>/dev/null)
+                        # Build status message with available info
+                        if [ -n "$CACHE_STATUS" ] && [ -n "$UPLOADING_FILES" ]; then
+                            echo "skypilot: cached mount is still uploading (elapsed: ${{ELAPSED}}s) [${{CACHE_STATUS}}] uploading: ${{UPLOADING_FILES}}"
+                        elif [ -n "$CACHE_STATUS" ]; then
+                            echo "skypilot: cached mount is still uploading (elapsed: ${{ELAPSED}}s) [${{CACHE_STATUS}}]"
+                        else
+                            # Fallback: show last non-empty line from log
+                            LAST_LINE=$(tac $file | grep -v "^$" | head -1 | sed 's/.*INFO  : //' | sed 's/.*ERROR : //' | sed 's/.*NOTICE: //' 2>/dev/null)
+                            if [ -n "$LAST_LINE" ]; then
+                                echo "skypilot: cached mount is still uploading (elapsed: ${{ELAPSED}}s) ${{LAST_LINE}}"
+                            else
+                                echo "skypilot: cached mount is still uploading (elapsed: ${{ELAPSED}}s)"
+                            fi
+                        fi
                         flushed=0
                         break
                     fi
                 done
             done
-            echo "skypilot: cached mount uploaded complete"
+            TOTAL_FLUSH_TIME=$(($(date +%s) - FLUSH_START_TIME))
+            echo "skypilot: cached mount upload complete (took ${{TOTAL_FLUSH_TIME}}s)"
         fi""")
 
     def add_prologue(self, job_id: int) -> None:
@@ -181,8 +202,8 @@ class TaskCodeGen:
         resources_dict: Dict[str, float],
         stable_cluster_internal_ips: List[str],
         env_vars: Dict[str, str],
+        log_dir: str,
         setup_cmd: Optional[str] = None,
-        setup_log_path: Optional[str] = None,
     ) -> None:
         """Generates code to set up the task on each node.
 
@@ -213,6 +234,9 @@ class TaskCodeGen:
         self._code += [
             textwrap.dedent(f"""\
             if sum(returncodes) != 0:
+                # Save exit codes to job metadata for potential recovery logic
+                if int(constants.SKYLET_VERSION) >= 28:
+                    job_lib.set_exit_codes({self.job_id!r}, returncodes)
                 job_lib.set_status({self.job_id!r}, job_lib.JobStatus.FAILED)
                 # Schedule the next pending job immediately to make the job
                 # scheduling more efficient.
@@ -379,13 +403,15 @@ class RayCodeGen(TaskCodeGen):
         resources_dict: Dict[str, float],
         stable_cluster_internal_ips: List[str],
         env_vars: Dict[str, str],
+        log_dir: str,
         setup_cmd: Optional[str] = None,
-        setup_log_path: Optional[str] = None,
     ) -> None:
         assert self._has_prologue, ('Call add_prologue() before '
                                     'add_setup().')
         self._has_setup = True
 
+        setup_log_path = os.path.join(log_dir, 'setup.log')
+
         bundles = [copy.copy(resources_dict) for _ in range(num_nodes)]
         # Set CPU to avoid ray hanging the resources allocation
         # for remote functions, since the task will request 1 CPU
@@ -480,6 +506,8 @@ class RayCodeGen(TaskCodeGen):
                     msg += f'Failed workers: ' + ', '.join([f'(pid={{pid}}, returncode={{returncode}})' for pid, returncode in failed_workers_and_returncodes])
                     msg += f'. See error logs above for more details.{colorama.Style.RESET_ALL}'
                     print(msg, flush=True)
+                    if int(constants.SKYLET_VERSION) >= 28:
+                        job_lib.set_exit_codes({self.job_id!r}, setup_returncodes)
                     job_lib.set_status({self.job_id!r}, job_lib.JobStatus.FAILED_SETUP)
                     # This waits for all streaming logs to finish.
                     time.sleep(1)
@@ -631,3 +659,351 @@ class RayCodeGen(TaskCodeGen):
         """Generates code that waits for all tasks, then exits."""
         self._code.append('returncodes, _ = get_or_fail(futures, pg)')
         super().add_epilogue()
+
+
+class SlurmCodeGen(TaskCodeGen):
+    """Code generator for task execution on Slurm using native srun."""
+
+    def __init__(self, slurm_job_id: str):
+        """Initialize SlurmCodeGen
+
+        Args:
+            slurm_job_id: The Slurm job ID, i.e. SLURM_JOB_ID
+        """
+        super().__init__()
+        self._slurm_job_id = slurm_job_id
+
+    def add_prologue(self, job_id: int) -> None:
+        assert not self._has_prologue, 'add_prologue() called twice?'
+        self._has_prologue = True
+        self.job_id = job_id
+
+        self._add_common_imports()
+
+        self._code.append(
+            textwrap.dedent("""\
+            import colorama
+            import copy
+            import json
+            import multiprocessing
+            import signal
+            import threading
+            from sky.backends import backend_utils
+            """))
+        self._add_skylet_imports()
+
+        self._add_constants()
+
+        self._add_logging_functions()
+
+        self._code.append(
+            textwrap.dedent(f"""\
+            def _cancel_slurm_job_steps():
+                slurm_job_id = {self._slurm_job_id!r}
+                assert slurm_job_id is not None, 'SLURM_JOB_ID is not set'
+                try:
+                    # Query steps for this job: squeue -s -j JOBID -h -o "%i %j"
+                    # Output format: "JOBID.STEPID STEPNAME"
+                    # TODO(kevin): This assumes that compute node is able
+                    # to run client commands against the controller.
+                    # Validate this assumption.
+                    result = subprocess.run(
+                        ['squeue', '-s', '-j', slurm_job_id, '-h', '-o', '%i %j'],
+                        capture_output=True, text=True, check=False)
+                    for line in result.stdout.strip().split('\\n'):
+                        if not line:
+                            continue
+                        parts = line.split()
+                        assert len(parts) >= 2, 'Expected at least 2 parts'
+                        step_id, step_name = parts[0], parts[1]
+                        if step_name == f'sky-{self.job_id}':
+                            subprocess.run(['scancel', step_id],
+                                            check=False, capture_output=True)
+                except Exception as e:
+                    print(f'Error in _cancel_slurm_job_steps: {{e}}', flush=True)
+                    pass
+
+            def _slurm_cleanup_handler(signum, _frame):
+                _cancel_slurm_job_steps()
+                # Re-raise to let default handler terminate.
+                signal.signal(signum, signal.SIG_DFL)
+                os.kill(os.getpid(), signum)
+
+            signal.signal(signal.SIGTERM, _slurm_cleanup_handler)
+            """))
+
+        self._code += [
+            'autostop_lib.set_last_active_time_to_now()',
+            f'job_lib.set_status({job_id!r}, job_lib.JobStatus.PENDING)',
+        ]
+
+        self._setup_cmd: Optional[str] = None
+        self._setup_envs: Optional[Dict[str, str]] = None
+        self._setup_log_dir: Optional[str] = None
+        self._setup_num_nodes: Optional[int] = None
+
+    def add_setup(
+        self,
+        num_nodes: int,
+        resources_dict: Dict[str, float],
+        stable_cluster_internal_ips: List[str],
+        env_vars: Dict[str, str],
+        log_dir: str,
+        setup_cmd: Optional[str] = None,
+    ) -> None:
+        assert self._has_prologue, ('Call add_prologue() before add_setup().')
+        self._has_setup = True
+        self._cluster_num_nodes = len(stable_cluster_internal_ips)
+        self._stable_cluster_ips = stable_cluster_internal_ips
+
+        self._add_waiting_for_resources_msg(num_nodes)
+
+        # Store setup information for use in add_task().
+        if setup_cmd is not None:
+            setup_envs = env_vars.copy()
+            setup_envs[constants.SKYPILOT_NUM_NODES] = str(num_nodes)
+            self._setup_cmd = setup_cmd
+            self._setup_envs = setup_envs
+            self._setup_log_dir = log_dir
+            self._setup_num_nodes = num_nodes
+
+    def add_task(
+        self,
+        num_nodes: int,
+        bash_script: Optional[str],
+        task_name: Optional[str],
+        resources_dict: Dict[str, float],
+        log_dir: str,
+        env_vars: Optional[Dict[str, str]] = None,
+    ) -> None:
+        """Generates code for invoking a bash command
+        using srun within sbatch allocation.
+        """
+        assert self._has_setup, 'Call add_setup() before add_task().'
+        env_vars = env_vars or {}
+        task_name = task_name if task_name is not None else 'task'
+
+        acc_name, acc_count = self._get_accelerator_details(resources_dict)
+        num_gpus = 0
+        if (acc_name is not None and
+                not accelerator_registry.is_schedulable_non_gpu_accelerator(
+                    acc_name)):
+            num_gpus = int(math.ceil(acc_count))
+
+        # Slurm does not support fractional CPUs.
+        task_cpu_demand = int(math.ceil(resources_dict.pop('CPU')))
+
+        sky_env_vars_dict_str = [
+            textwrap.dedent(f"""\
+            sky_env_vars_dict = {{}}
+            sky_env_vars_dict['SKYPILOT_INTERNAL_JOB_ID'] = {self.job_id}
+            """)
+        ]
+
+        if env_vars:
+            sky_env_vars_dict_str.extend(f'sky_env_vars_dict[{k!r}] = {v!r}'
+                                         for k, v in env_vars.items())
+        sky_env_vars_dict_str = '\n'.join(sky_env_vars_dict_str)
+
+        rclone_flush_script = self._get_rclone_flush_script()
+        streaming_msg = self._get_job_started_msg()
+        has_setup_cmd = self._setup_cmd is not None
+
+        self._code += [
+            sky_env_vars_dict_str,
+            textwrap.dedent(f"""\
+            script = {bash_script!r}
+            if script is None:
+                script = ''
+            rclone_flush_script = {rclone_flush_script!r}
+
+            if script or {has_setup_cmd!r}:
+                script += rclone_flush_script
+                sky_env_vars_dict['{constants.SKYPILOT_NUM_GPUS_PER_NODE}'] = {num_gpus}
+
+                # Signal files for setup/run synchronization:
+                # 1. alloc_signal_file: srun has acquired allocation
+                # 2. setup_done_signal_file: Driver has finished setup, run can proceed
+                #
+                # Signal files are stored in home directory, which is
+                # assumed to be on a shared NFS mount accessible by all nodes.
+                # To support clusters with non-NFS home directories, we would
+                # need to let users specify an NFS-backed "working directory"
+                # or use a different coordination mechanism.
+                alloc_signal_file = f'~/.sky_alloc_{self._slurm_job_id}_{self.job_id}'
+                alloc_signal_file = os.path.expanduser(alloc_signal_file)
+                setup_done_signal_file = f'~/.sky_setup_done_{self._slurm_job_id}_{self.job_id}'
+                setup_done_signal_file = os.path.expanduser(setup_done_signal_file)
+
+                # Start exclusive srun in a thread to reserve allocation (similar to ray.get(pg.ready()))
+                gpu_arg = f'--gpus-per-node={num_gpus}' if {num_gpus} > 0 else ''
+
+                def build_task_runner_cmd(user_script, extra_flags, log_dir, env_vars_dict,
+                                          task_name=None, is_setup=False,
+                                          alloc_signal=None, setup_done_signal=None):
+                    env_vars_json = json.dumps(env_vars_dict)
+
+                    log_dir = shlex.quote(log_dir)
+                    env_vars = shlex.quote(env_vars_json)
+                    cluster_ips = shlex.quote(",".join({self._stable_cluster_ips!r}))
+
+                    runner_args = f'--log-dir={{log_dir}} --env-vars={{env_vars}} --cluster-num-nodes={self._cluster_num_nodes} --cluster-ips={{cluster_ips}}'
+
+                    if task_name is not None:
+                        runner_args += f' --task-name={{shlex.quote(task_name)}}'
+
+                    if is_setup:
+                        runner_args += ' --is-setup'
+
+                    if alloc_signal is not None:
+                        runner_args += f' --alloc-signal-file={{shlex.quote(alloc_signal)}}'
+
+                    if setup_done_signal is not None:
+                        runner_args += f' --setup-done-signal-file={{shlex.quote(setup_done_signal)}}'
+
+                    script_path = None
+                    prefix = 'sky_setup_' if is_setup else 'sky_task_'
+                    if backend_utils.is_command_length_over_limit(user_script):
+                        with tempfile.NamedTemporaryFile('w', prefix=prefix, suffix='.sh', delete=False) as f:
+                            f.write(user_script)
+                            script_path = f.name
+                        runner_args += f' --script-path={{shlex.quote(script_path)}}'
+                    else:
+                        runner_args += f' --script={{shlex.quote(user_script)}}'
+
+                    # Use /usr/bin/env explicitly to work around a Slurm quirk where
+                    # srun's execvp() doesn't check execute permissions, failing when
+                    # $HOME/.local/bin/env (non-executable, from uv installation)
+                    # shadows /usr/bin/env.
+                    job_suffix = '-setup' if is_setup else ''
+                    # Unset SLURM_* environment variables before running srun.
+                    # When this srun runs inside another srun (from
+                    # SlurmCommandRunner.run), inherited variables like
+                    # SLURM_CPU_BIND, SLURM_NNODES, and SLURM_NODELIST constrain
+                    # the inner srun to the parent step's allocation. This causes
+                    # "CPU binding outside of job step allocation" errors.
+                    # Unsetting all SLURM_* variables allows this srun to access the full job
+                    # allocation. See:
+                    # https://support.schedmd.com/show_bug.cgi?id=14298
+                    # https://github.com/huggingface/datatrove/issues/248
+                    srun_cmd = (
+                        "unset $(env | awk -F= '/^SLURM_/ {{print $1}}') && "
+                        f'srun --export=ALL --quiet --unbuffered --kill-on-bad-exit --jobid={self._slurm_job_id} '
+                        f'--job-name=sky-{self.job_id}{{job_suffix}} --ntasks-per-node=1 {{extra_flags}} '
+                        f'{{constants.SKY_SLURM_PYTHON_CMD}} -m sky.skylet.executor.slurm {{runner_args}}'
+                    )
+                    return srun_cmd, script_path
+
+                def run_thread_func():
+                    # This blocks until Slurm allocates resources (--exclusive)
+                    # --mem=0 to match RayCodeGen's behavior where we don't explicitly request memory.
+                    run_flags = f'--nodes={num_nodes} --cpus-per-task={task_cpu_demand} --mem=0 {{gpu_arg}} --exclusive'
+                    srun_cmd, task_script_path = build_task_runner_cmd(
+                        script, run_flags, {log_dir!r}, sky_env_vars_dict,
+                        task_name={task_name!r},
+                        alloc_signal=alloc_signal_file,
+                        setup_done_signal=setup_done_signal_file
+                    )
+
+                    proc = subprocess.Popen(srun_cmd, shell=True,
+                                          stdout=subprocess.PIPE,
+                                          stderr=subprocess.STDOUT,
+                                          text=True)
+                    for line in proc.stdout:
+                        print(line, end='', flush=True)
+                    proc.wait()
+
+                    if task_script_path is not None:
+                        os.remove(task_script_path)
+                    return {{'return_code': proc.returncode, 'pid': proc.pid}}
+
+                run_thread_result = {{'result': None}}
+                def run_thread_wrapper():
+                    run_thread_result['result'] = run_thread_func()
+
+                run_thread = threading.Thread(target=run_thread_wrapper)
+                run_thread.start()
+
+                # Wait for allocation signal from inside srun
+                while not os.path.exists(alloc_signal_file):
+                    if not run_thread.is_alive():
+                        # srun failed before creating the signal file.
+                        run_thread.join()
+                        result = run_thread_result['result']
+                        returncode = int(result.get('return_code', 1))
+                        pid = result.get('pid', os.getpid())
+                        msg = f'ERROR: {colorama.Fore.RED}Job {self.job_id}\\'s setup failed with return code {{returncode}} (pid={{pid}}).'
+                        msg += f' See error logs above for more details.{colorama.Style.RESET_ALL}'
+                        print(msg, flush=True)
+                        returncodes = [returncode]
+                        if int(constants.SKYLET_VERSION) >= 28:
+                            job_lib.set_exit_codes({self.job_id!r}, returncodes)
+                        job_lib.set_status({self.job_id!r}, job_lib.JobStatus.FAILED_SETUP)
+                        sys.exit(1)
+                    time.sleep(0.1)
+
+                print({streaming_msg!r}, flush=True)
+
+                if {has_setup_cmd!r}:
+                    job_lib.set_status({self.job_id!r}, job_lib.JobStatus.SETTING_UP)
+
+                    # The schedule_step should be called after the job status is set to
+                    # non-PENDING, otherwise, the scheduler will think the current job
+                    # is not submitted yet, and skip the scheduling step.
+                    job_lib.scheduler.schedule_step()
+
+                    # --overlap as we have already secured allocation with the srun for the run section,
+                    # and otherwise this srun would get blocked and deadlock.
+                    setup_flags = f'--overlap --nodes={self._setup_num_nodes}'
+                    setup_srun, setup_script_path = build_task_runner_cmd(
+                        {self._setup_cmd!r}, setup_flags, {self._setup_log_dir!r}, {self._setup_envs!r},
+                        is_setup=True
+                    )
+
+                    # Run setup srun directly, streaming output to driver stdout
+                    setup_proc = subprocess.Popen(setup_srun, shell=True,
+                                                 stdout=subprocess.PIPE,
+                                                 stderr=subprocess.STDOUT,
+                                                 text=True)
+                    for line in setup_proc.stdout:
+                        print(line, end='', flush=True)
+                    setup_proc.wait()
+
+                    if setup_script_path is not None:
+                        os.remove(setup_script_path)
+
+                    setup_returncode = setup_proc.returncode
+                    if setup_returncode != 0:
+                        setup_pid = setup_proc.pid
+                        msg = f'ERROR: {colorama.Fore.RED}Job {self.job_id}\\'s setup failed with return code {{setup_returncode}} (pid={{setup_pid}}).'
+                        msg += f' See error logs above for more details.{colorama.Style.RESET_ALL}'
+                        print(msg, flush=True)
+                        job_lib.set_status({self.job_id!r}, job_lib.JobStatus.FAILED_SETUP)
+                        # Cancel the srun spawned by run_thread_func.
+                        _cancel_slurm_job_steps()
+                        sys.exit(1)
+
+                job_lib.set_job_started({self.job_id!r})
+                if not {has_setup_cmd!r}:
+                    # Need to call schedule_step() to make sure the scheduler
+                    # schedule the next pending job.
+                    job_lib.scheduler.schedule_step()
+
+                # Signal run thread to proceed.
+                pathlib.Path(setup_done_signal_file).touch()
+
+                # Wait for run thread to complete.
+                run_thread.join()
+                result = run_thread_result['result']
+
+                # Cleanup signal files
+                if os.path.exists(alloc_signal_file):
+                    os.remove(alloc_signal_file)
+                if os.path.exists(setup_done_signal_file):
+                    os.remove(setup_done_signal_file)
+
+                returncodes = [int(result.get('return_code', 1))]
+            else:
+                returncodes = [0]
+            """),
+        ]

sky/catalog/__init__.py

@@ -127,12 +127,9 @@ def list_accelerator_realtime(
     case_sensitive: bool = True,
 ) -> Tuple[Dict[str, List[int]], Dict[str, int], Dict[str, int]]:
     """Lists all accelerators offered by Sky with their realtime availability.
-
     Realtime availability is the total number of accelerators in the cluster
     and number of accelerators available at the time of the call.
-
     Used for fixed size cluster settings, such as Kubernetes.
-
     Returns:
         A tuple of three dictionaries mapping canonical accelerator names to:
         - A list of available counts. (e.g., [1, 2, 4])

sky/catalog/data_fetchers/fetch_gcp.py

@@ -189,6 +189,9 @@ SERIES_TO_DESCRIPTION = {
     'c2': 'Compute optimized',
     'c2d': 'C2D AMD Instance',
     'c3': 'C3 Instance',
+    'c3d': 'C3D Instance',
+    'c4': 'C4 Instance',
+    'c4d': 'C4D Instance',
     'e2': 'E2 Instance',
     'f1': 'Micro Instance with burstable CPU',
     'g1': 'Small Instance with 1 VCPU',
@@ -376,8 +379,13 @@ def get_vm_df(skus: List[Dict[str, Any]], region_prefix: str) -> 'pd.DataFrame':
                 is_cpu = True
             elif resource_group == 'RAM':
                 is_memory = True
+            elif resource_group == 'LocalSSD':
+                # Ignore local SSD pricing for now, as we do not include disk
+                # pricing for instances for now.
+                # TODO(zhwu): Handle local SSD pricing.
+                pass
             else:
-                assert resource_group == 'N1Standard'
+                assert resource_group == 'N1Standard', (resource_group, sku)
                 if 'Core' in description:
                     is_cpu = True
                 elif 'Ram' in description:

sky/catalog/data_fetchers/fetch_nebius.py

@@ -180,7 +180,7 @@ def _write_preset_prices(presets: List[PresetInfo], output_file: str) -> None:
         presets (List[PresetInfo]): A list of PresetInfo objects to write.
         output_file (str): The path to the output CSV file.
     """
-    os.makedirs(os.path.dirname(output_file))
+    os.makedirs(os.path.dirname(output_file), exist_ok=True)
     # Set up the CSV writer to output to stdout
     with open(output_file, 'w', encoding='utf-8') as out:
         header = [

sky/catalog/data_fetchers/fetch_vast.py

@@ -50,7 +50,7 @@ if __name__ == '__main__':
                    ('num_gpus', 'AcceleratorCount'), ('cpu_cores', 'vCPUs'),
                    ('cpu_ram', 'MemoryGiB'), ('gpu_name', 'GpuInfo'),
                    ('search.totalHour', 'Price'), ('min_bid', 'SpotPrice'),
-                   ('geolocation', 'Region'))
+                   ('geolocation', 'Region'), ('hosting_type', 'HostingType'))
 
     # Vast has a wide variety of machines, some of
     # which will have less diskspace and network
@@ -138,7 +138,9 @@ if __name__ == '__main__':
 
         maxBid = max([x.get('SpotPrice') for x in toList])
         for instance in toList:
-            stub = f'{instance["InstanceType"]} {instance["Region"][-2:]}'
+            hosting_type = instance.get('HostingType', 0)
+            stub = (f'{instance["InstanceType"]} '
+                    f'{instance["Region"][-2:]} {hosting_type}')
             if stub in seen:
                 printstub = f'{stub}#print'
                 if printstub not in seen:

sky/catalog/kubernetes_catalog.py

@@ -204,6 +204,9 @@ def _list_accelerators(
     min_quantity_filter = quantity_filter if quantity_filter else 1
 
     for node in nodes:
+        # Check if node is ready
+        node_is_ready = node.is_ready()
+
         for key in keys:
             if key in node.metadata.labels:
                 accelerator_name = lf.get_accelerator_from_label_value(
@@ -260,6 +263,15 @@ def _list_accelerators(
                         total_accelerators_capacity[
                             accelerator_name] += quantized_count
 
+                # Initialize the total_accelerators_available to make sure the
+                # key exists in the dictionary.
+                total_accelerators_available[accelerator_name] = (
+                    total_accelerators_available.get(accelerator_name, 0))
+
+                # Skip availability counting for not-ready nodes
+                if not node_is_ready:
+                    continue
+
                 if error_on_get_allocated_gpu_qty_by_node:
                     # If we can't get the allocated GPU quantity by each node,
                     # we can't get the GPU usage.
@@ -268,10 +280,6 @@ def _list_accelerators(
 
                 allocated_qty = allocated_qty_by_node[node.metadata.name]
                 accelerators_available = accelerator_count - allocated_qty
-                # Initialize the total_accelerators_available to make sure the
-                # key exists in the dictionary.
-                total_accelerators_available[accelerator_name] = (
-                    total_accelerators_available.get(accelerator_name, 0))
 
                 if accelerators_available >= min_quantity_filter:
                     quantized_availability = min_quantity_filter * (

sky/catalog/seeweb_catalog.py

@@ -7,22 +7,33 @@ query instance types and pricing information for Seeweb.
 import typing
 from typing import Dict, List, Optional, Tuple
 
-import pandas as pd
-
+from sky.adaptors import common as adaptors_common
 from sky.catalog import common
 from sky.utils import resources_utils
 from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
+    import pandas as pd
+
     from sky.clouds import cloud
+else:
+    pd = adaptors_common.LazyImport('pandas')
 
 _PULL_FREQUENCY_HOURS = 8
-_df = common.read_catalog('seeweb/vms.csv',
-                          pull_frequency_hours=_PULL_FREQUENCY_HOURS)
+_df = None
+
+
+def _get_df():
+    """Get the dataframe, loading it lazily if needed."""
+    global _df
+    if _df is None:
+        _df = common.read_catalog('seeweb/vms.csv',
+                                  pull_frequency_hours=_PULL_FREQUENCY_HOURS)
+    return _df
 
 
 def instance_type_exists(instance_type: str) -> bool:
-    result = common.instance_type_exists_impl(_df, instance_type)
+    result = common.instance_type_exists_impl(_get_df(), instance_type)
     return result
 
 
@@ -33,7 +44,7 @@ def validate_region_zone(
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Seeweb does not support zones.')
 
-    result = common.validate_region_zone_impl('Seeweb', _df, region, zone)
+    result = common.validate_region_zone_impl('Seeweb', _get_df(), region, zone)
     return result
 
 
@@ -46,14 +57,15 @@ def get_hourly_cost(instance_type: str,
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Seeweb does not support zones.')
 
-    result = common.get_hourly_cost_impl(_df, instance_type, use_spot, region,
-                                         zone)
+    result = common.get_hourly_cost_impl(_get_df(), instance_type, use_spot,
+                                         region, zone)
     return result
 
 
 def get_vcpus_mem_from_instance_type(
         instance_type: str) -> Tuple[Optional[float], Optional[float]]:
-    result = common.get_vcpus_mem_from_instance_type_impl(_df, instance_type)
+    result = common.get_vcpus_mem_from_instance_type_impl(
+        _get_df(), instance_type)
     return result
 
 
@@ -64,7 +76,7 @@ def get_default_instance_type(cpus: Optional[str] = None,
                               region: Optional[str] = None,
                               zone: Optional[str] = None) -> Optional[str]:
     del disk_tier  # unused
-    result = common.get_instance_type_for_cpus_mem_impl(_df, cpus, memory,
+    result = common.get_instance_type_for_cpus_mem_impl(_get_df(), cpus, memory,
                                                         region, zone)
     return result
 
@@ -72,7 +84,8 @@ def get_default_instance_type(cpus: Optional[str] = None,
 def get_accelerators_from_instance_type(
         instance_type: str) -> Optional[Dict[str, int]]:
     # Filter the dataframe for the specific instance type
-    df_filtered = _df[_df['InstanceType'] == instance_type]
+    df = _get_df()
+    df_filtered = df[df['InstanceType'] == instance_type]
     if df_filtered.empty:
         return None
 
@@ -114,7 +127,7 @@ def get_instance_type_for_accelerator(
         with ux_utils.print_exception_no_traceback():
             raise ValueError('Seeweb does not support zones.')
 
-    result = common.get_instance_type_for_accelerator_impl(df=_df,
+    result = common.get_instance_type_for_accelerator_impl(df=_get_df(),
                                                            acc_name=acc_name,
                                                            acc_count=acc_count,
                                                            cpus=cpus,
@@ -126,7 +139,7 @@ def get_instance_type_for_accelerator(
 
 
 def regions() -> List['cloud.Region']:
-    result = common.get_region_zones(_df, use_spot=False)
+    result = common.get_region_zones(_get_df(), use_spot=False)
     return result
 
 
@@ -135,7 +148,8 @@ def get_region_zones_for_instance_type(instance_type: str,
                                       ) -> List['cloud.Region']:
     """Returns a list of regions for a given instance type."""
     # Filter the dataframe for the specific instance type
-    df_filtered = _df[_df['InstanceType'] == instance_type]
+    df = _get_df()
+    df_filtered = df[df['InstanceType'] == instance_type]
     if df_filtered.empty:
         return []
 
@@ -174,7 +188,8 @@ def list_accelerators(
         require_price: bool = True) -> Dict[str, List[common.InstanceTypeInfo]]:
     """Lists accelerators offered in Seeweb."""
     # Filter out rows with empty or null regions (indicating unavailability)
-    df_filtered = _df.dropna(subset=['Region'])
+    df = _get_df()
+    df_filtered = df.dropna(subset=['Region'])
     df_filtered = df_filtered[df_filtered['Region'].str.strip() != '']
 
     result = common.list_accelerators_impl('Seeweb', df_filtered, gpus_only,