PyPI - skypilot-nightly - Versions diffs - 1.0.0.dev20251008__py3-none-any.whl → 1.0.0.dev20251011__py3-none-any.whl - Mend

skypilot-nightly 1.0.0.dev20251008py3-none-any.whl → 1.0.0.dev20251011py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of skypilot-nightly might be problematic. Click here for more details.

Files changed (86) hide show

sky/dashboard/out/workspaces.html CHANGED Viewed

	@@ -1 +1 @@
1	- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-~~3286453d56f3c0a0~~.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-~~7528cc0ef8c522c5~~.js" defer=""></script><script src="/dashboard/_next/static/~~MnvNdzHHpiZG1_oKSpbxF~~/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/~~MnvNdzHHpiZG1_oKSpbxF~~/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"~~MnvNdzHHpiZG1_oKSpbxF~~","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1	+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-66f23594d38c7f16.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-69c80d677d3c2949.js" defer=""></script><script src="/dashboard/_next/static/Xs6jdcfyNaUuBO8jmzU9_/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Xs6jdcfyNaUuBO8jmzU9_/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"Xs6jdcfyNaUuBO8jmzU9_","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>

sky/data/mounting_utils.py CHANGED Viewed

@@ -185,27 +185,63 @@ def get_gcs_mount_cmd(bucket_name: str,
 def get_az_mount_install_cmd() -> str:
     """Returns a command to install AZ Container mount utility blobfuse2."""
     install_cmd = (
-        'sudo apt-get update; '
-        'sudo apt-get install -y '
-        '-o Dpkg::Options::="--force-confdef" '
-        'fuse3 libfuse3-dev || { '
-        '  echo "fuse3 not available, falling back to fuse"; '
-        '  sudo apt-get install -y '
-        '  -o Dpkg::Options::="--force-confdef" '
-        '  fuse libfuse-dev; '
-        '} && '
+        # Check architecture first - blobfuse2 only supports x86_64
         'ARCH=$(uname -m) && '
         'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
         '  echo "blobfuse2 is not supported on $ARCH" && '
         f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'fi && '
+        # Try to install fuse3 from default repos
+        'sudo apt-get update && '
+        'FUSE3_INSTALLED=0 && '
+        'if sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        'fuse3 libfuse3-dev; then '
+        '  FUSE3_INSTALLED=1; '
+        '  echo "fuse3 installed from default repos"; '
         'else '
-        '  ARCH_SUFFIX="x86_64"; '
+        # If fuse3 not available, try focal for Ubuntu <= 20.04
+        '  DISTRO=$(grep "^ID=" /etc/os-release | cut -d= -f2 | '
+        'tr -d \'"\' | tr "[:upper:]" "[:lower:]") && '
+        '  VERSION=$(grep "^VERSION_ID=" /etc/os-release | cut -d= -f2 | '
+        'tr -d \'"\') && '
+        '  if [ "$DISTRO" = "ubuntu" ] && '
+        '[ "$(echo "$VERSION 20.04" | '
+        'awk \'{ print ($1 <= $2) }\')" = "1" ]; then '
+        '    echo "Trying to install fuse3 from focal for '
+        'Ubuntu $VERSION"; '
+        '    echo "deb http://archive.ubuntu.com/ubuntu '
+        'focal main universe" | '
+        'sudo tee /etc/apt/sources.list.d/focal-fuse3.list && '
+        '    sudo apt-get update && '
+        '    if sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        '-o Dpkg::Options::="--force-confold" '
+        'fuse3 libfuse3-3 libfuse3-dev; then '
+        '      FUSE3_INSTALLED=1; '
+        '      echo "fuse3 installed from focal"; '
+        '      sudo rm /etc/apt/sources.list.d/focal-fuse3.list; '
+        '      sudo apt-get update; '
+        '    else '
+        '      sudo rm -f /etc/apt/sources.list.d/focal-fuse3.list; '
+        '      sudo apt-get update; '
+        '    fi; '
+        '  fi; '
         'fi && '
-        'wget -nc https://github.com/Azure/azure-storage-fuse'
-        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
-        f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
+        # Install blobfuse2 only if fuse3 is available
+        'if [ "$FUSE3_INSTALLED" = "1" ]; then '
+        '  echo "Installing blobfuse2 with libfuse3 support"; '
+        '  wget -nc https://github.com/Azure/azure-storage-fuse'
+        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}/'
+        f'blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
         '-O /tmp/blobfuse2.deb && '
-        'sudo dpkg --install /tmp/blobfuse2.deb && '
+        '  sudo dpkg --install /tmp/blobfuse2.deb; '
+        'else '
+        '  echo "Error: libfuse3 is required for Azure storage '
+        'mounting with fusermount-wrapper."; '
+        '  echo "libfuse3 could not be installed on this system."; '
+        f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'fi && '
         f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
     return install_cmd
@@ -277,7 +313,10 @@ def get_az_mount_cmd(container_name: str,
                f'-- {blobfuse2_cmd} -o nonempty --foreground {{}}')
     original = f'{blobfuse2_cmd} {blobfuse2_options} {mount_path}'
     # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
-    # to avoid requiring root privilege.
+    # to avoid requiring privileged containers.
+    # fusermount-wrapper requires libfuse3;
+    # we install libfuse3 even on older distros like Ubuntu 18.04 by using
+    # Ubuntu 20.04 (focal) repositories.
     # TODO(aylei): feeling hacky, refactor this.
     get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
                      f'echo "{wrapped}" || echo "{original}"')

sky/jobs/controller.py CHANGED Viewed

@@ -870,8 +870,16 @@ class Controller:
             # because when SkyPilot API server machine sends the yaml config to
             # the controller machine, only storage metadata is sent, not the
             # storage object itself.
-            for storage in task.storage_mounts.values():
-                storage.construct()
+            try:
+                for storage in task.storage_mounts.values():
+                    storage.construct()
+            except (exceptions.StorageSpecError, exceptions.StorageError) as e:
+                job_logger.warning(
+                    f'Failed to construct storage object for teardown: {e}\n'
+                    'This may happen because storage construction already '
+                    'failed during launch, storage was deleted externally, '
+                    'credentials expired/changed, or network connectivity '
+                    'issues.')
             try:
                 backend.teardown_ephemeral_storage(task)
             except Exception as e:  # pylint: disable=broad-except

sky/jobs/recovery_strategy.py CHANGED Viewed

@@ -495,7 +495,9 @@ class StrategyExecutor:
                         self._logger.info('Managed job cluster launched.')
                     except (exceptions.InvalidClusterNameError,
                             exceptions.NoCloudAccessError,
-                            exceptions.ResourcesMismatchError) as e:
+                            exceptions.ResourcesMismatchError,
+                            exceptions.StorageSpecError,
+                            exceptions.StorageError) as e:
                         self._logger.error(
                             'Failure happened before provisioning. '
                             f'{common_utils.format_exception(e)}')

sky/jobs/server/server.py CHANGED Viewed

@@ -116,7 +116,7 @@ async def logs(
         # Cancel the coroutine after the request is done or client disconnects
         background_tasks.add_task(task.cancel)
-    return stream_utils.stream_response(
+    return stream_utils.stream_response_for_long_request(
         request_id=request_task.request_id,
         logs_path=request_task.log_path,
         background_tasks=background_tasks,
@@ -201,7 +201,7 @@ async def pool_tail_logs(
     request_task = api_requests.get_request(request.state.request_id)
-    return stream_utils.stream_response(
+    return stream_utils.stream_response_for_long_request(
         request_id=request_task.request_id,
         logs_path=request_task.log_path,
         background_tasks=background_tasks,

sky/metrics/utils.py CHANGED Viewed

@@ -195,6 +195,8 @@ def start_svc_port_forward(context: str, namespace: str, service: str,
     port_forward_process = None
     port_forward_exit = False
     local_port = None
+    poller = None
+    fd = None
     try:
         # start the port forward process
@@ -204,8 +206,13 @@ def start_svc_port_forward(context: str, namespace: str, service: str,
                                                 text=True,
                                                 env=env)
-        start_time = time.time()
+        # Use poll() instead of select() to avoid FD_SETSIZE limit
+        poller = select.poll()
+        assert port_forward_process.stdout is not None
+        fd = port_forward_process.stdout.fileno()
+        poller.register(fd, select.POLLIN)
+        start_time = time.time()
         buffer = ''
         # wait for the port forward to start and extract the local port
         while time.time() - start_time < start_port_forward_timeout:
@@ -215,22 +222,19 @@ def start_svc_port_forward(context: str, namespace: str, service: str,
                     port_forward_exit = True
                 break
-            # read output line by line to find the local port
-            if port_forward_process.stdout:
-                # Wait up to 1s for data to be available without blocking
-                r, _, _ = select.select([port_forward_process.stdout], [], [],
-                                        _SELECT_TIMEOUT)
-                if r:
-                    # Read available bytes from the FD without blocking
-                    fd = port_forward_process.stdout.fileno()
-                    raw = os.read(fd, _SELECT_BUFFER_SIZE)
-                    chunk = raw.decode(errors='ignore')
-                    buffer += chunk
-                    match = re.search(r'Forwarding from 127\.0\.0\.1:(\d+)',
-                                      buffer)
-                    if match:
-                        local_port = int(match.group(1))
-                        break
+            # Wait up to 1000ms for data to be available without blocking
+            # poll() takes timeout in milliseconds
+            events = poller.poll(_SELECT_TIMEOUT * 1000)
+            if events:
+                # Read available bytes from the FD without blocking
+                raw = os.read(fd, _SELECT_BUFFER_SIZE)
+                chunk = raw.decode(errors='ignore')
+                buffer += chunk
+                match = re.search(r'Forwarding from 127\.0\.0\.1:(\d+)', buffer)
+                if match:
+                    local_port = int(match.group(1))
+                    break
             # sleep for 100ms to avoid busy-waiting
             time.sleep(0.1)
@@ -239,6 +243,13 @@ def start_svc_port_forward(context: str, namespace: str, service: str,
             stop_svc_port_forward(port_forward_process,
                                   timeout=terminate_port_forward_timeout)
         raise
+    finally:
+        if poller is not None and fd is not None:
+            try:
+                poller.unregister(fd)
+            except (OSError, ValueError):
+                # FD may already be unregistered or invalid
+                pass
     if port_forward_exit:
         raise RuntimeError(f'Port forward failed for service {service} in '
                            f'namespace {namespace} on context {context}')

sky/provision/kubernetes/instance.py CHANGED Viewed

@@ -5,7 +5,7 @@ import json
 import re
 import sys
 import time
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 from sky import exceptions
 from sky import global_user_state
@@ -583,31 +583,6 @@ def _wait_for_pods_to_run(namespace, context, new_nodes):
         time.sleep(1)
-def _run_function_with_retries(func: Callable,
-                               operation_name: str,
-                               max_retries: int = _MAX_RETRIES,
-                               retry_delay: int = 5) -> Any:
-    """Runs a function with retries on Kubernetes errors.
-    Args:
-        func: Function to retry
-        operation_name: Name of the operation for logging
-        max_retries: Maximum number of retry attempts
-        retry_delay: Delay between retries in seconds
-    Raises:
-        The last exception encountered if all retries fail.
-    """
-    for attempt in range(max_retries + 1):
-        try:
-            return func()
-        except config_lib.KubernetesError:
-            if attempt < max_retries:
-                logger.warning(f'Failed to {operation_name} - '
-                               f'retrying in {retry_delay} seconds.')
-                time.sleep(retry_delay)
-            else:
-                raise
 @timeline.event
 def pre_init(namespace: str, context: Optional[str], new_nodes: List) -> None:
     """Pre-initialization step for SkyPilot pods.
@@ -934,8 +909,11 @@ def _create_pods(region: str, cluster_name: str, cluster_name_on_cloud: str,
     running_pods = kubernetes_utils.filter_pods(namespace, context, tags,
                                                 ['Pending', 'Running'])
     head_pod_name = _get_head_pod_name(running_pods)
+    running_pod_statuses = [{
+        pod.metadata.name: pod.status.phase
+    } for pod in running_pods.values()]
     logger.debug(f'Found {len(running_pods)} existing pods: '
-                 f'{list(running_pods.keys())}')
+                 f'{running_pod_statuses}')
     to_start_count = config.count - len(running_pods)
     if to_start_count < 0:
@@ -951,7 +929,7 @@ def _create_pods(region: str, cluster_name: str, cluster_name_on_cloud: str,
     nvidia_runtime_exists = False
     try:
         nvidia_runtime_exists = kubernetes_utils.check_nvidia_runtime_class(
-            context)
+            context=context)
     except kubernetes.kubernetes.client.ApiException as e:
         logger.warning('run_instances: Error occurred while checking for '
                        f'nvidia RuntimeClass - '
@@ -1142,10 +1120,21 @@ def _create_pods(region: str, cluster_name: str, cluster_name_on_cloud: str,
         pods = created_resources
     created_pods = {}
+    valid_pods = []
     for pod in pods:
+        # In case Pod is not created
+        if pod is None:
+            continue
+        valid_pods.append(pod)
         created_pods[pod.metadata.name] = pod
         if head_pod_name is None and _is_head(pod):
             head_pod_name = pod.metadata.name
+    pods = valid_pods
+    # The running_pods may include Pending Pods, so we add them to the pods
+    # list to wait for scheduling and running
+    if running_pods:
+        pods = pods + list(running_pods.values())
     provision_timeout = provider_config['timeout']
@@ -1369,8 +1358,9 @@ def get_cluster_info(
             assert head_spec is not None, pod
             cpu_request = head_spec.containers[0].resources.requests['cpu']
-    assert cpu_request is not None, ('cpu_request should not be None, check '
-                                     'the Pod status')
+    if cpu_request is None:
+        raise RuntimeError(f'Pod {cluster_name_on_cloud}-head not found'
+                           ' or not Running, check the Pod status')
     ssh_user = 'sky'
     # Use pattern matching to extract SSH user, handling MOTD contamination.

sky/provision/kubernetes/utils.py CHANGED Viewed

@@ -238,6 +238,40 @@ def normalize_tpu_accelerator_name(accelerator: str) -> Tuple[str, int]:
     return accelerator, 1
+def _is_cloudflare_403_error(exception: Exception) -> bool:
+    """Check if an exception is a transient CloudFlare 403 error.
+    CloudFlare proxy 403 errors with CF-specific headers are transient and
+    should be retried, unlike real RBAC 403 errors.
+    Args:
+        exception: The exception to check
+    Returns:
+        True if this is a CloudFlare 403 error that should be retried
+    """
+    if not isinstance(exception, kubernetes.api_exception()):
+        return False
+    # Only check for 403 errors
+    if exception.status != 403:
+        return False
+    # Check for CloudFlare-specific headers
+    headers = exception.headers if hasattr(exception, 'headers') else {}
+    if not headers:
+        return False
+    # CloudFlare errors have CF-RAY header and/or Server: cloudflare
+    for k, v in headers.items():
+        if 'cf-ray' in k.lower():
+            return True
+        if 'server' in k.lower() and 'cloudflare' in str(v).lower():
+            return True
+    return False
 def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
                     retry_interval=DEFAULT_RETRY_INTERVAL_SECONDS,
                     resource_type: Optional[str] = None):
@@ -272,19 +306,25 @@ def _retry_on_error(max_retries=DEFAULT_MAX_RETRIES,
                         kubernetes.api_exception(),
                         kubernetes.config_exception()) as e:
                     last_exception = e
+                    # Check if this is a CloudFlare transient 403 error
+                    is_cloudflare_403 = _is_cloudflare_403_error(e)
                     # Don't retry on permanent errors like 401 (Unauthorized)
-                    # or 403 (Forbidden)
+                    # or 403 (Forbidden), unless it's a CloudFlare transient 403
                     if (isinstance(e, kubernetes.api_exception()) and
-                            e.status in (401, 403)):
+                            e.status in (401, 403) and not is_cloudflare_403):
                         # Raise KubeAPIUnreachableError exception so that the
                         # optimizer/provisioner can failover to other clouds.
                         raise exceptions.KubeAPIUnreachableError(
                             f'Kubernetes API error: {str(e)}') from e
                     if attempt < max_retries - 1:
                         sleep_time = backoff.current_backoff()
-                        logger.debug(f'Kubernetes API call {func.__name__} '
-                                     f'failed with {str(e)}. Retrying in '
-                                     f'{sleep_time:.1f}s...')
+                        error_type = 'CloudFlare 403' if is_cloudflare_403 else 'error'
+                        logger.debug(
+                            f'Kubernetes API call {func.__name__} '
+                            f'failed with {error_type} {str(e)}. Retrying in '
+                            f'{sleep_time:.1f}s...')
                         time.sleep(sleep_time)
                         continue
@@ -2738,7 +2778,8 @@ def merge_custom_metadata(
     config_utils.merge_k8s_configs(original_metadata, custom_metadata)
-def check_nvidia_runtime_class(context: Optional[str] = None) -> bool:
+@_retry_on_error(resource_type='runtimeclass')
+def check_nvidia_runtime_class(*, context: Optional[str] = None) -> bool:
     """Checks if the 'nvidia' RuntimeClass exists in the cluster"""
     # Fetch the list of available RuntimeClasses
     runtime_classes = kubernetes.node_api(context).list_runtime_class()

sky/serve/constants.py CHANGED Viewed

@@ -76,6 +76,9 @@ CONTROLLER_AUTOSTOP = {
 # A period of time to initialize your service. Any readiness probe failures
 # during this period will be ignored.
 DEFAULT_INITIAL_DELAY_SECONDS = 1200
+# For pool, we shrink the initial delay to 300s to make the pool more
+# responsive to the failure that setup command starts a long-running server.
+DEFAULT_INITIAL_DELAY_SECONDS_POOL = 300
 DEFAULT_MIN_REPLICAS = 1
 # Default port range start for controller and load balancer. Ports will be

sky/serve/server/server.py CHANGED Viewed

@@ -109,7 +109,7 @@ async def tail_logs(
     task = executor.execute_request_in_coroutine(request_task)
     # Cancel the coroutine after the request is done or client disconnects
     background_tasks.add_task(task.cancel)
-    return stream_utils.stream_response(
+    return stream_utils.stream_response_for_long_request(
         request_id=request_task.request_id,
         logs_path=request_task.log_path,
         background_tasks=background_tasks,

sky/serve/service_spec.py CHANGED Viewed

@@ -125,6 +125,12 @@ class SkyServiceSpec:
                 self.base_ondemand_fallback_replicas is not None and
                 self.base_ondemand_fallback_replicas > 0)
+    @staticmethod
+    def _get_initial_delay_seconds(pool: bool) -> int:
+        if pool:
+            return constants.DEFAULT_INITIAL_DELAY_SECONDS_POOL
+        return constants.DEFAULT_INITIAL_DELAY_SECONDS
     @staticmethod
     def from_yaml_config(config: Dict[str, Any]) -> 'SkyServiceSpec':
         common_utils.validate_schema(config, schemas.get_service_schema(),
@@ -153,7 +159,8 @@ class SkyServiceSpec:
                 'timeout_seconds', None)
             readiness_headers = readiness_section.get('headers', None)
         if initial_delay_seconds is None:
-            initial_delay_seconds = constants.DEFAULT_INITIAL_DELAY_SECONDS
+            initial_delay_seconds = SkyServiceSpec._get_initial_delay_seconds(
+                config.get('pool', False))
         service_config['initial_delay_seconds'] = initial_delay_seconds
         if readiness_timeout_seconds is None:
             readiness_timeout_seconds = (

sky/server/requests/executor.py CHANGED Viewed

@@ -349,32 +349,6 @@ def override_request_env_and_config(
         os.environ.update(original_env)
-def _get_current_output() -> Tuple[int, int]:
-    """Get the current stdout and stderr file descriptors."""
-    return os.dup(sys.stdout.fileno()), os.dup(sys.stderr.fileno())
-def _redirect_output(file: TextIO) -> None:
-    """Redirect stdout and stderr to the log file."""
-    # Get the file descriptor from the file object
-    fd = file.fileno()
-    # Copy this fd to stdout and stderr
-    os.dup2(fd, sys.stdout.fileno())
-    os.dup2(fd, sys.stderr.fileno())
-def _restore_output(original_stdout: Optional[int],
-                    original_stderr: Optional[int]) -> None:
-    """Restore stdout and stderr to their original file descriptors."""
-    if original_stdout is not None:
-        os.dup2(original_stdout, sys.stdout.fileno())
-        os.close(original_stdout)
-    if original_stderr is not None:
-        os.dup2(original_stderr, sys.stderr.fileno())
-        os.close(original_stderr)
 def _sigterm_handler(signum: int, frame: Optional['types.FrameType']) -> None:
     raise KeyboardInterrupt
@@ -402,6 +376,34 @@ def _request_execution_wrapper(request_id: str,
     logger.info(f'Running request {request_id} with pid {pid}')
     original_stdout = original_stderr = None
+    def _save_current_output() -> None:
+        """Save the current stdout and stderr file descriptors."""
+        nonlocal original_stdout, original_stderr
+        original_stdout = os.dup(sys.stdout.fileno())
+        original_stderr = os.dup(sys.stderr.fileno())
+    def _redirect_output(file: TextIO) -> None:
+        """Redirect stdout and stderr to the log file."""
+        # Get the file descriptor from the file object
+        fd = file.fileno()
+        # Copy this fd to stdout and stderr
+        os.dup2(fd, sys.stdout.fileno())
+        os.dup2(fd, sys.stderr.fileno())
+    def _restore_output() -> None:
+        """Restore stdout and stderr to their original file descriptors."""
+        nonlocal original_stdout, original_stderr
+        if original_stdout is not None:
+            os.dup2(original_stdout, sys.stdout.fileno())
+            os.close(original_stdout)
+            original_stdout = None
+        if original_stderr is not None:
+            os.dup2(original_stderr, sys.stderr.fileno())
+            os.close(original_stderr)
+            original_stderr = None
     try:
         # As soon as the request is updated with the executor PID, we can
         # receive SIGTERM from cancellation. So, we update the request inside
@@ -422,7 +424,7 @@ def _request_execution_wrapper(request_id: str,
         # Store copies of the original stdout and stderr file descriptors
         # We do this in two steps because we should make sure to restore the
         # original values even if we are cancelled or fail during the redirect.
-        original_stdout, original_stderr = _get_current_output()
+        _save_current_output()
         # Append to the log file instead of overwriting it since there might be
         # logs from previous retries.
@@ -464,15 +466,14 @@ def _request_execution_wrapper(request_id: str,
             # clear the pid of the request.
             request_task.pid = None
         # Yield control to the scheduler for uniform handling of retries.
-        _restore_output(original_stdout, original_stderr)
+        _restore_output()
         raise
     except (Exception, SystemExit) as e:  # pylint: disable=broad-except
         api_requests.set_request_failed(request_id, e)
         # Manually reset the original stdout and stderr file descriptors early
         # so that the "Request xxxx failed due to ..." log message will be
         # written to the original stdout and stderr file descriptors.
-        _restore_output(original_stdout, original_stderr)
-        original_stdout = original_stderr = None
+        _restore_output()
         logger.info(f'Request {request_id} failed due to '
                     f'{common_utils.format_exception(e)}')
         return
@@ -482,11 +483,10 @@ def _request_execution_wrapper(request_id: str,
         # Manually reset the original stdout and stderr file descriptors early
         # so that the "Request xxxx failed due to ..." log message will be
         # written to the original stdout and stderr file descriptors.
-        _restore_output(original_stdout, original_stderr)
-        original_stdout = original_stderr = None
+        _restore_output()
         logger.info(f'Request {request_id} finished')
     finally:
-        _restore_output(original_stdout, original_stderr)
+        _restore_output()
         try:
             # Capture the peak RSS before GC.
             peak_rss = max(proc.memory_info().rss, metrics_lib.peak_rss_bytes)
@@ -580,11 +580,11 @@ async def _execute_request_coroutine(request: api_requests.Request):
                                                   **request_body.to_kwargs())
     async def poll_task(request_id: str) -> bool:
-        request = await api_requests.get_request_async(request_id)
-        if request is None:
+        req_status = await api_requests.get_request_status_async(request_id)
+        if req_status is None:
             raise RuntimeError('Request not found')
-        if request.status == api_requests.RequestStatus.CANCELLED:
+        if req_status.status == api_requests.RequestStatus.CANCELLED:
             ctx.cancel()
             return True

sky/server/server.py CHANGED Viewed

@@ -1243,7 +1243,7 @@ async def logs(
     background_tasks.add_task(task.cancel)
     # TODO(zhwu): This makes viewing logs in browser impossible. We should adopt
     # the same approach as /stream.
-    return stream_utils.stream_response(
+    return stream_utils.stream_response_for_long_request(
         request_id=request.state.request_id,
         logs_path=request_task.log_path,
         background_tasks=background_tasks,
@@ -1539,6 +1539,7 @@ async def stream(
                 'X-Accel-Buffering': 'no'
             })
+    polling_interval = stream_utils.DEFAULT_POLL_INTERVAL
     # Original plain text streaming logic
     if request_id is not None:
         request_task = await requests_lib.get_request_async(request_id)
@@ -1553,6 +1554,8 @@ async def stream(
             raise fastapi.HTTPException(
                 status_code=404,
                 detail=f'Log of request {request_id!r} has been deleted')
+        if request_task.schedule_type == requests_lib.ScheduleType.LONG:
+            polling_interval = stream_utils.LONG_REQUEST_POLL_INTERVAL
     else:
         assert log_path is not None, (request_id, log_path)
         if log_path == constants.API_SERVER_LOGS:
@@ -1600,7 +1603,8 @@ async def stream(
                                           log_path_to_stream,
                                           plain_logs=format == 'plain',
                                           tail=tail,
-                                          follow=follow),
+                                          follow=follow,
+                                          polling_interval=polling_interval),
         media_type='text/plain',
         headers=headers,
     )

skypilot-nightly 1.0.0.dev20251008__py3-none-any.whl → 1.0.0.dev20251011__py3-none-any.whl

Potentially problematic release.

skypilot-nightly 1.0.0.dev20251008py3-none-any.whl → 1.0.0.dev20251011py3-none-any.whl