skypilot-nightly 1.0.0.dev20250623__py3-none-any.whl → 1.0.0.dev20250624__py3-none-any.whl

sky/utils/command_runner.py

@@ -4,6 +4,7 @@ import hashlib
 import os
 import pathlib
 import shlex
+import sys
 import time
 from typing import Any, Callable, Iterable, List, Optional, Tuple, Type, Union
 
@@ -231,9 +232,9 @@ class CommandRunner:
             self,
             source: str,
             target: str,
-            node_destination: str,
+            node_destination: Optional[str],
             up: bool,
-            rsh_option: str,
+            rsh_option: Optional[str],
             # Advanced options.
             log_path: str = os.devnull,
             stream_logs: bool = True,
@@ -283,28 +284,43 @@ class CommandRunner:
                         RSYNC_EXCLUDE_OPTION.format(
                             shlex.quote(str(resolved_source / GIT_EXCLUDE))))
 
-        rsync_command.append(f'-e {shlex.quote(rsh_option)}')
+        if rsh_option is not None:
+            rsync_command.append(f'-e {shlex.quote(rsh_option)}')
+        maybe_dest_prefix = ('' if node_destination is None else
+                             f'{node_destination}:')
 
         if up:
             resolved_target = target
-            if target.startswith('~'):
-                remote_home_dir = _get_remote_home_dir_with_retry()
-                resolved_target = target.replace('~', remote_home_dir)
+            if node_destination is None:
+                # Is a local rsync. Directly resolve the target.
+                resolved_target = str(
+                    pathlib.Path(target).expanduser().resolve())
+            else:
+                if target.startswith('~'):
+                    remote_home_dir = _get_remote_home_dir_with_retry()
+                    resolved_target = target.replace('~', remote_home_dir)
             full_source_str = str(resolved_source)
             if resolved_source.is_dir():
                 full_source_str = os.path.join(full_source_str, '')
             rsync_command.extend([
                 f'{full_source_str!r}',
-                f'{node_destination}:{resolved_target!r}',
+                f'{maybe_dest_prefix}{resolved_target!r}',
             ])
         else:
             resolved_source = source
-            if source.startswith('~'):
-                remote_home_dir = _get_remote_home_dir_with_retry()
-                resolved_source = source.replace('~', remote_home_dir)
+            if node_destination is None:
+                resolved_target = str(
+                    pathlib.Path(target).expanduser().resolve())
+                resolved_source = str(
+                    pathlib.Path(source).expanduser().resolve())
+            else:
+                resolved_target = os.path.expanduser(target)
+                if source.startswith('~'):
+                    remote_home_dir = _get_remote_home_dir_with_retry()
+                    resolved_source = source.replace('~', remote_home_dir)
             rsync_command.extend([
-                f'{node_destination}:{resolved_source!r}',
-                f'{os.path.expanduser(target)!r}',
+                f'{maybe_dest_prefix}{resolved_source!r}',
+                f'{resolved_target!r}',
             ])
         command = ' '.join(rsync_command)
         logger.debug(f'Running rsync command: {command}')
@@ -964,3 +980,93 @@ class KubernetesCommandRunner(CommandRunner):
             # /~/xx, so we need to replace ~ with the remote home directory. We
             # only need to do this when ~ is at the beginning of the path.
             get_remote_home_dir=get_remote_home_dir)
+
+
+class LocalProcessCommandRunner(CommandRunner):
+    """Runner for local process commands."""
+
+    def __init__(self):
+        super().__init__('local')
+
+    @timeline.event
+    @context_utils.cancellation_guard
+    def run(
+            self,
+            cmd: Union[str, List[str]],
+            *,
+            require_outputs: bool = False,
+            port_forward: Optional[List[Tuple[int, int]]] = None,
+            # Advanced options.
+            log_path: str = os.devnull,
+            # If False, do not redirect stdout/stderr to optimize performance.
+            process_stream: bool = True,
+            stream_logs: bool = True,
+            ssh_mode: SshMode = SshMode.NON_INTERACTIVE,
+            separate_stderr: bool = False,
+            connect_timeout: Optional[int] = None,
+            source_bashrc: bool = False,
+            skip_num_lines: int = 0,
+            **kwargs) -> Union[int, Tuple[int, str, str]]:
+        """Use subprocess to run the command."""
+        del port_forward, ssh_mode, connect_timeout  # Unused.
+
+        command_str = self._get_command_to_run(cmd,
+                                               process_stream,
+                                               separate_stderr,
+                                               skip_num_lines=skip_num_lines,
+                                               source_bashrc=source_bashrc)
+
+        log_dir = os.path.expanduser(os.path.dirname(log_path))
+        os.makedirs(log_dir, exist_ok=True)
+
+        executable = None
+        command = [command_str]
+        if not process_stream:
+            if stream_logs:
+                command += [
+                    f'| tee {log_path}',
+                    # This also requires the executor to be '/bin/bash' instead
+                    # of the default '/bin/sh'.
+                    '; exit ${PIPESTATUS[0]}'
+                ]
+            else:
+                command += [f'> {log_path}']
+            executable = '/bin/bash'
+        command_str = ' '.join(command)
+        # For local process, the API server might not have this python path
+        # setup. But this command runner should only be triggered from the API
+        # server (in controller consolidation mode), so we can safely replace
+        # the python path with the executable of the API server.
+        command_str = command_str.replace(constants.SKY_PYTHON_CMD,
+                                          sys.executable)
+        logger.debug(f'Running command locally: {command_str}')
+        return log_lib.run_with_log(command_str,
+                                    log_path,
+                                    require_outputs=require_outputs,
+                                    stream_logs=stream_logs,
+                                    process_stream=process_stream,
+                                    shell=True,
+                                    executable=executable,
+                                    **kwargs)
+
+    @timeline.event
+    def rsync(
+        self,
+        source: str,
+        target: str,
+        *,
+        up: bool,
+        # Advanced options.
+        log_path: str = os.devnull,
+        stream_logs: bool = True,
+        max_retry: int = 1,
+    ) -> None:
+        """Use rsync to sync the source to the target."""
+        self._rsync(source,
+                    target,
+                    node_destination=None,
+                    up=up,
+                    rsh_option=None,
+                    log_path=log_path,
+                    stream_logs=stream_logs,
+                    max_retry=max_retry)

sky/utils/command_runner.pyi

@@ -271,3 +271,60 @@ class KubernetesCommandRunner(CommandRunner):
               stream_logs: bool = ...,
               max_retry: int = ...) -> None:
         ...
+
+
+class LocalProcessCommandRunner(CommandRunner):
+
+    def __init__(self) -> None:
+        ...
+
+    @typing.overload
+    def run(self,
+            cmd: Union[str, List[str]],
+            *,
+            port_forward: Optional[List[int]] = ...,
+            require_outputs: Literal[False] = ...,
+            log_path: str = ...,
+            process_stream: bool = ...,
+            stream_logs: bool = ...,
+            ssh_mode: SshMode = ...,
+            separate_stderr: bool = ...,
+            connect_timeout: Optional[int] = ...,
+            source_bashrc: bool = ...,
+            skip_lines: int = ...,
+            **kwargs) -> int:
+        ...
+
+    @typing.overload
+    def run(self,
+            cmd: Union[str, List[str]],
+            *,
+            port_forward: Optional[List[int]] = ...,
+            require_outputs: Literal[True],
+            log_path: str = ...,
+            process_stream: bool = ...,
+            stream_logs: bool = ...,
+            ssh_mode: SshMode = ...,
+            separate_stderr: bool = ...,
+            connect_timeout: Optional[int] = ...,
+            source_bashrc: bool = ...,
+            skip_lines: int = ...,
+            **kwargs) -> Tuple[int, str, str]:
+        ...
+
+    @typing.overload
+    def run(self,
+            cmd: Union[str, List[str]],
+            *,
+            port_forward: Optional[List[int]] = ...,
+            require_outputs: bool = ...,
+            log_path: str = ...,
+            process_stream: bool = ...,
+            stream_logs: bool = ...,
+            ssh_mode: SshMode = ...,
+            separate_stderr: bool = ...,
+            connect_timeout: Optional[int] = ...,
+            source_bashrc: bool = ...,
+            skip_lines: int = ...,
+            **kwargs) -> Union[Tuple[int, str, str], int]:
+        ...

sky/utils/common_utils.py

@@ -26,6 +26,7 @@ from sky.adaptors import common as adaptors_common
 from sky.skylet import constants
 from sky.usage import constants as usage_constants
 from sky.utils import annotations
+from sky.utils import common_utils
 from sky.utils import ux_utils
 from sky.utils import validator
 
@@ -298,6 +299,13 @@ def get_current_user() -> 'models.User':
     return models.User.get_current_user()
 
 
+def get_current_user_name() -> str:
+    """Returns the current user name."""
+    name = common_utils.get_current_user().name
+    assert name is not None
+    return name
+
+
 def set_current_user(user: 'models.User'):
     """Sets the current user."""
     global _current_user
@@ -754,7 +762,7 @@ def get_cleaned_username(username: str = '') -> str:
     Returns:
       A cleaned username.
     """
-    username = username or getpass.getuser()
+    username = username or common_utils.get_current_user_name()
     username = username.lower()
     username = re.sub(r'[^a-z0-9-_]', '', username)
     username = re.sub(r'^[0-9-]+', '', username)

sky/utils/controller_utils.py

@@ -2,7 +2,6 @@
 import copy
 import dataclasses
 import enum
-import getpass
 import os
 import tempfile
 import typing
@@ -498,7 +497,7 @@ def shared_controller_vars_to_fill(
     env_vars.update({
         # Should not use $USER here, as that env var can be empty when
         # running in a container.
-        constants.USER_ENV_VAR: getpass.getuser(),
+        constants.USER_ENV_VAR: common_utils.get_current_user_name(),
         constants.USER_ID_ENV_VAR: common_utils.get_user_hash(),
         # Skip cloud identity check to avoid the overhead.
         env_options.Options.SKIP_CLOUD_IDENTITY_CHECK.env_key: '1',

sky/utils/schemas.py

@@ -672,18 +672,6 @@ def get_task_schema():
             'service': {
                 'type': 'object',
             },
-            'job': {
-                'type': 'object',
-                'required': [],
-                'additionalProperties': False,
-                'properties': {
-                    'priority': {
-                        'type': 'integer',
-                        'minimum': 0,
-                        'maximum': 1000,
-                    },
-                },
-            },
             'setup': {
                 'type': 'string',
             },
@@ -899,30 +887,41 @@ def get_config_schema():
         if k != '$schema'
     }
     resources_schema['properties'].pop('ports')
-    controller_resources_schema = {
-        'type': 'object',
-        'required': [],
-        'additionalProperties': False,
-        'properties': {
-            'controller': {
-                'type': 'object',
-                'required': [],
-                'additionalProperties': False,
-                'properties': {
-                    'resources': resources_schema,
-                    'high_availability': {
-                        'type': 'boolean',
-                    },
-                    'autostop': _AUTOSTOP_SCHEMA,
-                }
+
+    def _get_controller_schema(add_consolidation_mode: bool = False):
+        controller_properties = {
+            'resources': resources_schema,
+            'high_availability': {
+                'type': 'boolean',
+                'default': False,
             },
-            'bucket': {
-                'type': 'string',
-                'pattern': '^(https|s3|gs|r2|cos)://.+',
-                'required': [],
+            'autostop': _AUTOSTOP_SCHEMA,
+        }
+        if add_consolidation_mode:
+            controller_properties['consolidation_mode'] = {
+                'type': 'boolean',
+                'default': False,
+            }
+
+        return {
+            'type': 'object',
+            'required': [],
+            'additionalProperties': False,
+            'properties': {
+                'controller': {
+                    'type': 'object',
+                    'required': [],
+                    'additionalProperties': False,
+                    'properties': controller_properties,
+                },
+                'bucket': {
+                    'type': 'string',
+                    'pattern': '^(https|s3|gs|r2|cos)://.+',
+                    'required': [],
+                }
             }
         }
-    }
+
     cloud_configs = {
         'aws': {
             'type': 'object',
@@ -1440,8 +1439,8 @@ def get_config_schema():
             'db': {
                 'type': 'string',
             },
-            'jobs': controller_resources_schema,
-            'serve': controller_resources_schema,
+            'jobs': _get_controller_schema(add_consolidation_mode=True),
+            'serve': _get_controller_schema(add_consolidation_mode=False),
             'allowed_clouds': allowed_clouds,
             'admin_policy': admin_policy_schema,
             'docker': docker_configs,

{skypilot_nightly-1.0.0.dev20250623.dist-info → skypilot_nightly-1.0.0.dev20250624.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250623
+Version: 1.0.0.dev20250624
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0