skypilot-nightly 1.0.0.dev20250808__py3-none-any.whl → 1.0.0.dev20250814__py3-none-any.whl

sky/templates/jobs-controller.yaml.j2

@@ -15,6 +15,12 @@ file_mounts:
   {{controller_file_mount_path}}: {{local_file_mount_path}}
   {%- endfor %}
 
+# NOTE(dev): This needs to be a subset of sky/templates/sky-serve-controller.yaml.j2.
+# It is because we use the --fast flag to submit jobs and no --fast flag to launch pools.
+# So when we launch a new pool, it will install the required dependencies.
+# TODO(tian): Add --fast to launch pools as well, and figure out the dependency installation.
+# Maybe in the --fast implementation, we can store the hash of setup commands that used to be
+# run and don't skip setup phase if the hash is different.
 setup: |
   {{ sky_activate_python_env }}
   # Disable the pip version check to avoid the warning message, which makes the

sky/users/permission.py

@@ -44,7 +44,7 @@ class PermissionService:
             if _enforcer_instance is None:
                 _enforcer_instance = self
                 engine = global_user_state.initialize_and_get_db()
-                db_utils.add_tables_to_db_sqlalchemy(
+                db_utils.add_all_tables_to_db_sqlalchemy(
                     sqlalchemy_adapter.Base.metadata, engine)
                 adapter = sqlalchemy_adapter.Adapter(engine)
                 model_path = os.path.join(os.path.dirname(__file__),

sky/utils/cli_utils/status_utils.py

@@ -81,6 +81,7 @@ def show_status_table(cluster_records: List[_ClusterRecord],
                          _get_command,
                          truncate=not show_all,
                          show_by_default=False),
+            StatusColumn('LAST_EVENT', _get_last_event, show_by_default=False),
         ]
 
     columns = []
@@ -314,6 +315,14 @@ def _get_head_ip(cluster_record: _ClusterRecord, truncate: bool = True) -> str:
     return handle.head_ip
 
 
+def _get_last_event(cluster_record: _ClusterRecord,
+                    truncate: bool = True) -> str:
+    del truncate
+    if cluster_record.get('last_event', None) is None:
+        return 'No recorded events.'
+    return cluster_record['last_event']
+
+
 def _is_pending_autostop(cluster_record: _ClusterRecord) -> bool:
     # autostop < 0 means nothing scheduled.
     return cluster_record['autostop'] >= 0 and _get_status(

sky/utils/command_runner.py

@@ -674,7 +674,7 @@ class SSHCommandRunner(CommandRunner):
             ssh += ['-tt']
         if port_forward is not None:
             for local, remote in port_forward:
-                logger.info(
+                logger.debug(
                     f'Forwarding local port {local} to remote port {remote}.')
                 ssh += ['-NL', f'{local}:localhost:{remote}']
         if self._docker_ssh_proxy_command is not None:

sky/utils/config_utils.py

@@ -8,6 +8,26 @@ logger = sky_logging.init_logger(__name__)
 
 _REGION_CONFIG_CLOUDS = ['nebius', 'oci']
 
+# Kubernetes API use list to represent dictionary fields with patch strategy
+# merge and each item is indexed by the patch merge key. The following map
+# maps the field name to the patch merge key.
+# pylint: disable=line-too-long
+# Ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podspec-v1-core
+# NOTE: field containers and imagePullSecrets are not included deliberately for
+# backward compatibility (we only support one container per pod now).
+_PATCH_MERGE_KEYS = {
+    'initContainers': 'name',
+    'ephemeralContainers': 'name',
+    'volumes': 'name',
+    'volumeMounts': 'name',
+    'resourceClaims': 'name',
+    'env': 'name',
+    'hostAliases': 'ip',
+    'topologySpreadConstraints': 'topologyKey',
+    'ports': 'containerPort',
+    'volumeDevices': 'devicePath',
+}
+
 
 class Config(Dict[str, Any]):
     """SkyPilot config that supports setting/getting values with nested keys."""
@@ -211,19 +231,23 @@ def merge_k8s_configs(
                 merge_k8s_configs(base_config[key][0], value[0],
                                   next_allowed_override_keys,
                                   next_disallowed_override_keys)
-            elif key in ['volumes', 'volumeMounts', 'initContainers']:
-                # If the key is 'volumes', 'volumeMounts', or 'initContainers',
-                # we search for item with the same name and merge it.
+            # For list fields with patch strategy "merge", we merge the list
+            # by the patch merge key.
+            elif key in _PATCH_MERGE_KEYS:
+                patch_merge_key = _PATCH_MERGE_KEYS[key]
                 for override_item in value:
-                    override_item_name = override_item.get('name')
+                    override_item_name = override_item.get(patch_merge_key)
                     if override_item_name is not None:
                         existing_base_item = next(
                             (v for v in base_config[key]
-                             if v.get('name') == override_item_name), None)
+                             if v.get(patch_merge_key) == override_item_name),
+                            None)
                         if existing_base_item is not None:
                             merge_k8s_configs(existing_base_item, override_item)
                         else:
                             base_config[key].append(override_item)
+                    else:
+                        base_config[key].append(override_item)
             else:
                 base_config[key].extend(value)
         else:

sky/utils/controller_utils.py

@@ -23,11 +23,14 @@ from sky.clouds import gcp
 from sky.data import data_utils
 from sky.data import storage as storage_lib
 from sky.jobs import constants as managed_job_constants
+from sky.jobs import state as managed_job_state
 from sky.provision.kubernetes import constants as kubernetes_constants
 from sky.serve import constants as serve_constants
+from sky.serve import serve_state
 from sky.setup_files import dependencies
 from sky.skylet import constants
 from sky.skylet import log_lib
+from sky.utils import annotations
 from sky.utils import common
 from sky.utils import common_utils
 from sky.utils import config_utils
@@ -37,8 +40,13 @@ from sky.utils import rich_utils
 from sky.utils import ux_utils
 
 if typing.TYPE_CHECKING:
+    import psutil
+
     from sky import task as task_lib
     from sky.backends import cloud_vm_ray_backend
+else:
+    from sky.adaptors import common as adaptors_common
+    psutil = adaptors_common.LazyImport('psutil')
 
 logger = sky_logging.init_logger(__name__)
 
@@ -1161,3 +1169,68 @@ def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task',
     task.update_storage_mounts(updated_mount_storages)
     if msg:
         logger.info(ux_utils.finishing_message('Uploaded local files/folders.'))
+
+
+# ======================= Resources Management Functions =======================
+
+# Based on testing, assume a running job process uses 350MB memory. We use the
+# same estimation for service controller process.
+JOB_MEMORY_MB = 350
+# Monitoring process for service is 1GB. This is based on an old estimation but
+# we keep it here for now.
+# TODO(tian): Remeasure this.
+SERVE_MONITORING_MEMORY_MB = 1024
+# The ratio of service controller process to job process. We will treat each
+# service as SERVE_PROC_RATIO job processes.
+SERVE_PROC_RATIO = SERVE_MONITORING_MEMORY_MB / JOB_MEMORY_MB
+# Past 2000 simultaneous jobs, we become unstable.
+# See https://github.com/skypilot-org/skypilot/issues/4649.
+MAX_JOB_LIMIT = 2000
+# Number of ongoing launches launches allowed per CPU, for managed jobs.
+JOB_LAUNCHES_PER_CPU = 4
+# Number of ongoing launches launches allowed per CPU, for services. This is
+# also based on an old estimation, but SKyServe indeed spawn a new process
+# for each launch operation, so it should be slightly more resources demanding
+# than managed jobs.
+SERVE_LAUNCHES_PER_CPU = 2
+# The ratio of service launch to job launch. This is inverted as the parallelism
+# is determined by 1 / LAUNCHES_PER_CPU.
+SERVE_LAUNCH_RATIO = JOB_LAUNCHES_PER_CPU / SERVE_LAUNCHES_PER_CPU
+
+# The _RESOURCES_LOCK should be held whenever we are checking the parallelism
+# control or updating the schedule_state of any job or service. Any code that
+# takes this lock must conclude by calling maybe_schedule_next_jobs.
+_RESOURCES_LOCK = '~/.sky/locks/controller_resources.lock'
+
+
+@annotations.lru_cache(scope='global', maxsize=1)
+def get_resources_lock_path() -> str:
+    path = os.path.expanduser(_RESOURCES_LOCK)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    return path
+
+
+@annotations.lru_cache(scope='request')
+def _get_job_parallelism() -> int:
+    job_memory = JOB_MEMORY_MB * 1024 * 1024
+    job_limit = min(psutil.virtual_memory().total // job_memory, MAX_JOB_LIMIT)
+    return max(job_limit, 1)
+
+
+@annotations.lru_cache(scope='request')
+def _get_launch_parallelism() -> int:
+    cpus = os.cpu_count()
+    return cpus * JOB_LAUNCHES_PER_CPU if cpus is not None else 1
+
+
+def can_provision() -> bool:
+    num_provision = (
+        serve_state.total_number_provisioning_replicas() * SERVE_LAUNCH_RATIO +
+        managed_job_state.get_num_launching_jobs())
+    return num_provision < _get_launch_parallelism()
+
+
+def can_start_new_process() -> bool:
+    num_procs = (serve_state.get_num_services() * SERVE_PROC_RATIO +
+                 managed_job_state.get_num_alive_jobs())
+    return num_procs < _get_job_parallelism()

sky/utils/db/db_utils.py

@@ -32,6 +32,23 @@ if typing.TYPE_CHECKING:
 _DB_TIMEOUT_S = 60
 
 
+class UniqueConstraintViolationError(Exception):
+    """Exception raised for unique constraint violation.
+    Attributes:
+        value -- the input value that caused the error
+        message -- explanation of the error
+    """
+
+    def __init__(self, value, message='Unique constraint violation'):
+        self.value = value
+        self.message = message
+        super().__init__(self.message)
+
+    def __str__(self):
+        return (f'UniqueConstraintViolationError: {self.message} '
+                f'(Value: {self.value})')
+
+
 class SQLAlchemyDialect(enum.Enum):
     SQLITE = 'sqlite'
     POSTGRESQL = 'postgresql'
@@ -87,7 +104,7 @@ def add_column_to_table(
     conn.commit()
 
 
-def add_tables_to_db_sqlalchemy(
+def add_all_tables_to_db_sqlalchemy(
     metadata: sqlalchemy.MetaData,
     engine: sqlalchemy.Engine,
 ):
@@ -103,6 +120,27 @@ def add_tables_to_db_sqlalchemy(
                 raise
 
 
+def add_table_to_db_sqlalchemy(
+    metadata: sqlalchemy.MetaData,
+    engine: sqlalchemy.Engine,
+    table_name: str,
+):
+    """Add a specific table to the database."""
+    try:
+        table = metadata.tables[table_name]
+    except KeyError as e:
+        raise e
+
+    try:
+        table.create(bind=engine, checkfirst=True)
+    except (sqlalchemy_exc.OperationalError,
+            sqlalchemy_exc.ProgrammingError) as e:
+        if 'already exists' in str(e):
+            pass
+        else:
+            raise
+
+
 def add_column_to_table_sqlalchemy(
     session: 'Session',
     table_name: str,

sky/utils/db/migration_utils.py

@@ -19,7 +19,7 @@ logger = sky_logging.init_logger(__name__)
 DB_INIT_LOCK_TIMEOUT_SECONDS = 10
 
 GLOBAL_USER_STATE_DB_NAME = 'state_db'
-GLOBAL_USER_STATE_VERSION = '004'
+GLOBAL_USER_STATE_VERSION = '005'
 GLOBAL_USER_STATE_LOCK_PATH = '~/.sky/locks/.state_db.lock'
 
 SPOT_JOBS_DB_NAME = 'spot_jobs_db'

sky/utils/schemas.py

@@ -1205,6 +1205,9 @@ def get_config_schema():
                 'disk_encrypted': {
                     'type': 'boolean',
                 },
+                'ssh_user': {
+                    'type': 'string',
+                },
                 'security_group_name':
                     (_PROPERTY_NAME_OR_CLUSTER_NAME_TO_PROPERTY),
                 'vpc_name': {

sky/volumes/server/core.py

@@ -7,12 +7,12 @@ import uuid
 
 import filelock
 
-import sky
 from sky import global_user_state
 from sky import models
 from sky import provision
 from sky import sky_logging
 from sky.utils import common_utils
+from sky.utils import registry
 from sky.utils import rich_utils
 from sky.utils import status_lib
 from sky.utils import ux_utils
@@ -180,7 +180,7 @@ def volume_apply(name: str, volume_type: str, cloud: str, region: Optional[str],
     with rich_utils.safe_status(ux_utils.spinner_message('Creating volume')):
         # Reuse the method for cluster name on cloud to
         # generate the storage name on cloud.
-        cloud_obj = sky.CLOUD_REGISTRY.from_str(cloud)
+        cloud_obj = registry.CLOUD_REGISTRY.from_str(cloud)
         assert cloud_obj is not None
         name_uuid = str(uuid.uuid4())[:6]
         name_on_cloud = common_utils.make_cluster_name_on_cloud(

sky/volumes/server/server.py

@@ -2,12 +2,12 @@
 
 import fastapi
 
-import sky
 from sky import clouds
 from sky import sky_logging
 from sky.server.requests import executor
 from sky.server.requests import payloads
 from sky.server.requests import requests as requests_lib
+from sky.utils import registry
 from sky.utils import volume
 from sky.volumes.server import core
 
@@ -55,7 +55,7 @@ async def volume_apply(request: fastapi.Request,
     if volume_type not in supported_volume_types:
         raise fastapi.HTTPException(
             status_code=400, detail=f'Invalid volume type: {volume_type}')
-    cloud = sky.CLOUD_REGISTRY.from_str(volume_cloud)
+    cloud = registry.CLOUD_REGISTRY.from_str(volume_cloud)
     if cloud is None:
         raise fastapi.HTTPException(status_code=400,
                                     detail=f'Invalid cloud: {volume_cloud}')

{skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: skypilot-nightly
-Version: 1.0.0.dev20250808
+Version: 1.0.0.dev20250814
 Summary: SkyPilot: Run AI on Any Infra — Unified, Faster, Cheaper.
 Author: SkyPilot Team
 License: Apache 2.0
@@ -104,9 +104,8 @@ Provides-Extra: ssh
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "ssh"
 Requires-Dist: websockets; extra == "ssh"
 Provides-Extra: remote
-Requires-Dist: grpcio!=1.48.0,>=1.32.0; python_version < "3.10" and extra == "remote"
-Requires-Dist: grpcio!=1.48.0,>=1.42.0; python_version >= "3.10" and extra == "remote"
-Requires-Dist: protobuf!=3.19.5,>=3.15.3; extra == "remote"
+Requires-Dist: grpcio>=1.63.0; extra == "remote"
+Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "remote"
 Provides-Extra: runpod
 Requires-Dist: runpod>=1.6.1; extra == "runpod"
 Provides-Extra: fluidstack
@@ -169,9 +168,8 @@ Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
 Requires-Dist: websockets; extra == "all"
 Requires-Dist: kubernetes!=32.0.0,>=20.0.0; extra == "all"
 Requires-Dist: websockets; extra == "all"
-Requires-Dist: grpcio!=1.48.0,>=1.32.0; python_version < "3.10" and extra == "all"
-Requires-Dist: grpcio!=1.48.0,>=1.42.0; python_version >= "3.10" and extra == "all"
-Requires-Dist: protobuf!=3.19.5,>=3.15.3; extra == "all"
+Requires-Dist: grpcio>=1.63.0; extra == "all"
+Requires-Dist: protobuf<7.0.0,>=5.26.1; extra == "all"
 Requires-Dist: runpod>=1.6.1; extra == "all"
 Requires-Dist: cudo-compute>=0.1.10; extra == "all"
 Requires-Dist: pydo>=0.3.0; extra == "all"