skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of skypilot-nightly might be problematic. Click here for more details.
- sky/__init__.py +22 -6
- sky/adaptors/aws.py +25 -7
- sky/adaptors/common.py +24 -1
- sky/adaptors/coreweave.py +278 -0
- sky/adaptors/do.py +8 -2
- sky/adaptors/hyperbolic.py +8 -0
- sky/adaptors/kubernetes.py +149 -18
- sky/adaptors/nebius.py +170 -17
- sky/adaptors/primeintellect.py +1 -0
- sky/adaptors/runpod.py +68 -0
- sky/adaptors/seeweb.py +167 -0
- sky/adaptors/shadeform.py +89 -0
- sky/admin_policy.py +187 -4
- sky/authentication.py +179 -225
- sky/backends/__init__.py +4 -2
- sky/backends/backend.py +22 -9
- sky/backends/backend_utils.py +1299 -380
- sky/backends/cloud_vm_ray_backend.py +1715 -518
- sky/backends/docker_utils.py +1 -1
- sky/backends/local_docker_backend.py +11 -6
- sky/backends/wheel_utils.py +37 -9
- sky/{clouds/service_catalog → catalog}/__init__.py +21 -19
- sky/{clouds/service_catalog → catalog}/aws_catalog.py +27 -8
- sky/{clouds/service_catalog → catalog}/azure_catalog.py +10 -7
- sky/{clouds/service_catalog → catalog}/common.py +89 -48
- sky/{clouds/service_catalog → catalog}/cudo_catalog.py +8 -5
- sky/{clouds/service_catalog → catalog}/data_fetchers/analyze.py +1 -1
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_aws.py +30 -40
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_cudo.py +38 -38
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_gcp.py +42 -15
- sky/catalog/data_fetchers/fetch_hyperbolic.py +136 -0
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_lambda_cloud.py +1 -0
- sky/catalog/data_fetchers/fetch_nebius.py +335 -0
- sky/catalog/data_fetchers/fetch_runpod.py +698 -0
- sky/catalog/data_fetchers/fetch_seeweb.py +329 -0
- sky/catalog/data_fetchers/fetch_shadeform.py +142 -0
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_vast.py +1 -1
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_vsphere.py +1 -1
- sky/{clouds/service_catalog → catalog}/do_catalog.py +5 -2
- sky/{clouds/service_catalog → catalog}/fluidstack_catalog.py +6 -3
- sky/{clouds/service_catalog → catalog}/gcp_catalog.py +41 -15
- sky/catalog/hyperbolic_catalog.py +136 -0
- sky/{clouds/service_catalog → catalog}/ibm_catalog.py +9 -6
- sky/{clouds/service_catalog → catalog}/kubernetes_catalog.py +36 -24
- sky/{clouds/service_catalog → catalog}/lambda_catalog.py +9 -6
- sky/{clouds/service_catalog → catalog}/nebius_catalog.py +9 -7
- sky/{clouds/service_catalog → catalog}/oci_catalog.py +9 -6
- sky/{clouds/service_catalog → catalog}/paperspace_catalog.py +5 -2
- sky/catalog/primeintellect_catalog.py +95 -0
- sky/{clouds/service_catalog → catalog}/runpod_catalog.py +11 -4
- sky/{clouds/service_catalog → catalog}/scp_catalog.py +9 -6
- sky/catalog/seeweb_catalog.py +184 -0
- sky/catalog/shadeform_catalog.py +165 -0
- sky/catalog/ssh_catalog.py +167 -0
- sky/{clouds/service_catalog → catalog}/vast_catalog.py +6 -3
- sky/{clouds/service_catalog → catalog}/vsphere_catalog.py +5 -2
- sky/check.py +491 -203
- sky/cli.py +5 -6005
- sky/client/{cli.py → cli/command.py} +2477 -1885
- sky/client/cli/deprecation_utils.py +99 -0
- sky/client/cli/flags.py +359 -0
- sky/client/cli/table_utils.py +320 -0
- sky/client/common.py +70 -32
- sky/client/oauth.py +82 -0
- sky/client/sdk.py +1203 -297
- sky/client/sdk_async.py +833 -0
- sky/client/service_account_auth.py +47 -0
- sky/cloud_stores.py +73 -0
- sky/clouds/__init__.py +13 -0
- sky/clouds/aws.py +358 -93
- sky/clouds/azure.py +105 -83
- sky/clouds/cloud.py +127 -36
- sky/clouds/cudo.py +68 -50
- sky/clouds/do.py +66 -48
- sky/clouds/fluidstack.py +63 -44
- sky/clouds/gcp.py +339 -110
- sky/clouds/hyperbolic.py +293 -0
- sky/clouds/ibm.py +70 -49
- sky/clouds/kubernetes.py +563 -162
- sky/clouds/lambda_cloud.py +74 -54
- sky/clouds/nebius.py +206 -80
- sky/clouds/oci.py +88 -66
- sky/clouds/paperspace.py +61 -44
- sky/clouds/primeintellect.py +317 -0
- sky/clouds/runpod.py +164 -74
- sky/clouds/scp.py +89 -83
- sky/clouds/seeweb.py +466 -0
- sky/clouds/shadeform.py +400 -0
- sky/clouds/ssh.py +263 -0
- sky/clouds/utils/aws_utils.py +10 -4
- sky/clouds/utils/gcp_utils.py +87 -11
- sky/clouds/utils/oci_utils.py +38 -14
- sky/clouds/utils/scp_utils.py +177 -124
- sky/clouds/vast.py +99 -77
- sky/clouds/vsphere.py +51 -40
- sky/core.py +349 -139
- sky/dag.py +15 -0
- sky/dashboard/out/404.html +1 -1
- sky/dashboard/out/_next/static/chunks/1141-e6aa9ab418717c59.js +11 -0
- sky/dashboard/out/_next/static/chunks/1272-1ef0bf0237faccdb.js +1 -0
- sky/dashboard/out/_next/static/chunks/1871-74503c8e80fd253b.js +6 -0
- sky/dashboard/out/_next/static/chunks/2260-7703229c33c5ebd5.js +1 -0
- sky/dashboard/out/_next/static/chunks/2350.fab69e61bac57b23.js +1 -0
- sky/dashboard/out/_next/static/chunks/2369.fc20f0c2c8ed9fe7.js +15 -0
- sky/dashboard/out/_next/static/chunks/2755.fff53c4a3fcae910.js +26 -0
- sky/dashboard/out/_next/static/chunks/3294.72362fa129305b19.js +1 -0
- sky/dashboard/out/_next/static/chunks/3785.ad6adaa2a0fa9768.js +1 -0
- sky/dashboard/out/_next/static/chunks/3850-ff4a9a69d978632b.js +1 -0
- sky/dashboard/out/_next/static/chunks/3937.210053269f121201.js +1 -0
- sky/dashboard/out/_next/static/chunks/4725.a830b5c9e7867c92.js +1 -0
- sky/dashboard/out/_next/static/chunks/4937.a2baa2df5572a276.js +15 -0
- sky/dashboard/out/_next/static/chunks/5739-d67458fcb1386c92.js +8 -0
- sky/dashboard/out/_next/static/chunks/6130-2be46d70a38f1e82.js +1 -0
- sky/dashboard/out/_next/static/chunks/616-3d59f75e2ccf9321.js +39 -0
- sky/dashboard/out/_next/static/chunks/6212-7bd06f60ba693125.js +13 -0
- sky/dashboard/out/_next/static/chunks/6601-06114c982db410b6.js +1 -0
- sky/dashboard/out/_next/static/chunks/6856-ef8ba11f96d8c4a3.js +1 -0
- sky/dashboard/out/_next/static/chunks/6989-01359c57e018caa4.js +1 -0
- sky/dashboard/out/_next/static/chunks/6990-32b6e2d3822301fa.js +1 -0
- sky/dashboard/out/_next/static/chunks/7359-c8d04e06886000b3.js +30 -0
- sky/dashboard/out/_next/static/chunks/7411-b15471acd2cba716.js +41 -0
- sky/dashboard/out/_next/static/chunks/7615-3301e838e5f25772.js +1 -0
- sky/dashboard/out/_next/static/chunks/8640.5b9475a2d18c5416.js +16 -0
- sky/dashboard/out/_next/static/chunks/8969-1e4613c651bf4051.js +1 -0
- sky/dashboard/out/_next/static/chunks/9025.fa408f3242e9028d.js +6 -0
- sky/dashboard/out/_next/static/chunks/9353-cff34f7e773b2e2b.js +1 -0
- sky/dashboard/out/_next/static/chunks/9360.7310982cf5a0dc79.js +31 -0
- sky/dashboard/out/_next/static/chunks/9847.3aaca6bb33455140.js +30 -0
- sky/dashboard/out/_next/static/chunks/fd9d1056-86323a29a8f7e46a.js +1 -0
- sky/dashboard/out/_next/static/chunks/framework-cf60a09ccd051a10.js +33 -0
- sky/dashboard/out/_next/static/chunks/main-app-587214043926b3cc.js +1 -0
- sky/dashboard/out/_next/static/chunks/main-f15ccb73239a3bf1.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/_app-bde01e4a2beec258.js +34 -0
- sky/dashboard/out/_next/static/chunks/pages/_error-c66a4e8afc46f17b.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-c736ead69c2d86ec.js +16 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-a37d2063af475a1c.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters-d44859594e6f8064.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/config-dfb9bf07b13045f4.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/index-444f1804401f04ea.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/infra/[context]-c0b5935149902e6f.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/infra-aed0ea19df7cf961.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-5796e8d6aea291a0.js +16 -0
- sky/dashboard/out/_next/static/chunks/pages/jobs/pools/[pool]-6edeb7d06032adfc.js +21 -0
- sky/dashboard/out/_next/static/chunks/pages/jobs-479dde13399cf270.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/users-5ab3b907622cf0fe.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/volumes-b84b948ff357c43e.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/workspaces/[name]-c5a3eeee1c218af1.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/workspaces-22b23febb3e89ce1.js +1 -0
- sky/dashboard/out/_next/static/chunks/webpack-2679be77fc08a2f8.js +1 -0
- sky/dashboard/out/_next/static/css/0748ce22df867032.css +3 -0
- sky/dashboard/out/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js +1 -0
- sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
- sky/dashboard/out/clusters/[cluster].html +1 -1
- sky/dashboard/out/clusters.html +1 -1
- sky/dashboard/out/config.html +1 -0
- sky/dashboard/out/index.html +1 -1
- sky/dashboard/out/infra/[context].html +1 -0
- sky/dashboard/out/infra.html +1 -0
- sky/dashboard/out/jobs/[job].html +1 -1
- sky/dashboard/out/jobs/pools/[pool].html +1 -0
- sky/dashboard/out/jobs.html +1 -1
- sky/dashboard/out/users.html +1 -0
- sky/dashboard/out/volumes.html +1 -0
- sky/dashboard/out/workspace/new.html +1 -0
- sky/dashboard/out/workspaces/[name].html +1 -0
- sky/dashboard/out/workspaces.html +1 -0
- sky/data/data_utils.py +137 -1
- sky/data/mounting_utils.py +269 -84
- sky/data/storage.py +1451 -1807
- sky/data/storage_utils.py +43 -57
- sky/exceptions.py +132 -2
- sky/execution.py +206 -63
- sky/global_user_state.py +2374 -586
- sky/jobs/__init__.py +5 -0
- sky/jobs/client/sdk.py +242 -65
- sky/jobs/client/sdk_async.py +143 -0
- sky/jobs/constants.py +9 -8
- sky/jobs/controller.py +839 -277
- sky/jobs/file_content_utils.py +80 -0
- sky/jobs/log_gc.py +201 -0
- sky/jobs/recovery_strategy.py +398 -152
- sky/jobs/scheduler.py +315 -189
- sky/jobs/server/core.py +829 -255
- sky/jobs/server/server.py +156 -115
- sky/jobs/server/utils.py +136 -0
- sky/jobs/state.py +2092 -701
- sky/jobs/utils.py +1242 -160
- sky/logs/__init__.py +21 -0
- sky/logs/agent.py +108 -0
- sky/logs/aws.py +243 -0
- sky/logs/gcp.py +91 -0
- sky/metrics/__init__.py +0 -0
- sky/metrics/utils.py +443 -0
- sky/models.py +78 -1
- sky/optimizer.py +164 -70
- sky/provision/__init__.py +90 -4
- sky/provision/aws/config.py +147 -26
- sky/provision/aws/instance.py +135 -50
- sky/provision/azure/instance.py +10 -5
- sky/provision/common.py +13 -1
- sky/provision/cudo/cudo_machine_type.py +1 -1
- sky/provision/cudo/cudo_utils.py +14 -8
- sky/provision/cudo/cudo_wrapper.py +72 -71
- sky/provision/cudo/instance.py +10 -6
- sky/provision/do/instance.py +10 -6
- sky/provision/do/utils.py +4 -3
- sky/provision/docker_utils.py +114 -23
- sky/provision/fluidstack/instance.py +13 -8
- sky/provision/gcp/__init__.py +1 -0
- sky/provision/gcp/config.py +301 -19
- sky/provision/gcp/constants.py +218 -0
- sky/provision/gcp/instance.py +36 -8
- sky/provision/gcp/instance_utils.py +18 -4
- sky/provision/gcp/volume_utils.py +247 -0
- sky/provision/hyperbolic/__init__.py +12 -0
- sky/provision/hyperbolic/config.py +10 -0
- sky/provision/hyperbolic/instance.py +437 -0
- sky/provision/hyperbolic/utils.py +373 -0
- sky/provision/instance_setup.py +93 -14
- sky/provision/kubernetes/__init__.py +5 -0
- sky/provision/kubernetes/config.py +9 -52
- sky/provision/kubernetes/constants.py +17 -0
- sky/provision/kubernetes/instance.py +789 -247
- sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml +1 -2
- sky/provision/kubernetes/network.py +27 -17
- sky/provision/kubernetes/network_utils.py +40 -43
- sky/provision/kubernetes/utils.py +1192 -531
- sky/provision/kubernetes/volume.py +282 -0
- sky/provision/lambda_cloud/instance.py +22 -16
- sky/provision/nebius/constants.py +50 -0
- sky/provision/nebius/instance.py +19 -6
- sky/provision/nebius/utils.py +196 -91
- sky/provision/oci/instance.py +10 -5
- sky/provision/paperspace/instance.py +10 -7
- sky/provision/paperspace/utils.py +1 -1
- sky/provision/primeintellect/__init__.py +10 -0
- sky/provision/primeintellect/config.py +11 -0
- sky/provision/primeintellect/instance.py +454 -0
- sky/provision/primeintellect/utils.py +398 -0
- sky/provision/provisioner.py +110 -36
- sky/provision/runpod/__init__.py +5 -0
- sky/provision/runpod/instance.py +27 -6
- sky/provision/runpod/utils.py +51 -18
- sky/provision/runpod/volume.py +180 -0
- sky/provision/scp/__init__.py +15 -0
- sky/provision/scp/config.py +93 -0
- sky/provision/scp/instance.py +531 -0
- sky/provision/seeweb/__init__.py +11 -0
- sky/provision/seeweb/config.py +13 -0
- sky/provision/seeweb/instance.py +807 -0
- sky/provision/shadeform/__init__.py +11 -0
- sky/provision/shadeform/config.py +12 -0
- sky/provision/shadeform/instance.py +351 -0
- sky/provision/shadeform/shadeform_utils.py +83 -0
- sky/provision/ssh/__init__.py +18 -0
- sky/provision/vast/instance.py +13 -8
- sky/provision/vast/utils.py +10 -7
- sky/provision/vsphere/common/vim_utils.py +1 -2
- sky/provision/vsphere/instance.py +15 -10
- sky/provision/vsphere/vsphere_utils.py +9 -19
- sky/py.typed +0 -0
- sky/resources.py +844 -118
- sky/schemas/__init__.py +0 -0
- sky/schemas/api/__init__.py +0 -0
- sky/schemas/api/responses.py +225 -0
- sky/schemas/db/README +4 -0
- sky/schemas/db/env.py +90 -0
- sky/schemas/db/global_user_state/001_initial_schema.py +124 -0
- sky/schemas/db/global_user_state/002_add_workspace_to_cluster_history.py +35 -0
- sky/schemas/db/global_user_state/003_fix_initial_revision.py +61 -0
- sky/schemas/db/global_user_state/004_is_managed.py +34 -0
- sky/schemas/db/global_user_state/005_cluster_event.py +32 -0
- sky/schemas/db/global_user_state/006_provision_log.py +41 -0
- sky/schemas/db/global_user_state/007_cluster_event_request_id.py +34 -0
- sky/schemas/db/global_user_state/008_skylet_ssh_tunnel_metadata.py +34 -0
- sky/schemas/db/global_user_state/009_last_activity_and_launched_at.py +89 -0
- sky/schemas/db/global_user_state/010_save_ssh_key.py +66 -0
- sky/schemas/db/script.py.mako +28 -0
- sky/schemas/db/serve_state/001_initial_schema.py +67 -0
- sky/schemas/db/skypilot_config/001_initial_schema.py +30 -0
- sky/schemas/db/spot_jobs/001_initial_schema.py +97 -0
- sky/schemas/db/spot_jobs/002_cluster_pool.py +42 -0
- sky/schemas/db/spot_jobs/003_pool_hash.py +34 -0
- sky/schemas/db/spot_jobs/004_job_file_contents.py +42 -0
- sky/schemas/db/spot_jobs/005_logs_gc.py +38 -0
- sky/schemas/generated/__init__.py +0 -0
- sky/schemas/generated/autostopv1_pb2.py +36 -0
- sky/schemas/generated/autostopv1_pb2.pyi +43 -0
- sky/schemas/generated/autostopv1_pb2_grpc.py +146 -0
- sky/schemas/generated/jobsv1_pb2.py +86 -0
- sky/schemas/generated/jobsv1_pb2.pyi +254 -0
- sky/schemas/generated/jobsv1_pb2_grpc.py +542 -0
- sky/schemas/generated/managed_jobsv1_pb2.py +74 -0
- sky/schemas/generated/managed_jobsv1_pb2.pyi +278 -0
- sky/schemas/generated/managed_jobsv1_pb2_grpc.py +278 -0
- sky/schemas/generated/servev1_pb2.py +58 -0
- sky/schemas/generated/servev1_pb2.pyi +115 -0
- sky/schemas/generated/servev1_pb2_grpc.py +322 -0
- sky/serve/autoscalers.py +357 -5
- sky/serve/client/impl.py +310 -0
- sky/serve/client/sdk.py +47 -139
- sky/serve/client/sdk_async.py +130 -0
- sky/serve/constants.py +10 -8
- sky/serve/controller.py +64 -19
- sky/serve/load_balancer.py +106 -60
- sky/serve/load_balancing_policies.py +115 -1
- sky/serve/replica_managers.py +273 -162
- sky/serve/serve_rpc_utils.py +179 -0
- sky/serve/serve_state.py +554 -251
- sky/serve/serve_utils.py +733 -220
- sky/serve/server/core.py +66 -711
- sky/serve/server/impl.py +1093 -0
- sky/serve/server/server.py +21 -18
- sky/serve/service.py +133 -48
- sky/serve/service_spec.py +135 -16
- sky/serve/spot_placer.py +3 -0
- sky/server/auth/__init__.py +0 -0
- sky/server/auth/authn.py +50 -0
- sky/server/auth/loopback.py +38 -0
- sky/server/auth/oauth2_proxy.py +200 -0
- sky/server/common.py +475 -181
- sky/server/config.py +81 -23
- sky/server/constants.py +44 -6
- sky/server/daemons.py +229 -0
- sky/server/html/token_page.html +185 -0
- sky/server/metrics.py +160 -0
- sky/server/requests/executor.py +528 -138
- sky/server/requests/payloads.py +351 -17
- sky/server/requests/preconditions.py +21 -17
- sky/server/requests/process.py +112 -29
- sky/server/requests/request_names.py +120 -0
- sky/server/requests/requests.py +817 -224
- sky/server/requests/serializers/decoders.py +82 -31
- sky/server/requests/serializers/encoders.py +140 -22
- sky/server/requests/threads.py +106 -0
- sky/server/rest.py +417 -0
- sky/server/server.py +1290 -284
- sky/server/state.py +20 -0
- sky/server/stream_utils.py +345 -57
- sky/server/uvicorn.py +217 -3
- sky/server/versions.py +270 -0
- sky/setup_files/MANIFEST.in +5 -0
- sky/setup_files/alembic.ini +156 -0
- sky/setup_files/dependencies.py +136 -31
- sky/setup_files/setup.py +44 -42
- sky/sky_logging.py +102 -5
- sky/skylet/attempt_skylet.py +1 -0
- sky/skylet/autostop_lib.py +129 -8
- sky/skylet/configs.py +27 -20
- sky/skylet/constants.py +171 -19
- sky/skylet/events.py +105 -21
- sky/skylet/job_lib.py +335 -104
- sky/skylet/log_lib.py +297 -18
- sky/skylet/log_lib.pyi +44 -1
- sky/skylet/ray_patches/__init__.py +17 -3
- sky/skylet/ray_patches/autoscaler.py.diff +18 -0
- sky/skylet/ray_patches/cli.py.diff +19 -0
- sky/skylet/ray_patches/command_runner.py.diff +17 -0
- sky/skylet/ray_patches/log_monitor.py.diff +20 -0
- sky/skylet/ray_patches/resource_demand_scheduler.py.diff +32 -0
- sky/skylet/ray_patches/updater.py.diff +18 -0
- sky/skylet/ray_patches/worker.py.diff +41 -0
- sky/skylet/services.py +564 -0
- sky/skylet/skylet.py +63 -4
- sky/skylet/subprocess_daemon.py +103 -29
- sky/skypilot_config.py +506 -99
- sky/ssh_node_pools/__init__.py +1 -0
- sky/ssh_node_pools/core.py +135 -0
- sky/ssh_node_pools/server.py +233 -0
- sky/task.py +621 -137
- sky/templates/aws-ray.yml.j2 +10 -3
- sky/templates/azure-ray.yml.j2 +1 -1
- sky/templates/do-ray.yml.j2 +1 -1
- sky/templates/gcp-ray.yml.j2 +57 -0
- sky/templates/hyperbolic-ray.yml.j2 +67 -0
- sky/templates/jobs-controller.yaml.j2 +27 -24
- sky/templates/kubernetes-loadbalancer.yml.j2 +2 -0
- sky/templates/kubernetes-ray.yml.j2 +607 -51
- sky/templates/lambda-ray.yml.j2 +1 -1
- sky/templates/nebius-ray.yml.j2 +33 -12
- sky/templates/paperspace-ray.yml.j2 +1 -1
- sky/templates/primeintellect-ray.yml.j2 +71 -0
- sky/templates/runpod-ray.yml.j2 +9 -1
- sky/templates/scp-ray.yml.j2 +3 -50
- sky/templates/seeweb-ray.yml.j2 +108 -0
- sky/templates/shadeform-ray.yml.j2 +72 -0
- sky/templates/sky-serve-controller.yaml.j2 +22 -2
- sky/templates/websocket_proxy.py +178 -18
- sky/usage/usage_lib.py +18 -11
- sky/users/__init__.py +0 -0
- sky/users/model.conf +15 -0
- sky/users/permission.py +387 -0
- sky/users/rbac.py +121 -0
- sky/users/server.py +720 -0
- sky/users/token_service.py +218 -0
- sky/utils/accelerator_registry.py +34 -5
- sky/utils/admin_policy_utils.py +84 -38
- sky/utils/annotations.py +16 -5
- sky/utils/asyncio_utils.py +78 -0
- sky/utils/auth_utils.py +153 -0
- sky/utils/benchmark_utils.py +60 -0
- sky/utils/cli_utils/status_utils.py +159 -86
- sky/utils/cluster_utils.py +31 -9
- sky/utils/command_runner.py +354 -68
- sky/utils/command_runner.pyi +93 -3
- sky/utils/common.py +35 -8
- sky/utils/common_utils.py +310 -87
- sky/utils/config_utils.py +87 -5
- sky/utils/context.py +402 -0
- sky/utils/context_utils.py +222 -0
- sky/utils/controller_utils.py +264 -89
- sky/utils/dag_utils.py +31 -12
- sky/utils/db/__init__.py +0 -0
- sky/utils/db/db_utils.py +470 -0
- sky/utils/db/migration_utils.py +133 -0
- sky/utils/directory_utils.py +12 -0
- sky/utils/env_options.py +13 -0
- sky/utils/git.py +567 -0
- sky/utils/git_clone.sh +460 -0
- sky/utils/infra_utils.py +195 -0
- sky/utils/kubernetes/cleanup-tunnel.sh +62 -0
- sky/utils/kubernetes/config_map_utils.py +133 -0
- sky/utils/kubernetes/create_cluster.sh +13 -27
- sky/utils/kubernetes/delete_cluster.sh +10 -7
- sky/utils/kubernetes/deploy_remote_cluster.py +1299 -0
- sky/utils/kubernetes/exec_kubeconfig_converter.py +22 -31
- sky/utils/kubernetes/generate_kind_config.py +6 -66
- sky/utils/kubernetes/generate_kubeconfig.sh +4 -1
- sky/utils/kubernetes/gpu_labeler.py +5 -5
- sky/utils/kubernetes/kubernetes_deploy_utils.py +354 -47
- sky/utils/kubernetes/ssh-tunnel.sh +379 -0
- sky/utils/kubernetes/ssh_utils.py +221 -0
- sky/utils/kubernetes_enums.py +8 -15
- sky/utils/lock_events.py +94 -0
- sky/utils/locks.py +368 -0
- sky/utils/log_utils.py +300 -6
- sky/utils/perf_utils.py +22 -0
- sky/utils/resource_checker.py +298 -0
- sky/utils/resources_utils.py +249 -32
- sky/utils/rich_utils.py +213 -37
- sky/utils/schemas.py +905 -147
- sky/utils/serialize_utils.py +16 -0
- sky/utils/status_lib.py +10 -0
- sky/utils/subprocess_utils.py +38 -15
- sky/utils/tempstore.py +70 -0
- sky/utils/timeline.py +24 -52
- sky/utils/ux_utils.py +84 -15
- sky/utils/validator.py +11 -1
- sky/utils/volume.py +86 -0
- sky/utils/yaml_utils.py +111 -0
- sky/volumes/__init__.py +13 -0
- sky/volumes/client/__init__.py +0 -0
- sky/volumes/client/sdk.py +149 -0
- sky/volumes/server/__init__.py +0 -0
- sky/volumes/server/core.py +258 -0
- sky/volumes/server/server.py +122 -0
- sky/volumes/volume.py +212 -0
- sky/workspaces/__init__.py +0 -0
- sky/workspaces/core.py +655 -0
- sky/workspaces/server.py +101 -0
- sky/workspaces/utils.py +56 -0
- skypilot_nightly-1.0.0.dev20251107.dist-info/METADATA +675 -0
- skypilot_nightly-1.0.0.dev20251107.dist-info/RECORD +594 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/WHEEL +1 -1
- sky/benchmark/benchmark_state.py +0 -256
- sky/benchmark/benchmark_utils.py +0 -641
- sky/clouds/service_catalog/constants.py +0 -7
- sky/dashboard/out/_next/static/LksQgChY5izXjokL3LcEu/_buildManifest.js +0 -1
- sky/dashboard/out/_next/static/chunks/236-f49500b82ad5392d.js +0 -6
- sky/dashboard/out/_next/static/chunks/312-c3c8845990db8ffc.js +0 -15
- sky/dashboard/out/_next/static/chunks/37-0a572fe0dbb89c4d.js +0 -6
- sky/dashboard/out/_next/static/chunks/678-206dddca808e6d16.js +0 -59
- sky/dashboard/out/_next/static/chunks/845-0f8017370869e269.js +0 -1
- sky/dashboard/out/_next/static/chunks/979-7bf73a4c7cea0f5c.js +0 -1
- sky/dashboard/out/_next/static/chunks/fd9d1056-2821b0f0cabcd8bd.js +0 -1
- sky/dashboard/out/_next/static/chunks/framework-87d061ee6ed71b28.js +0 -33
- sky/dashboard/out/_next/static/chunks/main-app-241eb28595532291.js +0 -1
- sky/dashboard/out/_next/static/chunks/main-e0e2335212e72357.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/_app-e6b013bc3f77ad60.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/_error-1be831200e60c5c0.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-e15db85d0ea1fbe1.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-f383db7389368ea7.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters-a93b93e10b8b074e.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/index-f9f039532ca8cbc4.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-03f279c6741fb48b.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs-a75029b67aab6a2e.js +0 -1
- sky/dashboard/out/_next/static/chunks/webpack-830f59b8404e96b8.js +0 -1
- sky/dashboard/out/_next/static/css/c6933bbb2ce7f4dd.css +0 -3
- sky/jobs/dashboard/dashboard.py +0 -223
- sky/jobs/dashboard/static/favicon.ico +0 -0
- sky/jobs/dashboard/templates/index.html +0 -831
- sky/jobs/server/dashboard_utils.py +0 -69
- sky/skylet/providers/scp/__init__.py +0 -2
- sky/skylet/providers/scp/config.py +0 -149
- sky/skylet/providers/scp/node_provider.py +0 -578
- sky/templates/kubernetes-ssh-jump.yml.j2 +0 -94
- sky/utils/db_utils.py +0 -100
- sky/utils/kubernetes/deploy_remote_cluster.sh +0 -308
- sky/utils/kubernetes/ssh_jump_lifecycle_manager.py +0 -191
- skypilot_nightly-1.0.0.dev20250509.dist-info/METADATA +0 -361
- skypilot_nightly-1.0.0.dev20250509.dist-info/RECORD +0 -396
- /sky/{clouds/service_catalog → catalog}/config.py +0 -0
- /sky/{benchmark → catalog/data_fetchers}/__init__.py +0 -0
- /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_azure.py +0 -0
- /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_fluidstack.py +0 -0
- /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_ibm.py +0 -0
- /sky/{clouds/service_catalog/data_fetchers → client/cli}/__init__.py +0 -0
- /sky/dashboard/out/_next/static/{LksQgChY5izXjokL3LcEu → zB0ed6ge_W1MDszVHhijS}/_ssgManifest.js +0 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/licenses/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Add skylet_ssh_tunnel_metadata to clusters.
|
|
2
|
+
|
|
3
|
+
Revision ID: 008
|
|
4
|
+
Revises: 007
|
|
5
|
+
Create Date: 2025-09-09
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
from typing import Sequence, Union
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.utils.db import db_utils
|
|
15
|
+
|
|
16
|
+
# revision identifiers, used by Alembic.
|
|
17
|
+
revision: str = '008'
|
|
18
|
+
down_revision: Union[str, Sequence[str], None] = '007'
|
|
19
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
20
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def upgrade():
|
|
24
|
+
"""Add skylet_ssh_tunnel_metadata column to clusters."""
|
|
25
|
+
with op.get_context().autocommit_block():
|
|
26
|
+
db_utils.add_column_to_table_alembic('clusters',
|
|
27
|
+
'skylet_ssh_tunnel_metadata',
|
|
28
|
+
sa.LargeBinary(),
|
|
29
|
+
server_default=None)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def downgrade():
|
|
33
|
+
"""No-op for backward compatibility."""
|
|
34
|
+
pass
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Add last_activity_time and launched_at to cluster history.
|
|
2
|
+
|
|
3
|
+
Revision ID: 009
|
|
4
|
+
Revises: 008
|
|
5
|
+
Create Date: 2025-09-24
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
import pickle
|
|
10
|
+
from typing import Sequence, Union
|
|
11
|
+
|
|
12
|
+
from alembic import op
|
|
13
|
+
import sqlalchemy as sa
|
|
14
|
+
|
|
15
|
+
from sky.utils.db import db_utils
|
|
16
|
+
|
|
17
|
+
# revision identifiers, used by Alembic.
|
|
18
|
+
revision: str = '009'
|
|
19
|
+
down_revision: Union[str, Sequence[str], None] = '008'
|
|
20
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
21
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def upgrade():
|
|
25
|
+
"""Add last_activity_time and launched_at columns to cluster history."""
|
|
26
|
+
with op.get_context().autocommit_block():
|
|
27
|
+
# Add the columns with indices
|
|
28
|
+
db_utils.add_column_to_table_alembic('cluster_history',
|
|
29
|
+
'last_activity_time',
|
|
30
|
+
sa.Integer(),
|
|
31
|
+
server_default=None,
|
|
32
|
+
index=True)
|
|
33
|
+
|
|
34
|
+
db_utils.add_column_to_table_alembic('cluster_history',
|
|
35
|
+
'launched_at',
|
|
36
|
+
sa.Integer(),
|
|
37
|
+
server_default=None,
|
|
38
|
+
index=True)
|
|
39
|
+
|
|
40
|
+
# Populate the columns for existing rows
|
|
41
|
+
_populate_cluster_history_columns()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _populate_cluster_history_columns():
|
|
45
|
+
"""Populate last_activity_time and launched_at for existing rows using
|
|
46
|
+
usage_intervals logic."""
|
|
47
|
+
connection = op.get_bind()
|
|
48
|
+
|
|
49
|
+
# Get all existing rows with usage_intervals
|
|
50
|
+
result = connection.execute(
|
|
51
|
+
sa.text('SELECT cluster_hash, usage_intervals FROM cluster_history '
|
|
52
|
+
'WHERE usage_intervals IS NOT NULL'))
|
|
53
|
+
|
|
54
|
+
for row in result:
|
|
55
|
+
cluster_hash = row[0]
|
|
56
|
+
usage_intervals_blob = row[1]
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
# Deserialize the usage_intervals
|
|
60
|
+
usage_intervals = pickle.loads(usage_intervals_blob)
|
|
61
|
+
|
|
62
|
+
if usage_intervals:
|
|
63
|
+
# Calculate last_activity_time: end time of last interval
|
|
64
|
+
# or start time if still running
|
|
65
|
+
last_interval = usage_intervals[-1]
|
|
66
|
+
last_activity_time = (last_interval[1] if last_interval[1]
|
|
67
|
+
is not None else last_interval[0])
|
|
68
|
+
|
|
69
|
+
# Calculate launched_at: start time of first interval
|
|
70
|
+
launched_at = usage_intervals[0][0]
|
|
71
|
+
|
|
72
|
+
# Update the row with both calculated values
|
|
73
|
+
connection.execute(
|
|
74
|
+
sa.text('UPDATE cluster_history '
|
|
75
|
+
'SET last_activity_time = :last_activity_time, '
|
|
76
|
+
'launched_at = :launched_at '
|
|
77
|
+
'WHERE cluster_hash = :cluster_hash'), {
|
|
78
|
+
'last_activity_time': last_activity_time,
|
|
79
|
+
'launched_at': launched_at,
|
|
80
|
+
'cluster_hash': cluster_hash
|
|
81
|
+
})
|
|
82
|
+
except (pickle.PickleError, AttributeError, IndexError):
|
|
83
|
+
# Skip rows with corrupted or invalid usage_intervals
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def downgrade():
|
|
88
|
+
"""No-op for backward compatibility."""
|
|
89
|
+
pass
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Add ssh keys in filesystem to global user state.
|
|
2
|
+
|
|
3
|
+
Revision ID: 010
|
|
4
|
+
Revises: 009
|
|
5
|
+
Create Date: 2025-10-07
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
import glob
|
|
9
|
+
# pylint: disable=invalid-name
|
|
10
|
+
import os
|
|
11
|
+
from typing import Sequence, Union
|
|
12
|
+
|
|
13
|
+
from alembic import op
|
|
14
|
+
import sqlalchemy as sa
|
|
15
|
+
|
|
16
|
+
# revision identifiers, used by Alembic.
|
|
17
|
+
revision: str = '010'
|
|
18
|
+
down_revision: Union[str, Sequence[str], None] = '009'
|
|
19
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
20
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def upgrade():
|
|
24
|
+
"""Add last_activity_time and launched_at columns to cluster history."""
|
|
25
|
+
connection = op.get_bind()
|
|
26
|
+
|
|
27
|
+
match_dirs = glob.glob(os.path.expanduser('~/.sky/clients/*/ssh'))
|
|
28
|
+
file_user_hashes = set()
|
|
29
|
+
for match_dir in match_dirs:
|
|
30
|
+
user_hash = match_dir.split('/')[-2]
|
|
31
|
+
file_user_hashes.add(user_hash)
|
|
32
|
+
|
|
33
|
+
# Get all existing ssh keys
|
|
34
|
+
existing_user_hashes = set()
|
|
35
|
+
result = connection.execute(sa.text('SELECT user_hash FROM ssh_key'))
|
|
36
|
+
for row in result:
|
|
37
|
+
existing_user_hashes.add(row[0])
|
|
38
|
+
|
|
39
|
+
user_hashes_to_add = file_user_hashes - existing_user_hashes
|
|
40
|
+
for user_hash in user_hashes_to_add:
|
|
41
|
+
match_dir = os.path.join(os.path.expanduser('~/.sky/clients'),
|
|
42
|
+
user_hash, 'ssh')
|
|
43
|
+
public_key_path = os.path.join(match_dir, 'sky-key.pub')
|
|
44
|
+
private_key_path = os.path.join(match_dir, 'sky-key')
|
|
45
|
+
try:
|
|
46
|
+
with open(public_key_path, 'r', encoding='utf-8') as f:
|
|
47
|
+
public_key = f.read().strip()
|
|
48
|
+
with open(private_key_path, 'r', encoding='utf-8') as f:
|
|
49
|
+
private_key = f.read().strip()
|
|
50
|
+
except FileNotFoundError:
|
|
51
|
+
# Skip if the key files are not found
|
|
52
|
+
continue
|
|
53
|
+
connection.execute(
|
|
54
|
+
sa.text('INSERT INTO ssh_key '
|
|
55
|
+
'(user_hash, ssh_public_key, ssh_private_key) '
|
|
56
|
+
'VALUES (:user_hash, :ssh_public_key, :ssh_private_key) '
|
|
57
|
+
'ON CONFLICT DO NOTHING'), {
|
|
58
|
+
'user_hash': user_hash,
|
|
59
|
+
'ssh_public_key': public_key,
|
|
60
|
+
'ssh_private_key': private_key
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def downgrade():
|
|
65
|
+
"""No-op for backward compatibility."""
|
|
66
|
+
pass
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""${message}
|
|
2
|
+
|
|
3
|
+
Revision ID: ${up_revision}
|
|
4
|
+
Revises: ${down_revision | comma,n}
|
|
5
|
+
Create Date: ${create_date}
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
from typing import Sequence, Union
|
|
9
|
+
|
|
10
|
+
from alembic import op
|
|
11
|
+
import sqlalchemy as sa
|
|
12
|
+
${imports if imports else ""}
|
|
13
|
+
|
|
14
|
+
# revision identifiers, used by Alembic.
|
|
15
|
+
revision: str = ${repr(up_revision)}
|
|
16
|
+
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
|
17
|
+
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
|
18
|
+
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def upgrade() -> None:
|
|
22
|
+
"""Upgrade schema."""
|
|
23
|
+
${upgrades if upgrades else "pass"}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def downgrade() -> None:
|
|
27
|
+
"""Downgrade schema."""
|
|
28
|
+
${downgrades if downgrades else "pass"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Initial schema for sky serve state database with backwards compatibility
|
|
2
|
+
|
|
3
|
+
Revision ID: 001
|
|
4
|
+
Revises:
|
|
5
|
+
Create Date: 2024-01-01 12:00:00.000000
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
import json
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.serve import constants
|
|
15
|
+
from sky.serve.serve_state import Base
|
|
16
|
+
from sky.utils.db import db_utils
|
|
17
|
+
|
|
18
|
+
# revision identifiers, used by Alembic.
|
|
19
|
+
revision = '001'
|
|
20
|
+
down_revision = None
|
|
21
|
+
branch_labels = None
|
|
22
|
+
depends_on = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def upgrade():
|
|
26
|
+
"""Create initial schema and add all backwards compatibility columns"""
|
|
27
|
+
with op.get_context().autocommit_block():
|
|
28
|
+
# Create all tables with their current schema
|
|
29
|
+
db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
|
|
30
|
+
|
|
31
|
+
# Add backwards compatibility columns using helper function that matches
|
|
32
|
+
# original add_column_to_table_sqlalchemy behavior exactly
|
|
33
|
+
db_utils.add_column_to_table_alembic('services',
|
|
34
|
+
'requested_resources_str',
|
|
35
|
+
sa.Text())
|
|
36
|
+
db_utils.add_column_to_table_alembic(
|
|
37
|
+
'services',
|
|
38
|
+
'current_version',
|
|
39
|
+
sa.Integer(),
|
|
40
|
+
server_default=f'{constants.INITIAL_VERSION}')
|
|
41
|
+
db_utils.add_column_to_table_alembic('services',
|
|
42
|
+
'active_versions',
|
|
43
|
+
sa.Text(),
|
|
44
|
+
server_default=json.dumps([]))
|
|
45
|
+
db_utils.add_column_to_table_alembic('services',
|
|
46
|
+
'load_balancing_policy', sa.Text())
|
|
47
|
+
db_utils.add_column_to_table_alembic('services',
|
|
48
|
+
'tls_encrypted',
|
|
49
|
+
sa.Integer(),
|
|
50
|
+
server_default='0')
|
|
51
|
+
db_utils.add_column_to_table_alembic('services',
|
|
52
|
+
'pool',
|
|
53
|
+
sa.Integer(),
|
|
54
|
+
server_default='0')
|
|
55
|
+
db_utils.add_column_to_table_alembic(
|
|
56
|
+
'services',
|
|
57
|
+
'controller_pid',
|
|
58
|
+
sa.Integer(),
|
|
59
|
+
value_to_replace_existing_entries=-1)
|
|
60
|
+
db_utils.add_column_to_table_alembic('services', 'hash', sa.Text())
|
|
61
|
+
db_utils.add_column_to_table_alembic('services', 'entrypoint',
|
|
62
|
+
sa.Text())
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def downgrade():
|
|
66
|
+
"""Drop all tables"""
|
|
67
|
+
Base.metadata.drop_all(bind=op.get_bind())
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Initial schema for sky config database
|
|
2
|
+
|
|
3
|
+
Revision ID: 001
|
|
4
|
+
Revises:
|
|
5
|
+
Create Date: 2025-10-21
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
from alembic import op
|
|
10
|
+
|
|
11
|
+
from sky.skypilot_config import Base
|
|
12
|
+
from sky.utils.db import db_utils
|
|
13
|
+
|
|
14
|
+
# revision identifiers, used by Alembic.
|
|
15
|
+
revision = '001'
|
|
16
|
+
down_revision = None
|
|
17
|
+
branch_labels = None
|
|
18
|
+
depends_on = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def upgrade():
|
|
22
|
+
"""Create initial schema for config_yaml table"""
|
|
23
|
+
with op.get_context().autocommit_block():
|
|
24
|
+
# Create all tables with their current schema
|
|
25
|
+
db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def downgrade():
|
|
29
|
+
"""Drop all tables"""
|
|
30
|
+
Base.metadata.drop_all(bind=op.get_bind())
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Initial schema for spot jobs database with backwards compatibility columns
|
|
2
|
+
|
|
3
|
+
Revision ID: 001
|
|
4
|
+
Revises:
|
|
5
|
+
Create Date: 2024-01-01 12:00:00.000000
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
import json
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.jobs.state import Base
|
|
15
|
+
from sky.skylet import constants
|
|
16
|
+
from sky.utils.db import db_utils
|
|
17
|
+
|
|
18
|
+
# revision identifiers, used by Alembic.
|
|
19
|
+
revision = '001'
|
|
20
|
+
down_revision = None
|
|
21
|
+
branch_labels = None
|
|
22
|
+
depends_on = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def upgrade():
|
|
26
|
+
"""Create initial schema and add all backwards compatibility columns"""
|
|
27
|
+
with op.get_context().autocommit_block():
|
|
28
|
+
# Create all tables with their current schema
|
|
29
|
+
db_utils.add_all_tables_to_db_sqlalchemy(Base.metadata, op.get_bind())
|
|
30
|
+
|
|
31
|
+
# Add backwards compatibility columns using helper function that matches
|
|
32
|
+
# original add_column_to_table_sqlalchemy behavior exactly
|
|
33
|
+
|
|
34
|
+
# Spot table columns
|
|
35
|
+
db_utils.add_column_to_table_alembic('spot', 'failure_reason',
|
|
36
|
+
sa.Text())
|
|
37
|
+
db_utils.add_column_to_table_alembic('spot',
|
|
38
|
+
'spot_job_id',
|
|
39
|
+
sa.Integer(),
|
|
40
|
+
copy_from='job_id')
|
|
41
|
+
db_utils.add_column_to_table_alembic(
|
|
42
|
+
'spot',
|
|
43
|
+
'task_id',
|
|
44
|
+
sa.Integer(),
|
|
45
|
+
server_default='0',
|
|
46
|
+
value_to_replace_existing_entries=0)
|
|
47
|
+
db_utils.add_column_to_table_alembic('spot',
|
|
48
|
+
'task_name',
|
|
49
|
+
sa.Text(),
|
|
50
|
+
copy_from='job_name')
|
|
51
|
+
db_utils.add_column_to_table_alembic(
|
|
52
|
+
'spot',
|
|
53
|
+
'specs',
|
|
54
|
+
sa.Text(),
|
|
55
|
+
value_to_replace_existing_entries=json.dumps(
|
|
56
|
+
{'max_restarts_on_errors': 0}))
|
|
57
|
+
db_utils.add_column_to_table_alembic('spot', 'local_log_file',
|
|
58
|
+
sa.Text())
|
|
59
|
+
db_utils.add_column_to_table_alembic(
|
|
60
|
+
'spot',
|
|
61
|
+
'metadata',
|
|
62
|
+
sa.Text(),
|
|
63
|
+
server_default='{}',
|
|
64
|
+
value_to_replace_existing_entries='{}')
|
|
65
|
+
|
|
66
|
+
# Job info table columns
|
|
67
|
+
db_utils.add_column_to_table_alembic('job_info', 'schedule_state',
|
|
68
|
+
sa.Text())
|
|
69
|
+
db_utils.add_column_to_table_alembic('job_info', 'controller_pid',
|
|
70
|
+
sa.Integer())
|
|
71
|
+
db_utils.add_column_to_table_alembic('job_info', 'dag_yaml_path',
|
|
72
|
+
sa.Text())
|
|
73
|
+
db_utils.add_column_to_table_alembic('job_info', 'env_file_path',
|
|
74
|
+
sa.Text())
|
|
75
|
+
db_utils.add_column_to_table_alembic('job_info', 'user_hash', sa.Text())
|
|
76
|
+
db_utils.add_column_to_table_alembic(
|
|
77
|
+
'job_info',
|
|
78
|
+
'workspace',
|
|
79
|
+
sa.Text(),
|
|
80
|
+
value_to_replace_existing_entries=constants.
|
|
81
|
+
SKYPILOT_DEFAULT_WORKSPACE)
|
|
82
|
+
db_utils.add_column_to_table_alembic(
|
|
83
|
+
'job_info',
|
|
84
|
+
'priority',
|
|
85
|
+
sa.Integer(),
|
|
86
|
+
server_default=str(constants.DEFAULT_PRIORITY),
|
|
87
|
+
value_to_replace_existing_entries=constants.DEFAULT_PRIORITY)
|
|
88
|
+
db_utils.add_column_to_table_alembic('job_info', 'entrypoint',
|
|
89
|
+
sa.Text())
|
|
90
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
91
|
+
'original_user_yaml_path',
|
|
92
|
+
sa.Text())
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def downgrade():
|
|
96
|
+
"""Drop all tables"""
|
|
97
|
+
Base.metadata.drop_all(bind=op.get_bind())
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Columns for cluster pool.
|
|
2
|
+
|
|
3
|
+
Revision ID: 002
|
|
4
|
+
Revises: 001
|
|
5
|
+
Create Date: 2025-07-18
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
from typing import Sequence, Union
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.utils.db import db_utils
|
|
15
|
+
|
|
16
|
+
# revision identifiers, used by Alembic.
|
|
17
|
+
revision: str = '002'
|
|
18
|
+
down_revision: Union[str, Sequence[str], None] = '001'
|
|
19
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
20
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def upgrade():
|
|
24
|
+
"""Add columns for cluster pool."""
|
|
25
|
+
with op.get_context().autocommit_block():
|
|
26
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
27
|
+
'pool',
|
|
28
|
+
sa.Text(),
|
|
29
|
+
server_default=None)
|
|
30
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
31
|
+
'current_cluster_name',
|
|
32
|
+
sa.Text(),
|
|
33
|
+
server_default=None)
|
|
34
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
35
|
+
'job_id_on_pool_cluster',
|
|
36
|
+
sa.Integer(),
|
|
37
|
+
server_default=None)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def downgrade():
|
|
41
|
+
"""Remove columns for cluster pool."""
|
|
42
|
+
pass
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Adding a hash column for pool.
|
|
2
|
+
|
|
3
|
+
Revision ID: 003
|
|
4
|
+
Revises: 002
|
|
5
|
+
Create Date: 2025-07-18
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
from typing import Sequence, Union
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.utils.db import db_utils
|
|
15
|
+
|
|
16
|
+
# revision identifiers, used by Alembic.
|
|
17
|
+
revision: str = '003'
|
|
18
|
+
down_revision: Union[str, Sequence[str], None] = '002'
|
|
19
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
20
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def upgrade():
|
|
24
|
+
"""Add columns for pool hash."""
|
|
25
|
+
with op.get_context().autocommit_block():
|
|
26
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
27
|
+
'pool_hash',
|
|
28
|
+
sa.Text(),
|
|
29
|
+
server_default=None)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def downgrade():
|
|
33
|
+
"""Remove columns for pool hash."""
|
|
34
|
+
pass
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Add columns for stored DAG/env file contents.
|
|
2
|
+
|
|
3
|
+
Revision ID: 004
|
|
4
|
+
Revises: 003
|
|
5
|
+
Create Date: 2025-10-27
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
from typing import Sequence, Union
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.utils.db import db_utils
|
|
15
|
+
|
|
16
|
+
# revision identifiers, used by Alembic.
|
|
17
|
+
revision: str = '004'
|
|
18
|
+
down_revision: Union[str, Sequence[str], None] = '003'
|
|
19
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
20
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def upgrade():
|
|
24
|
+
"""Add columns to persist job file contents in the database."""
|
|
25
|
+
with op.get_context().autocommit_block():
|
|
26
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
27
|
+
'dag_yaml_content',
|
|
28
|
+
sa.Text(),
|
|
29
|
+
server_default=None)
|
|
30
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
31
|
+
'original_user_yaml_content',
|
|
32
|
+
sa.Text(),
|
|
33
|
+
server_default=None)
|
|
34
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
35
|
+
'env_file_content',
|
|
36
|
+
sa.Text(),
|
|
37
|
+
server_default=None)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def downgrade():
|
|
41
|
+
"""No downgrade logic."""
|
|
42
|
+
pass
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Adding columns for the GC time of task logs and controller logs.
|
|
2
|
+
|
|
3
|
+
Revision ID: 005
|
|
4
|
+
Revises: 004
|
|
5
|
+
Create Date: 2025-10-20
|
|
6
|
+
|
|
7
|
+
"""
|
|
8
|
+
# pylint: disable=invalid-name
|
|
9
|
+
from typing import Sequence, Union
|
|
10
|
+
|
|
11
|
+
from alembic import op
|
|
12
|
+
import sqlalchemy as sa
|
|
13
|
+
|
|
14
|
+
from sky.utils.db import db_utils
|
|
15
|
+
|
|
16
|
+
# revision identifiers, used by Alembic.
|
|
17
|
+
revision: str = '005'
|
|
18
|
+
down_revision: Union[str, Sequence[str], None] = '004'
|
|
19
|
+
branch_labels: Union[str, Sequence[str], None] = None
|
|
20
|
+
depends_on: Union[str, Sequence[str], None] = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def upgrade():
|
|
24
|
+
"""Add columns for logs gc."""
|
|
25
|
+
with op.get_context().autocommit_block():
|
|
26
|
+
db_utils.add_column_to_table_alembic('job_info',
|
|
27
|
+
'controller_logs_cleaned_at',
|
|
28
|
+
sa.Float(),
|
|
29
|
+
server_default=None)
|
|
30
|
+
db_utils.add_column_to_table_alembic('spot',
|
|
31
|
+
'logs_cleaned_at',
|
|
32
|
+
sa.Float(),
|
|
33
|
+
server_default=None)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def downgrade():
|
|
37
|
+
"""Remove columns for logs gc."""
|
|
38
|
+
pass
|
|
File without changes
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# source: sky/schemas/generated/autostopv1.proto
|
|
4
|
+
# Protobuf Python Version: 5.26.1
|
|
5
|
+
"""Generated protocol buffer code."""
|
|
6
|
+
from google.protobuf import descriptor as _descriptor
|
|
7
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
8
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
9
|
+
from google.protobuf.internal import builder as _builder
|
|
10
|
+
# @@protoc_insertion_point(imports)
|
|
11
|
+
|
|
12
|
+
_sym_db = _symbol_database.Default()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&sky/schemas/generated/autostopv1.proto\x12\x0b\x61utostop.v1\"y\n\x12SetAutostopRequest\x12\x14\n\x0cidle_minutes\x18\x01 \x01(\x05\x12\x0f\n\x07\x62\x61\x63kend\x18\x02 \x01(\t\x12.\n\x08wait_for\x18\x03 \x01(\x0e\x32\x1c.autostop.v1.AutostopWaitFor\x12\x0c\n\x04\x64own\x18\x04 \x01(\x08\"\x15\n\x13SetAutostopResponse\"\x17\n\x15IsAutostoppingRequest\"1\n\x16IsAutostoppingResponse\x12\x17\n\x0fis_autostopping\x18\x01 \x01(\x08*\x90\x01\n\x0f\x41utostopWaitFor\x12!\n\x1d\x41UTOSTOP_WAIT_FOR_UNSPECIFIED\x10\x00\x12\"\n\x1e\x41UTOSTOP_WAIT_FOR_JOBS_AND_SSH\x10\x01\x12\x1a\n\x16\x41UTOSTOP_WAIT_FOR_JOBS\x10\x02\x12\x1a\n\x16\x41UTOSTOP_WAIT_FOR_NONE\x10\x03\x32\xbe\x01\n\x0f\x41utostopService\x12P\n\x0bSetAutostop\x12\x1f.autostop.v1.SetAutostopRequest\x1a .autostop.v1.SetAutostopResponse\x12Y\n\x0eIsAutostopping\x12\".autostop.v1.IsAutostoppingRequest\x1a#.autostop.v1.IsAutostoppingResponseb\x06proto3')
|
|
18
|
+
|
|
19
|
+
_globals = globals()
|
|
20
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
21
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sky.schemas.generated.autostopv1_pb2', _globals)
|
|
22
|
+
if not _descriptor._USE_C_DESCRIPTORS:
|
|
23
|
+
DESCRIPTOR._loaded_options = None
|
|
24
|
+
_globals['_AUTOSTOPWAITFOR']._serialized_start=278
|
|
25
|
+
_globals['_AUTOSTOPWAITFOR']._serialized_end=422
|
|
26
|
+
_globals['_SETAUTOSTOPREQUEST']._serialized_start=55
|
|
27
|
+
_globals['_SETAUTOSTOPREQUEST']._serialized_end=176
|
|
28
|
+
_globals['_SETAUTOSTOPRESPONSE']._serialized_start=178
|
|
29
|
+
_globals['_SETAUTOSTOPRESPONSE']._serialized_end=199
|
|
30
|
+
_globals['_ISAUTOSTOPPINGREQUEST']._serialized_start=201
|
|
31
|
+
_globals['_ISAUTOSTOPPINGREQUEST']._serialized_end=224
|
|
32
|
+
_globals['_ISAUTOSTOPPINGRESPONSE']._serialized_start=226
|
|
33
|
+
_globals['_ISAUTOSTOPPINGRESPONSE']._serialized_end=275
|
|
34
|
+
_globals['_AUTOSTOPSERVICE']._serialized_start=425
|
|
35
|
+
_globals['_AUTOSTOPSERVICE']._serialized_end=615
|
|
36
|
+
# @@protoc_insertion_point(module_scope)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
|
|
2
|
+
from google.protobuf import descriptor as _descriptor
|
|
3
|
+
from google.protobuf import message as _message
|
|
4
|
+
from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
|
|
5
|
+
|
|
6
|
+
DESCRIPTOR: _descriptor.FileDescriptor
|
|
7
|
+
|
|
8
|
+
class AutostopWaitFor(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
|
9
|
+
__slots__ = ()
|
|
10
|
+
AUTOSTOP_WAIT_FOR_UNSPECIFIED: _ClassVar[AutostopWaitFor]
|
|
11
|
+
AUTOSTOP_WAIT_FOR_JOBS_AND_SSH: _ClassVar[AutostopWaitFor]
|
|
12
|
+
AUTOSTOP_WAIT_FOR_JOBS: _ClassVar[AutostopWaitFor]
|
|
13
|
+
AUTOSTOP_WAIT_FOR_NONE: _ClassVar[AutostopWaitFor]
|
|
14
|
+
AUTOSTOP_WAIT_FOR_UNSPECIFIED: AutostopWaitFor
|
|
15
|
+
AUTOSTOP_WAIT_FOR_JOBS_AND_SSH: AutostopWaitFor
|
|
16
|
+
AUTOSTOP_WAIT_FOR_JOBS: AutostopWaitFor
|
|
17
|
+
AUTOSTOP_WAIT_FOR_NONE: AutostopWaitFor
|
|
18
|
+
|
|
19
|
+
class SetAutostopRequest(_message.Message):
|
|
20
|
+
__slots__ = ("idle_minutes", "backend", "wait_for", "down")
|
|
21
|
+
IDLE_MINUTES_FIELD_NUMBER: _ClassVar[int]
|
|
22
|
+
BACKEND_FIELD_NUMBER: _ClassVar[int]
|
|
23
|
+
WAIT_FOR_FIELD_NUMBER: _ClassVar[int]
|
|
24
|
+
DOWN_FIELD_NUMBER: _ClassVar[int]
|
|
25
|
+
idle_minutes: int
|
|
26
|
+
backend: str
|
|
27
|
+
wait_for: AutostopWaitFor
|
|
28
|
+
down: bool
|
|
29
|
+
def __init__(self, idle_minutes: _Optional[int] = ..., backend: _Optional[str] = ..., wait_for: _Optional[_Union[AutostopWaitFor, str]] = ..., down: bool = ...) -> None: ...
|
|
30
|
+
|
|
31
|
+
class SetAutostopResponse(_message.Message):
|
|
32
|
+
__slots__ = ()
|
|
33
|
+
def __init__(self) -> None: ...
|
|
34
|
+
|
|
35
|
+
class IsAutostoppingRequest(_message.Message):
|
|
36
|
+
__slots__ = ()
|
|
37
|
+
def __init__(self) -> None: ...
|
|
38
|
+
|
|
39
|
+
class IsAutostoppingResponse(_message.Message):
|
|
40
|
+
__slots__ = ("is_autostopping",)
|
|
41
|
+
IS_AUTOSTOPPING_FIELD_NUMBER: _ClassVar[int]
|
|
42
|
+
is_autostopping: bool
|
|
43
|
+
def __init__(self, is_autostopping: bool = ...) -> None: ...
|