skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of skypilot-nightly might be problematic. Click here for more details.
- sky/__init__.py +22 -6
- sky/adaptors/aws.py +25 -7
- sky/adaptors/common.py +24 -1
- sky/adaptors/coreweave.py +278 -0
- sky/adaptors/do.py +8 -2
- sky/adaptors/hyperbolic.py +8 -0
- sky/adaptors/kubernetes.py +149 -18
- sky/adaptors/nebius.py +170 -17
- sky/adaptors/primeintellect.py +1 -0
- sky/adaptors/runpod.py +68 -0
- sky/adaptors/seeweb.py +167 -0
- sky/adaptors/shadeform.py +89 -0
- sky/admin_policy.py +187 -4
- sky/authentication.py +179 -225
- sky/backends/__init__.py +4 -2
- sky/backends/backend.py +22 -9
- sky/backends/backend_utils.py +1299 -380
- sky/backends/cloud_vm_ray_backend.py +1715 -518
- sky/backends/docker_utils.py +1 -1
- sky/backends/local_docker_backend.py +11 -6
- sky/backends/wheel_utils.py +37 -9
- sky/{clouds/service_catalog → catalog}/__init__.py +21 -19
- sky/{clouds/service_catalog → catalog}/aws_catalog.py +27 -8
- sky/{clouds/service_catalog → catalog}/azure_catalog.py +10 -7
- sky/{clouds/service_catalog → catalog}/common.py +89 -48
- sky/{clouds/service_catalog → catalog}/cudo_catalog.py +8 -5
- sky/{clouds/service_catalog → catalog}/data_fetchers/analyze.py +1 -1
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_aws.py +30 -40
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_cudo.py +38 -38
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_gcp.py +42 -15
- sky/catalog/data_fetchers/fetch_hyperbolic.py +136 -0
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_lambda_cloud.py +1 -0
- sky/catalog/data_fetchers/fetch_nebius.py +335 -0
- sky/catalog/data_fetchers/fetch_runpod.py +698 -0
- sky/catalog/data_fetchers/fetch_seeweb.py +329 -0
- sky/catalog/data_fetchers/fetch_shadeform.py +142 -0
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_vast.py +1 -1
- sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_vsphere.py +1 -1
- sky/{clouds/service_catalog → catalog}/do_catalog.py +5 -2
- sky/{clouds/service_catalog → catalog}/fluidstack_catalog.py +6 -3
- sky/{clouds/service_catalog → catalog}/gcp_catalog.py +41 -15
- sky/catalog/hyperbolic_catalog.py +136 -0
- sky/{clouds/service_catalog → catalog}/ibm_catalog.py +9 -6
- sky/{clouds/service_catalog → catalog}/kubernetes_catalog.py +36 -24
- sky/{clouds/service_catalog → catalog}/lambda_catalog.py +9 -6
- sky/{clouds/service_catalog → catalog}/nebius_catalog.py +9 -7
- sky/{clouds/service_catalog → catalog}/oci_catalog.py +9 -6
- sky/{clouds/service_catalog → catalog}/paperspace_catalog.py +5 -2
- sky/catalog/primeintellect_catalog.py +95 -0
- sky/{clouds/service_catalog → catalog}/runpod_catalog.py +11 -4
- sky/{clouds/service_catalog → catalog}/scp_catalog.py +9 -6
- sky/catalog/seeweb_catalog.py +184 -0
- sky/catalog/shadeform_catalog.py +165 -0
- sky/catalog/ssh_catalog.py +167 -0
- sky/{clouds/service_catalog → catalog}/vast_catalog.py +6 -3
- sky/{clouds/service_catalog → catalog}/vsphere_catalog.py +5 -2
- sky/check.py +491 -203
- sky/cli.py +5 -6005
- sky/client/{cli.py → cli/command.py} +2477 -1885
- sky/client/cli/deprecation_utils.py +99 -0
- sky/client/cli/flags.py +359 -0
- sky/client/cli/table_utils.py +320 -0
- sky/client/common.py +70 -32
- sky/client/oauth.py +82 -0
- sky/client/sdk.py +1203 -297
- sky/client/sdk_async.py +833 -0
- sky/client/service_account_auth.py +47 -0
- sky/cloud_stores.py +73 -0
- sky/clouds/__init__.py +13 -0
- sky/clouds/aws.py +358 -93
- sky/clouds/azure.py +105 -83
- sky/clouds/cloud.py +127 -36
- sky/clouds/cudo.py +68 -50
- sky/clouds/do.py +66 -48
- sky/clouds/fluidstack.py +63 -44
- sky/clouds/gcp.py +339 -110
- sky/clouds/hyperbolic.py +293 -0
- sky/clouds/ibm.py +70 -49
- sky/clouds/kubernetes.py +563 -162
- sky/clouds/lambda_cloud.py +74 -54
- sky/clouds/nebius.py +206 -80
- sky/clouds/oci.py +88 -66
- sky/clouds/paperspace.py +61 -44
- sky/clouds/primeintellect.py +317 -0
- sky/clouds/runpod.py +164 -74
- sky/clouds/scp.py +89 -83
- sky/clouds/seeweb.py +466 -0
- sky/clouds/shadeform.py +400 -0
- sky/clouds/ssh.py +263 -0
- sky/clouds/utils/aws_utils.py +10 -4
- sky/clouds/utils/gcp_utils.py +87 -11
- sky/clouds/utils/oci_utils.py +38 -14
- sky/clouds/utils/scp_utils.py +177 -124
- sky/clouds/vast.py +99 -77
- sky/clouds/vsphere.py +51 -40
- sky/core.py +349 -139
- sky/dag.py +15 -0
- sky/dashboard/out/404.html +1 -1
- sky/dashboard/out/_next/static/chunks/1141-e6aa9ab418717c59.js +11 -0
- sky/dashboard/out/_next/static/chunks/1272-1ef0bf0237faccdb.js +1 -0
- sky/dashboard/out/_next/static/chunks/1871-74503c8e80fd253b.js +6 -0
- sky/dashboard/out/_next/static/chunks/2260-7703229c33c5ebd5.js +1 -0
- sky/dashboard/out/_next/static/chunks/2350.fab69e61bac57b23.js +1 -0
- sky/dashboard/out/_next/static/chunks/2369.fc20f0c2c8ed9fe7.js +15 -0
- sky/dashboard/out/_next/static/chunks/2755.fff53c4a3fcae910.js +26 -0
- sky/dashboard/out/_next/static/chunks/3294.72362fa129305b19.js +1 -0
- sky/dashboard/out/_next/static/chunks/3785.ad6adaa2a0fa9768.js +1 -0
- sky/dashboard/out/_next/static/chunks/3850-ff4a9a69d978632b.js +1 -0
- sky/dashboard/out/_next/static/chunks/3937.210053269f121201.js +1 -0
- sky/dashboard/out/_next/static/chunks/4725.a830b5c9e7867c92.js +1 -0
- sky/dashboard/out/_next/static/chunks/4937.a2baa2df5572a276.js +15 -0
- sky/dashboard/out/_next/static/chunks/5739-d67458fcb1386c92.js +8 -0
- sky/dashboard/out/_next/static/chunks/6130-2be46d70a38f1e82.js +1 -0
- sky/dashboard/out/_next/static/chunks/616-3d59f75e2ccf9321.js +39 -0
- sky/dashboard/out/_next/static/chunks/6212-7bd06f60ba693125.js +13 -0
- sky/dashboard/out/_next/static/chunks/6601-06114c982db410b6.js +1 -0
- sky/dashboard/out/_next/static/chunks/6856-ef8ba11f96d8c4a3.js +1 -0
- sky/dashboard/out/_next/static/chunks/6989-01359c57e018caa4.js +1 -0
- sky/dashboard/out/_next/static/chunks/6990-32b6e2d3822301fa.js +1 -0
- sky/dashboard/out/_next/static/chunks/7359-c8d04e06886000b3.js +30 -0
- sky/dashboard/out/_next/static/chunks/7411-b15471acd2cba716.js +41 -0
- sky/dashboard/out/_next/static/chunks/7615-3301e838e5f25772.js +1 -0
- sky/dashboard/out/_next/static/chunks/8640.5b9475a2d18c5416.js +16 -0
- sky/dashboard/out/_next/static/chunks/8969-1e4613c651bf4051.js +1 -0
- sky/dashboard/out/_next/static/chunks/9025.fa408f3242e9028d.js +6 -0
- sky/dashboard/out/_next/static/chunks/9353-cff34f7e773b2e2b.js +1 -0
- sky/dashboard/out/_next/static/chunks/9360.7310982cf5a0dc79.js +31 -0
- sky/dashboard/out/_next/static/chunks/9847.3aaca6bb33455140.js +30 -0
- sky/dashboard/out/_next/static/chunks/fd9d1056-86323a29a8f7e46a.js +1 -0
- sky/dashboard/out/_next/static/chunks/framework-cf60a09ccd051a10.js +33 -0
- sky/dashboard/out/_next/static/chunks/main-app-587214043926b3cc.js +1 -0
- sky/dashboard/out/_next/static/chunks/main-f15ccb73239a3bf1.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/_app-bde01e4a2beec258.js +34 -0
- sky/dashboard/out/_next/static/chunks/pages/_error-c66a4e8afc46f17b.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-c736ead69c2d86ec.js +16 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-a37d2063af475a1c.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters-d44859594e6f8064.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/config-dfb9bf07b13045f4.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/index-444f1804401f04ea.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/infra/[context]-c0b5935149902e6f.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/infra-aed0ea19df7cf961.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-5796e8d6aea291a0.js +16 -0
- sky/dashboard/out/_next/static/chunks/pages/jobs/pools/[pool]-6edeb7d06032adfc.js +21 -0
- sky/dashboard/out/_next/static/chunks/pages/jobs-479dde13399cf270.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/users-5ab3b907622cf0fe.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/volumes-b84b948ff357c43e.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/workspaces/[name]-c5a3eeee1c218af1.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/workspaces-22b23febb3e89ce1.js +1 -0
- sky/dashboard/out/_next/static/chunks/webpack-2679be77fc08a2f8.js +1 -0
- sky/dashboard/out/_next/static/css/0748ce22df867032.css +3 -0
- sky/dashboard/out/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js +1 -0
- sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
- sky/dashboard/out/clusters/[cluster].html +1 -1
- sky/dashboard/out/clusters.html +1 -1
- sky/dashboard/out/config.html +1 -0
- sky/dashboard/out/index.html +1 -1
- sky/dashboard/out/infra/[context].html +1 -0
- sky/dashboard/out/infra.html +1 -0
- sky/dashboard/out/jobs/[job].html +1 -1
- sky/dashboard/out/jobs/pools/[pool].html +1 -0
- sky/dashboard/out/jobs.html +1 -1
- sky/dashboard/out/users.html +1 -0
- sky/dashboard/out/volumes.html +1 -0
- sky/dashboard/out/workspace/new.html +1 -0
- sky/dashboard/out/workspaces/[name].html +1 -0
- sky/dashboard/out/workspaces.html +1 -0
- sky/data/data_utils.py +137 -1
- sky/data/mounting_utils.py +269 -84
- sky/data/storage.py +1451 -1807
- sky/data/storage_utils.py +43 -57
- sky/exceptions.py +132 -2
- sky/execution.py +206 -63
- sky/global_user_state.py +2374 -586
- sky/jobs/__init__.py +5 -0
- sky/jobs/client/sdk.py +242 -65
- sky/jobs/client/sdk_async.py +143 -0
- sky/jobs/constants.py +9 -8
- sky/jobs/controller.py +839 -277
- sky/jobs/file_content_utils.py +80 -0
- sky/jobs/log_gc.py +201 -0
- sky/jobs/recovery_strategy.py +398 -152
- sky/jobs/scheduler.py +315 -189
- sky/jobs/server/core.py +829 -255
- sky/jobs/server/server.py +156 -115
- sky/jobs/server/utils.py +136 -0
- sky/jobs/state.py +2092 -701
- sky/jobs/utils.py +1242 -160
- sky/logs/__init__.py +21 -0
- sky/logs/agent.py +108 -0
- sky/logs/aws.py +243 -0
- sky/logs/gcp.py +91 -0
- sky/metrics/__init__.py +0 -0
- sky/metrics/utils.py +443 -0
- sky/models.py +78 -1
- sky/optimizer.py +164 -70
- sky/provision/__init__.py +90 -4
- sky/provision/aws/config.py +147 -26
- sky/provision/aws/instance.py +135 -50
- sky/provision/azure/instance.py +10 -5
- sky/provision/common.py +13 -1
- sky/provision/cudo/cudo_machine_type.py +1 -1
- sky/provision/cudo/cudo_utils.py +14 -8
- sky/provision/cudo/cudo_wrapper.py +72 -71
- sky/provision/cudo/instance.py +10 -6
- sky/provision/do/instance.py +10 -6
- sky/provision/do/utils.py +4 -3
- sky/provision/docker_utils.py +114 -23
- sky/provision/fluidstack/instance.py +13 -8
- sky/provision/gcp/__init__.py +1 -0
- sky/provision/gcp/config.py +301 -19
- sky/provision/gcp/constants.py +218 -0
- sky/provision/gcp/instance.py +36 -8
- sky/provision/gcp/instance_utils.py +18 -4
- sky/provision/gcp/volume_utils.py +247 -0
- sky/provision/hyperbolic/__init__.py +12 -0
- sky/provision/hyperbolic/config.py +10 -0
- sky/provision/hyperbolic/instance.py +437 -0
- sky/provision/hyperbolic/utils.py +373 -0
- sky/provision/instance_setup.py +93 -14
- sky/provision/kubernetes/__init__.py +5 -0
- sky/provision/kubernetes/config.py +9 -52
- sky/provision/kubernetes/constants.py +17 -0
- sky/provision/kubernetes/instance.py +789 -247
- sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml +1 -2
- sky/provision/kubernetes/network.py +27 -17
- sky/provision/kubernetes/network_utils.py +40 -43
- sky/provision/kubernetes/utils.py +1192 -531
- sky/provision/kubernetes/volume.py +282 -0
- sky/provision/lambda_cloud/instance.py +22 -16
- sky/provision/nebius/constants.py +50 -0
- sky/provision/nebius/instance.py +19 -6
- sky/provision/nebius/utils.py +196 -91
- sky/provision/oci/instance.py +10 -5
- sky/provision/paperspace/instance.py +10 -7
- sky/provision/paperspace/utils.py +1 -1
- sky/provision/primeintellect/__init__.py +10 -0
- sky/provision/primeintellect/config.py +11 -0
- sky/provision/primeintellect/instance.py +454 -0
- sky/provision/primeintellect/utils.py +398 -0
- sky/provision/provisioner.py +110 -36
- sky/provision/runpod/__init__.py +5 -0
- sky/provision/runpod/instance.py +27 -6
- sky/provision/runpod/utils.py +51 -18
- sky/provision/runpod/volume.py +180 -0
- sky/provision/scp/__init__.py +15 -0
- sky/provision/scp/config.py +93 -0
- sky/provision/scp/instance.py +531 -0
- sky/provision/seeweb/__init__.py +11 -0
- sky/provision/seeweb/config.py +13 -0
- sky/provision/seeweb/instance.py +807 -0
- sky/provision/shadeform/__init__.py +11 -0
- sky/provision/shadeform/config.py +12 -0
- sky/provision/shadeform/instance.py +351 -0
- sky/provision/shadeform/shadeform_utils.py +83 -0
- sky/provision/ssh/__init__.py +18 -0
- sky/provision/vast/instance.py +13 -8
- sky/provision/vast/utils.py +10 -7
- sky/provision/vsphere/common/vim_utils.py +1 -2
- sky/provision/vsphere/instance.py +15 -10
- sky/provision/vsphere/vsphere_utils.py +9 -19
- sky/py.typed +0 -0
- sky/resources.py +844 -118
- sky/schemas/__init__.py +0 -0
- sky/schemas/api/__init__.py +0 -0
- sky/schemas/api/responses.py +225 -0
- sky/schemas/db/README +4 -0
- sky/schemas/db/env.py +90 -0
- sky/schemas/db/global_user_state/001_initial_schema.py +124 -0
- sky/schemas/db/global_user_state/002_add_workspace_to_cluster_history.py +35 -0
- sky/schemas/db/global_user_state/003_fix_initial_revision.py +61 -0
- sky/schemas/db/global_user_state/004_is_managed.py +34 -0
- sky/schemas/db/global_user_state/005_cluster_event.py +32 -0
- sky/schemas/db/global_user_state/006_provision_log.py +41 -0
- sky/schemas/db/global_user_state/007_cluster_event_request_id.py +34 -0
- sky/schemas/db/global_user_state/008_skylet_ssh_tunnel_metadata.py +34 -0
- sky/schemas/db/global_user_state/009_last_activity_and_launched_at.py +89 -0
- sky/schemas/db/global_user_state/010_save_ssh_key.py +66 -0
- sky/schemas/db/script.py.mako +28 -0
- sky/schemas/db/serve_state/001_initial_schema.py +67 -0
- sky/schemas/db/skypilot_config/001_initial_schema.py +30 -0
- sky/schemas/db/spot_jobs/001_initial_schema.py +97 -0
- sky/schemas/db/spot_jobs/002_cluster_pool.py +42 -0
- sky/schemas/db/spot_jobs/003_pool_hash.py +34 -0
- sky/schemas/db/spot_jobs/004_job_file_contents.py +42 -0
- sky/schemas/db/spot_jobs/005_logs_gc.py +38 -0
- sky/schemas/generated/__init__.py +0 -0
- sky/schemas/generated/autostopv1_pb2.py +36 -0
- sky/schemas/generated/autostopv1_pb2.pyi +43 -0
- sky/schemas/generated/autostopv1_pb2_grpc.py +146 -0
- sky/schemas/generated/jobsv1_pb2.py +86 -0
- sky/schemas/generated/jobsv1_pb2.pyi +254 -0
- sky/schemas/generated/jobsv1_pb2_grpc.py +542 -0
- sky/schemas/generated/managed_jobsv1_pb2.py +74 -0
- sky/schemas/generated/managed_jobsv1_pb2.pyi +278 -0
- sky/schemas/generated/managed_jobsv1_pb2_grpc.py +278 -0
- sky/schemas/generated/servev1_pb2.py +58 -0
- sky/schemas/generated/servev1_pb2.pyi +115 -0
- sky/schemas/generated/servev1_pb2_grpc.py +322 -0
- sky/serve/autoscalers.py +357 -5
- sky/serve/client/impl.py +310 -0
- sky/serve/client/sdk.py +47 -139
- sky/serve/client/sdk_async.py +130 -0
- sky/serve/constants.py +10 -8
- sky/serve/controller.py +64 -19
- sky/serve/load_balancer.py +106 -60
- sky/serve/load_balancing_policies.py +115 -1
- sky/serve/replica_managers.py +273 -162
- sky/serve/serve_rpc_utils.py +179 -0
- sky/serve/serve_state.py +554 -251
- sky/serve/serve_utils.py +733 -220
- sky/serve/server/core.py +66 -711
- sky/serve/server/impl.py +1093 -0
- sky/serve/server/server.py +21 -18
- sky/serve/service.py +133 -48
- sky/serve/service_spec.py +135 -16
- sky/serve/spot_placer.py +3 -0
- sky/server/auth/__init__.py +0 -0
- sky/server/auth/authn.py +50 -0
- sky/server/auth/loopback.py +38 -0
- sky/server/auth/oauth2_proxy.py +200 -0
- sky/server/common.py +475 -181
- sky/server/config.py +81 -23
- sky/server/constants.py +44 -6
- sky/server/daemons.py +229 -0
- sky/server/html/token_page.html +185 -0
- sky/server/metrics.py +160 -0
- sky/server/requests/executor.py +528 -138
- sky/server/requests/payloads.py +351 -17
- sky/server/requests/preconditions.py +21 -17
- sky/server/requests/process.py +112 -29
- sky/server/requests/request_names.py +120 -0
- sky/server/requests/requests.py +817 -224
- sky/server/requests/serializers/decoders.py +82 -31
- sky/server/requests/serializers/encoders.py +140 -22
- sky/server/requests/threads.py +106 -0
- sky/server/rest.py +417 -0
- sky/server/server.py +1290 -284
- sky/server/state.py +20 -0
- sky/server/stream_utils.py +345 -57
- sky/server/uvicorn.py +217 -3
- sky/server/versions.py +270 -0
- sky/setup_files/MANIFEST.in +5 -0
- sky/setup_files/alembic.ini +156 -0
- sky/setup_files/dependencies.py +136 -31
- sky/setup_files/setup.py +44 -42
- sky/sky_logging.py +102 -5
- sky/skylet/attempt_skylet.py +1 -0
- sky/skylet/autostop_lib.py +129 -8
- sky/skylet/configs.py +27 -20
- sky/skylet/constants.py +171 -19
- sky/skylet/events.py +105 -21
- sky/skylet/job_lib.py +335 -104
- sky/skylet/log_lib.py +297 -18
- sky/skylet/log_lib.pyi +44 -1
- sky/skylet/ray_patches/__init__.py +17 -3
- sky/skylet/ray_patches/autoscaler.py.diff +18 -0
- sky/skylet/ray_patches/cli.py.diff +19 -0
- sky/skylet/ray_patches/command_runner.py.diff +17 -0
- sky/skylet/ray_patches/log_monitor.py.diff +20 -0
- sky/skylet/ray_patches/resource_demand_scheduler.py.diff +32 -0
- sky/skylet/ray_patches/updater.py.diff +18 -0
- sky/skylet/ray_patches/worker.py.diff +41 -0
- sky/skylet/services.py +564 -0
- sky/skylet/skylet.py +63 -4
- sky/skylet/subprocess_daemon.py +103 -29
- sky/skypilot_config.py +506 -99
- sky/ssh_node_pools/__init__.py +1 -0
- sky/ssh_node_pools/core.py +135 -0
- sky/ssh_node_pools/server.py +233 -0
- sky/task.py +621 -137
- sky/templates/aws-ray.yml.j2 +10 -3
- sky/templates/azure-ray.yml.j2 +1 -1
- sky/templates/do-ray.yml.j2 +1 -1
- sky/templates/gcp-ray.yml.j2 +57 -0
- sky/templates/hyperbolic-ray.yml.j2 +67 -0
- sky/templates/jobs-controller.yaml.j2 +27 -24
- sky/templates/kubernetes-loadbalancer.yml.j2 +2 -0
- sky/templates/kubernetes-ray.yml.j2 +607 -51
- sky/templates/lambda-ray.yml.j2 +1 -1
- sky/templates/nebius-ray.yml.j2 +33 -12
- sky/templates/paperspace-ray.yml.j2 +1 -1
- sky/templates/primeintellect-ray.yml.j2 +71 -0
- sky/templates/runpod-ray.yml.j2 +9 -1
- sky/templates/scp-ray.yml.j2 +3 -50
- sky/templates/seeweb-ray.yml.j2 +108 -0
- sky/templates/shadeform-ray.yml.j2 +72 -0
- sky/templates/sky-serve-controller.yaml.j2 +22 -2
- sky/templates/websocket_proxy.py +178 -18
- sky/usage/usage_lib.py +18 -11
- sky/users/__init__.py +0 -0
- sky/users/model.conf +15 -0
- sky/users/permission.py +387 -0
- sky/users/rbac.py +121 -0
- sky/users/server.py +720 -0
- sky/users/token_service.py +218 -0
- sky/utils/accelerator_registry.py +34 -5
- sky/utils/admin_policy_utils.py +84 -38
- sky/utils/annotations.py +16 -5
- sky/utils/asyncio_utils.py +78 -0
- sky/utils/auth_utils.py +153 -0
- sky/utils/benchmark_utils.py +60 -0
- sky/utils/cli_utils/status_utils.py +159 -86
- sky/utils/cluster_utils.py +31 -9
- sky/utils/command_runner.py +354 -68
- sky/utils/command_runner.pyi +93 -3
- sky/utils/common.py +35 -8
- sky/utils/common_utils.py +310 -87
- sky/utils/config_utils.py +87 -5
- sky/utils/context.py +402 -0
- sky/utils/context_utils.py +222 -0
- sky/utils/controller_utils.py +264 -89
- sky/utils/dag_utils.py +31 -12
- sky/utils/db/__init__.py +0 -0
- sky/utils/db/db_utils.py +470 -0
- sky/utils/db/migration_utils.py +133 -0
- sky/utils/directory_utils.py +12 -0
- sky/utils/env_options.py +13 -0
- sky/utils/git.py +567 -0
- sky/utils/git_clone.sh +460 -0
- sky/utils/infra_utils.py +195 -0
- sky/utils/kubernetes/cleanup-tunnel.sh +62 -0
- sky/utils/kubernetes/config_map_utils.py +133 -0
- sky/utils/kubernetes/create_cluster.sh +13 -27
- sky/utils/kubernetes/delete_cluster.sh +10 -7
- sky/utils/kubernetes/deploy_remote_cluster.py +1299 -0
- sky/utils/kubernetes/exec_kubeconfig_converter.py +22 -31
- sky/utils/kubernetes/generate_kind_config.py +6 -66
- sky/utils/kubernetes/generate_kubeconfig.sh +4 -1
- sky/utils/kubernetes/gpu_labeler.py +5 -5
- sky/utils/kubernetes/kubernetes_deploy_utils.py +354 -47
- sky/utils/kubernetes/ssh-tunnel.sh +379 -0
- sky/utils/kubernetes/ssh_utils.py +221 -0
- sky/utils/kubernetes_enums.py +8 -15
- sky/utils/lock_events.py +94 -0
- sky/utils/locks.py +368 -0
- sky/utils/log_utils.py +300 -6
- sky/utils/perf_utils.py +22 -0
- sky/utils/resource_checker.py +298 -0
- sky/utils/resources_utils.py +249 -32
- sky/utils/rich_utils.py +213 -37
- sky/utils/schemas.py +905 -147
- sky/utils/serialize_utils.py +16 -0
- sky/utils/status_lib.py +10 -0
- sky/utils/subprocess_utils.py +38 -15
- sky/utils/tempstore.py +70 -0
- sky/utils/timeline.py +24 -52
- sky/utils/ux_utils.py +84 -15
- sky/utils/validator.py +11 -1
- sky/utils/volume.py +86 -0
- sky/utils/yaml_utils.py +111 -0
- sky/volumes/__init__.py +13 -0
- sky/volumes/client/__init__.py +0 -0
- sky/volumes/client/sdk.py +149 -0
- sky/volumes/server/__init__.py +0 -0
- sky/volumes/server/core.py +258 -0
- sky/volumes/server/server.py +122 -0
- sky/volumes/volume.py +212 -0
- sky/workspaces/__init__.py +0 -0
- sky/workspaces/core.py +655 -0
- sky/workspaces/server.py +101 -0
- sky/workspaces/utils.py +56 -0
- skypilot_nightly-1.0.0.dev20251107.dist-info/METADATA +675 -0
- skypilot_nightly-1.0.0.dev20251107.dist-info/RECORD +594 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/WHEEL +1 -1
- sky/benchmark/benchmark_state.py +0 -256
- sky/benchmark/benchmark_utils.py +0 -641
- sky/clouds/service_catalog/constants.py +0 -7
- sky/dashboard/out/_next/static/LksQgChY5izXjokL3LcEu/_buildManifest.js +0 -1
- sky/dashboard/out/_next/static/chunks/236-f49500b82ad5392d.js +0 -6
- sky/dashboard/out/_next/static/chunks/312-c3c8845990db8ffc.js +0 -15
- sky/dashboard/out/_next/static/chunks/37-0a572fe0dbb89c4d.js +0 -6
- sky/dashboard/out/_next/static/chunks/678-206dddca808e6d16.js +0 -59
- sky/dashboard/out/_next/static/chunks/845-0f8017370869e269.js +0 -1
- sky/dashboard/out/_next/static/chunks/979-7bf73a4c7cea0f5c.js +0 -1
- sky/dashboard/out/_next/static/chunks/fd9d1056-2821b0f0cabcd8bd.js +0 -1
- sky/dashboard/out/_next/static/chunks/framework-87d061ee6ed71b28.js +0 -33
- sky/dashboard/out/_next/static/chunks/main-app-241eb28595532291.js +0 -1
- sky/dashboard/out/_next/static/chunks/main-e0e2335212e72357.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/_app-e6b013bc3f77ad60.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/_error-1be831200e60c5c0.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-e15db85d0ea1fbe1.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-f383db7389368ea7.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters-a93b93e10b8b074e.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/index-f9f039532ca8cbc4.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-03f279c6741fb48b.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs-a75029b67aab6a2e.js +0 -1
- sky/dashboard/out/_next/static/chunks/webpack-830f59b8404e96b8.js +0 -1
- sky/dashboard/out/_next/static/css/c6933bbb2ce7f4dd.css +0 -3
- sky/jobs/dashboard/dashboard.py +0 -223
- sky/jobs/dashboard/static/favicon.ico +0 -0
- sky/jobs/dashboard/templates/index.html +0 -831
- sky/jobs/server/dashboard_utils.py +0 -69
- sky/skylet/providers/scp/__init__.py +0 -2
- sky/skylet/providers/scp/config.py +0 -149
- sky/skylet/providers/scp/node_provider.py +0 -578
- sky/templates/kubernetes-ssh-jump.yml.j2 +0 -94
- sky/utils/db_utils.py +0 -100
- sky/utils/kubernetes/deploy_remote_cluster.sh +0 -308
- sky/utils/kubernetes/ssh_jump_lifecycle_manager.py +0 -191
- skypilot_nightly-1.0.0.dev20250509.dist-info/METADATA +0 -361
- skypilot_nightly-1.0.0.dev20250509.dist-info/RECORD +0 -396
- /sky/{clouds/service_catalog → catalog}/config.py +0 -0
- /sky/{benchmark → catalog/data_fetchers}/__init__.py +0 -0
- /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_azure.py +0 -0
- /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_fluidstack.py +0 -0
- /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_ibm.py +0 -0
- /sky/{clouds/service_catalog/data_fetchers → client/cli}/__init__.py +0 -0
- /sky/dashboard/out/_next/static/{LksQgChY5izXjokL3LcEu → zB0ed6ge_W1MDszVHhijS}/_ssgManifest.js +0 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/licenses/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/top_level.txt +0 -0
sky/clouds/utils/gcp_utils.py
CHANGED
|
@@ -10,7 +10,7 @@ import dataclasses
|
|
|
10
10
|
import json
|
|
11
11
|
import time
|
|
12
12
|
import typing
|
|
13
|
-
from typing import List, Optional, Set
|
|
13
|
+
from typing import Any, Dict, List, Optional, Set, Tuple
|
|
14
14
|
|
|
15
15
|
import cachetools
|
|
16
16
|
|
|
@@ -18,6 +18,7 @@ from sky import sky_logging
|
|
|
18
18
|
from sky import skypilot_config
|
|
19
19
|
from sky.provision.gcp import constants
|
|
20
20
|
from sky.provision.kubernetes import utils as kubernetes_utils
|
|
21
|
+
from sky.utils import resources_utils
|
|
21
22
|
from sky.utils import subprocess_utils
|
|
22
23
|
|
|
23
24
|
if typing.TYPE_CHECKING:
|
|
@@ -36,9 +37,10 @@ def is_tpu(resources: Optional['resources_lib.Resources']) -> bool:
|
|
|
36
37
|
def is_tpu_vm(resources: Optional['resources_lib.Resources']) -> bool:
|
|
37
38
|
if not is_tpu(resources):
|
|
38
39
|
return False
|
|
39
|
-
assert (resources is not None and
|
|
40
|
+
assert (resources is not None and resources.accelerators is not None and
|
|
41
|
+
len(resources.accelerators) == 1)
|
|
40
42
|
acc, _ = list(resources.accelerators.items())[0]
|
|
41
|
-
if kubernetes_utils.is_tpu_on_gke(acc):
|
|
43
|
+
if kubernetes_utils.is_tpu_on_gke(acc, normalize=False):
|
|
42
44
|
return False
|
|
43
45
|
if resources.accelerator_args is None:
|
|
44
46
|
return True
|
|
@@ -48,7 +50,7 @@ def is_tpu_vm(resources: Optional['resources_lib.Resources']) -> bool:
|
|
|
48
50
|
def is_tpu_vm_pod(resources: Optional['resources_lib.Resources']) -> bool:
|
|
49
51
|
if not is_tpu_vm(resources):
|
|
50
52
|
return False
|
|
51
|
-
assert resources is not None
|
|
53
|
+
assert resources is not None and resources.accelerators is not None
|
|
52
54
|
acc, _ = list(resources.accelerators.items())[0]
|
|
53
55
|
return not acc.endswith('-8')
|
|
54
56
|
|
|
@@ -136,10 +138,16 @@ def _list_reservations_for_instance_type(
|
|
|
136
138
|
For example, if we have a specific reservation with n1-highmem-8
|
|
137
139
|
in us-central1-c. `sky launch --gpus V100` will fail.
|
|
138
140
|
"""
|
|
139
|
-
prioritize_reservations = skypilot_config.
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
('
|
|
141
|
+
prioritize_reservations = skypilot_config.get_effective_region_config(
|
|
142
|
+
cloud='gcp',
|
|
143
|
+
region=None,
|
|
144
|
+
keys=('prioritize_reservations',),
|
|
145
|
+
default_value=False)
|
|
146
|
+
specific_reservations = skypilot_config.get_effective_region_config(
|
|
147
|
+
cloud='gcp',
|
|
148
|
+
region=None,
|
|
149
|
+
keys=('specific_reservations',),
|
|
150
|
+
default_value=[])
|
|
143
151
|
if not prioritize_reservations and not specific_reservations:
|
|
144
152
|
return []
|
|
145
153
|
logger.debug(f'Querying GCP reservations for instance {instance_type!r}')
|
|
@@ -169,14 +177,23 @@ def _list_reservations_for_instance_type(
|
|
|
169
177
|
|
|
170
178
|
def get_minimal_compute_permissions() -> List[str]:
|
|
171
179
|
permissions = copy.copy(constants.VM_MINIMAL_PERMISSIONS)
|
|
172
|
-
if skypilot_config.
|
|
180
|
+
if skypilot_config.get_effective_region_config(
|
|
181
|
+
cloud='gcp', region=None, keys=('vpc_name',),
|
|
182
|
+
default_value=None) is None:
|
|
173
183
|
# If custom VPC is not specified, permissions to modify network are
|
|
174
184
|
# required to ensure SkyPilot to be able to setup the network, and
|
|
175
185
|
# allow opening ports (e.g., via `resources.ports`).
|
|
176
186
|
permissions += constants.FIREWALL_PERMISSIONS
|
|
177
187
|
|
|
178
|
-
if (skypilot_config.
|
|
179
|
-
|
|
188
|
+
if (skypilot_config.get_effective_region_config(
|
|
189
|
+
cloud='gcp',
|
|
190
|
+
region=None,
|
|
191
|
+
keys=('prioritize_reservations',),
|
|
192
|
+
default_value=False) or skypilot_config.get_effective_region_config(
|
|
193
|
+
cloud='gcp',
|
|
194
|
+
region=None,
|
|
195
|
+
keys=('specific_reservations',),
|
|
196
|
+
default_value=[])):
|
|
180
197
|
permissions += constants.RESERVATION_PERMISSIONS
|
|
181
198
|
|
|
182
199
|
permissions += constants.GCP_MINIMAL_PERMISSIONS
|
|
@@ -190,3 +207,62 @@ def get_minimal_storage_permissions() -> List[str]:
|
|
|
190
207
|
permissions += constants.GCP_MINIMAL_PERMISSIONS
|
|
191
208
|
|
|
192
209
|
return permissions
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# Get the DWS configuration for the given context in GKE.
|
|
213
|
+
def get_dws_config(
|
|
214
|
+
context: str,
|
|
215
|
+
k8s_kueue_local_queue_name: Optional[str],
|
|
216
|
+
cluster_config_overrides: Optional[Dict[str, Any]] = None,
|
|
217
|
+
) -> Tuple[bool, bool, Optional[int]]:
|
|
218
|
+
"""Get the DWS configuration for the given context.
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
context: The context to get the DWS configuration for.
|
|
222
|
+
k8s_kueue_local_queue_name: The name of the Kueue local queue.
|
|
223
|
+
cluster_config_overrides: The cluster config overrides.
|
|
224
|
+
|
|
225
|
+
Returns:
|
|
226
|
+
A tuple of (enable_flex_start,
|
|
227
|
+
enable_flex_start_queued_provisioning,
|
|
228
|
+
max_run_duration_seconds).
|
|
229
|
+
|
|
230
|
+
Raises:
|
|
231
|
+
ValueError: If k8s_kueue_local_queue_name is missing to enable
|
|
232
|
+
flex start queued provisioning for the given context.
|
|
233
|
+
"""
|
|
234
|
+
dws_config = skypilot_config.get_effective_region_config(
|
|
235
|
+
cloud='kubernetes',
|
|
236
|
+
region=context,
|
|
237
|
+
keys=('dws',),
|
|
238
|
+
default_value={},
|
|
239
|
+
override_configs=cluster_config_overrides)
|
|
240
|
+
if not dws_config:
|
|
241
|
+
return False, False, None
|
|
242
|
+
|
|
243
|
+
enabled = dws_config.get('enabled', False)
|
|
244
|
+
if not enabled:
|
|
245
|
+
return False, False, None
|
|
246
|
+
|
|
247
|
+
enable_flex_start = False
|
|
248
|
+
enable_flex_start_queued_provisioning = False
|
|
249
|
+
max_run_duration_seconds = None
|
|
250
|
+
# If users already use Kueue, use the flex start with queued
|
|
251
|
+
# provisioning mode.
|
|
252
|
+
if k8s_kueue_local_queue_name:
|
|
253
|
+
enable_flex_start_queued_provisioning = True
|
|
254
|
+
else:
|
|
255
|
+
enable_flex_start = True
|
|
256
|
+
|
|
257
|
+
if not enable_flex_start_queued_provisioning:
|
|
258
|
+
return (enable_flex_start, enable_flex_start_queued_provisioning,
|
|
259
|
+
max_run_duration_seconds)
|
|
260
|
+
|
|
261
|
+
# Max run duration is only used in the flex start with queued
|
|
262
|
+
# provisioning mode.
|
|
263
|
+
max_run_duration = dws_config.get('max_run_duration', None)
|
|
264
|
+
if max_run_duration:
|
|
265
|
+
max_run_duration_seconds = resources_utils.parse_time_minutes(
|
|
266
|
+
max_run_duration) * 60
|
|
267
|
+
return (enable_flex_start, enable_flex_start_queued_provisioning,
|
|
268
|
+
max_run_duration_seconds)
|
sky/clouds/utils/oci_utils.py
CHANGED
|
@@ -105,22 +105,34 @@ class OCIConfig:
|
|
|
105
105
|
@classmethod
|
|
106
106
|
def get_compartment(cls, region):
|
|
107
107
|
# Allow task(cluster)-specific compartment/VCN parameters.
|
|
108
|
-
default_compartment_ocid = skypilot_config.
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
('
|
|
108
|
+
default_compartment_ocid = skypilot_config.get_effective_region_config(
|
|
109
|
+
cloud='oci',
|
|
110
|
+
region='default',
|
|
111
|
+
keys=('compartment_ocid',),
|
|
112
|
+
default_value=None)
|
|
113
|
+
compartment = skypilot_config.get_effective_region_config(
|
|
114
|
+
cloud='oci',
|
|
115
|
+
region=region,
|
|
116
|
+
keys=('compartment_ocid',),
|
|
117
|
+
default_value=default_compartment_ocid)
|
|
112
118
|
return compartment
|
|
113
119
|
|
|
114
120
|
@classmethod
|
|
115
121
|
def get_vcn_ocid(cls, region):
|
|
116
122
|
# Will reuse the regional VCN if specified.
|
|
117
|
-
vcn = skypilot_config.
|
|
123
|
+
vcn = skypilot_config.get_effective_region_config(cloud='oci',
|
|
124
|
+
region=region,
|
|
125
|
+
keys=('vcn_ocid',),
|
|
126
|
+
default_value=None)
|
|
118
127
|
return vcn
|
|
119
128
|
|
|
120
129
|
@classmethod
|
|
121
130
|
def get_vcn_subnet(cls, region):
|
|
122
131
|
# Will reuse the subnet if specified.
|
|
123
|
-
vcn = skypilot_config.
|
|
132
|
+
vcn = skypilot_config.get_effective_region_config(cloud='oci',
|
|
133
|
+
region=region,
|
|
134
|
+
keys=('vcn_subnet',),
|
|
135
|
+
default_value=None)
|
|
124
136
|
return vcn
|
|
125
137
|
|
|
126
138
|
@classmethod
|
|
@@ -129,16 +141,22 @@ class OCIConfig:
|
|
|
129
141
|
# we give a choice to set the default image tag (for gpu instances) in
|
|
130
142
|
# the sky's user-config file (if not specified, use the hardcode one at
|
|
131
143
|
# last)
|
|
132
|
-
return skypilot_config.
|
|
133
|
-
|
|
144
|
+
return skypilot_config.get_effective_region_config(
|
|
145
|
+
cloud='oci',
|
|
146
|
+
region='default',
|
|
147
|
+
keys=('image_tag_gpu',),
|
|
148
|
+
default_value='skypilot:gpu-ubuntu-2204')
|
|
134
149
|
|
|
135
150
|
@classmethod
|
|
136
151
|
def get_default_image_tag(cls) -> str:
|
|
137
152
|
# Get the default image tag. Instead of hardcoding, we give a choice to
|
|
138
153
|
# set the default image tag in the sky's user-config file. (if not
|
|
139
154
|
# specified, use the hardcode one at last)
|
|
140
|
-
return skypilot_config.
|
|
141
|
-
|
|
155
|
+
return skypilot_config.get_effective_region_config(
|
|
156
|
+
cloud='oci',
|
|
157
|
+
region='default',
|
|
158
|
+
keys=('image_tag_general',),
|
|
159
|
+
default_value='skypilot:cpu-ubuntu-2204')
|
|
142
160
|
|
|
143
161
|
@classmethod
|
|
144
162
|
def get_sky_user_config_file(cls) -> str:
|
|
@@ -152,16 +170,22 @@ class OCIConfig:
|
|
|
152
170
|
|
|
153
171
|
@classmethod
|
|
154
172
|
def get_profile(cls) -> str:
|
|
155
|
-
return skypilot_config.
|
|
156
|
-
|
|
173
|
+
return skypilot_config.get_effective_region_config(
|
|
174
|
+
cloud='oci',
|
|
175
|
+
region='default',
|
|
176
|
+
keys=('oci_config_profile',),
|
|
177
|
+
default_value='DEFAULT')
|
|
157
178
|
|
|
158
179
|
@classmethod
|
|
159
180
|
def get_default_image_os(cls) -> str:
|
|
160
181
|
# Get the default image OS. Instead of hardcoding, we give a choice to
|
|
161
182
|
# set the default image OS type in the sky's user-config file. (if not
|
|
162
183
|
# specified, use the hardcode one at last)
|
|
163
|
-
return skypilot_config.
|
|
164
|
-
|
|
184
|
+
return skypilot_config.get_effective_region_config(
|
|
185
|
+
cloud='oci',
|
|
186
|
+
region='default',
|
|
187
|
+
keys=('image_os_type',),
|
|
188
|
+
default_value='ubuntu')
|
|
165
189
|
|
|
166
190
|
|
|
167
191
|
oci_config = OCIConfig()
|
sky/clouds/utils/scp_utils.py
CHANGED
|
@@ -184,7 +184,7 @@ class SCPClient:
|
|
|
184
184
|
|
|
185
185
|
def create_instance(self, instance_config):
|
|
186
186
|
"""Launch new instances."""
|
|
187
|
-
url = f'{API_ENDPOINT}/virtual-server/
|
|
187
|
+
url = f'{API_ENDPOINT}/virtual-server/v4/virtual-servers'
|
|
188
188
|
return self._post(url, instance_config)
|
|
189
189
|
|
|
190
190
|
@_retry
|
|
@@ -226,105 +226,113 @@ class SCPClient:
|
|
|
226
226
|
raise_scp_error(response)
|
|
227
227
|
return response.json()
|
|
228
228
|
|
|
229
|
-
def create_security_group(self, zone_id,
|
|
229
|
+
def create_security_group(self, zone_id, vpc_id, sg_name):
|
|
230
230
|
url = f'{API_ENDPOINT}/security-group/v3/security-groups'
|
|
231
231
|
request_body = {
|
|
232
232
|
'loggable': False,
|
|
233
233
|
'securityGroupName': sg_name,
|
|
234
234
|
'serviceZoneId': zone_id,
|
|
235
|
-
'vpcId':
|
|
236
|
-
'securityGroupDescription': '
|
|
235
|
+
'vpcId': vpc_id,
|
|
236
|
+
'securityGroupDescription': 'sky security group'
|
|
237
237
|
}
|
|
238
238
|
return self._post(url, request_body)
|
|
239
239
|
|
|
240
|
-
def
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
'
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
'
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
'
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
'
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
240
|
+
def _security_group_rule_not_exist(self, sg_id, direction, ports):
|
|
241
|
+
response = self.get_security_group_rules(sg_id)
|
|
242
|
+
rules = []
|
|
243
|
+
for rule in response:
|
|
244
|
+
rule_direction = rule['ruleDirection']
|
|
245
|
+
if rule_direction == direction:
|
|
246
|
+
rules.append(rule)
|
|
247
|
+
for rule in rules:
|
|
248
|
+
port_list = ','.join(rule['tcpServices'])
|
|
249
|
+
port = ','.join(ports)
|
|
250
|
+
if port == port_list:
|
|
251
|
+
return False
|
|
252
|
+
return True
|
|
253
|
+
|
|
254
|
+
def add_security_group_rule(self, sg_id, direction,
|
|
255
|
+
ports: Optional[List[str]]):
|
|
256
|
+
if ports is None:
|
|
257
|
+
if direction == 'IN':
|
|
258
|
+
ports = ['22']
|
|
259
|
+
else:
|
|
260
|
+
ports = ['21', '22', '80', '443']
|
|
261
|
+
services = []
|
|
262
|
+
for port in ports:
|
|
263
|
+
services.append({'serviceType': 'TCP', 'serviceValue': port})
|
|
264
|
+
if self._security_group_rule_not_exist(sg_id, direction, ports):
|
|
265
|
+
url = f'{API_ENDPOINT}/security-group/v2/security-groups/{sg_id}/rules' # pylint: disable=line-too-long
|
|
266
|
+
if direction == 'IN':
|
|
267
|
+
target_address = 'sourceIpAddresses'
|
|
268
|
+
else:
|
|
269
|
+
target_address = 'destinationIpAddresses'
|
|
270
|
+
request_body = {
|
|
271
|
+
'ruleDirection': direction,
|
|
272
|
+
'services': services,
|
|
273
|
+
target_address: ['0.0.0.0/0'],
|
|
274
|
+
'ruleDescription': 'sky security group rule'
|
|
275
|
+
}
|
|
276
|
+
return self._post(url, request_body)
|
|
277
|
+
|
|
278
|
+
def _firewall_rule_not_exist(self, firewall_id, internal_ip, direction,
|
|
279
|
+
ports):
|
|
280
|
+
response = self.get_firewall_rules(firewall_id)
|
|
281
|
+
rules = []
|
|
282
|
+
for rule in response:
|
|
283
|
+
if direction == 'IN':
|
|
284
|
+
if internal_ip == rule['destinationIpAddresses'][0]:
|
|
285
|
+
rules.append(rule)
|
|
286
|
+
else:
|
|
287
|
+
if internal_ip == rule['sourceIpAddresses'][0]:
|
|
288
|
+
rules.append(rule)
|
|
289
|
+
for rule in rules:
|
|
290
|
+
port_list = ','.join(rule['tcpServices'])
|
|
291
|
+
port = ','.join(ports)
|
|
292
|
+
if port == port_list:
|
|
293
|
+
return False
|
|
294
|
+
return True
|
|
295
|
+
|
|
296
|
+
def add_firewall_rule(self, firewall_id, internal_ip, direction, ports):
|
|
297
|
+
if ports is None:
|
|
298
|
+
if direction == 'IN':
|
|
299
|
+
ports = ['22']
|
|
300
|
+
else:
|
|
301
|
+
ports = ['21', '22', '80', '443']
|
|
302
|
+
services = []
|
|
303
|
+
for port in ports:
|
|
304
|
+
services.append({'serviceType': 'TCP', 'serviceValue': port})
|
|
305
|
+
if self._firewall_rule_not_exist(firewall_id, internal_ip, direction,
|
|
306
|
+
ports):
|
|
307
|
+
url = f'{API_ENDPOINT}/firewall/v2/firewalls/{firewall_id}/rules'
|
|
308
|
+
if direction == 'IN':
|
|
309
|
+
source_ip = '0.0.0.0/0'
|
|
310
|
+
destination_ip = internal_ip
|
|
311
|
+
else:
|
|
312
|
+
source_ip = internal_ip
|
|
313
|
+
destination_ip = '0.0.0.0/0'
|
|
314
|
+
request_body = {
|
|
315
|
+
'sourceIpAddresses': [source_ip],
|
|
316
|
+
'destinationIpAddresses': [destination_ip],
|
|
317
|
+
'services': services,
|
|
318
|
+
'ruleDirection': direction,
|
|
319
|
+
'ruleAction': 'ALLOW',
|
|
320
|
+
'isRuleEnabled': True,
|
|
321
|
+
'ruleLocationType': 'FIRST',
|
|
322
|
+
'ruleDescription': 'sky firewall rule'
|
|
323
|
+
}
|
|
324
|
+
return self._post(url, request_body)
|
|
317
325
|
|
|
318
|
-
def terminate_instance(self,
|
|
319
|
-
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{
|
|
326
|
+
def terminate_instance(self, instance_id):
|
|
327
|
+
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{instance_id}'
|
|
320
328
|
return self._delete(url)
|
|
321
329
|
|
|
322
|
-
def
|
|
330
|
+
def get_instances(self) -> List[dict]:
|
|
323
331
|
"""List existing instances."""
|
|
324
332
|
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers'
|
|
325
333
|
return self._get(url)
|
|
326
334
|
|
|
327
|
-
def
|
|
335
|
+
def get_catalog(self) -> Dict[str, Any]:
|
|
328
336
|
"""List offered instances and their availability."""
|
|
329
337
|
response = requests.get(f'{API_ENDPOINT}/instance-types',
|
|
330
338
|
headers=self.headers)
|
|
@@ -362,89 +370,134 @@ class SCPClient:
|
|
|
362
370
|
self.headers['X-Cmp-Timestamp'] = self.timestamp
|
|
363
371
|
|
|
364
372
|
def set_signature(self, method: str, url: str) -> None:
|
|
365
|
-
|
|
366
373
|
self.signature = self.get_signature(url=url, method=method)
|
|
367
374
|
self.headers['X-Cmp-Signature'] = self.signature
|
|
368
375
|
|
|
369
|
-
def
|
|
370
|
-
|
|
371
|
-
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{virtual_server_id}/nics' # pylint: disable=line-too-long
|
|
376
|
+
def get_nic(self, instance_id) -> List[dict]:
|
|
377
|
+
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{instance_id}/nics' # pylint: disable=line-too-long
|
|
372
378
|
return self._get(url)
|
|
373
379
|
|
|
374
|
-
def get_external_ip(self,
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
nic_details['subnetType'] == 'PUBLIC'):
|
|
380
|
-
return nic_details['natIp']
|
|
380
|
+
def get_external_ip(self, instance_id, ip):
|
|
381
|
+
nics = self.get_nic(instance_id=instance_id)
|
|
382
|
+
for nic in nics:
|
|
383
|
+
if (nic['ip'] == ip and nic['subnetType'] == 'PUBLIC'):
|
|
384
|
+
return nic['natIp']
|
|
381
385
|
return None
|
|
382
386
|
|
|
383
|
-
def
|
|
387
|
+
def get_zones(self) -> List[dict]:
|
|
384
388
|
url = f'{API_ENDPOINT}/project/v3/projects/{self.project_id}/zones'
|
|
385
389
|
return self._get(url)
|
|
386
390
|
|
|
387
|
-
def
|
|
388
|
-
url = f'{API_ENDPOINT}/product/v2/zones/{service_zone_id}/products'
|
|
389
|
-
return self._get(url)
|
|
390
|
-
|
|
391
|
-
def list_product_groups(self, service_zone_id) -> List[dict]:
|
|
392
|
-
url = f'{API_ENDPOINT}/product/v2/zones/{service_zone_id}/product-groups' # pylint: disable=line-too-long
|
|
393
|
-
return self._get(url)
|
|
394
|
-
|
|
395
|
-
def list_vpcs(self, service_zone_id) -> List[dict]:
|
|
391
|
+
def get_vpcs(self, service_zone_id) -> List[dict]:
|
|
396
392
|
url = f'{API_ENDPOINT}/vpc/v2/vpcs?serviceZoneId={service_zone_id}'
|
|
397
393
|
return self._get(url)
|
|
398
394
|
|
|
399
|
-
def
|
|
395
|
+
def get_subnets(self) -> List[dict]:
|
|
400
396
|
url = f'{API_ENDPOINT}/subnet/v2/subnets?subnetTypes=PUBLIC'
|
|
401
397
|
return self._get(url)
|
|
402
398
|
|
|
403
|
-
def
|
|
399
|
+
def delete_security_group(self, sg_id):
|
|
404
400
|
url = f'{API_ENDPOINT}/security-group/v2/security-groups/{sg_id}'
|
|
405
401
|
return self._delete(url)
|
|
406
402
|
|
|
407
|
-
def
|
|
403
|
+
def delete_firewall_rule(self, firewall_id, rule_ids):
|
|
408
404
|
url = f'{API_ENDPOINT}/firewall/v2/firewalls/{firewall_id}/rules'
|
|
409
|
-
request_body = {'ruleDeletionType': 'PARTIAL', 'ruleIds':
|
|
405
|
+
request_body = {'ruleDeletionType': 'PARTIAL', 'ruleIds': rule_ids}
|
|
410
406
|
return self._delete(url, request_body=request_body)
|
|
411
407
|
|
|
412
|
-
def
|
|
408
|
+
def get_security_groups(self, vpc_id=None, sg_name=None):
|
|
413
409
|
url = f'{API_ENDPOINT}/security-group/v2/security-groups'
|
|
414
410
|
parameter = []
|
|
415
411
|
if vpc_id is not None:
|
|
416
412
|
parameter.append('vpcId=' + vpc_id)
|
|
417
413
|
if sg_name is not None:
|
|
418
414
|
parameter.append('securityGroupName=' + sg_name)
|
|
419
|
-
if parameter:
|
|
415
|
+
if len(parameter) > 0:
|
|
420
416
|
url = url + '?' + '&'.join(parameter)
|
|
421
417
|
return self._get(url)
|
|
422
418
|
|
|
423
|
-
def
|
|
419
|
+
def get_internet_gateway(self):
|
|
424
420
|
url = f'{API_ENDPOINT}/internet-gateway/v2/internet-gateways'
|
|
425
421
|
return self._get(url)
|
|
426
422
|
|
|
427
|
-
def
|
|
428
|
-
url = f'{API_ENDPOINT}/virtual-server/v3/virtual-servers/{vm_id}'
|
|
429
|
-
return self._get(url, contents_key=None)
|
|
430
|
-
|
|
431
|
-
def get_firewal_rule_info(self, firewall_id, rule_id):
|
|
423
|
+
def get_firewall_rule_info(self, firewall_id, rule_id):
|
|
432
424
|
url = f'{API_ENDPOINT}/firewall/v2/firewalls/{firewall_id}/rules/{rule_id}' # pylint: disable=line-too-long
|
|
433
425
|
return self._get(url, contents_key=None)
|
|
434
426
|
|
|
435
|
-
def
|
|
427
|
+
def get_firewalls(self):
|
|
436
428
|
url = f'{API_ENDPOINT}/firewall/v2/firewalls'
|
|
437
429
|
return self._get(url)
|
|
438
430
|
|
|
439
|
-
def
|
|
431
|
+
def get_service_zone_names(self):
|
|
440
432
|
url = f'{API_ENDPOINT}/project/v3/projects/{self.project_id}/zones'
|
|
441
433
|
zone_contents = self._get(url)
|
|
442
434
|
return [content['serviceZoneName'] for content in zone_contents]
|
|
443
435
|
|
|
444
|
-
def start_instance(self,
|
|
445
|
-
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{
|
|
436
|
+
def start_instance(self, instance_id):
|
|
437
|
+
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{instance_id}/start' # pylint: disable=line-too-long
|
|
446
438
|
return self._post(url=url, request_body={})
|
|
447
439
|
|
|
448
|
-
def stop_instance(self,
|
|
449
|
-
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{
|
|
440
|
+
def stop_instance(self, instance_id):
|
|
441
|
+
url = f'{API_ENDPOINT}/virtual-server/v2/virtual-servers/{instance_id}/stop' # pylint: disable=line-too-long
|
|
450
442
|
return self._post(url=url, request_body={})
|
|
443
|
+
|
|
444
|
+
def get_security_group_rules(self, sg_id):
|
|
445
|
+
url = f'{API_ENDPOINT}/security-group/v2/security-groups/{sg_id}/rules'
|
|
446
|
+
return self._get(url)
|
|
447
|
+
|
|
448
|
+
def get_firewall_rules(self, firewall_id):
|
|
449
|
+
url = f'{API_ENDPOINT}/firewall/v2/firewalls/{firewall_id}/rules'
|
|
450
|
+
return self._get(url)
|
|
451
|
+
|
|
452
|
+
def get_instance_info(self, instance_id):
|
|
453
|
+
url = f'{API_ENDPOINT}/virtual-server/v3/virtual-servers/{instance_id}'
|
|
454
|
+
return self._get(url=url, contents_key=None)
|
|
455
|
+
|
|
456
|
+
def create_vpc(self, zone_id):
|
|
457
|
+
vpc_name = 'skyvpc' + zone_id[5:10]
|
|
458
|
+
request_body = {
|
|
459
|
+
'serviceZoneId': zone_id,
|
|
460
|
+
'vpcName': vpc_name,
|
|
461
|
+
'vpcDescription': 'sky vpc'
|
|
462
|
+
}
|
|
463
|
+
url = f'{API_ENDPOINT}/vpc/v3/vpcs'
|
|
464
|
+
return self._post(url, request_body)
|
|
465
|
+
|
|
466
|
+
def create_subnet(self, vpc_id, zone_id):
|
|
467
|
+
subnet_name = 'skysubnet' + zone_id[5:10]
|
|
468
|
+
request_body = {
|
|
469
|
+
'subnetCidrBlock': '192.168.0.0/24',
|
|
470
|
+
'subnetName': subnet_name,
|
|
471
|
+
'subnetType': 'PUBLIC',
|
|
472
|
+
'vpcId': vpc_id,
|
|
473
|
+
'subnetDescription': 'sky subnet'
|
|
474
|
+
}
|
|
475
|
+
url = f'{API_ENDPOINT}/subnet/v2/subnets'
|
|
476
|
+
return self._post(url, request_body)
|
|
477
|
+
|
|
478
|
+
def create_internet_gateway(self, vpc_id):
|
|
479
|
+
request_body = {
|
|
480
|
+
'firewallEnabled': True,
|
|
481
|
+
'firewallLoggable': False,
|
|
482
|
+
'internetGatewayType': 'SHARED',
|
|
483
|
+
'vpcId': vpc_id,
|
|
484
|
+
'internetGatewayDescription': 'sky internet gateway'
|
|
485
|
+
}
|
|
486
|
+
url = f'{API_ENDPOINT}/internet-gateway/v4/internet-gateways'
|
|
487
|
+
return self._post(url, request_body)
|
|
488
|
+
|
|
489
|
+
def get_vpc_info(self, vpc_id):
|
|
490
|
+
url = f'{API_ENDPOINT}/vpc/v2/vpcs/{vpc_id}'
|
|
491
|
+
return self._get(url=url, contents_key=None)
|
|
492
|
+
|
|
493
|
+
def get_subnet_info(self, subnet_id):
|
|
494
|
+
url = f'{API_ENDPOINT}/subnet/v2/subnets/{subnet_id}'
|
|
495
|
+
return self._get(url=url, contents_key=None)
|
|
496
|
+
|
|
497
|
+
def get_internet_gateway_info(self, internet_gateway_id):
|
|
498
|
+
url = f'{API_ENDPOINT}/internet-gateway/v2/internet-gateways/{internet_gateway_id}' # pylint: disable=line-too-long
|
|
499
|
+
return self._get(url=url, contents_key=None)
|
|
500
|
+
|
|
501
|
+
def get_key_pairs(self):
|
|
502
|
+
url = f'{API_ENDPOINT}/key-pair/v1/key-pairs'
|
|
503
|
+
return self._get(url=url, contents_key=None)
|