skypilot-nightly 1.0.0.dev20250529__py3-none-any.whl → 1.0.0.dev20250531__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sky/__init__.py +2 -2
- sky/adaptors/nebius.py +99 -16
- sky/authentication.py +54 -7
- sky/backends/backend_utils.py +37 -24
- sky/backends/cloud_vm_ray_backend.py +33 -17
- sky/check.py +1 -1
- sky/cli.py +43 -15
- sky/client/cli.py +43 -15
- sky/clouds/cloud.py +20 -0
- sky/clouds/cudo.py +2 -0
- sky/clouds/do.py +3 -0
- sky/clouds/fluidstack.py +3 -0
- sky/clouds/gcp.py +10 -3
- sky/clouds/kubernetes.py +70 -4
- sky/clouds/lambda_cloud.py +3 -0
- sky/clouds/nebius.py +57 -14
- sky/clouds/paperspace.py +3 -0
- sky/clouds/runpod.py +2 -0
- sky/clouds/scp.py +3 -0
- sky/clouds/vast.py +3 -0
- sky/clouds/vsphere.py +3 -0
- sky/dashboard/out/404.html +1 -1
- sky/dashboard/out/_next/static/bdeJWb62qu7L7FOq1dbXX/_buildManifest.js +1 -0
- sky/dashboard/out/_next/static/chunks/236-7458fda7b295f305.js +6 -0
- sky/dashboard/out/_next/static/chunks/37-b638675d511d58b4.js +6 -0
- sky/dashboard/out/_next/static/chunks/{470-4d003c441839094d.js → 470-9e7a479cc8303baa.js} +1 -1
- sky/dashboard/out/_next/static/chunks/{173-7db8607cefc20f70.js → 614-3d29f98e0634b179.js} +2 -2
- sky/dashboard/out/_next/static/chunks/682-5c12535476a21ce3.js +6 -0
- sky/dashboard/out/_next/static/chunks/798-c0525dc3f21e488d.js +1 -0
- sky/dashboard/out/_next/static/chunks/843-786c36624d5ff61f.js +11 -0
- sky/dashboard/out/_next/static/chunks/856-ab9627e7e8ac35e8.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-8f270e2c9c59fa1a.js +6 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-25edb867a41b6b20.js +6 -0
- sky/dashboard/out/_next/static/chunks/pages/{clusters-943992b84fd6f4ee.js → clusters-f37ff20f0af29aae.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/{config-7c48919fe030bc43.js → config-3c6a2dabf56e8cd6.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/infra/{[context]-909f1ceb0fcf1b99.js → [context]-342bc15bb78ab2e5.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/{infra-d4c6875c88771e17.js → infra-7b4b8e7fa9fa0827.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-c0c1dff3cd463d9e.js +11 -0
- sky/dashboard/out/_next/static/chunks/pages/{jobs-a4efc09e61988f8d.js → jobs-78a6c5ba3e24c0cf.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/{users-b2634885d67c49a6.js → users-89f9212b81d8897e.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/workspace/{new-579b3203c7c19d84.js → new-198b6e00d7d724c5.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/workspaces/{[name]-9388e38fac73ee8f.js → [name]-2ce792183b03c341.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/workspaces-17d41826537196e7.js +1 -0
- sky/dashboard/out/_next/static/chunks/webpack-f27c9a32aa3d9c6d.js +1 -0
- sky/dashboard/out/_next/static/css/2b3ee34e586949a3.css +3 -0
- sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
- sky/dashboard/out/clusters/[cluster].html +1 -1
- sky/dashboard/out/clusters.html +1 -1
- sky/dashboard/out/config.html +1 -1
- sky/dashboard/out/index.html +1 -1
- sky/dashboard/out/infra/[context].html +1 -1
- sky/dashboard/out/infra.html +1 -1
- sky/dashboard/out/jobs/[job].html +1 -1
- sky/dashboard/out/jobs.html +1 -1
- sky/dashboard/out/users.html +1 -1
- sky/dashboard/out/workspace/new.html +1 -1
- sky/dashboard/out/workspaces/[name].html +1 -1
- sky/dashboard/out/workspaces.html +1 -1
- sky/exceptions.py +10 -0
- sky/global_user_state.py +149 -1
- sky/jobs/client/sdk.py +3 -0
- sky/jobs/constants.py +1 -1
- sky/jobs/server/core.py +8 -3
- sky/jobs/state.py +24 -5
- sky/jobs/utils.py +34 -11
- sky/provision/gcp/config.py +3 -1
- sky/provision/gcp/constants.py +10 -0
- sky/provision/kubernetes/utils.py +2 -1
- sky/provision/provisioner.py +15 -10
- sky/resources.py +44 -3
- sky/serve/controller.py +10 -7
- sky/serve/replica_managers.py +22 -18
- sky/serve/service.py +5 -4
- sky/server/common.py +5 -2
- sky/server/constants.py +1 -1
- sky/server/requests/payloads.py +1 -0
- sky/server/stream_utils.py +21 -0
- sky/templates/kubernetes-ray.yml.j2 +26 -1
- sky/utils/common_utils.py +66 -0
- sky/utils/resources_utils.py +26 -0
- sky/utils/rich_utils.py +5 -0
- sky/utils/schemas.py +23 -1
- {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/METADATA +1 -1
- {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/RECORD +90 -91
- sky/dashboard/out/_next/static/HvNkg7hqKM1p0ptAcdDcF/_buildManifest.js +0 -1
- sky/dashboard/out/_next/static/chunks/236-90e5498a5b00ec29.js +0 -6
- sky/dashboard/out/_next/static/chunks/303-2c7b0f7af571710b.js +0 -6
- sky/dashboard/out/_next/static/chunks/320-afea3ddcc5bd1c6c.js +0 -6
- sky/dashboard/out/_next/static/chunks/578-9146658cead92981.js +0 -6
- sky/dashboard/out/_next/static/chunks/843-256ec920f6d5f41f.js +0 -11
- sky/dashboard/out/_next/static/chunks/856-59a1760784c9e770.js +0 -1
- sky/dashboard/out/_next/static/chunks/9f96d65d-5a3e4af68c26849e.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-159bffb2fa34ed54.js +0 -6
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-9506c00257d10dbd.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-6b80e9e0c6aa16a1.js +0 -6
- sky/dashboard/out/_next/static/chunks/pages/workspaces-610c49ae3619ee85.js +0 -1
- sky/dashboard/out/_next/static/chunks/webpack-deda68c926e8d0bc.js +0 -1
- sky/dashboard/out/_next/static/css/ffd1cd601648c303.css +0 -3
- /sky/dashboard/out/_next/static/{HvNkg7hqKM1p0ptAcdDcF → bdeJWb62qu7L7FOq1dbXX}/_ssgManifest.js +0 -0
- /sky/dashboard/out/_next/static/chunks/pages/{_app-a631df412d8172de.js → _app-ad1edd7fe17ea796.js} +0 -0
- {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/WHEEL +0 -0
- {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/licenses/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/top_level.txt +0 -0
@@ -1 +1 @@
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><link rel="preload" href="/dashboard/skypilot.svg" as="image" fetchpriority="high"/><meta name="next-head-count" content="3"/><link rel="preload" href="/dashboard/_next/static/css/ffd1cd601648c303.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/ffd1cd601648c303.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-deda68c926e8d0bc.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-87d061ee6ed71b28.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e0e2335212e72357.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-a631df412d8172de.js" defer=""></script><script src="/dashboard/_next/static/chunks/9f96d65d-5a3e4af68c26849e.js" defer=""></script><script src="/dashboard/_next/static/chunks/173-7db8607cefc20f70.js" defer=""></script><script src="/dashboard/_next/static/chunks/121-8f55ee3fa6301784.js" defer=""></script><script src="/dashboard/_next/static/chunks/320-afea3ddcc5bd1c6c.js" defer=""></script><script src="/dashboard/_next/static/chunks/470-4d003c441839094d.js" defer=""></script><script src="/dashboard/_next/static/chunks/293-351268365226d251.js" defer=""></script><script src="/dashboard/_next/static/chunks/856-59a1760784c9e770.js" defer=""></script><script src="/dashboard/_next/static/chunks/973-1a09cac61cfcc1e1.js" defer=""></script><script src="/dashboard/_next/static/chunks/236-90e5498a5b00ec29.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-610c49ae3619ee85.js" defer=""></script><script src="/dashboard/_next/static/HvNkg7hqKM1p0ptAcdDcF/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/HvNkg7hqKM1p0ptAcdDcF/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div class="min-h-screen bg-gray-50"><div class="fixed top-0 left-0 right-0 z-50 shadow-sm"><div class="fixed top-0 left-0 right-0 bg-white z-30 h-14 px-4 border-b border-gray-200 shadow-sm"><div class="flex items-center h-full"><div class="flex items-center space-x-4 mr-6"><a class="flex items-center px-1 pt-1 h-full" href="/dashboard"><div class="h-20 w-20 flex items-center justify-center"><img alt="SkyPilot Logo" fetchpriority="high" width="80" height="80" decoding="async" data-nimg="1" class="w-full h-full object-contain" style="color:transparent" src="/dashboard/skypilot.svg"/></div></a></div><div class="flex items-center space-x-2 md:space-x-4 mr-6"><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/clusters"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="20" height="8" x="2" y="2" rx="2" ry="2"></rect><rect width="20" height="8" x="2" y="14" rx="2" ry="2"></rect><line x1="6" x2="6.01" y1="6" y2="6"></line><line x1="6" x2="6.01" y1="18" y2="18"></line></svg><span>Clusters</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/jobs"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 20V4a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"></path><rect width="20" height="14" x="2" y="6" rx="2"></rect></svg><span>Jobs</span></a><div class="border-l border-gray-200 h-6 mx-1"></div><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/infra"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2" ry="2"></rect><rect x="9" y="9" width="6" height="6"></rect><line x1="9" y1="1" x2="9" y2="4"></line><line x1="15" y1="1" x2="15" y2="4"></line><line x1="9" y1="20" x2="9" y2="23"></line><line x1="15" y1="20" x2="15" y2="23"></line><line x1="20" y1="9" x2="23" y2="9"></line><line x1="20" y1="14" x2="23" y2="14"></line><line x1="1" y1="9" x2="4" y2="9"></line><line x1="1" y1="14" x2="4" y2="14"></line></svg><span>Infra</span></a><a class="inline-flex items-center border-b-2 border-transparent text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/workspaces"><svg class="w-4 h-4" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><g><path fill="none" d="M0 0h24v24H0z"></path><path d="M3 18.5V5a3 3 0 0 1 3-3h14a1 1 0 0 1 1 1v18a1 1 0 0 1-1 1H6.5A3.5 3.5 0 0 1 3 18.5zM19 20v-3H6.5a1.5 1.5 0 0 0 0 3H19zM10 4H6a1 1 0 0 0-1 1v10.337A3.486 3.486 0 0 1 6.5 15H19V4h-2v8l-3.5-2-3.5 2V4z"></path></g></svg><span>Workspaces</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/users"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-users w-4 h-4"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"></path><circle cx="9" cy="7" r="4"></circle><path d="M22 21v-2a4 4 0 0 0-3-3.87"></path><path d="M16 3.13a4 4 0 0 1 0 7.75"></path></svg><span>Users</span></a></div><div class="flex items-center space-x-1 ml-auto"><a href="https://skypilot.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center px-2 py-1 text-gray-600 hover:text-blue-600 transition-colors duration-150 cursor-pointer" title="Docs" tabindex="0"><span class="mr-1">Docs</span><svg class="w-3.5 h-3.5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a><a href="https://github.com/skypilot-org/skypilot" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="GitHub" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://slack.skypilot.co/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Slack" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path transform="scale(0.85) translate(1.8, 1.8)" d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"></path></svg></a><a href="https://github.com/skypilot-org/skypilot/issues/new" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Leave Feedback" tabindex="0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-square w-5 h-5"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"></path></svg></a><div class="border-l border-gray-200 h-6"></div><a class="inline-flex items-center justify-center p-2 rounded-full transition-colors duration-150 cursor-pointer text-gray-600 hover:bg-gray-100" title="Configuration" tabindex="0" href="/dashboard/config"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-settings w-5 h-5"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"></path><circle cx="12" cy="12" r="3"></circle></svg></a></div></div></div></div><div class="transition-all duration-200 ease-in-out min-h-screen" style="padding-top:56px"><main class="p-6"><div class="flex justify-center items-center h-64"><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:40px;height:40px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span><span class="ml-2 text-gray-500">Loading workspaces...</span></div></main></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"HvNkg7hqKM1p0ptAcdDcF","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>Workspaces | SkyPilot Dashboard</title><link rel="preload" href="/dashboard/skypilot.svg" as="image" fetchpriority="high"/><meta name="next-head-count" content="4"/><link rel="preload" href="/dashboard/_next/static/css/2b3ee34e586949a3.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/2b3ee34e586949a3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-f27c9a32aa3d9c6d.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-87d061ee6ed71b28.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e0e2335212e72357.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ad1edd7fe17ea796.js" defer=""></script><script src="/dashboard/_next/static/chunks/614-3d29f98e0634b179.js" defer=""></script><script src="/dashboard/_next/static/chunks/798-c0525dc3f21e488d.js" defer=""></script><script src="/dashboard/_next/static/chunks/121-8f55ee3fa6301784.js" defer=""></script><script src="/dashboard/_next/static/chunks/470-9e7a479cc8303baa.js" defer=""></script><script src="/dashboard/_next/static/chunks/293-351268365226d251.js" defer=""></script><script src="/dashboard/_next/static/chunks/856-ab9627e7e8ac35e8.js" defer=""></script><script src="/dashboard/_next/static/chunks/973-1a09cac61cfcc1e1.js" defer=""></script><script src="/dashboard/_next/static/chunks/236-7458fda7b295f305.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-17d41826537196e7.js" defer=""></script><script src="/dashboard/_next/static/bdeJWb62qu7L7FOq1dbXX/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/bdeJWb62qu7L7FOq1dbXX/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div class="min-h-screen bg-gray-50"><div class="fixed top-0 left-0 right-0 z-50 shadow-sm"><div class="fixed top-0 left-0 right-0 bg-white z-30 h-14 px-4 border-b border-gray-200 shadow-sm"><div class="flex items-center h-full"><div class="flex items-center space-x-4 mr-6"><a class="flex items-center px-1 pt-1 h-full" href="/dashboard"><div class="h-20 w-20 flex items-center justify-center"><img alt="SkyPilot Logo" fetchpriority="high" width="80" height="80" decoding="async" data-nimg="1" class="w-full h-full object-contain" style="color:transparent" src="/dashboard/skypilot.svg"/></div></a></div><div class="flex items-center space-x-2 md:space-x-4 mr-6"><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/clusters"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="20" height="8" x="2" y="2" rx="2" ry="2"></rect><rect width="20" height="8" x="2" y="14" rx="2" ry="2"></rect><line x1="6" x2="6.01" y1="6" y2="6"></line><line x1="6" x2="6.01" y1="18" y2="18"></line></svg><span>Clusters</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/jobs"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 20V4a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"></path><rect width="20" height="14" x="2" y="6" rx="2"></rect></svg><span>Jobs</span></a><div class="border-l border-gray-200 h-6 mx-1"></div><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/infra"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2" ry="2"></rect><rect x="9" y="9" width="6" height="6"></rect><line x1="9" y1="1" x2="9" y2="4"></line><line x1="15" y1="1" x2="15" y2="4"></line><line x1="9" y1="20" x2="9" y2="23"></line><line x1="15" y1="20" x2="15" y2="23"></line><line x1="20" y1="9" x2="23" y2="9"></line><line x1="20" y1="14" x2="23" y2="14"></line><line x1="1" y1="9" x2="4" y2="9"></line><line x1="1" y1="14" x2="4" y2="14"></line></svg><span>Infra</span></a><a class="inline-flex items-center border-b-2 border-transparent text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/workspaces"><svg class="w-4 h-4" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><g><path fill="none" d="M0 0h24v24H0z"></path><path d="M3 18.5V5a3 3 0 0 1 3-3h14a1 1 0 0 1 1 1v18a1 1 0 0 1-1 1H6.5A3.5 3.5 0 0 1 3 18.5zM19 20v-3H6.5a1.5 1.5 0 0 0 0 3H19zM10 4H6a1 1 0 0 0-1 1v10.337A3.486 3.486 0 0 1 6.5 15H19V4h-2v8l-3.5-2-3.5 2V4z"></path></g></svg><span>Workspaces</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/users"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-users w-4 h-4"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"></path><circle cx="9" cy="7" r="4"></circle><path d="M22 21v-2a4 4 0 0 0-3-3.87"></path><path d="M16 3.13a4 4 0 0 1 0 7.75"></path></svg><span>Users</span></a></div><div class="flex items-center space-x-1 ml-auto"><a href="https://skypilot.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center px-2 py-1 text-gray-600 hover:text-blue-600 transition-colors duration-150 cursor-pointer" title="Docs" tabindex="0"><span class="mr-1">Docs</span><svg class="w-3.5 h-3.5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a><a href="https://github.com/skypilot-org/skypilot" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="GitHub" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://slack.skypilot.co/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Slack" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path transform="scale(0.85) translate(1.8, 1.8)" d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"></path></svg></a><a href="https://github.com/skypilot-org/skypilot/issues/new" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Leave Feedback" tabindex="0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-square w-5 h-5"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"></path></svg></a><div class="border-l border-gray-200 h-6"></div><a class="inline-flex items-center justify-center p-2 rounded-full transition-colors duration-150 cursor-pointer text-gray-600 hover:bg-gray-100" title="Configuration" tabindex="0" href="/dashboard/config"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-settings w-5 h-5"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"></path><circle cx="12" cy="12" r="3"></circle></svg></a></div></div></div></div><div class="transition-all duration-200 ease-in-out min-h-screen" style="padding-top:56px"><main class="p-6"><div class="flex justify-center items-center h-64"><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:40px;height:40px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span><span class="ml-2 text-gray-500">Loading workspaces...</span></div></main></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"bdeJWb62qu7L7FOq1dbXX","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/exceptions.py
CHANGED
@@ -488,6 +488,16 @@ class ApiServerConnectionError(RuntimeError):
|
|
488
488
|
f'Try: curl {server_url}/api/health')
|
489
489
|
|
490
490
|
|
491
|
+
class ApiServerAuthenticationError(RuntimeError):
|
492
|
+
"""Raised when authentication is required for the API server."""
|
493
|
+
|
494
|
+
def __init__(self, server_url: str):
|
495
|
+
super().__init__(
|
496
|
+
f'Authentication required for SkyPilot API server at {server_url}. '
|
497
|
+
f'Please run:\n'
|
498
|
+
f' sky api login -e {server_url}')
|
499
|
+
|
500
|
+
|
491
501
|
class APIVersionMismatchError(RuntimeError):
|
492
502
|
"""Raised when the API version mismatch."""
|
493
503
|
pass
|
sky/global_user_state.py
CHANGED
@@ -22,6 +22,7 @@ from sqlalchemy import orm
|
|
22
22
|
from sqlalchemy.dialects import postgresql
|
23
23
|
from sqlalchemy.dialects import sqlite
|
24
24
|
from sqlalchemy.ext import declarative
|
25
|
+
import yaml
|
25
26
|
|
26
27
|
from sky import models
|
27
28
|
from sky import sky_logging
|
@@ -96,6 +97,12 @@ cluster_table = sqlalchemy.Table(
|
|
96
97
|
sqlalchemy.Column('workspace',
|
97
98
|
sqlalchemy.Text,
|
98
99
|
server_default=constants.SKYPILOT_DEFAULT_WORKSPACE),
|
100
|
+
sqlalchemy.Column('last_creation_yaml',
|
101
|
+
sqlalchemy.Text,
|
102
|
+
server_default=None),
|
103
|
+
sqlalchemy.Column('last_creation_command',
|
104
|
+
sqlalchemy.Text,
|
105
|
+
server_default=None),
|
99
106
|
)
|
100
107
|
|
101
108
|
storage_table = sqlalchemy.Table(
|
@@ -133,6 +140,21 @@ cluster_history_table = sqlalchemy.Table(
|
|
133
140
|
sqlalchemy.Column('user_hash', sqlalchemy.Text),
|
134
141
|
)
|
135
142
|
|
143
|
+
ssh_key_table = sqlalchemy.Table(
|
144
|
+
'ssh_key',
|
145
|
+
Base.metadata,
|
146
|
+
sqlalchemy.Column('user_hash', sqlalchemy.Text, primary_key=True),
|
147
|
+
sqlalchemy.Column('ssh_public_key', sqlalchemy.Text),
|
148
|
+
sqlalchemy.Column('ssh_private_key', sqlalchemy.Text),
|
149
|
+
)
|
150
|
+
|
151
|
+
cluster_yaml_table = sqlalchemy.Table(
|
152
|
+
'cluster_yaml',
|
153
|
+
Base.metadata,
|
154
|
+
sqlalchemy.Column('cluster_name', sqlalchemy.Text, primary_key=True),
|
155
|
+
sqlalchemy.Column('yaml', sqlalchemy.Text),
|
156
|
+
)
|
157
|
+
|
136
158
|
|
137
159
|
def _glob_to_similar(glob_pattern):
|
138
160
|
"""Converts a glob pattern to a PostgreSQL LIKE pattern."""
|
@@ -270,6 +292,19 @@ def create_table():
|
|
270
292
|
default_statement='DEFAULT \'default\'',
|
271
293
|
value_to_replace_existing_entries=constants.
|
272
294
|
SKYPILOT_DEFAULT_WORKSPACE)
|
295
|
+
db_utils.add_column_to_table_sqlalchemy(
|
296
|
+
session,
|
297
|
+
'clusters',
|
298
|
+
'last_creation_yaml',
|
299
|
+
sqlalchemy.Text(),
|
300
|
+
default_statement='DEFAULT NULL',
|
301
|
+
)
|
302
|
+
db_utils.add_column_to_table_sqlalchemy(
|
303
|
+
session,
|
304
|
+
'clusters',
|
305
|
+
'last_creation_command',
|
306
|
+
sqlalchemy.Text(),
|
307
|
+
default_statement='DEFAULT NULL')
|
273
308
|
session.commit()
|
274
309
|
|
275
310
|
|
@@ -318,7 +353,8 @@ def add_or_update_cluster(cluster_name: str,
|
|
318
353
|
requested_resources: Optional[Set[Any]],
|
319
354
|
ready: bool,
|
320
355
|
is_launch: bool = True,
|
321
|
-
config_hash: Optional[str] = None
|
356
|
+
config_hash: Optional[str] = None,
|
357
|
+
task_config: Optional[Dict[str, Any]] = None):
|
322
358
|
"""Adds or updates cluster_name -> cluster_handle mapping.
|
323
359
|
|
324
360
|
Args:
|
@@ -329,6 +365,8 @@ def add_or_update_cluster(cluster_name: str,
|
|
329
365
|
be marked as INIT, otherwise it will be marked as UP.
|
330
366
|
is_launch: if the cluster is firstly launched. If True, the launched_at
|
331
367
|
and last_use will be updated. Otherwise, use the old value.
|
368
|
+
config_hash: Configuration hash for the cluster.
|
369
|
+
task_config: The config of the task being launched.
|
332
370
|
"""
|
333
371
|
# TODO(zhwu): have to be imported here to avoid circular import.
|
334
372
|
from sky import skypilot_config # pylint: disable=import-outside-toplevel
|
@@ -404,6 +442,13 @@ def add_or_update_cluster(cluster_name: str,
|
|
404
442
|
conditional_values.update({
|
405
443
|
'workspace': active_workspace,
|
406
444
|
})
|
445
|
+
if (is_launch and not cluster_row or
|
446
|
+
cluster_row.status != status_lib.ClusterStatus.UP.value):
|
447
|
+
conditional_values.update({
|
448
|
+
'last_creation_yaml': common_utils.dump_yaml_str(task_config)
|
449
|
+
if task_config else None,
|
450
|
+
'last_creation_command': last_use,
|
451
|
+
})
|
407
452
|
|
408
453
|
if (_SQLALCHEMY_ENGINE.dialect.name ==
|
409
454
|
db_utils.SQLAlchemyDialect.SQLITE.value):
|
@@ -790,6 +835,8 @@ def get_cluster_from_name(
|
|
790
835
|
'user_name': get_user(user_hash).name,
|
791
836
|
'config_hash': row.config_hash,
|
792
837
|
'workspace': row.workspace,
|
838
|
+
'last_creation_yaml': row.last_creation_yaml,
|
839
|
+
'last_creation_command': row.last_creation_command,
|
793
840
|
}
|
794
841
|
|
795
842
|
return record
|
@@ -822,6 +869,8 @@ def get_clusters() -> List[Dict[str, Any]]:
|
|
822
869
|
'user_name': get_user(user_hash).name,
|
823
870
|
'config_hash': row.config_hash,
|
824
871
|
'workspace': row.workspace,
|
872
|
+
'last_creation_yaml': row.last_creation_yaml,
|
873
|
+
'last_creation_command': row.last_creation_command,
|
825
874
|
}
|
826
875
|
|
827
876
|
records.append(record)
|
@@ -1049,3 +1098,102 @@ def get_storage() -> List[Dict[str, Any]]:
|
|
1049
1098
|
'status': status_lib.StorageStatus[row.status],
|
1050
1099
|
})
|
1051
1100
|
return records
|
1101
|
+
|
1102
|
+
|
1103
|
+
def get_ssh_keys(user_hash: str) -> Tuple[str, str, bool]:
|
1104
|
+
with orm.Session(_SQLALCHEMY_ENGINE) as session:
|
1105
|
+
row = session.query(ssh_key_table).filter_by(
|
1106
|
+
user_hash=user_hash).first()
|
1107
|
+
if row:
|
1108
|
+
return row.ssh_public_key, row.ssh_private_key, True
|
1109
|
+
return '', '', False
|
1110
|
+
|
1111
|
+
|
1112
|
+
def set_ssh_keys(user_hash: str, ssh_public_key: str, ssh_private_key: str):
|
1113
|
+
with orm.Session(_SQLALCHEMY_ENGINE) as session:
|
1114
|
+
if (_SQLALCHEMY_ENGINE.dialect.name ==
|
1115
|
+
db_utils.SQLAlchemyDialect.SQLITE.value):
|
1116
|
+
insert_func = sqlite.insert
|
1117
|
+
elif (_SQLALCHEMY_ENGINE.dialect.name ==
|
1118
|
+
db_utils.SQLAlchemyDialect.POSTGRESQL.value):
|
1119
|
+
insert_func = postgresql.insert
|
1120
|
+
else:
|
1121
|
+
raise ValueError('Unsupported database dialect')
|
1122
|
+
insert_stmnt = insert_func(ssh_key_table).values(
|
1123
|
+
user_hash=user_hash,
|
1124
|
+
ssh_public_key=ssh_public_key,
|
1125
|
+
ssh_private_key=ssh_private_key)
|
1126
|
+
do_update_stmt = insert_stmnt.on_conflict_do_update(
|
1127
|
+
index_elements=[ssh_key_table.c.user_hash],
|
1128
|
+
set_={
|
1129
|
+
ssh_key_table.c.ssh_public_key: ssh_public_key,
|
1130
|
+
ssh_key_table.c.ssh_private_key: ssh_private_key
|
1131
|
+
})
|
1132
|
+
session.execute(do_update_stmt)
|
1133
|
+
session.commit()
|
1134
|
+
|
1135
|
+
|
1136
|
+
def get_cluster_yaml_str(cluster_yaml_path: Optional[str]) -> Optional[str]:
|
1137
|
+
"""Get the cluster yaml from the database or the local file system.
|
1138
|
+
If the cluster yaml is not in the database, check if it exists on the
|
1139
|
+
local file system and migrate it to the database.
|
1140
|
+
|
1141
|
+
It is assumed that the cluster yaml file is named as <cluster_name>.yml.
|
1142
|
+
"""
|
1143
|
+
if cluster_yaml_path is None:
|
1144
|
+
raise ValueError('Attempted to read a None YAML.')
|
1145
|
+
cluster_file_name = os.path.basename(cluster_yaml_path)
|
1146
|
+
cluster_name, _ = os.path.splitext(cluster_file_name)
|
1147
|
+
with orm.Session(_SQLALCHEMY_ENGINE) as session:
|
1148
|
+
row = session.query(cluster_yaml_table).filter_by(
|
1149
|
+
cluster_name=cluster_name).first()
|
1150
|
+
if row is None:
|
1151
|
+
# If the cluster yaml is not in the database, check if it exists
|
1152
|
+
# on the local file system and migrate it to the database.
|
1153
|
+
# TODO(syang): remove this check once we have a way to migrate the
|
1154
|
+
# cluster from file to database. Remove on v0.12.0.
|
1155
|
+
if cluster_yaml_path is not None and os.path.exists(cluster_yaml_path):
|
1156
|
+
with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
|
1157
|
+
yaml_str = f.read()
|
1158
|
+
set_cluster_yaml(cluster_name, yaml_str)
|
1159
|
+
return yaml_str
|
1160
|
+
return None
|
1161
|
+
return row.yaml
|
1162
|
+
|
1163
|
+
|
1164
|
+
def get_cluster_yaml_dict(cluster_yaml_path: Optional[str]) -> Dict[str, Any]:
|
1165
|
+
"""Get the cluster yaml as a dictionary from the database.
|
1166
|
+
|
1167
|
+
It is assumed that the cluster yaml file is named as <cluster_name>.yml.
|
1168
|
+
"""
|
1169
|
+
yaml_str = get_cluster_yaml_str(cluster_yaml_path)
|
1170
|
+
if yaml_str is None:
|
1171
|
+
raise ValueError(f'Cluster yaml {cluster_yaml_path} not found.')
|
1172
|
+
return yaml.safe_load(yaml_str)
|
1173
|
+
|
1174
|
+
|
1175
|
+
def set_cluster_yaml(cluster_name: str, yaml_str: str) -> None:
|
1176
|
+
"""Set the cluster yaml in the database."""
|
1177
|
+
with orm.Session(_SQLALCHEMY_ENGINE) as session:
|
1178
|
+
if (_SQLALCHEMY_ENGINE.dialect.name ==
|
1179
|
+
db_utils.SQLAlchemyDialect.SQLITE.value):
|
1180
|
+
insert_func = sqlite.insert
|
1181
|
+
elif (_SQLALCHEMY_ENGINE.dialect.name ==
|
1182
|
+
db_utils.SQLAlchemyDialect.POSTGRESQL.value):
|
1183
|
+
insert_func = postgresql.insert
|
1184
|
+
else:
|
1185
|
+
raise ValueError('Unsupported database dialect')
|
1186
|
+
insert_stmnt = insert_func(cluster_yaml_table).values(
|
1187
|
+
cluster_name=cluster_name, yaml=yaml_str)
|
1188
|
+
do_update_stmt = insert_stmnt.on_conflict_do_update(
|
1189
|
+
index_elements=[cluster_yaml_table.c.cluster_name],
|
1190
|
+
set_={cluster_yaml_table.c.yaml: yaml_str})
|
1191
|
+
session.execute(do_update_stmt)
|
1192
|
+
session.commit()
|
1193
|
+
|
1194
|
+
|
1195
|
+
def remove_cluster_yaml(cluster_name: str):
|
1196
|
+
with orm.Session(_SQLALCHEMY_ENGINE) as session:
|
1197
|
+
session.query(cluster_yaml_table).filter_by(
|
1198
|
+
cluster_name=cluster_name).delete()
|
1199
|
+
session.commit()
|
sky/jobs/client/sdk.py
CHANGED
@@ -192,6 +192,7 @@ def tail_logs(name: Optional[str] = None,
|
|
192
192
|
follow: bool = True,
|
193
193
|
controller: bool = False,
|
194
194
|
refresh: bool = False,
|
195
|
+
tail: Optional[int] = None,
|
195
196
|
output_stream: Optional['io.TextIOBase'] = None) -> int:
|
196
197
|
"""Tails logs of managed jobs.
|
197
198
|
|
@@ -204,6 +205,7 @@ def tail_logs(name: Optional[str] = None,
|
|
204
205
|
follow: Whether to follow the logs.
|
205
206
|
controller: Whether to tail logs from the jobs controller.
|
206
207
|
refresh: Whether to restart the jobs controller if it is stopped.
|
208
|
+
tail: Number of lines to tail from the end of the log file.
|
207
209
|
output_stream: The stream to write the logs to. If None, print to the
|
208
210
|
console.
|
209
211
|
|
@@ -222,6 +224,7 @@ def tail_logs(name: Optional[str] = None,
|
|
222
224
|
follow=follow,
|
223
225
|
controller=controller,
|
224
226
|
refresh=refresh,
|
227
|
+
tail=tail,
|
225
228
|
)
|
226
229
|
response = requests.post(
|
227
230
|
f'{server_common.get_server_url()}/jobs/logs',
|
sky/jobs/constants.py
CHANGED
@@ -47,7 +47,7 @@ JOBS_CLUSTER_NAME_PREFIX_LENGTH = 25
|
|
47
47
|
# The version of the lib files that jobs/utils use. Whenever there is an API
|
48
48
|
# change for the jobs/utils, we need to bump this version and update
|
49
49
|
# job.utils.ManagedJobCodeGen to handle the version update.
|
50
|
-
MANAGED_JOBS_VERSION =
|
50
|
+
MANAGED_JOBS_VERSION = 6
|
51
51
|
|
52
52
|
# The command for setting up the jobs dashboard on the controller. It firstly
|
53
53
|
# checks if the systemd services are available, and if not (e.g., Kubernetes
|
sky/jobs/server/core.py
CHANGED
@@ -521,8 +521,12 @@ def cancel(name: Optional[str] = None,
|
|
521
521
|
|
522
522
|
|
523
523
|
@usage_lib.entrypoint
|
524
|
-
def tail_logs(name: Optional[str],
|
525
|
-
|
524
|
+
def tail_logs(name: Optional[str],
|
525
|
+
job_id: Optional[int],
|
526
|
+
follow: bool,
|
527
|
+
controller: bool,
|
528
|
+
refresh: bool,
|
529
|
+
tail: Optional[int] = None) -> int:
|
526
530
|
# NOTE(dev): Keep the docstring consistent between the Python API and CLI.
|
527
531
|
"""Tail logs of managed jobs.
|
528
532
|
|
@@ -565,7 +569,8 @@ def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
|
|
565
569
|
job_id=job_id,
|
566
570
|
job_name=name,
|
567
571
|
follow=follow,
|
568
|
-
controller=controller
|
572
|
+
controller=controller,
|
573
|
+
tail=tail)
|
569
574
|
|
570
575
|
|
571
576
|
def start_dashboard_forwarding(refresh: bool = False) -> Tuple[int, int]:
|
sky/jobs/state.py
CHANGED
@@ -121,7 +121,8 @@ def create_table(cursor, conn):
|
|
121
121
|
env_file_path TEXT,
|
122
122
|
user_hash TEXT,
|
123
123
|
workspace TEXT DEFAULT NULL,
|
124
|
-
priority INTEGER DEFAULT 500
|
124
|
+
priority INTEGER DEFAULT 500,
|
125
|
+
entrypoint TEXT DEFAULT NULL)""")
|
125
126
|
|
126
127
|
db_utils.add_column_to_table(cursor, conn, 'job_info', 'schedule_state',
|
127
128
|
'TEXT')
|
@@ -151,6 +152,7 @@ def create_table(cursor, conn):
|
|
151
152
|
'INTEGER',
|
152
153
|
value_to_replace_existing_entries=500)
|
153
154
|
|
155
|
+
db_utils.add_column_to_table(cursor, conn, 'job_info', 'entrypoint', 'TEXT')
|
154
156
|
conn.commit()
|
155
157
|
|
156
158
|
|
@@ -209,6 +211,7 @@ columns = [
|
|
209
211
|
'user_hash',
|
210
212
|
'workspace',
|
211
213
|
'priority',
|
214
|
+
'entrypoint',
|
212
215
|
]
|
213
216
|
|
214
217
|
|
@@ -412,14 +415,15 @@ class ManagedJobScheduleState(enum.Enum):
|
|
412
415
|
|
413
416
|
|
414
417
|
# === Status transition functions ===
|
415
|
-
def set_job_info(job_id: int, name: str, workspace: str):
|
418
|
+
def set_job_info(job_id: int, name: str, workspace: str, entrypoint: str):
|
416
419
|
with db_utils.safe_cursor(_DB_PATH) as cursor:
|
417
420
|
cursor.execute(
|
418
421
|
"""\
|
419
422
|
INSERT INTO job_info
|
420
|
-
(spot_job_id, name, schedule_state, workspace)
|
421
|
-
VALUES (?, ?, ?, ?)""",
|
422
|
-
(job_id, name, ManagedJobScheduleState.INACTIVE.value, workspace
|
423
|
+
(spot_job_id, name, schedule_state, workspace, entrypoint)
|
424
|
+
VALUES (?, ?, ?, ?, ?)""",
|
425
|
+
(job_id, name, ManagedJobScheduleState.INACTIVE.value, workspace,
|
426
|
+
entrypoint))
|
423
427
|
|
424
428
|
|
425
429
|
def set_pending(job_id: int, task_id: int, task_name: str, resources_str: str):
|
@@ -1008,6 +1012,21 @@ def get_managed_jobs(job_id: Optional[int] = None) -> List[Dict[str, Any]]:
|
|
1008
1012
|
job_dict['schedule_state'])
|
1009
1013
|
if job_dict['job_name'] is None:
|
1010
1014
|
job_dict['job_name'] = job_dict['task_name']
|
1015
|
+
|
1016
|
+
# Add YAML content and command for managed jobs
|
1017
|
+
dag_yaml_path = job_dict.get('dag_yaml_path')
|
1018
|
+
if dag_yaml_path:
|
1019
|
+
try:
|
1020
|
+
with open(dag_yaml_path, 'r', encoding='utf-8') as f:
|
1021
|
+
job_dict['dag_yaml'] = f.read()
|
1022
|
+
except (FileNotFoundError, IOError, OSError):
|
1023
|
+
job_dict['dag_yaml'] = None
|
1024
|
+
|
1025
|
+
# Generate a command that could be used to launch this job
|
1026
|
+
# Format: sky jobs launch <yaml_path>
|
1027
|
+
else:
|
1028
|
+
job_dict['dag_yaml'] = None
|
1029
|
+
|
1011
1030
|
jobs.append(job_dict)
|
1012
1031
|
return jobs
|
1013
1032
|
|
sky/jobs/utils.py
CHANGED
@@ -13,7 +13,7 @@ import textwrap
|
|
13
13
|
import time
|
14
14
|
import traceback
|
15
15
|
import typing
|
16
|
-
from typing import Any, Dict, List, Optional, Set, Tuple, Union
|
16
|
+
from typing import Any, Deque, Dict, List, Optional, Set, TextIO, Tuple, Union
|
17
17
|
|
18
18
|
import colorama
|
19
19
|
import filelock
|
@@ -546,7 +546,9 @@ def cancel_job_by_name(job_name: str,
|
|
546
546
|
return f'{job_name!r} {msg}'
|
547
547
|
|
548
548
|
|
549
|
-
def stream_logs_by_id(job_id: int,
|
549
|
+
def stream_logs_by_id(job_id: int,
|
550
|
+
follow: bool = True,
|
551
|
+
tail: Optional[int] = None) -> Tuple[str, int]:
|
550
552
|
"""Stream logs by job id.
|
551
553
|
|
552
554
|
Returns:
|
@@ -583,7 +585,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
|
|
583
585
|
# Stream the logs to the console without reading the whole
|
584
586
|
# file into memory.
|
585
587
|
start_streaming = False
|
586
|
-
|
588
|
+
read_from: Union[TextIO, Deque[str]] = f
|
589
|
+
if tail is not None:
|
590
|
+
assert tail > 0
|
591
|
+
# Read only the last 'tail' lines using deque
|
592
|
+
read_from = collections.deque(f, maxlen=tail)
|
593
|
+
for line in read_from:
|
587
594
|
if log_lib.LOG_FILE_START_STREAMING_AT in line:
|
588
595
|
start_streaming = True
|
589
596
|
if start_streaming:
|
@@ -644,10 +651,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
|
|
644
651
|
managed_job_state.ManagedJobStatus.RUNNING)
|
645
652
|
assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
|
646
653
|
status_display.stop()
|
654
|
+
tail_param = tail if tail is not None else 0
|
647
655
|
returncode = backend.tail_logs(handle,
|
648
656
|
job_id=None,
|
649
657
|
managed_job_id=job_id,
|
650
|
-
follow=follow
|
658
|
+
follow=follow,
|
659
|
+
tail=tail_param)
|
651
660
|
if returncode in [rc.value for rc in exceptions.JobExitCode]:
|
652
661
|
# If the log tailing exits with a known exit code we can safely
|
653
662
|
# break the loop because it indicates the tailing process
|
@@ -784,7 +793,8 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
|
|
784
793
|
def stream_logs(job_id: Optional[int],
|
785
794
|
job_name: Optional[str],
|
786
795
|
controller: bool = False,
|
787
|
-
follow: bool = True
|
796
|
+
follow: bool = True,
|
797
|
+
tail: Optional[int] = None) -> Tuple[str, int]:
|
788
798
|
"""Stream logs by job id or job name.
|
789
799
|
|
790
800
|
Returns:
|
@@ -855,7 +865,12 @@ def stream_logs(job_id: Optional[int],
|
|
855
865
|
with open(controller_log_path, 'r', newline='', encoding='utf-8') as f:
|
856
866
|
# Note: we do not need to care about start_stream_at here, since
|
857
867
|
# that should be in the job log printed above.
|
858
|
-
|
868
|
+
read_from: Union[TextIO, Deque[str]] = f
|
869
|
+
if tail is not None:
|
870
|
+
assert tail > 0
|
871
|
+
# Read only the last 'tail' lines efficiently using deque
|
872
|
+
read_from = collections.deque(f, maxlen=tail)
|
873
|
+
for line in read_from:
|
859
874
|
print(line, end='')
|
860
875
|
# Flush.
|
861
876
|
print(end='', flush=True)
|
@@ -907,7 +922,7 @@ def stream_logs(job_id: Optional[int],
|
|
907
922
|
f'Multiple running jobs found with name {job_name!r}.')
|
908
923
|
job_id = job_ids[0]
|
909
924
|
|
910
|
-
return stream_logs_by_id(job_id, follow)
|
925
|
+
return stream_logs_by_id(job_id, follow, tail)
|
911
926
|
|
912
927
|
|
913
928
|
def dump_managed_job_queue() -> str:
|
@@ -1370,10 +1385,16 @@ class ManagedJobCodeGen:
|
|
1370
1385
|
job_name: Optional[str],
|
1371
1386
|
job_id: Optional[int],
|
1372
1387
|
follow: bool = True,
|
1373
|
-
controller: bool = False
|
1388
|
+
controller: bool = False,
|
1389
|
+
tail: Optional[int] = None) -> str:
|
1374
1390
|
code = textwrap.dedent(f"""\
|
1375
|
-
|
1376
|
-
|
1391
|
+
if managed_job_version < 6:
|
1392
|
+
# Versions before 5 did not support tail parameter
|
1393
|
+
result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
|
1394
|
+
follow={follow}, controller={controller})
|
1395
|
+
else:
|
1396
|
+
result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
|
1397
|
+
follow={follow}, controller={controller}, tail={tail!r})
|
1377
1398
|
if managed_job_version < 3:
|
1378
1399
|
# Versions 2 and older did not return a retcode, so we just print
|
1379
1400
|
# the result.
|
@@ -1388,13 +1409,15 @@ class ManagedJobCodeGen:
|
|
1388
1409
|
|
1389
1410
|
@classmethod
|
1390
1411
|
def set_pending(cls, job_id: int, managed_job_dag: 'dag_lib.Dag',
|
1391
|
-
workspace) -> str:
|
1412
|
+
workspace: str, entrypoint: str) -> str:
|
1392
1413
|
dag_name = managed_job_dag.name
|
1393
1414
|
# Add the managed job to queue table.
|
1394
1415
|
code = textwrap.dedent(f"""\
|
1395
1416
|
set_job_info_kwargs = {{'workspace': {workspace!r}}}
|
1396
1417
|
if managed_job_version < 4:
|
1397
1418
|
set_job_info_kwargs = {{}}
|
1419
|
+
if managed_job_version >= 5:
|
1420
|
+
set_job_info_kwargs['entrypoint'] = {entrypoint!r}
|
1398
1421
|
managed_job_state.set_job_info(
|
1399
1422
|
{job_id}, {dag_name!r}, **set_job_info_kwargs)
|
1400
1423
|
""")
|
sky/provision/gcp/config.py
CHANGED
@@ -10,6 +10,7 @@ from sky.clouds.utils import gcp_utils
|
|
10
10
|
from sky.provision import common
|
11
11
|
from sky.provision.gcp import constants
|
12
12
|
from sky.provision.gcp import instance_utils
|
13
|
+
from sky.utils import resources_utils
|
13
14
|
|
14
15
|
logger = logging.getLogger(__name__)
|
15
16
|
|
@@ -788,7 +789,8 @@ def _configure_subnet(region: str, cluster_name: str,
|
|
788
789
|
default_interfaces = []
|
789
790
|
enable_gpu_direct = config.provider_config.get('enable_gpu_direct', False)
|
790
791
|
enable_gvnic = config.provider_config.get('enable_gvnic', False)
|
791
|
-
|
792
|
+
network_tier = config.provider_config.get('network_tier', 'standard')
|
793
|
+
if enable_gpu_direct or network_tier == resources_utils.NetworkTier.BEST:
|
792
794
|
if not enable_gvnic:
|
793
795
|
logger.warning(
|
794
796
|
'Enable GPU Direct requires gvnic to be enabled, enabling gvnic'
|
sky/provision/gcp/constants.py
CHANGED
@@ -84,6 +84,8 @@ GPU_DIRECT_TCPX_USER_DATA = textwrap.dedent("""
|
|
84
84
|
echo "GPU Direct TCPX installed"
|
85
85
|
""")
|
86
86
|
|
87
|
+
# Some NCCL options are from the following link.
|
88
|
+
# https://docs.nvidia.com/dgx-cloud/run-ai/latest/appendix-gcp.html
|
87
89
|
GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
|
88
90
|
'--cap-add=IPC_LOCK',
|
89
91
|
'--userns=host',
|
@@ -103,6 +105,14 @@ GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
|
|
103
105
|
'--device /dev/nvidia-uvm:/dev/nvidia-uvm',
|
104
106
|
'--device /dev/nvidiactl:/dev/nvidiactl',
|
105
107
|
'--env LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/tcpx/lib64',
|
108
|
+
'--env NCCL_GPUDIRECTTCPX_SOCKET_IFNAME=eth1,eth2,eth3,eth4',
|
109
|
+
'--env NCCL_GPUDIRECTTCPX_CTRL_DEV=eth0',
|
110
|
+
'--env NCCL_GPUDIRECTTCPX_TX_BINDINGS="eth1:8-21,112-125;eth2:8-21,112-125;eth3:60-73,164-177;eth4:60-73,164-177"',
|
111
|
+
'--env NCCL_GPUDIRECTTCPX_RX_BINDINGS="eth1:22-35,126-139;eth2:22-35,126-139;eth3:74-87,178-191;eth4:74-87,178-191"',
|
112
|
+
'--env NCCL_GPUDIRECTTCPX_PROGRAM_FLOW_STEERING_WAIT_MICROS=50000',
|
113
|
+
'--env NCCL_GPUDIRECTTCPX_UNIX_CLIENT_PREFIX="/run/tcpx"',
|
114
|
+
'--env NCCL_GPUDIRECTTCPX_FORCE_ACK=0',
|
115
|
+
'--env NCCL_SOCKET_IFNAME=eth0',
|
106
116
|
]
|
107
117
|
|
108
118
|
PD_EXTREME_IOPS = 20000
|
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
|
|
15
15
|
import sky
|
16
16
|
from sky import clouds
|
17
17
|
from sky import exceptions
|
18
|
+
from sky import global_user_state
|
18
19
|
from sky import models
|
19
20
|
from sky import sky_logging
|
20
21
|
from sky import skypilot_config
|
@@ -2810,7 +2811,7 @@ def set_autodown_annotations(handle: 'backends.CloudVmRayResourceHandle',
|
|
2810
2811
|
tags = {
|
2811
2812
|
provision_constants.TAG_RAY_CLUSTER_NAME: handle.cluster_name_on_cloud,
|
2812
2813
|
}
|
2813
|
-
ray_config =
|
2814
|
+
ray_config = global_user_state.get_cluster_yaml_dict(handle.cluster_yaml)
|
2814
2815
|
provider_config = ray_config['provider']
|
2815
2816
|
namespace = get_namespace_from_config(provider_config)
|
2816
2817
|
context = get_context_from_config(provider_config)
|