skypilot-nightly 1.0.0.dev20250602__py3-none-any.whl → 1.0.0.dev20250603__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sky/__init__.py +2 -2
- sky/client/sdk.py +3 -4
- sky/clouds/gcp.py +3 -6
- sky/dashboard/out/404.html +1 -1
- sky/dashboard/out/_next/static/chunks/236-fef38aa6e5639300.js +6 -0
- sky/dashboard/out/_next/static/chunks/37-947904ccc5687bac.js +6 -0
- sky/dashboard/out/_next/static/chunks/682-2be9b0f169727f2f.js +6 -0
- sky/dashboard/out/_next/static/chunks/856-f1b1f7f47edde2e8.js +1 -0
- sky/dashboard/out/_next/static/chunks/969-d7b6fb7f602bfcb3.js +1 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-158b70da336d8607.js +6 -0
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-62c9982dc3675725.js +6 -0
- sky/dashboard/out/_next/static/chunks/pages/{clusters-f37ff20f0af29aae.js → clusters-5549a350f97d7ef3.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/config-35383adcb0edb5e2.js +6 -0
- sky/dashboard/out/_next/static/chunks/pages/infra/{[context]-342bc15bb78ab2e5.js → [context]-b68ddeed712d45b5.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/{infra-7b4b8e7fa9fa0827.js → infra-13b117a831702196.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-a62a3c65dc9bc57c.js +11 -0
- sky/dashboard/out/_next/static/chunks/pages/{jobs-78a6c5ba3e24c0cf.js → jobs-a76b2700eca236f7.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/{users-89f9212b81d8897e.js → users-07b523ccb19317ad.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/workspace/{new-198b6e00d7d724c5.js → new-c7516f2b4c3727c0.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/workspaces/{[name]-2ce792183b03c341.js → [name]-7799de9e691e35d8.js} +1 -1
- sky/dashboard/out/_next/static/chunks/pages/workspaces-f54921ec9eb20965.js +1 -0
- sky/dashboard/out/_next/static/css/63d3995d8b528eb1.css +3 -0
- sky/dashboard/out/_next/static/zTAFq_Iv6_yxQj3fXvJWR/_buildManifest.js +1 -0
- sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
- sky/dashboard/out/clusters/[cluster].html +1 -1
- sky/dashboard/out/clusters.html +1 -1
- sky/dashboard/out/config.html +1 -1
- sky/dashboard/out/index.html +1 -1
- sky/dashboard/out/infra/[context].html +1 -1
- sky/dashboard/out/infra.html +1 -1
- sky/dashboard/out/jobs/[job].html +1 -1
- sky/dashboard/out/jobs.html +1 -1
- sky/dashboard/out/users.html +1 -1
- sky/dashboard/out/workspace/new.html +1 -1
- sky/dashboard/out/workspaces/[name].html +1 -1
- sky/dashboard/out/workspaces.html +1 -1
- sky/execution.py +1 -2
- sky/global_user_state.py +1 -1
- sky/jobs/utils.py +31 -1
- sky/provision/gcp/constants.py +4 -0
- sky/resources.py +31 -0
- sky/server/common.py +86 -53
- sky/server/stream_utils.py +38 -2
- sky/utils/controller_utils.py +4 -14
- {skypilot_nightly-1.0.0.dev20250602.dist-info → skypilot_nightly-1.0.0.dev20250603.dist-info}/METADATA +1 -1
- {skypilot_nightly-1.0.0.dev20250602.dist-info → skypilot_nightly-1.0.0.dev20250603.dist-info}/RECORD +53 -52
- sky/dashboard/out/_next/static/chunks/236-7458fda7b295f305.js +0 -6
- sky/dashboard/out/_next/static/chunks/37-b638675d511d58b4.js +0 -6
- sky/dashboard/out/_next/static/chunks/682-5c12535476a21ce3.js +0 -6
- sky/dashboard/out/_next/static/chunks/856-ab9627e7e8ac35e8.js +0 -1
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-8f270e2c9c59fa1a.js +0 -6
- sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-25edb867a41b6b20.js +0 -6
- sky/dashboard/out/_next/static/chunks/pages/config-3c6a2dabf56e8cd6.js +0 -6
- sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-c0c1dff3cd463d9e.js +0 -11
- sky/dashboard/out/_next/static/chunks/pages/workspaces-17d41826537196e7.js +0 -1
- sky/dashboard/out/_next/static/css/2b3ee34e586949a3.css +0 -3
- sky/dashboard/out/_next/static/dev-ndwjPgd_uQ4dcXXiv/_buildManifest.js +0 -1
- /sky/dashboard/out/_next/static/chunks/{843-786c36624d5ff61f.js → 843-a097338acb89b7d7.js} +0 -0
- /sky/dashboard/out/_next/static/chunks/pages/{_app-ad1edd7fe17ea796.js → _app-67925f5e6382e22f.js} +0 -0
- /sky/dashboard/out/_next/static/{dev-ndwjPgd_uQ4dcXXiv → zTAFq_Iv6_yxQj3fXvJWR}/_ssgManifest.js +0 -0
- {skypilot_nightly-1.0.0.dev20250602.dist-info → skypilot_nightly-1.0.0.dev20250603.dist-info}/WHEEL +0 -0
- {skypilot_nightly-1.0.0.dev20250602.dist-info → skypilot_nightly-1.0.0.dev20250603.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250602.dist-info → skypilot_nightly-1.0.0.dev20250603.dist-info}/licenses/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250602.dist-info → skypilot_nightly-1.0.0.dev20250603.dist-info}/top_level.txt +0 -0
@@ -1 +1 @@
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>Workspaces | SkyPilot Dashboard</title><link rel="preload" href="/dashboard/skypilot.svg" as="image" fetchpriority="high"/><meta name="next-head-count" content="4"/><link rel="preload" href="/dashboard/_next/static/css/2b3ee34e586949a3.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/2b3ee34e586949a3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-f27c9a32aa3d9c6d.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-87d061ee6ed71b28.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e0e2335212e72357.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ad1edd7fe17ea796.js" defer=""></script><script src="/dashboard/_next/static/chunks/614-3d29f98e0634b179.js" defer=""></script><script src="/dashboard/_next/static/chunks/798-c0525dc3f21e488d.js" defer=""></script><script src="/dashboard/_next/static/chunks/121-8f55ee3fa6301784.js" defer=""></script><script src="/dashboard/_next/static/chunks/470-9e7a479cc8303baa.js" defer=""></script><script src="/dashboard/_next/static/chunks/293-351268365226d251.js" defer=""></script><script src="/dashboard/_next/static/chunks/856-ab9627e7e8ac35e8.js" defer=""></script><script src="/dashboard/_next/static/chunks/973-1a09cac61cfcc1e1.js" defer=""></script><script src="/dashboard/_next/static/chunks/236-7458fda7b295f305.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-17d41826537196e7.js" defer=""></script><script src="/dashboard/_next/static/dev-ndwjPgd_uQ4dcXXiv/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/dev-ndwjPgd_uQ4dcXXiv/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div class="min-h-screen bg-gray-50"><div class="fixed top-0 left-0 right-0 z-50 shadow-sm"><div class="fixed top-0 left-0 right-0 bg-white z-30 h-14 px-4 border-b border-gray-200 shadow-sm"><div class="flex items-center h-full"><div class="flex items-center space-x-4 mr-6"><a class="flex items-center px-1 pt-1 h-full" href="/dashboard"><div class="h-20 w-20 flex items-center justify-center"><img alt="SkyPilot Logo" fetchpriority="high" width="80" height="80" decoding="async" data-nimg="1" class="w-full h-full object-contain" style="color:transparent" src="/dashboard/skypilot.svg"/></div></a></div><div class="flex items-center space-x-2 md:space-x-4 mr-6"><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/clusters"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="20" height="8" x="2" y="2" rx="2" ry="2"></rect><rect width="20" height="8" x="2" y="14" rx="2" ry="2"></rect><line x1="6" x2="6.01" y1="6" y2="6"></line><line x1="6" x2="6.01" y1="18" y2="18"></line></svg><span>Clusters</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/jobs"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 20V4a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"></path><rect width="20" height="14" x="2" y="6" rx="2"></rect></svg><span>Jobs</span></a><div class="border-l border-gray-200 h-6 mx-1"></div><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/infra"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2" ry="2"></rect><rect x="9" y="9" width="6" height="6"></rect><line x1="9" y1="1" x2="9" y2="4"></line><line x1="15" y1="1" x2="15" y2="4"></line><line x1="9" y1="20" x2="9" y2="23"></line><line x1="15" y1="20" x2="15" y2="23"></line><line x1="20" y1="9" x2="23" y2="9"></line><line x1="20" y1="14" x2="23" y2="14"></line><line x1="1" y1="9" x2="4" y2="9"></line><line x1="1" y1="14" x2="4" y2="14"></line></svg><span>Infra</span></a><a class="inline-flex items-center border-b-2 border-transparent text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/workspaces"><svg class="w-4 h-4" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><g><path fill="none" d="M0 0h24v24H0z"></path><path d="M3 18.5V5a3 3 0 0 1 3-3h14a1 1 0 0 1 1 1v18a1 1 0 0 1-1 1H6.5A3.5 3.5 0 0 1 3 18.5zM19 20v-3H6.5a1.5 1.5 0 0 0 0 3H19zM10 4H6a1 1 0 0 0-1 1v10.337A3.486 3.486 0 0 1 6.5 15H19V4h-2v8l-3.5-2-3.5 2V4z"></path></g></svg><span>Workspaces</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/users"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-users w-4 h-4"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"></path><circle cx="9" cy="7" r="4"></circle><path d="M22 21v-2a4 4 0 0 0-3-3.87"></path><path d="M16 3.13a4 4 0 0 1 0 7.75"></path></svg><span>Users</span></a></div><div class="flex items-center space-x-1 ml-auto"><a href="https://skypilot.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center px-2 py-1 text-gray-600 hover:text-blue-600 transition-colors duration-150 cursor-pointer" title="Docs" tabindex="0"><span class="mr-1">Docs</span><svg class="w-3.5 h-3.5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a><a href="https://github.com/skypilot-org/skypilot" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="GitHub" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://slack.skypilot.co/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Slack" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path transform="scale(0.85) translate(1.8, 1.8)" d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"></path></svg></a><a href="https://github.com/skypilot-org/skypilot/issues/new" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Leave Feedback" tabindex="0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-square w-5 h-5"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"></path></svg></a><div class="border-l border-gray-200 h-6"></div><a class="inline-flex items-center justify-center p-2 rounded-full transition-colors duration-150 cursor-pointer text-gray-600 hover:bg-gray-100" title="Configuration" tabindex="0" href="/dashboard/config"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-settings w-5 h-5"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"></path><circle cx="12" cy="12" r="3"></circle></svg></a></div></div></div></div><div class="transition-all duration-200 ease-in-out min-h-screen" style="padding-top:56px"><main class="p-6"><div class="flex justify-center items-center h-64"><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:40px;height:40px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span><span class="ml-2 text-gray-500">Loading workspaces...</span></div></main></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"dev-ndwjPgd_uQ4dcXXiv","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>Workspaces | SkyPilot Dashboard</title><link rel="preload" href="/dashboard/skypilot.svg" as="image" fetchpriority="high"/><meta name="next-head-count" content="4"/><link rel="preload" href="/dashboard/_next/static/css/63d3995d8b528eb1.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/63d3995d8b528eb1.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-f27c9a32aa3d9c6d.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-87d061ee6ed71b28.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e0e2335212e72357.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-67925f5e6382e22f.js" defer=""></script><script src="/dashboard/_next/static/chunks/614-3d29f98e0634b179.js" defer=""></script><script src="/dashboard/_next/static/chunks/798-c0525dc3f21e488d.js" defer=""></script><script src="/dashboard/_next/static/chunks/121-8f55ee3fa6301784.js" defer=""></script><script src="/dashboard/_next/static/chunks/470-9e7a479cc8303baa.js" defer=""></script><script src="/dashboard/_next/static/chunks/293-351268365226d251.js" defer=""></script><script src="/dashboard/_next/static/chunks/969-d7b6fb7f602bfcb3.js" defer=""></script><script src="/dashboard/_next/static/chunks/856-f1b1f7f47edde2e8.js" defer=""></script><script src="/dashboard/_next/static/chunks/973-1a09cac61cfcc1e1.js" defer=""></script><script src="/dashboard/_next/static/chunks/236-fef38aa6e5639300.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-f54921ec9eb20965.js" defer=""></script><script src="/dashboard/_next/static/zTAFq_Iv6_yxQj3fXvJWR/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/zTAFq_Iv6_yxQj3fXvJWR/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div class="min-h-screen bg-gray-50"><div class="fixed top-0 left-0 right-0 z-50 shadow-sm"><div class="fixed top-0 left-0 right-0 bg-white z-30 h-14 px-4 border-b border-gray-200 shadow-sm"><div class="flex items-center h-full"><div class="flex items-center space-x-4 mr-6"><a class="flex items-center px-1 pt-1 h-full" href="/dashboard"><div class="h-20 w-20 flex items-center justify-center"><img alt="SkyPilot Logo" fetchpriority="high" width="80" height="80" decoding="async" data-nimg="1" class="w-full h-full object-contain" style="color:transparent" src="/dashboard/skypilot.svg"/></div></a></div><div class="flex items-center space-x-2 md:space-x-4 mr-6"><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/clusters"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="20" height="8" x="2" y="2" rx="2" ry="2"></rect><rect width="20" height="8" x="2" y="14" rx="2" ry="2"></rect><line x1="6" x2="6.01" y1="6" y2="6"></line><line x1="6" x2="6.01" y1="18" y2="18"></line></svg><span>Clusters</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/jobs"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 20V4a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"></path><rect width="20" height="14" x="2" y="6" rx="2"></rect></svg><span>Jobs</span></a><div class="border-l border-gray-200 h-6 mx-1"></div><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/infra"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2" ry="2"></rect><rect x="9" y="9" width="6" height="6"></rect><line x1="9" y1="1" x2="9" y2="4"></line><line x1="15" y1="1" x2="15" y2="4"></line><line x1="9" y1="20" x2="9" y2="23"></line><line x1="15" y1="20" x2="15" y2="23"></line><line x1="20" y1="9" x2="23" y2="9"></line><line x1="20" y1="14" x2="23" y2="14"></line><line x1="1" y1="9" x2="4" y2="9"></line><line x1="1" y1="14" x2="4" y2="14"></line></svg><span>Infra</span></a><a class="inline-flex items-center border-b-2 border-transparent text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/workspaces"><svg class="w-4 h-4" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><g><path fill="none" d="M0 0h24v24H0z"></path><path d="M3 18.5V5a3 3 0 0 1 3-3h14a1 1 0 0 1 1 1v18a1 1 0 0 1-1 1H6.5A3.5 3.5 0 0 1 3 18.5zM19 20v-3H6.5a1.5 1.5 0 0 0 0 3H19zM10 4H6a1 1 0 0 0-1 1v10.337A3.486 3.486 0 0 1 6.5 15H19V4h-2v8l-3.5-2-3.5 2V4z"></path></g></svg><span>Workspaces</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/users"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-users w-4 h-4"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"></path><circle cx="9" cy="7" r="4"></circle><path d="M22 21v-2a4 4 0 0 0-3-3.87"></path><path d="M16 3.13a4 4 0 0 1 0 7.75"></path></svg><span>Users</span></a></div><div class="flex items-center space-x-1 ml-auto"><a href="https://skypilot.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center px-2 py-1 text-gray-600 hover:text-blue-600 transition-colors duration-150 cursor-pointer" title="Docs" tabindex="0"><span class="mr-1">Docs</span><svg class="w-3.5 h-3.5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a><a href="https://github.com/skypilot-org/skypilot" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="GitHub" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://slack.skypilot.co/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Slack" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path transform="scale(0.85) translate(1.8, 1.8)" d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"></path></svg></a><a href="https://github.com/skypilot-org/skypilot/issues/new" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Leave Feedback" tabindex="0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-square w-5 h-5"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"></path></svg></a><div class="border-l border-gray-200 h-6"></div><a class="inline-flex items-center justify-center p-2 rounded-full transition-colors duration-150 cursor-pointer text-gray-600 hover:bg-gray-100" title="Configuration" tabindex="0" href="/dashboard/config"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-settings w-5 h-5"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"></path><circle cx="12" cy="12" r="3"></circle></svg></a></div></div></div></div><div class="transition-all duration-200 ease-in-out min-h-screen" style="padding-top:56px"><main class="p-6"><div class="flex justify-center items-center h-64"><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:40px;height:40px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span><span class="ml-2 text-gray-500">Loading workspaces...</span></div></main></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"zTAFq_Iv6_yxQj3fXvJWR","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/execution.py
CHANGED
@@ -263,8 +263,7 @@ def _execute_dag(
|
|
263
263
|
if controller is not None:
|
264
264
|
requested_features.add(
|
265
265
|
clouds.CloudImplementationFeatures.HOST_CONTROLLERS)
|
266
|
-
if controller_utils.high_availability_specified(cluster_name
|
267
|
-
skip_warning=False):
|
266
|
+
if controller_utils.high_availability_specified(cluster_name):
|
268
267
|
requested_features.add(clouds.CloudImplementationFeatures.
|
269
268
|
HIGH_AVAILABILITY_CONTROLLERS)
|
270
269
|
# If we provision a cluster that supports high availability
|
sky/global_user_state.py
CHANGED
@@ -266,7 +266,7 @@ def create_table():
|
|
266
266
|
db_utils.add_column_to_table_sqlalchemy(
|
267
267
|
session,
|
268
268
|
'clusters',
|
269
|
-
'
|
269
|
+
'user_hash',
|
270
270
|
sqlalchemy.Text(),
|
271
271
|
default_statement='DEFAULT NULL',
|
272
272
|
value_to_replace_existing_entries=common_utils.get_user_hash())
|
sky/jobs/utils.py
CHANGED
@@ -926,8 +926,32 @@ def stream_logs(job_id: Optional[int],
|
|
926
926
|
|
927
927
|
|
928
928
|
def dump_managed_job_queue() -> str:
|
929
|
+
# Make sure to get all jobs - some logic below (e.g. high priority job
|
930
|
+
# detection) requires a full view of the jobs table.
|
929
931
|
jobs = managed_job_state.get_managed_jobs()
|
930
932
|
|
933
|
+
# Figure out what the highest priority blocking job is. We need to know in
|
934
|
+
# order to determine if other jobs are blocked by a higher priority job, or
|
935
|
+
# just by the limited controller resources.
|
936
|
+
lowest_blocking_priority_value = 1000
|
937
|
+
for job in jobs:
|
938
|
+
if job['schedule_state'] not in (
|
939
|
+
# LAUNCHING and ALIVE_BACKOFF jobs will block other jobs with
|
940
|
+
# lower priority.
|
941
|
+
managed_job_state.ManagedJobScheduleState.LAUNCHING,
|
942
|
+
managed_job_state.ManagedJobScheduleState.ALIVE_BACKOFF,
|
943
|
+
# It's possible for a WAITING/ALIVE_WAITING job to be ready to
|
944
|
+
# launch, but the scheduler just hasn't run yet.
|
945
|
+
managed_job_state.ManagedJobScheduleState.WAITING,
|
946
|
+
managed_job_state.ManagedJobScheduleState.ALIVE_WAITING,
|
947
|
+
):
|
948
|
+
# This job will not block others.
|
949
|
+
continue
|
950
|
+
|
951
|
+
priority = job.get('priority')
|
952
|
+
if priority is not None and priority < lowest_blocking_priority_value:
|
953
|
+
lowest_blocking_priority_value = priority
|
954
|
+
|
931
955
|
for job in jobs:
|
932
956
|
end_at = job['end_at']
|
933
957
|
if end_at is None:
|
@@ -973,7 +997,13 @@ def dump_managed_job_queue() -> str:
|
|
973
997
|
if job['schedule_state'] == 'ALIVE_BACKOFF':
|
974
998
|
state_details = 'In backoff, waiting for resources'
|
975
999
|
elif job['schedule_state'] in ('WAITING', 'ALIVE_WAITING'):
|
976
|
-
|
1000
|
+
priority = job.get('priority')
|
1001
|
+
if (priority is not None and
|
1002
|
+
priority > lowest_blocking_priority_value):
|
1003
|
+
# Job is lower priority than some other blocking job.
|
1004
|
+
state_details = 'Waiting for higher priority jobs to launch'
|
1005
|
+
else:
|
1006
|
+
state_details = 'Waiting for other jobs to launch'
|
977
1007
|
|
978
1008
|
if state_details and job['failure_reason']:
|
979
1009
|
job['details'] = f'{state_details} - {job["failure_reason"]}'
|
sky/provision/gcp/constants.py
CHANGED
@@ -54,6 +54,10 @@ CLUSTER_PREFIX_LENGTH = 10
|
|
54
54
|
|
55
55
|
COMPACT_GROUP_PLACEMENT_POLICY = 'compact'
|
56
56
|
COLLOCATED_COLLOCATION = 'COLLOCATED'
|
57
|
+
|
58
|
+
# From https://cloud.google.com/compute/docs/gpus/gpudirect
|
59
|
+
# A specific image is used to ensure that the the GPU is configured with TCPX support.
|
60
|
+
GCP_GPU_DIRECT_IMAGE_ID = 'docker:us-docker.pkg.dev/gce-ai-infra/gpudirect-tcpx/nccl-plugin-gpudirecttcpx'
|
57
61
|
GPU_DIRECT_TCPX_USER_DATA = textwrap.dedent("""
|
58
62
|
# Install GPU Direct TCPX
|
59
63
|
cos-extensions install gpu -- --version=latest;
|
sky/resources.py
CHANGED
@@ -15,6 +15,7 @@ from sky import skypilot_config
|
|
15
15
|
from sky.clouds import cloud as sky_cloud
|
16
16
|
from sky.clouds import service_catalog
|
17
17
|
from sky.provision import docker_utils
|
18
|
+
from sky.provision.gcp import constants as gcp_constants
|
18
19
|
from sky.provision.kubernetes import utils as kubernetes_utils
|
19
20
|
from sky.skylet import constants
|
20
21
|
from sky.utils import accelerator_registry
|
@@ -1162,6 +1163,36 @@ class Resources:
|
|
1162
1163
|
Raises:
|
1163
1164
|
ValueError: if the attribute is invalid.
|
1164
1165
|
"""
|
1166
|
+
|
1167
|
+
if (self._network_tier == resources_utils.NetworkTier.BEST and
|
1168
|
+
isinstance(self._cloud, clouds.GCP)):
|
1169
|
+
# Handle GPU Direct TCPX requirement for docker images
|
1170
|
+
if self._image_id is None:
|
1171
|
+
# No custom image specified - use the default GPU Direct image
|
1172
|
+
self._image_id = {
|
1173
|
+
self._region: gcp_constants.GCP_GPU_DIRECT_IMAGE_ID
|
1174
|
+
}
|
1175
|
+
else:
|
1176
|
+
# Custom image specified - validate it's a docker image
|
1177
|
+
# Check if any of the specified images are not docker images
|
1178
|
+
non_docker_images = []
|
1179
|
+
for region, image_id in self._image_id.items():
|
1180
|
+
if not image_id.startswith('docker:'):
|
1181
|
+
non_docker_images.append(
|
1182
|
+
f'{image_id} (region: {region})')
|
1183
|
+
|
1184
|
+
if non_docker_images:
|
1185
|
+
with ux_utils.print_exception_no_traceback():
|
1186
|
+
raise ValueError(
|
1187
|
+
f'When using network_tier=BEST on GCP, image_id '
|
1188
|
+
f'must be a docker image. '
|
1189
|
+
f'Found non-docker images: '
|
1190
|
+
f'{", ".join(non_docker_images)}. '
|
1191
|
+
f'Please either: (1) use a docker image '
|
1192
|
+
f'(prefix with "docker:"), or '
|
1193
|
+
f'(2) leave image_id empty to use the default '
|
1194
|
+
f'GPU Direct TCPX image.')
|
1195
|
+
|
1165
1196
|
if self._image_id is None:
|
1166
1197
|
return
|
1167
1198
|
|
sky/server/common.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
import dataclasses
|
4
4
|
import enum
|
5
5
|
import functools
|
6
|
+
from http.cookiejar import CookieJar
|
6
7
|
from http.cookiejar import MozillaCookieJar
|
7
8
|
import json
|
8
9
|
import os
|
@@ -128,24 +129,56 @@ class ApiServerInfo:
|
|
128
129
|
commit: Optional[str] = None
|
129
130
|
|
130
131
|
|
131
|
-
def get_api_cookie_jar_path() ->
|
132
|
-
|
133
|
-
|
132
|
+
def get_api_cookie_jar_path() -> pathlib.Path:
|
133
|
+
"""Returns the Path to the API cookie jar file."""
|
134
|
+
return pathlib.Path(
|
135
|
+
os.environ.get(server_constants.API_COOKIE_FILE_ENV_VAR,
|
136
|
+
server_constants.API_COOKIE_FILE_DEFAULT_LOCATION)
|
137
|
+
).expanduser().resolve()
|
134
138
|
|
135
139
|
|
136
140
|
def get_api_cookie_jar() -> requests.cookies.RequestsCookieJar:
|
137
141
|
"""Returns the cookie jar used by the client to access the API server."""
|
138
142
|
cookie_jar = requests.cookies.RequestsCookieJar()
|
139
|
-
|
140
|
-
if
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
file_cookie_jar.load()
|
145
|
-
cookie_jar.update(file_cookie_jar)
|
143
|
+
cookie_path = get_api_cookie_jar_path()
|
144
|
+
if cookie_path.exists():
|
145
|
+
file_cookie_jar = MozillaCookieJar(cookie_path)
|
146
|
+
file_cookie_jar.load()
|
147
|
+
cookie_jar.update(file_cookie_jar)
|
146
148
|
return cookie_jar
|
147
149
|
|
148
150
|
|
151
|
+
def set_api_cookie_jar(cookie_jar: CookieJar,
|
152
|
+
create_if_not_exists: bool = True) -> None:
|
153
|
+
"""Updates the file cookie jar with the given cookie jar."""
|
154
|
+
cookie_path = get_api_cookie_jar_path()
|
155
|
+
if not cookie_path.exists() and not create_if_not_exists:
|
156
|
+
# if the file doesn't exist and we don't want to create it, do nothing
|
157
|
+
return
|
158
|
+
if not cookie_path.parent.exists():
|
159
|
+
cookie_path.parent.mkdir(parents=True, exist_ok=True)
|
160
|
+
|
161
|
+
file_cookie_jar = MozillaCookieJar(cookie_path)
|
162
|
+
if cookie_path.exists():
|
163
|
+
file_cookie_jar.load()
|
164
|
+
|
165
|
+
for cookie in cookie_jar:
|
166
|
+
file_cookie_jar.set_cookie(cookie)
|
167
|
+
file_cookie_jar.save()
|
168
|
+
|
169
|
+
|
170
|
+
def get_cookies_from_response(
|
171
|
+
response: 'requests.Response') -> requests.cookies.RequestsCookieJar:
|
172
|
+
"""Returns the cookies from the API server response."""
|
173
|
+
server_url = get_server_url()
|
174
|
+
cookies = response.cookies
|
175
|
+
for prev_resp in response.history:
|
176
|
+
for cookie in prev_resp.cookies:
|
177
|
+
if cookie.domain in server_url:
|
178
|
+
cookies.set_cookie(cookie)
|
179
|
+
return cookies
|
180
|
+
|
181
|
+
|
149
182
|
@annotations.lru_cache(scope='global')
|
150
183
|
def get_server_url(host: Optional[str] = None) -> str:
|
151
184
|
endpoint = DEFAULT_SERVER_URL
|
@@ -207,48 +240,6 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
|
|
207
240
|
response = requests.get(f'{server_url}/api/health',
|
208
241
|
timeout=2.5,
|
209
242
|
cookies=get_api_cookie_jar())
|
210
|
-
logger.debug(f'Health check status: {response.status_code}')
|
211
|
-
if response.status_code == 200:
|
212
|
-
try:
|
213
|
-
result = response.json()
|
214
|
-
api_version = result.get('api_version')
|
215
|
-
version = result.get('version')
|
216
|
-
version_on_disk = result.get('version_on_disk')
|
217
|
-
commit = result.get('commit')
|
218
|
-
server_info = ApiServerInfo(status=ApiServerStatus.HEALTHY,
|
219
|
-
api_version=api_version,
|
220
|
-
version=version,
|
221
|
-
version_on_disk=version_on_disk,
|
222
|
-
commit=commit)
|
223
|
-
if api_version is None or version is None or commit is None:
|
224
|
-
logger.warning(f'API server response missing '
|
225
|
-
f'version info. {server_url} may '
|
226
|
-
f'not be running SkyPilot API server.')
|
227
|
-
server_info.status = ApiServerStatus.UNHEALTHY
|
228
|
-
elif api_version != server_constants.API_VERSION:
|
229
|
-
server_info.status = ApiServerStatus.VERSION_MISMATCH
|
230
|
-
return server_info
|
231
|
-
except (json.JSONDecodeError, AttributeError) as e:
|
232
|
-
# Try to check if we got redirected to a login page.
|
233
|
-
for prev_response in response.history:
|
234
|
-
logger.debug(f'Previous response: {prev_response.url}')
|
235
|
-
# Heuristic: check if the url looks like a login page or
|
236
|
-
# oauth flow.
|
237
|
-
if any(key in prev_response.url
|
238
|
-
for key in ['login', 'oauth2']):
|
239
|
-
logger.debug(
|
240
|
-
f'URL {prev_response.url} looks like '
|
241
|
-
'a login page or oauth flow, so try to '
|
242
|
-
'get the cookie.')
|
243
|
-
return ApiServerInfo(
|
244
|
-
status=ApiServerStatus.NEEDS_AUTH)
|
245
|
-
logger.warning('Failed to parse API server response: '
|
246
|
-
f'{str(e)}')
|
247
|
-
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
248
|
-
elif response.status_code == 401:
|
249
|
-
return ApiServerInfo(status=ApiServerStatus.NEEDS_AUTH)
|
250
|
-
else:
|
251
|
-
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
252
243
|
except requests.exceptions.Timeout:
|
253
244
|
if time_out_try_count == RETRY_COUNT_ON_TIMEOUT:
|
254
245
|
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
@@ -257,6 +248,48 @@ def get_api_server_status(endpoint: Optional[str] = None) -> ApiServerInfo:
|
|
257
248
|
except requests.exceptions.ConnectionError:
|
258
249
|
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
259
250
|
|
251
|
+
logger.debug(f'Health check status: {response.status_code}')
|
252
|
+
if response.status_code == 401:
|
253
|
+
return ApiServerInfo(status=ApiServerStatus.NEEDS_AUTH)
|
254
|
+
elif response.status_code != 200:
|
255
|
+
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
256
|
+
# The response is 200, so we can parse the response.
|
257
|
+
try:
|
258
|
+
result = response.json()
|
259
|
+
api_version = result.get('api_version')
|
260
|
+
version = result.get('version')
|
261
|
+
version_on_disk = result.get('version_on_disk')
|
262
|
+
commit = result.get('commit')
|
263
|
+
server_info = ApiServerInfo(status=ApiServerStatus.HEALTHY,
|
264
|
+
api_version=api_version,
|
265
|
+
version=version,
|
266
|
+
version_on_disk=version_on_disk,
|
267
|
+
commit=commit)
|
268
|
+
if api_version is None or version is None or commit is None:
|
269
|
+
logger.warning(f'API server response missing '
|
270
|
+
f'version info. {server_url} may '
|
271
|
+
f'not be running SkyPilot API server.')
|
272
|
+
server_info.status = ApiServerStatus.UNHEALTHY
|
273
|
+
elif api_version != server_constants.API_VERSION:
|
274
|
+
server_info.status = ApiServerStatus.VERSION_MISMATCH
|
275
|
+
cookies = get_cookies_from_response(response)
|
276
|
+
set_api_cookie_jar(cookies, create_if_not_exists=False)
|
277
|
+
return server_info
|
278
|
+
except (json.JSONDecodeError, AttributeError) as e:
|
279
|
+
# Try to check if we got redirected to a login page.
|
280
|
+
for prev_response in response.history:
|
281
|
+
logger.debug(f'Previous response: {prev_response.url}')
|
282
|
+
# Heuristic: check if the url looks like a login page or
|
283
|
+
# oauth flow.
|
284
|
+
if any(key in prev_response.url for key in ['login', 'oauth2']):
|
285
|
+
logger.debug(f'URL {prev_response.url} looks like '
|
286
|
+
'a login page or oauth flow, so try to '
|
287
|
+
'get the cookie.')
|
288
|
+
return ApiServerInfo(status=ApiServerStatus.NEEDS_AUTH)
|
289
|
+
logger.warning('Failed to parse API server response: '
|
290
|
+
f'{str(e)}')
|
291
|
+
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
292
|
+
|
260
293
|
return ApiServerInfo(status=ApiServerStatus.UNHEALTHY)
|
261
294
|
|
262
295
|
|
@@ -265,7 +298,7 @@ def handle_request_error(response: 'requests.Response') -> None:
|
|
265
298
|
with ux_utils.print_exception_no_traceback():
|
266
299
|
raise RuntimeError(
|
267
300
|
'Failed to process response from SkyPilot API server at '
|
268
|
-
f'{
|
301
|
+
f'{response.url}. '
|
269
302
|
f'Response: {response.status_code} '
|
270
303
|
f'{response.text}')
|
271
304
|
|
sky/server/stream_utils.py
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
import asyncio
|
4
4
|
import collections
|
5
5
|
import pathlib
|
6
|
-
from typing import AsyncGenerator, Deque, Optional
|
6
|
+
from typing import AsyncGenerator, Deque, List, Optional
|
7
7
|
|
8
8
|
import aiofiles
|
9
9
|
import fastapi
|
@@ -15,6 +15,12 @@ from sky.utils import rich_utils
|
|
15
15
|
|
16
16
|
logger = sky_logging.init_logger(__name__)
|
17
17
|
|
18
|
+
# When streaming log lines, buffer the lines in memory and flush them in chunks
|
19
|
+
# to improve log tailing throughput. Buffer size is the max size bytes of each
|
20
|
+
# chunk and the timeout threshold for flushing the buffer to ensure
|
21
|
+
# responsiveness.
|
22
|
+
_BUFFER_SIZE = 8 * 1024 # 8KB
|
23
|
+
_BUFFER_TIMEOUT = 0.02 # 20ms
|
18
24
|
_HEARTBEAT_INTERVAL = 30
|
19
25
|
|
20
26
|
|
@@ -38,6 +44,20 @@ async def log_streamer(request_id: Optional[str],
|
|
38
44
|
follow: bool = True) -> AsyncGenerator[str, None]:
|
39
45
|
"""Streams the logs of a request."""
|
40
46
|
|
47
|
+
# Buffer the lines in memory and flush them in chunks to improve log tailing
|
48
|
+
# throughput.
|
49
|
+
buffer: List[str] = []
|
50
|
+
buffer_bytes = 0
|
51
|
+
last_flush_time = asyncio.get_event_loop().time()
|
52
|
+
|
53
|
+
async def flush_buffer() -> AsyncGenerator[str, None]:
|
54
|
+
nonlocal buffer, buffer_bytes, last_flush_time
|
55
|
+
if buffer:
|
56
|
+
yield ''.join(buffer)
|
57
|
+
buffer.clear()
|
58
|
+
buffer_bytes = 0
|
59
|
+
last_flush_time = asyncio.get_event_loop().time()
|
60
|
+
|
41
61
|
if request_id is not None:
|
42
62
|
status_msg = rich_utils.EncodedStatusMessage(
|
43
63
|
f'[dim]Checking request: {request_id}[/dim]')
|
@@ -99,7 +119,14 @@ async def log_streamer(request_id: Optional[str],
|
|
99
119
|
# while keeps the loop tight to make log stream responsive.
|
100
120
|
await asyncio.sleep(0)
|
101
121
|
line: Optional[bytes] = await f.readline()
|
122
|
+
|
123
|
+
current_time = asyncio.get_event_loop().time()
|
102
124
|
if not line:
|
125
|
+
if buffer and (current_time -
|
126
|
+
last_flush_time) >= _BUFFER_TIMEOUT:
|
127
|
+
async for chunk in flush_buffer():
|
128
|
+
yield chunk
|
129
|
+
|
103
130
|
if request_id is not None:
|
104
131
|
request_task = requests_lib.get_request(request_id)
|
105
132
|
if request_task.status > requests_lib.RequestStatus.RUNNING:
|
@@ -138,7 +165,16 @@ async def log_streamer(request_id: Optional[str],
|
|
138
165
|
# sending invisible characters might be okay.
|
139
166
|
if is_payload:
|
140
167
|
continue
|
141
|
-
|
168
|
+
|
169
|
+
# Add to buffer
|
170
|
+
buffer.append(line_str)
|
171
|
+
buffer_bytes += len(line_str.encode('utf-8'))
|
172
|
+
|
173
|
+
# Check if we should flush the buffer
|
174
|
+
if (buffer_bytes >= _BUFFER_SIZE or
|
175
|
+
(current_time - last_flush_time) >= _BUFFER_TIMEOUT):
|
176
|
+
async for chunk in flush_buffer():
|
177
|
+
yield chunk
|
142
178
|
|
143
179
|
|
144
180
|
def stream_response(
|
sky/utils/controller_utils.py
CHANGED
@@ -206,8 +206,7 @@ class Controllers(enum.Enum):
|
|
206
206
|
return None
|
207
207
|
|
208
208
|
|
209
|
-
def high_availability_specified(cluster_name: Optional[str]
|
210
|
-
skip_warning: bool = True) -> bool:
|
209
|
+
def high_availability_specified(cluster_name: Optional[str]) -> bool:
|
211
210
|
"""Check if the controller high availability is specified in user config.
|
212
211
|
"""
|
213
212
|
controller = Controllers.from_name(cluster_name)
|
@@ -215,18 +214,9 @@ def high_availability_specified(cluster_name: Optional[str],
|
|
215
214
|
return False
|
216
215
|
|
217
216
|
if skypilot_config.loaded():
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
if high_availability:
|
222
|
-
if controller.value.controller_type != 'serve':
|
223
|
-
if not skip_warning:
|
224
|
-
print(f'{colorama.Fore.RED}High availability controller is'
|
225
|
-
'only supported for SkyServe controller. It cannot'
|
226
|
-
f'be enabled for {controller.value.name}.'
|
227
|
-
f'Skipping this flag.{colorama.Style.RESET_ALL}')
|
228
|
-
else:
|
229
|
-
return True
|
217
|
+
return skypilot_config.get_nested((controller.value.controller_type,
|
218
|
+
'controller', 'high_availability'),
|
219
|
+
False)
|
230
220
|
return False
|
231
221
|
|
232
222
|
|