skypilot-nightly 1.0.0.dev20250529__py3-none-any.whl → 1.0.0.dev20250531__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. sky/__init__.py +2 -2
  2. sky/adaptors/nebius.py +99 -16
  3. sky/authentication.py +54 -7
  4. sky/backends/backend_utils.py +37 -24
  5. sky/backends/cloud_vm_ray_backend.py +33 -17
  6. sky/check.py +1 -1
  7. sky/cli.py +43 -15
  8. sky/client/cli.py +43 -15
  9. sky/clouds/cloud.py +20 -0
  10. sky/clouds/cudo.py +2 -0
  11. sky/clouds/do.py +3 -0
  12. sky/clouds/fluidstack.py +3 -0
  13. sky/clouds/gcp.py +10 -3
  14. sky/clouds/kubernetes.py +70 -4
  15. sky/clouds/lambda_cloud.py +3 -0
  16. sky/clouds/nebius.py +57 -14
  17. sky/clouds/paperspace.py +3 -0
  18. sky/clouds/runpod.py +2 -0
  19. sky/clouds/scp.py +3 -0
  20. sky/clouds/vast.py +3 -0
  21. sky/clouds/vsphere.py +3 -0
  22. sky/dashboard/out/404.html +1 -1
  23. sky/dashboard/out/_next/static/bdeJWb62qu7L7FOq1dbXX/_buildManifest.js +1 -0
  24. sky/dashboard/out/_next/static/chunks/236-7458fda7b295f305.js +6 -0
  25. sky/dashboard/out/_next/static/chunks/37-b638675d511d58b4.js +6 -0
  26. sky/dashboard/out/_next/static/chunks/{470-4d003c441839094d.js → 470-9e7a479cc8303baa.js} +1 -1
  27. sky/dashboard/out/_next/static/chunks/{173-7db8607cefc20f70.js → 614-3d29f98e0634b179.js} +2 -2
  28. sky/dashboard/out/_next/static/chunks/682-5c12535476a21ce3.js +6 -0
  29. sky/dashboard/out/_next/static/chunks/798-c0525dc3f21e488d.js +1 -0
  30. sky/dashboard/out/_next/static/chunks/843-786c36624d5ff61f.js +11 -0
  31. sky/dashboard/out/_next/static/chunks/856-ab9627e7e8ac35e8.js +1 -0
  32. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-8f270e2c9c59fa1a.js +6 -0
  33. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-25edb867a41b6b20.js +6 -0
  34. sky/dashboard/out/_next/static/chunks/pages/{clusters-943992b84fd6f4ee.js → clusters-f37ff20f0af29aae.js} +1 -1
  35. sky/dashboard/out/_next/static/chunks/pages/{config-7c48919fe030bc43.js → config-3c6a2dabf56e8cd6.js} +1 -1
  36. sky/dashboard/out/_next/static/chunks/pages/infra/{[context]-909f1ceb0fcf1b99.js → [context]-342bc15bb78ab2e5.js} +1 -1
  37. sky/dashboard/out/_next/static/chunks/pages/{infra-d4c6875c88771e17.js → infra-7b4b8e7fa9fa0827.js} +1 -1
  38. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-c0c1dff3cd463d9e.js +11 -0
  39. sky/dashboard/out/_next/static/chunks/pages/{jobs-a4efc09e61988f8d.js → jobs-78a6c5ba3e24c0cf.js} +1 -1
  40. sky/dashboard/out/_next/static/chunks/pages/{users-b2634885d67c49a6.js → users-89f9212b81d8897e.js} +1 -1
  41. sky/dashboard/out/_next/static/chunks/pages/workspace/{new-579b3203c7c19d84.js → new-198b6e00d7d724c5.js} +1 -1
  42. sky/dashboard/out/_next/static/chunks/pages/workspaces/{[name]-9388e38fac73ee8f.js → [name]-2ce792183b03c341.js} +1 -1
  43. sky/dashboard/out/_next/static/chunks/pages/workspaces-17d41826537196e7.js +1 -0
  44. sky/dashboard/out/_next/static/chunks/webpack-f27c9a32aa3d9c6d.js +1 -0
  45. sky/dashboard/out/_next/static/css/2b3ee34e586949a3.css +3 -0
  46. sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
  47. sky/dashboard/out/clusters/[cluster].html +1 -1
  48. sky/dashboard/out/clusters.html +1 -1
  49. sky/dashboard/out/config.html +1 -1
  50. sky/dashboard/out/index.html +1 -1
  51. sky/dashboard/out/infra/[context].html +1 -1
  52. sky/dashboard/out/infra.html +1 -1
  53. sky/dashboard/out/jobs/[job].html +1 -1
  54. sky/dashboard/out/jobs.html +1 -1
  55. sky/dashboard/out/users.html +1 -1
  56. sky/dashboard/out/workspace/new.html +1 -1
  57. sky/dashboard/out/workspaces/[name].html +1 -1
  58. sky/dashboard/out/workspaces.html +1 -1
  59. sky/exceptions.py +10 -0
  60. sky/global_user_state.py +149 -1
  61. sky/jobs/client/sdk.py +3 -0
  62. sky/jobs/constants.py +1 -1
  63. sky/jobs/server/core.py +8 -3
  64. sky/jobs/state.py +24 -5
  65. sky/jobs/utils.py +34 -11
  66. sky/provision/gcp/config.py +3 -1
  67. sky/provision/gcp/constants.py +10 -0
  68. sky/provision/kubernetes/utils.py +2 -1
  69. sky/provision/provisioner.py +15 -10
  70. sky/resources.py +44 -3
  71. sky/serve/controller.py +10 -7
  72. sky/serve/replica_managers.py +22 -18
  73. sky/serve/service.py +5 -4
  74. sky/server/common.py +5 -2
  75. sky/server/constants.py +1 -1
  76. sky/server/requests/payloads.py +1 -0
  77. sky/server/stream_utils.py +21 -0
  78. sky/templates/kubernetes-ray.yml.j2 +26 -1
  79. sky/utils/common_utils.py +66 -0
  80. sky/utils/resources_utils.py +26 -0
  81. sky/utils/rich_utils.py +5 -0
  82. sky/utils/schemas.py +23 -1
  83. {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/METADATA +1 -1
  84. {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/RECORD +90 -91
  85. sky/dashboard/out/_next/static/HvNkg7hqKM1p0ptAcdDcF/_buildManifest.js +0 -1
  86. sky/dashboard/out/_next/static/chunks/236-90e5498a5b00ec29.js +0 -6
  87. sky/dashboard/out/_next/static/chunks/303-2c7b0f7af571710b.js +0 -6
  88. sky/dashboard/out/_next/static/chunks/320-afea3ddcc5bd1c6c.js +0 -6
  89. sky/dashboard/out/_next/static/chunks/578-9146658cead92981.js +0 -6
  90. sky/dashboard/out/_next/static/chunks/843-256ec920f6d5f41f.js +0 -11
  91. sky/dashboard/out/_next/static/chunks/856-59a1760784c9e770.js +0 -1
  92. sky/dashboard/out/_next/static/chunks/9f96d65d-5a3e4af68c26849e.js +0 -1
  93. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-159bffb2fa34ed54.js +0 -6
  94. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-9506c00257d10dbd.js +0 -1
  95. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-6b80e9e0c6aa16a1.js +0 -6
  96. sky/dashboard/out/_next/static/chunks/pages/workspaces-610c49ae3619ee85.js +0 -1
  97. sky/dashboard/out/_next/static/chunks/webpack-deda68c926e8d0bc.js +0 -1
  98. sky/dashboard/out/_next/static/css/ffd1cd601648c303.css +0 -3
  99. /sky/dashboard/out/_next/static/{HvNkg7hqKM1p0ptAcdDcF → bdeJWb62qu7L7FOq1dbXX}/_ssgManifest.js +0 -0
  100. /sky/dashboard/out/_next/static/chunks/pages/{_app-a631df412d8172de.js → _app-ad1edd7fe17ea796.js} +0 -0
  101. {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/WHEEL +0 -0
  102. {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/entry_points.txt +0 -0
  103. {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/licenses/LICENSE +0 -0
  104. {skypilot_nightly-1.0.0.dev20250529.dist-info → skypilot_nightly-1.0.0.dev20250531.dist-info}/top_level.txt +0 -0
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><link rel="preload" href="/dashboard/skypilot.svg" as="image" fetchpriority="high"/><meta name="next-head-count" content="3"/><link rel="preload" href="/dashboard/_next/static/css/ffd1cd601648c303.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/ffd1cd601648c303.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-deda68c926e8d0bc.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-87d061ee6ed71b28.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e0e2335212e72357.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-a631df412d8172de.js" defer=""></script><script src="/dashboard/_next/static/chunks/9f96d65d-5a3e4af68c26849e.js" defer=""></script><script src="/dashboard/_next/static/chunks/173-7db8607cefc20f70.js" defer=""></script><script src="/dashboard/_next/static/chunks/121-8f55ee3fa6301784.js" defer=""></script><script src="/dashboard/_next/static/chunks/320-afea3ddcc5bd1c6c.js" defer=""></script><script src="/dashboard/_next/static/chunks/470-4d003c441839094d.js" defer=""></script><script src="/dashboard/_next/static/chunks/293-351268365226d251.js" defer=""></script><script src="/dashboard/_next/static/chunks/856-59a1760784c9e770.js" defer=""></script><script src="/dashboard/_next/static/chunks/973-1a09cac61cfcc1e1.js" defer=""></script><script src="/dashboard/_next/static/chunks/236-90e5498a5b00ec29.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-610c49ae3619ee85.js" defer=""></script><script src="/dashboard/_next/static/HvNkg7hqKM1p0ptAcdDcF/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/HvNkg7hqKM1p0ptAcdDcF/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div class="min-h-screen bg-gray-50"><div class="fixed top-0 left-0 right-0 z-50 shadow-sm"><div class="fixed top-0 left-0 right-0 bg-white z-30 h-14 px-4 border-b border-gray-200 shadow-sm"><div class="flex items-center h-full"><div class="flex items-center space-x-4 mr-6"><a class="flex items-center px-1 pt-1 h-full" href="/dashboard"><div class="h-20 w-20 flex items-center justify-center"><img alt="SkyPilot Logo" fetchpriority="high" width="80" height="80" decoding="async" data-nimg="1" class="w-full h-full object-contain" style="color:transparent" src="/dashboard/skypilot.svg"/></div></a></div><div class="flex items-center space-x-2 md:space-x-4 mr-6"><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/clusters"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="20" height="8" x="2" y="2" rx="2" ry="2"></rect><rect width="20" height="8" x="2" y="14" rx="2" ry="2"></rect><line x1="6" x2="6.01" y1="6" y2="6"></line><line x1="6" x2="6.01" y1="18" y2="18"></line></svg><span>Clusters</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/jobs"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 20V4a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"></path><rect width="20" height="14" x="2" y="6" rx="2"></rect></svg><span>Jobs</span></a><div class="border-l border-gray-200 h-6 mx-1"></div><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/infra"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2" ry="2"></rect><rect x="9" y="9" width="6" height="6"></rect><line x1="9" y1="1" x2="9" y2="4"></line><line x1="15" y1="1" x2="15" y2="4"></line><line x1="9" y1="20" x2="9" y2="23"></line><line x1="15" y1="20" x2="15" y2="23"></line><line x1="20" y1="9" x2="23" y2="9"></line><line x1="20" y1="14" x2="23" y2="14"></line><line x1="1" y1="9" x2="4" y2="9"></line><line x1="1" y1="14" x2="4" y2="14"></line></svg><span>Infra</span></a><a class="inline-flex items-center border-b-2 border-transparent text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/workspaces"><svg class="w-4 h-4" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><g><path fill="none" d="M0 0h24v24H0z"></path><path d="M3 18.5V5a3 3 0 0 1 3-3h14a1 1 0 0 1 1 1v18a1 1 0 0 1-1 1H6.5A3.5 3.5 0 0 1 3 18.5zM19 20v-3H6.5a1.5 1.5 0 0 0 0 3H19zM10 4H6a1 1 0 0 0-1 1v10.337A3.486 3.486 0 0 1 6.5 15H19V4h-2v8l-3.5-2-3.5 2V4z"></path></g></svg><span>Workspaces</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/users"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-users w-4 h-4"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"></path><circle cx="9" cy="7" r="4"></circle><path d="M22 21v-2a4 4 0 0 0-3-3.87"></path><path d="M16 3.13a4 4 0 0 1 0 7.75"></path></svg><span>Users</span></a></div><div class="flex items-center space-x-1 ml-auto"><a href="https://skypilot.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center px-2 py-1 text-gray-600 hover:text-blue-600 transition-colors duration-150 cursor-pointer" title="Docs" tabindex="0"><span class="mr-1">Docs</span><svg class="w-3.5 h-3.5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a><a href="https://github.com/skypilot-org/skypilot" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="GitHub" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://slack.skypilot.co/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Slack" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path transform="scale(0.85) translate(1.8, 1.8)" d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"></path></svg></a><a href="https://github.com/skypilot-org/skypilot/issues/new" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Leave Feedback" tabindex="0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-square w-5 h-5"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"></path></svg></a><div class="border-l border-gray-200 h-6"></div><a class="inline-flex items-center justify-center p-2 rounded-full transition-colors duration-150 cursor-pointer text-gray-600 hover:bg-gray-100" title="Configuration" tabindex="0" href="/dashboard/config"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-settings w-5 h-5"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"></path><circle cx="12" cy="12" r="3"></circle></svg></a></div></div></div></div><div class="transition-all duration-200 ease-in-out min-h-screen" style="padding-top:56px"><main class="p-6"><div class="flex justify-center items-center h-64"><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:40px;height:40px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span><span class="ml-2 text-gray-500">Loading workspaces...</span></div></main></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"HvNkg7hqKM1p0ptAcdDcF","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>Workspaces | SkyPilot Dashboard</title><link rel="preload" href="/dashboard/skypilot.svg" as="image" fetchpriority="high"/><meta name="next-head-count" content="4"/><link rel="preload" href="/dashboard/_next/static/css/2b3ee34e586949a3.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/2b3ee34e586949a3.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-f27c9a32aa3d9c6d.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-87d061ee6ed71b28.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-e0e2335212e72357.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ad1edd7fe17ea796.js" defer=""></script><script src="/dashboard/_next/static/chunks/614-3d29f98e0634b179.js" defer=""></script><script src="/dashboard/_next/static/chunks/798-c0525dc3f21e488d.js" defer=""></script><script src="/dashboard/_next/static/chunks/121-8f55ee3fa6301784.js" defer=""></script><script src="/dashboard/_next/static/chunks/470-9e7a479cc8303baa.js" defer=""></script><script src="/dashboard/_next/static/chunks/293-351268365226d251.js" defer=""></script><script src="/dashboard/_next/static/chunks/856-ab9627e7e8ac35e8.js" defer=""></script><script src="/dashboard/_next/static/chunks/973-1a09cac61cfcc1e1.js" defer=""></script><script src="/dashboard/_next/static/chunks/236-7458fda7b295f305.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-17d41826537196e7.js" defer=""></script><script src="/dashboard/_next/static/bdeJWb62qu7L7FOq1dbXX/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/bdeJWb62qu7L7FOq1dbXX/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div class="min-h-screen bg-gray-50"><div class="fixed top-0 left-0 right-0 z-50 shadow-sm"><div class="fixed top-0 left-0 right-0 bg-white z-30 h-14 px-4 border-b border-gray-200 shadow-sm"><div class="flex items-center h-full"><div class="flex items-center space-x-4 mr-6"><a class="flex items-center px-1 pt-1 h-full" href="/dashboard"><div class="h-20 w-20 flex items-center justify-center"><img alt="SkyPilot Logo" fetchpriority="high" width="80" height="80" decoding="async" data-nimg="1" class="w-full h-full object-contain" style="color:transparent" src="/dashboard/skypilot.svg"/></div></a></div><div class="flex items-center space-x-2 md:space-x-4 mr-6"><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/clusters"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect width="20" height="8" x="2" y="2" rx="2" ry="2"></rect><rect width="20" height="8" x="2" y="14" rx="2" ry="2"></rect><line x1="6" x2="6.01" y1="6" y2="6"></line><line x1="6" x2="6.01" y1="18" y2="18"></line></svg><span>Clusters</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/jobs"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M16 20V4a2 2 0 0 0-2-2h-4a2 2 0 0 0-2 2v16"></path><rect width="20" height="14" x="2" y="6" rx="2"></rect></svg><span>Jobs</span></a><div class="border-l border-gray-200 h-6 mx-1"></div><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/infra"><svg class="w-4 h-4" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="4" y="4" width="16" height="16" rx="2" ry="2"></rect><rect x="9" y="9" width="6" height="6"></rect><line x1="9" y1="1" x2="9" y2="4"></line><line x1="15" y1="1" x2="15" y2="4"></line><line x1="9" y1="20" x2="9" y2="23"></line><line x1="15" y1="20" x2="15" y2="23"></line><line x1="20" y1="9" x2="23" y2="9"></line><line x1="20" y1="14" x2="23" y2="14"></line><line x1="1" y1="9" x2="4" y2="9"></line><line x1="1" y1="14" x2="4" y2="14"></line></svg><span>Infra</span></a><a class="inline-flex items-center border-b-2 border-transparent text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/workspaces"><svg class="w-4 h-4" stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><g><path fill="none" d="M0 0h24v24H0z"></path><path d="M3 18.5V5a3 3 0 0 1 3-3h14a1 1 0 0 1 1 1v18a1 1 0 0 1-1 1H6.5A3.5 3.5 0 0 1 3 18.5zM19 20v-3H6.5a1.5 1.5 0 0 0 0 3H19zM10 4H6a1 1 0 0 0-1 1v10.337A3.486 3.486 0 0 1 6.5 15H19V4h-2v8l-3.5-2-3.5 2V4z"></path></g></svg><span>Workspaces</span></a><a class="inline-flex items-center border-b-2 border-transparent hover:text-blue-600 px-1 pt-1 space-x-2" href="/dashboard/users"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-users w-4 h-4"><path d="M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2"></path><circle cx="9" cy="7" r="4"></circle><path d="M22 21v-2a4 4 0 0 0-3-3.87"></path><path d="M16 3.13a4 4 0 0 1 0 7.75"></path></svg><span>Users</span></a></div><div class="flex items-center space-x-1 ml-auto"><a href="https://skypilot.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center px-2 py-1 text-gray-600 hover:text-blue-600 transition-colors duration-150 cursor-pointer" title="Docs" tabindex="0"><span class="mr-1">Docs</span><svg class="w-3.5 h-3.5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"></path><polyline points="15 3 21 3 21 9"></polyline><line x1="10" y1="14" x2="21" y2="3"></line></svg></a><a href="https://github.com/skypilot-org/skypilot" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="GitHub" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z"></path></svg></a><a href="https://slack.skypilot.co/" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Slack" tabindex="0"><svg class="w-5 h-5" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="currentColor"><path transform="scale(0.85) translate(1.8, 1.8)" d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52zM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313zM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834zM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312zM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834zM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312zM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52zM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313z"></path></svg></a><a href="https://github.com/skypilot-org/skypilot/issues/new" target="_blank" rel="noopener noreferrer" class="inline-flex items-center justify-center p-2 rounded-full text-gray-600 hover:bg-gray-100 transition-colors duration-150 cursor-pointer" title="Leave Feedback" tabindex="0"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-square w-5 h-5"><path d="M21 15a2 2 0 0 1-2 2H7l-4 4V5a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2z"></path></svg></a><div class="border-l border-gray-200 h-6"></div><a class="inline-flex items-center justify-center p-2 rounded-full transition-colors duration-150 cursor-pointer text-gray-600 hover:bg-gray-100" title="Configuration" tabindex="0" href="/dashboard/config"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-settings w-5 h-5"><path d="M12.22 2h-.44a2 2 0 0 0-2 2v.18a2 2 0 0 1-1 1.73l-.43.25a2 2 0 0 1-2 0l-.15-.08a2 2 0 0 0-2.73.73l-.22.38a2 2 0 0 0 .73 2.73l.15.1a2 2 0 0 1 1 1.72v.51a2 2 0 0 1-1 1.74l-.15.09a2 2 0 0 0-.73 2.73l.22.38a2 2 0 0 0 2.73.73l.15-.08a2 2 0 0 1 2 0l.43.25a2 2 0 0 1 1 1.73V20a2 2 0 0 0 2 2h.44a2 2 0 0 0 2-2v-.18a2 2 0 0 1 1-1.73l.43-.25a2 2 0 0 1 2 0l.15.08a2 2 0 0 0 2.73-.73l.22-.39a2 2 0 0 0-.73-2.73l-.15-.08a2 2 0 0 1-1-1.74v-.5a2 2 0 0 1 1-1.74l.15-.09a2 2 0 0 0 .73-2.73l-.22-.38a2 2 0 0 0-2.73-.73l-.15.08a2 2 0 0 1-2 0l-.43-.25a2 2 0 0 1-1-1.73V4a2 2 0 0 0-2-2z"></path><circle cx="12" cy="12" r="3"></circle></svg></a></div></div></div></div><div class="transition-all duration-200 ease-in-out min-h-screen" style="padding-top:56px"><main class="p-6"><div class="flex justify-center items-center h-64"><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:40px;height:40px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span><span class="ml-2 text-gray-500">Loading workspaces...</span></div></main></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"bdeJWb62qu7L7FOq1dbXX","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/exceptions.py CHANGED
@@ -488,6 +488,16 @@ class ApiServerConnectionError(RuntimeError):
488
488
  f'Try: curl {server_url}/api/health')
489
489
 
490
490
 
491
+ class ApiServerAuthenticationError(RuntimeError):
492
+ """Raised when authentication is required for the API server."""
493
+
494
+ def __init__(self, server_url: str):
495
+ super().__init__(
496
+ f'Authentication required for SkyPilot API server at {server_url}. '
497
+ f'Please run:\n'
498
+ f' sky api login -e {server_url}')
499
+
500
+
491
501
  class APIVersionMismatchError(RuntimeError):
492
502
  """Raised when the API version mismatch."""
493
503
  pass
sky/global_user_state.py CHANGED
@@ -22,6 +22,7 @@ from sqlalchemy import orm
22
22
  from sqlalchemy.dialects import postgresql
23
23
  from sqlalchemy.dialects import sqlite
24
24
  from sqlalchemy.ext import declarative
25
+ import yaml
25
26
 
26
27
  from sky import models
27
28
  from sky import sky_logging
@@ -96,6 +97,12 @@ cluster_table = sqlalchemy.Table(
96
97
  sqlalchemy.Column('workspace',
97
98
  sqlalchemy.Text,
98
99
  server_default=constants.SKYPILOT_DEFAULT_WORKSPACE),
100
+ sqlalchemy.Column('last_creation_yaml',
101
+ sqlalchemy.Text,
102
+ server_default=None),
103
+ sqlalchemy.Column('last_creation_command',
104
+ sqlalchemy.Text,
105
+ server_default=None),
99
106
  )
100
107
 
101
108
  storage_table = sqlalchemy.Table(
@@ -133,6 +140,21 @@ cluster_history_table = sqlalchemy.Table(
133
140
  sqlalchemy.Column('user_hash', sqlalchemy.Text),
134
141
  )
135
142
 
143
+ ssh_key_table = sqlalchemy.Table(
144
+ 'ssh_key',
145
+ Base.metadata,
146
+ sqlalchemy.Column('user_hash', sqlalchemy.Text, primary_key=True),
147
+ sqlalchemy.Column('ssh_public_key', sqlalchemy.Text),
148
+ sqlalchemy.Column('ssh_private_key', sqlalchemy.Text),
149
+ )
150
+
151
+ cluster_yaml_table = sqlalchemy.Table(
152
+ 'cluster_yaml',
153
+ Base.metadata,
154
+ sqlalchemy.Column('cluster_name', sqlalchemy.Text, primary_key=True),
155
+ sqlalchemy.Column('yaml', sqlalchemy.Text),
156
+ )
157
+
136
158
 
137
159
  def _glob_to_similar(glob_pattern):
138
160
  """Converts a glob pattern to a PostgreSQL LIKE pattern."""
@@ -270,6 +292,19 @@ def create_table():
270
292
  default_statement='DEFAULT \'default\'',
271
293
  value_to_replace_existing_entries=constants.
272
294
  SKYPILOT_DEFAULT_WORKSPACE)
295
+ db_utils.add_column_to_table_sqlalchemy(
296
+ session,
297
+ 'clusters',
298
+ 'last_creation_yaml',
299
+ sqlalchemy.Text(),
300
+ default_statement='DEFAULT NULL',
301
+ )
302
+ db_utils.add_column_to_table_sqlalchemy(
303
+ session,
304
+ 'clusters',
305
+ 'last_creation_command',
306
+ sqlalchemy.Text(),
307
+ default_statement='DEFAULT NULL')
273
308
  session.commit()
274
309
 
275
310
 
@@ -318,7 +353,8 @@ def add_or_update_cluster(cluster_name: str,
318
353
  requested_resources: Optional[Set[Any]],
319
354
  ready: bool,
320
355
  is_launch: bool = True,
321
- config_hash: Optional[str] = None):
356
+ config_hash: Optional[str] = None,
357
+ task_config: Optional[Dict[str, Any]] = None):
322
358
  """Adds or updates cluster_name -> cluster_handle mapping.
323
359
 
324
360
  Args:
@@ -329,6 +365,8 @@ def add_or_update_cluster(cluster_name: str,
329
365
  be marked as INIT, otherwise it will be marked as UP.
330
366
  is_launch: if the cluster is firstly launched. If True, the launched_at
331
367
  and last_use will be updated. Otherwise, use the old value.
368
+ config_hash: Configuration hash for the cluster.
369
+ task_config: The config of the task being launched.
332
370
  """
333
371
  # TODO(zhwu): have to be imported here to avoid circular import.
334
372
  from sky import skypilot_config # pylint: disable=import-outside-toplevel
@@ -404,6 +442,13 @@ def add_or_update_cluster(cluster_name: str,
404
442
  conditional_values.update({
405
443
  'workspace': active_workspace,
406
444
  })
445
+ if (is_launch and not cluster_row or
446
+ cluster_row.status != status_lib.ClusterStatus.UP.value):
447
+ conditional_values.update({
448
+ 'last_creation_yaml': common_utils.dump_yaml_str(task_config)
449
+ if task_config else None,
450
+ 'last_creation_command': last_use,
451
+ })
407
452
 
408
453
  if (_SQLALCHEMY_ENGINE.dialect.name ==
409
454
  db_utils.SQLAlchemyDialect.SQLITE.value):
@@ -790,6 +835,8 @@ def get_cluster_from_name(
790
835
  'user_name': get_user(user_hash).name,
791
836
  'config_hash': row.config_hash,
792
837
  'workspace': row.workspace,
838
+ 'last_creation_yaml': row.last_creation_yaml,
839
+ 'last_creation_command': row.last_creation_command,
793
840
  }
794
841
 
795
842
  return record
@@ -822,6 +869,8 @@ def get_clusters() -> List[Dict[str, Any]]:
822
869
  'user_name': get_user(user_hash).name,
823
870
  'config_hash': row.config_hash,
824
871
  'workspace': row.workspace,
872
+ 'last_creation_yaml': row.last_creation_yaml,
873
+ 'last_creation_command': row.last_creation_command,
825
874
  }
826
875
 
827
876
  records.append(record)
@@ -1049,3 +1098,102 @@ def get_storage() -> List[Dict[str, Any]]:
1049
1098
  'status': status_lib.StorageStatus[row.status],
1050
1099
  })
1051
1100
  return records
1101
+
1102
+
1103
+ def get_ssh_keys(user_hash: str) -> Tuple[str, str, bool]:
1104
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
1105
+ row = session.query(ssh_key_table).filter_by(
1106
+ user_hash=user_hash).first()
1107
+ if row:
1108
+ return row.ssh_public_key, row.ssh_private_key, True
1109
+ return '', '', False
1110
+
1111
+
1112
+ def set_ssh_keys(user_hash: str, ssh_public_key: str, ssh_private_key: str):
1113
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
1114
+ if (_SQLALCHEMY_ENGINE.dialect.name ==
1115
+ db_utils.SQLAlchemyDialect.SQLITE.value):
1116
+ insert_func = sqlite.insert
1117
+ elif (_SQLALCHEMY_ENGINE.dialect.name ==
1118
+ db_utils.SQLAlchemyDialect.POSTGRESQL.value):
1119
+ insert_func = postgresql.insert
1120
+ else:
1121
+ raise ValueError('Unsupported database dialect')
1122
+ insert_stmnt = insert_func(ssh_key_table).values(
1123
+ user_hash=user_hash,
1124
+ ssh_public_key=ssh_public_key,
1125
+ ssh_private_key=ssh_private_key)
1126
+ do_update_stmt = insert_stmnt.on_conflict_do_update(
1127
+ index_elements=[ssh_key_table.c.user_hash],
1128
+ set_={
1129
+ ssh_key_table.c.ssh_public_key: ssh_public_key,
1130
+ ssh_key_table.c.ssh_private_key: ssh_private_key
1131
+ })
1132
+ session.execute(do_update_stmt)
1133
+ session.commit()
1134
+
1135
+
1136
+ def get_cluster_yaml_str(cluster_yaml_path: Optional[str]) -> Optional[str]:
1137
+ """Get the cluster yaml from the database or the local file system.
1138
+ If the cluster yaml is not in the database, check if it exists on the
1139
+ local file system and migrate it to the database.
1140
+
1141
+ It is assumed that the cluster yaml file is named as <cluster_name>.yml.
1142
+ """
1143
+ if cluster_yaml_path is None:
1144
+ raise ValueError('Attempted to read a None YAML.')
1145
+ cluster_file_name = os.path.basename(cluster_yaml_path)
1146
+ cluster_name, _ = os.path.splitext(cluster_file_name)
1147
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
1148
+ row = session.query(cluster_yaml_table).filter_by(
1149
+ cluster_name=cluster_name).first()
1150
+ if row is None:
1151
+ # If the cluster yaml is not in the database, check if it exists
1152
+ # on the local file system and migrate it to the database.
1153
+ # TODO(syang): remove this check once we have a way to migrate the
1154
+ # cluster from file to database. Remove on v0.12.0.
1155
+ if cluster_yaml_path is not None and os.path.exists(cluster_yaml_path):
1156
+ with open(cluster_yaml_path, 'r', encoding='utf-8') as f:
1157
+ yaml_str = f.read()
1158
+ set_cluster_yaml(cluster_name, yaml_str)
1159
+ return yaml_str
1160
+ return None
1161
+ return row.yaml
1162
+
1163
+
1164
+ def get_cluster_yaml_dict(cluster_yaml_path: Optional[str]) -> Dict[str, Any]:
1165
+ """Get the cluster yaml as a dictionary from the database.
1166
+
1167
+ It is assumed that the cluster yaml file is named as <cluster_name>.yml.
1168
+ """
1169
+ yaml_str = get_cluster_yaml_str(cluster_yaml_path)
1170
+ if yaml_str is None:
1171
+ raise ValueError(f'Cluster yaml {cluster_yaml_path} not found.')
1172
+ return yaml.safe_load(yaml_str)
1173
+
1174
+
1175
+ def set_cluster_yaml(cluster_name: str, yaml_str: str) -> None:
1176
+ """Set the cluster yaml in the database."""
1177
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
1178
+ if (_SQLALCHEMY_ENGINE.dialect.name ==
1179
+ db_utils.SQLAlchemyDialect.SQLITE.value):
1180
+ insert_func = sqlite.insert
1181
+ elif (_SQLALCHEMY_ENGINE.dialect.name ==
1182
+ db_utils.SQLAlchemyDialect.POSTGRESQL.value):
1183
+ insert_func = postgresql.insert
1184
+ else:
1185
+ raise ValueError('Unsupported database dialect')
1186
+ insert_stmnt = insert_func(cluster_yaml_table).values(
1187
+ cluster_name=cluster_name, yaml=yaml_str)
1188
+ do_update_stmt = insert_stmnt.on_conflict_do_update(
1189
+ index_elements=[cluster_yaml_table.c.cluster_name],
1190
+ set_={cluster_yaml_table.c.yaml: yaml_str})
1191
+ session.execute(do_update_stmt)
1192
+ session.commit()
1193
+
1194
+
1195
+ def remove_cluster_yaml(cluster_name: str):
1196
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
1197
+ session.query(cluster_yaml_table).filter_by(
1198
+ cluster_name=cluster_name).delete()
1199
+ session.commit()
sky/jobs/client/sdk.py CHANGED
@@ -192,6 +192,7 @@ def tail_logs(name: Optional[str] = None,
192
192
  follow: bool = True,
193
193
  controller: bool = False,
194
194
  refresh: bool = False,
195
+ tail: Optional[int] = None,
195
196
  output_stream: Optional['io.TextIOBase'] = None) -> int:
196
197
  """Tails logs of managed jobs.
197
198
 
@@ -204,6 +205,7 @@ def tail_logs(name: Optional[str] = None,
204
205
  follow: Whether to follow the logs.
205
206
  controller: Whether to tail logs from the jobs controller.
206
207
  refresh: Whether to restart the jobs controller if it is stopped.
208
+ tail: Number of lines to tail from the end of the log file.
207
209
  output_stream: The stream to write the logs to. If None, print to the
208
210
  console.
209
211
 
@@ -222,6 +224,7 @@ def tail_logs(name: Optional[str] = None,
222
224
  follow=follow,
223
225
  controller=controller,
224
226
  refresh=refresh,
227
+ tail=tail,
225
228
  )
226
229
  response = requests.post(
227
230
  f'{server_common.get_server_url()}/jobs/logs',
sky/jobs/constants.py CHANGED
@@ -47,7 +47,7 @@ JOBS_CLUSTER_NAME_PREFIX_LENGTH = 25
47
47
  # The version of the lib files that jobs/utils use. Whenever there is an API
48
48
  # change for the jobs/utils, we need to bump this version and update
49
49
  # job.utils.ManagedJobCodeGen to handle the version update.
50
- MANAGED_JOBS_VERSION = 4
50
+ MANAGED_JOBS_VERSION = 6
51
51
 
52
52
  # The command for setting up the jobs dashboard on the controller. It firstly
53
53
  # checks if the systemd services are available, and if not (e.g., Kubernetes
sky/jobs/server/core.py CHANGED
@@ -521,8 +521,12 @@ def cancel(name: Optional[str] = None,
521
521
 
522
522
 
523
523
  @usage_lib.entrypoint
524
- def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
525
- controller: bool, refresh: bool) -> int:
524
+ def tail_logs(name: Optional[str],
525
+ job_id: Optional[int],
526
+ follow: bool,
527
+ controller: bool,
528
+ refresh: bool,
529
+ tail: Optional[int] = None) -> int:
526
530
  # NOTE(dev): Keep the docstring consistent between the Python API and CLI.
527
531
  """Tail logs of managed jobs.
528
532
 
@@ -565,7 +569,8 @@ def tail_logs(name: Optional[str], job_id: Optional[int], follow: bool,
565
569
  job_id=job_id,
566
570
  job_name=name,
567
571
  follow=follow,
568
- controller=controller)
572
+ controller=controller,
573
+ tail=tail)
569
574
 
570
575
 
571
576
  def start_dashboard_forwarding(refresh: bool = False) -> Tuple[int, int]:
sky/jobs/state.py CHANGED
@@ -121,7 +121,8 @@ def create_table(cursor, conn):
121
121
  env_file_path TEXT,
122
122
  user_hash TEXT,
123
123
  workspace TEXT DEFAULT NULL,
124
- priority INTEGER DEFAULT 500)""")
124
+ priority INTEGER DEFAULT 500,
125
+ entrypoint TEXT DEFAULT NULL)""")
125
126
 
126
127
  db_utils.add_column_to_table(cursor, conn, 'job_info', 'schedule_state',
127
128
  'TEXT')
@@ -151,6 +152,7 @@ def create_table(cursor, conn):
151
152
  'INTEGER',
152
153
  value_to_replace_existing_entries=500)
153
154
 
155
+ db_utils.add_column_to_table(cursor, conn, 'job_info', 'entrypoint', 'TEXT')
154
156
  conn.commit()
155
157
 
156
158
 
@@ -209,6 +211,7 @@ columns = [
209
211
  'user_hash',
210
212
  'workspace',
211
213
  'priority',
214
+ 'entrypoint',
212
215
  ]
213
216
 
214
217
 
@@ -412,14 +415,15 @@ class ManagedJobScheduleState(enum.Enum):
412
415
 
413
416
 
414
417
  # === Status transition functions ===
415
- def set_job_info(job_id: int, name: str, workspace: str):
418
+ def set_job_info(job_id: int, name: str, workspace: str, entrypoint: str):
416
419
  with db_utils.safe_cursor(_DB_PATH) as cursor:
417
420
  cursor.execute(
418
421
  """\
419
422
  INSERT INTO job_info
420
- (spot_job_id, name, schedule_state, workspace)
421
- VALUES (?, ?, ?, ?)""",
422
- (job_id, name, ManagedJobScheduleState.INACTIVE.value, workspace))
423
+ (spot_job_id, name, schedule_state, workspace, entrypoint)
424
+ VALUES (?, ?, ?, ?, ?)""",
425
+ (job_id, name, ManagedJobScheduleState.INACTIVE.value, workspace,
426
+ entrypoint))
423
427
 
424
428
 
425
429
  def set_pending(job_id: int, task_id: int, task_name: str, resources_str: str):
@@ -1008,6 +1012,21 @@ def get_managed_jobs(job_id: Optional[int] = None) -> List[Dict[str, Any]]:
1008
1012
  job_dict['schedule_state'])
1009
1013
  if job_dict['job_name'] is None:
1010
1014
  job_dict['job_name'] = job_dict['task_name']
1015
+
1016
+ # Add YAML content and command for managed jobs
1017
+ dag_yaml_path = job_dict.get('dag_yaml_path')
1018
+ if dag_yaml_path:
1019
+ try:
1020
+ with open(dag_yaml_path, 'r', encoding='utf-8') as f:
1021
+ job_dict['dag_yaml'] = f.read()
1022
+ except (FileNotFoundError, IOError, OSError):
1023
+ job_dict['dag_yaml'] = None
1024
+
1025
+ # Generate a command that could be used to launch this job
1026
+ # Format: sky jobs launch <yaml_path>
1027
+ else:
1028
+ job_dict['dag_yaml'] = None
1029
+
1011
1030
  jobs.append(job_dict)
1012
1031
  return jobs
1013
1032
 
sky/jobs/utils.py CHANGED
@@ -13,7 +13,7 @@ import textwrap
13
13
  import time
14
14
  import traceback
15
15
  import typing
16
- from typing import Any, Dict, List, Optional, Set, Tuple, Union
16
+ from typing import Any, Deque, Dict, List, Optional, Set, TextIO, Tuple, Union
17
17
 
18
18
  import colorama
19
19
  import filelock
@@ -546,7 +546,9 @@ def cancel_job_by_name(job_name: str,
546
546
  return f'{job_name!r} {msg}'
547
547
 
548
548
 
549
- def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
549
+ def stream_logs_by_id(job_id: int,
550
+ follow: bool = True,
551
+ tail: Optional[int] = None) -> Tuple[str, int]:
550
552
  """Stream logs by job id.
551
553
 
552
554
  Returns:
@@ -583,7 +585,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
583
585
  # Stream the logs to the console without reading the whole
584
586
  # file into memory.
585
587
  start_streaming = False
586
- for line in f:
588
+ read_from: Union[TextIO, Deque[str]] = f
589
+ if tail is not None:
590
+ assert tail > 0
591
+ # Read only the last 'tail' lines using deque
592
+ read_from = collections.deque(f, maxlen=tail)
593
+ for line in read_from:
587
594
  if log_lib.LOG_FILE_START_STREAMING_AT in line:
588
595
  start_streaming = True
589
596
  if start_streaming:
@@ -644,10 +651,12 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
644
651
  managed_job_state.ManagedJobStatus.RUNNING)
645
652
  assert isinstance(handle, backends.CloudVmRayResourceHandle), handle
646
653
  status_display.stop()
654
+ tail_param = tail if tail is not None else 0
647
655
  returncode = backend.tail_logs(handle,
648
656
  job_id=None,
649
657
  managed_job_id=job_id,
650
- follow=follow)
658
+ follow=follow,
659
+ tail=tail_param)
651
660
  if returncode in [rc.value for rc in exceptions.JobExitCode]:
652
661
  # If the log tailing exits with a known exit code we can safely
653
662
  # break the loop because it indicates the tailing process
@@ -784,7 +793,8 @@ def stream_logs_by_id(job_id: int, follow: bool = True) -> Tuple[str, int]:
784
793
  def stream_logs(job_id: Optional[int],
785
794
  job_name: Optional[str],
786
795
  controller: bool = False,
787
- follow: bool = True) -> Tuple[str, int]:
796
+ follow: bool = True,
797
+ tail: Optional[int] = None) -> Tuple[str, int]:
788
798
  """Stream logs by job id or job name.
789
799
 
790
800
  Returns:
@@ -855,7 +865,12 @@ def stream_logs(job_id: Optional[int],
855
865
  with open(controller_log_path, 'r', newline='', encoding='utf-8') as f:
856
866
  # Note: we do not need to care about start_stream_at here, since
857
867
  # that should be in the job log printed above.
858
- for line in f:
868
+ read_from: Union[TextIO, Deque[str]] = f
869
+ if tail is not None:
870
+ assert tail > 0
871
+ # Read only the last 'tail' lines efficiently using deque
872
+ read_from = collections.deque(f, maxlen=tail)
873
+ for line in read_from:
859
874
  print(line, end='')
860
875
  # Flush.
861
876
  print(end='', flush=True)
@@ -907,7 +922,7 @@ def stream_logs(job_id: Optional[int],
907
922
  f'Multiple running jobs found with name {job_name!r}.')
908
923
  job_id = job_ids[0]
909
924
 
910
- return stream_logs_by_id(job_id, follow)
925
+ return stream_logs_by_id(job_id, follow, tail)
911
926
 
912
927
 
913
928
  def dump_managed_job_queue() -> str:
@@ -1370,10 +1385,16 @@ class ManagedJobCodeGen:
1370
1385
  job_name: Optional[str],
1371
1386
  job_id: Optional[int],
1372
1387
  follow: bool = True,
1373
- controller: bool = False) -> str:
1388
+ controller: bool = False,
1389
+ tail: Optional[int] = None) -> str:
1374
1390
  code = textwrap.dedent(f"""\
1375
- result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
1376
- follow={follow}, controller={controller})
1391
+ if managed_job_version < 6:
1392
+ # Versions before 5 did not support tail parameter
1393
+ result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
1394
+ follow={follow}, controller={controller})
1395
+ else:
1396
+ result = utils.stream_logs(job_id={job_id!r}, job_name={job_name!r},
1397
+ follow={follow}, controller={controller}, tail={tail!r})
1377
1398
  if managed_job_version < 3:
1378
1399
  # Versions 2 and older did not return a retcode, so we just print
1379
1400
  # the result.
@@ -1388,13 +1409,15 @@ class ManagedJobCodeGen:
1388
1409
 
1389
1410
  @classmethod
1390
1411
  def set_pending(cls, job_id: int, managed_job_dag: 'dag_lib.Dag',
1391
- workspace) -> str:
1412
+ workspace: str, entrypoint: str) -> str:
1392
1413
  dag_name = managed_job_dag.name
1393
1414
  # Add the managed job to queue table.
1394
1415
  code = textwrap.dedent(f"""\
1395
1416
  set_job_info_kwargs = {{'workspace': {workspace!r}}}
1396
1417
  if managed_job_version < 4:
1397
1418
  set_job_info_kwargs = {{}}
1419
+ if managed_job_version >= 5:
1420
+ set_job_info_kwargs['entrypoint'] = {entrypoint!r}
1398
1421
  managed_job_state.set_job_info(
1399
1422
  {job_id}, {dag_name!r}, **set_job_info_kwargs)
1400
1423
  """)
@@ -10,6 +10,7 @@ from sky.clouds.utils import gcp_utils
10
10
  from sky.provision import common
11
11
  from sky.provision.gcp import constants
12
12
  from sky.provision.gcp import instance_utils
13
+ from sky.utils import resources_utils
13
14
 
14
15
  logger = logging.getLogger(__name__)
15
16
 
@@ -788,7 +789,8 @@ def _configure_subnet(region: str, cluster_name: str,
788
789
  default_interfaces = []
789
790
  enable_gpu_direct = config.provider_config.get('enable_gpu_direct', False)
790
791
  enable_gvnic = config.provider_config.get('enable_gvnic', False)
791
- if enable_gpu_direct:
792
+ network_tier = config.provider_config.get('network_tier', 'standard')
793
+ if enable_gpu_direct or network_tier == resources_utils.NetworkTier.BEST:
792
794
  if not enable_gvnic:
793
795
  logger.warning(
794
796
  'Enable GPU Direct requires gvnic to be enabled, enabling gvnic'
@@ -84,6 +84,8 @@ GPU_DIRECT_TCPX_USER_DATA = textwrap.dedent("""
84
84
  echo "GPU Direct TCPX installed"
85
85
  """)
86
86
 
87
+ # Some NCCL options are from the following link.
88
+ # https://docs.nvidia.com/dgx-cloud/run-ai/latest/appendix-gcp.html
87
89
  GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
88
90
  '--cap-add=IPC_LOCK',
89
91
  '--userns=host',
@@ -103,6 +105,14 @@ GPU_DIRECT_TCPX_SPECIFIC_OPTIONS = [
103
105
  '--device /dev/nvidia-uvm:/dev/nvidia-uvm',
104
106
  '--device /dev/nvidiactl:/dev/nvidiactl',
105
107
  '--env LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/tcpx/lib64',
108
+ '--env NCCL_GPUDIRECTTCPX_SOCKET_IFNAME=eth1,eth2,eth3,eth4',
109
+ '--env NCCL_GPUDIRECTTCPX_CTRL_DEV=eth0',
110
+ '--env NCCL_GPUDIRECTTCPX_TX_BINDINGS="eth1:8-21,112-125;eth2:8-21,112-125;eth3:60-73,164-177;eth4:60-73,164-177"',
111
+ '--env NCCL_GPUDIRECTTCPX_RX_BINDINGS="eth1:22-35,126-139;eth2:22-35,126-139;eth3:74-87,178-191;eth4:74-87,178-191"',
112
+ '--env NCCL_GPUDIRECTTCPX_PROGRAM_FLOW_STEERING_WAIT_MICROS=50000',
113
+ '--env NCCL_GPUDIRECTTCPX_UNIX_CLIENT_PREFIX="/run/tcpx"',
114
+ '--env NCCL_GPUDIRECTTCPX_FORCE_ACK=0',
115
+ '--env NCCL_SOCKET_IFNAME=eth0',
106
116
  ]
107
117
 
108
118
  PD_EXTREME_IOPS = 20000
@@ -15,6 +15,7 @@ from urllib.parse import urlparse
15
15
  import sky
16
16
  from sky import clouds
17
17
  from sky import exceptions
18
+ from sky import global_user_state
18
19
  from sky import models
19
20
  from sky import sky_logging
20
21
  from sky import skypilot_config
@@ -2810,7 +2811,7 @@ def set_autodown_annotations(handle: 'backends.CloudVmRayResourceHandle',
2810
2811
  tags = {
2811
2812
  provision_constants.TAG_RAY_CLUSTER_NAME: handle.cluster_name_on_cloud,
2812
2813
  }
2813
- ray_config = common_utils.read_yaml(handle.cluster_yaml)
2814
+ ray_config = global_user_state.get_cluster_yaml_dict(handle.cluster_yaml)
2814
2815
  provider_config = ray_config['provider']
2815
2816
  namespace = get_namespace_from_config(provider_config)
2816
2817
  context = get_context_from_config(provider_config)