skypilot-nightly 1.0.0.dev20250808__py3-none-any.whl → 1.0.0.dev20250814__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of skypilot-nightly might be problematic. Click here for more details.

Files changed (120) hide show
  1. sky/__init__.py +4 -2
  2. sky/adaptors/kubernetes.py +5 -2
  3. sky/backends/backend_utils.py +102 -8
  4. sky/backends/cloud_vm_ray_backend.py +197 -31
  5. sky/catalog/cudo_catalog.py +1 -1
  6. sky/catalog/data_fetchers/fetch_cudo.py +1 -1
  7. sky/catalog/data_fetchers/fetch_nebius.py +6 -3
  8. sky/client/cli/command.py +60 -77
  9. sky/client/common.py +1 -1
  10. sky/client/sdk.py +19 -19
  11. sky/client/sdk_async.py +5 -4
  12. sky/clouds/aws.py +52 -1
  13. sky/clouds/kubernetes.py +14 -0
  14. sky/core.py +5 -0
  15. sky/dag.py +1 -0
  16. sky/dashboard/out/404.html +1 -1
  17. sky/dashboard/out/_next/static/{-DXZksWqf2waNHeU9YTQe → Y0eNlwi85qGRecLTin11y}/_buildManifest.js +1 -1
  18. sky/dashboard/out/_next/static/chunks/{6989-6129c1cfbcf51063.js → 6989-37611fe6b86d274d.js} +1 -1
  19. sky/dashboard/out/_next/static/chunks/8056-5bdeda81199c0def.js +1 -0
  20. sky/dashboard/out/_next/static/chunks/pages/{_app-491a4d699d95e808.js → _app-c2ea34fda4f1f8c8.js} +1 -1
  21. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-078751bad714c017.js +11 -0
  22. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-da9cc0901349c2e9.js +1 -0
  23. sky/dashboard/out/_next/static/chunks/pages/jobs/pools/{[pool]-f5ccf5d39d87aebe.js → [pool]-664c36eda967b1ba.js} +1 -1
  24. sky/dashboard/out/_next/static/chunks/{webpack-339efec49c0cc7d0.js → webpack-00c0a51d21157453.js} +1 -1
  25. sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
  26. sky/dashboard/out/clusters/[cluster].html +1 -1
  27. sky/dashboard/out/clusters.html +1 -1
  28. sky/dashboard/out/config.html +1 -1
  29. sky/dashboard/out/index.html +1 -1
  30. sky/dashboard/out/infra/[context].html +1 -1
  31. sky/dashboard/out/infra.html +1 -1
  32. sky/dashboard/out/jobs/[job].html +1 -1
  33. sky/dashboard/out/jobs/pools/[pool].html +1 -1
  34. sky/dashboard/out/jobs.html +1 -1
  35. sky/dashboard/out/users.html +1 -1
  36. sky/dashboard/out/volumes.html +1 -1
  37. sky/dashboard/out/workspace/new.html +1 -1
  38. sky/dashboard/out/workspaces/[name].html +1 -1
  39. sky/dashboard/out/workspaces.html +1 -1
  40. sky/data/storage.py +11 -1
  41. sky/exceptions.py +5 -0
  42. sky/execution.py +15 -0
  43. sky/global_user_state.py +160 -2
  44. sky/jobs/constants.py +1 -1
  45. sky/jobs/controller.py +0 -1
  46. sky/jobs/recovery_strategy.py +6 -3
  47. sky/jobs/scheduler.py +23 -68
  48. sky/jobs/server/core.py +22 -12
  49. sky/jobs/state.py +6 -2
  50. sky/jobs/utils.py +17 -2
  51. sky/provision/__init__.py +4 -2
  52. sky/provision/aws/config.py +9 -0
  53. sky/provision/aws/instance.py +41 -17
  54. sky/provision/azure/instance.py +7 -4
  55. sky/provision/cudo/cudo_wrapper.py +1 -1
  56. sky/provision/cudo/instance.py +7 -4
  57. sky/provision/do/instance.py +7 -4
  58. sky/provision/fluidstack/instance.py +7 -4
  59. sky/provision/gcp/instance.py +7 -4
  60. sky/provision/hyperbolic/instance.py +7 -5
  61. sky/provision/kubernetes/instance.py +169 -6
  62. sky/provision/lambda_cloud/instance.py +7 -4
  63. sky/provision/nebius/instance.py +7 -4
  64. sky/provision/oci/instance.py +7 -4
  65. sky/provision/paperspace/instance.py +7 -5
  66. sky/provision/paperspace/utils.py +1 -1
  67. sky/provision/provisioner.py +6 -0
  68. sky/provision/runpod/instance.py +7 -4
  69. sky/provision/runpod/utils.py +1 -1
  70. sky/provision/scp/instance.py +7 -5
  71. sky/provision/vast/instance.py +7 -5
  72. sky/provision/vsphere/instance.py +7 -4
  73. sky/resources.py +1 -2
  74. sky/schemas/__init__.py +0 -0
  75. sky/schemas/api/__init__.py +0 -0
  76. sky/schemas/api/responses.py +70 -0
  77. sky/schemas/db/global_user_state/001_initial_schema.py +1 -1
  78. sky/schemas/db/global_user_state/005_cluster_event.py +32 -0
  79. sky/schemas/db/serve_state/001_initial_schema.py +1 -1
  80. sky/schemas/db/spot_jobs/001_initial_schema.py +1 -1
  81. sky/schemas/generated/__init__.py +0 -0
  82. sky/schemas/generated/autostopv1_pb2.py +36 -0
  83. sky/schemas/generated/autostopv1_pb2.pyi +43 -0
  84. sky/schemas/generated/autostopv1_pb2_grpc.py +146 -0
  85. sky/serve/constants.py +3 -7
  86. sky/serve/replica_managers.py +15 -16
  87. sky/serve/serve_state.py +10 -0
  88. sky/serve/serve_utils.py +58 -23
  89. sky/serve/server/impl.py +15 -19
  90. sky/serve/service.py +31 -16
  91. sky/server/server.py +20 -14
  92. sky/setup_files/dependencies.py +11 -10
  93. sky/skylet/autostop_lib.py +38 -5
  94. sky/skylet/constants.py +3 -1
  95. sky/skylet/services.py +44 -0
  96. sky/skylet/skylet.py +49 -4
  97. sky/skypilot_config.py +4 -4
  98. sky/task.py +19 -16
  99. sky/templates/aws-ray.yml.j2 +2 -2
  100. sky/templates/jobs-controller.yaml.j2 +6 -0
  101. sky/users/permission.py +1 -1
  102. sky/utils/cli_utils/status_utils.py +9 -0
  103. sky/utils/command_runner.py +1 -1
  104. sky/utils/config_utils.py +29 -5
  105. sky/utils/controller_utils.py +73 -0
  106. sky/utils/db/db_utils.py +39 -1
  107. sky/utils/db/migration_utils.py +1 -1
  108. sky/utils/schemas.py +3 -0
  109. sky/volumes/server/core.py +2 -2
  110. sky/volumes/server/server.py +2 -2
  111. {skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/METADATA +5 -7
  112. {skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/RECORD +117 -108
  113. sky/dashboard/out/_next/static/chunks/8056-34d27f51e6d1c631.js +0 -1
  114. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-ae17cec0fc6483d9.js +0 -11
  115. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-155d477a6c3e04e2.js +0 -1
  116. /sky/dashboard/out/_next/static/{-DXZksWqf2waNHeU9YTQe → Y0eNlwi85qGRecLTin11y}/_ssgManifest.js +0 -0
  117. {skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/WHEEL +0 -0
  118. {skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/entry_points.txt +0 -0
  119. {skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/licenses/LICENSE +0 -0
  120. {skypilot_nightly-1.0.0.dev20250808.dist-info → skypilot_nightly-1.0.0.dev20250814.dist-info}/top_level.txt +0 -0
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-6129c1cfbcf51063.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D/%5Bjob%5D-ae17cec0fc6483d9.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]/[job]","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-37611fe6b86d274d.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D/%5Bjob%5D-078751bad714c017.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]/[job]","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/1559-6c00e20454194859.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-6129c1cfbcf51063.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-0f886f16e0d55ff8.js" defer=""></script><script src="/dashboard/_next/static/chunks/8056-34d27f51e6d1c631.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/9159-11421c0f2909236f.js" defer=""></script><script src="/dashboard/_next/static/chunks/1871-980a395e92633a5c.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D-155d477a6c3e04e2.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/1559-6c00e20454194859.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-37611fe6b86d274d.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-0f886f16e0d55ff8.js" defer=""></script><script src="/dashboard/_next/static/chunks/8056-5bdeda81199c0def.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/9159-11421c0f2909236f.js" defer=""></script><script src="/dashboard/_next/static/chunks/1871-980a395e92633a5c.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D-da9cc0901349c2e9.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters-b30460f683e6ba96.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters-b30460f683e6ba96.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/config-dfb9bf07b13045f4.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/config","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/config-dfb9bf07b13045f4.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/config","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/index-444f1804401f04ea.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/index-444f1804401f04ea.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/infra/%5Bcontext%5D-13d53fffc03ccb52.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/infra/[context]","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/infra/%5Bcontext%5D-13d53fffc03ccb52.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/infra/[context]","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/infra-fc9222e26c8e2f0d.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/infra","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/infra-fc9222e26c8e2f0d.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/infra","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-6129c1cfbcf51063.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/%5Bjob%5D-154f55cf8af55be5.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/[job]","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-37611fe6b86d274d.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/%5Bjob%5D-154f55cf8af55be5.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/[job]","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-6129c1cfbcf51063.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/pools/%5Bpool%5D-f5ccf5d39d87aebe.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/pools/[pool]","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-37611fe6b86d274d.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/pools/%5Bpool%5D-664c36eda967b1ba.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/pools/[pool]","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs-cdc60fb5d371e16a.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs-cdc60fb5d371e16a.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/users-7ed36e44e779d5c7.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/users","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/users-7ed36e44e779d5c7.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/users","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/volumes-c9695d657f78b5dc.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/volumes","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/volumes-c9695d657f78b5dc.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/volumes","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/1559-6c00e20454194859.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-6129c1cfbcf51063.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-0f886f16e0d55ff8.js" defer=""></script><script src="/dashboard/_next/static/chunks/8056-34d27f51e6d1c631.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/9159-11421c0f2909236f.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-a8a8f1adba34c892.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-f72f73bcef9541dc.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/1559-6c00e20454194859.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-37611fe6b86d274d.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-c9686994ddafcf01.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-0f886f16e0d55ff8.js" defer=""></script><script src="/dashboard/_next/static/chunks/8056-5bdeda81199c0def.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-85426374db04811e.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/9159-11421c0f2909236f.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-a8a8f1adba34c892.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-f72f73bcef9541dc.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-339efec49c0cc7d0.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-491a4d699d95e808.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-8f67be60165724cc.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/-DXZksWqf2waNHeU9YTQe/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"-DXZksWqf2waNHeU9YTQe","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-00c0a51d21157453.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-c2ea34fda4f1f8c8.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-8f67be60165724cc.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/Y0eNlwi85qGRecLTin11y/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"Y0eNlwi85qGRecLTin11y","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/data/storage.py CHANGED
@@ -4510,9 +4510,19 @@ class R2Store(S3CompatibleStore):
4510
4510
  extra_cli_args=['--checksum-algorithm', 'CRC32'], # R2 specific
4511
4511
  cloud_name=cloudflare.NAME,
4512
4512
  default_region='auto',
4513
- mount_cmd_factory=mounting_utils.get_r2_mount_cmd,
4513
+ mount_cmd_factory=cls._get_r2_mount_cmd,
4514
4514
  )
4515
4515
 
4516
+ @classmethod
4517
+ def _get_r2_mount_cmd(cls, bucket_name: str, mount_path: str,
4518
+ bucket_sub_path: Optional[str]) -> str:
4519
+ """Factory method for R2 mount command."""
4520
+ endpoint_url = cloudflare.create_endpoint()
4521
+ return mounting_utils.get_r2_mount_cmd(cloudflare.R2_CREDENTIALS_PATH,
4522
+ cloudflare.R2_PROFILE_NAME,
4523
+ endpoint_url, bucket_name,
4524
+ mount_path, bucket_sub_path)
4525
+
4516
4526
  def mount_cached_command(self, mount_path: str) -> str:
4517
4527
  """R2-specific cached mount implementation using rclone."""
4518
4528
  install_cmd = mounting_utils.get_rclone_install_cmd()
sky/exceptions.py CHANGED
@@ -651,3 +651,8 @@ class RequestInterruptedError(Exception):
651
651
  this error is raised.
652
652
  """
653
653
  pass
654
+
655
+
656
+ class SkyletInternalError(Exception):
657
+ """Raised when a Skylet internal error occurs."""
658
+ pass
sky/execution.py CHANGED
@@ -436,9 +436,19 @@ def _execute_dag(
436
436
  logger.info(ux_utils.starting_message('Syncing files.'))
437
437
 
438
438
  if do_workdir:
439
+ if cluster_name is not None:
440
+ global_user_state.add_cluster_event(
441
+ cluster_name, status_lib.ClusterStatus.INIT,
442
+ 'Syncing files to cluster',
443
+ global_user_state.ClusterEventType.STATUS_CHANGE)
439
444
  backend.sync_workdir(handle, task.workdir, task.envs_and_secrets)
440
445
 
441
446
  if do_file_mounts:
447
+ if cluster_name is not None:
448
+ global_user_state.add_cluster_event(
449
+ cluster_name, status_lib.ClusterStatus.UP,
450
+ 'Syncing file mounts',
451
+ global_user_state.ClusterEventType.STATUS_CHANGE)
442
452
  backend.sync_file_mounts(handle, task.file_mounts,
443
453
  task.storage_mounts)
444
454
 
@@ -449,6 +459,11 @@ def _execute_dag(
449
459
  logger.debug('Unnecessary provisioning was skipped, so '
450
460
  'skipping setup as well.')
451
461
  else:
462
+ if cluster_name is not None:
463
+ global_user_state.add_cluster_event(
464
+ cluster_name, status_lib.ClusterStatus.UP,
465
+ 'Running setup commands to install dependencies',
466
+ global_user_state.ClusterEventType.STATUS_CHANGE)
452
467
  backend.setup(handle, task, detach_setup=detach_setup)
453
468
 
454
469
  if Stage.PRE_EXEC in stages and not dryrun:
sky/global_user_state.py CHANGED
@@ -6,6 +6,7 @@ Concepts:
6
6
  - Cluster handle: (non-user facing) an opaque backend handle for us to
7
7
  interact with a cluster.
8
8
  """
9
+ import enum
9
10
  import functools
10
11
  import json
11
12
  import os
@@ -162,6 +163,33 @@ cluster_history_table = sqlalchemy.Table(
162
163
  sqlalchemy.Column('workspace', sqlalchemy.Text, server_default=None),
163
164
  )
164
165
 
166
+
167
+ class ClusterEventType(enum.Enum):
168
+ """Type of cluster event."""
169
+ DEBUG = 'DEBUG'
170
+ """Used to denote events that are not related to cluster status."""
171
+
172
+ STATUS_CHANGE = 'STATUS_CHANGE'
173
+ """Used to denote events that modify cluster status."""
174
+
175
+
176
+ # Table for cluster status change events.
177
+ # starting_status: Status of the cluster at the start of the event.
178
+ # ending_status: Status of the cluster at the end of the event.
179
+ # reason: Reason for the transition.
180
+ # transitioned_at: Timestamp of the transition.
181
+ cluster_event_table = sqlalchemy.Table(
182
+ 'cluster_events',
183
+ Base.metadata,
184
+ sqlalchemy.Column('cluster_hash', sqlalchemy.Text, primary_key=True),
185
+ sqlalchemy.Column('name', sqlalchemy.Text),
186
+ sqlalchemy.Column('starting_status', sqlalchemy.Text),
187
+ sqlalchemy.Column('ending_status', sqlalchemy.Text),
188
+ sqlalchemy.Column('reason', sqlalchemy.Text, primary_key=True),
189
+ sqlalchemy.Column('transitioned_at', sqlalchemy.Integer, primary_key=True),
190
+ sqlalchemy.Column('type', sqlalchemy.Text),
191
+ )
192
+
165
193
  ssh_key_table = sqlalchemy.Table(
166
194
  'ssh_key',
167
195
  Base.metadata,
@@ -612,6 +640,126 @@ def add_or_update_cluster(cluster_name: str,
612
640
  session.commit()
613
641
 
614
642
 
643
+ @_init_db
644
+ def add_cluster_event(cluster_name: str,
645
+ new_status: Optional[status_lib.ClusterStatus],
646
+ reason: str,
647
+ event_type: ClusterEventType,
648
+ nop_if_duplicate: bool = False,
649
+ duplicate_regex: Optional[str] = None,
650
+ expose_duplicate_error: bool = False,
651
+ transitioned_at: Optional[int] = None) -> None:
652
+ """Add a cluster event.
653
+
654
+ Args:
655
+ cluster_name: Name of the cluster.
656
+ new_status: New status of the cluster.
657
+ reason: Reason for the event.
658
+ event_type: Type of the event.
659
+ nop_if_duplicate: If True, do not add the event if it is a duplicate.
660
+ duplicate_regex: If provided, do not add the event if it matches the
661
+ regex. Only used if nop_if_duplicate is True.
662
+ expose_duplicate_error: If True, raise an error if the event is a
663
+ duplicate. Only used if nop_if_duplicate is True.
664
+ transitioned_at: If provided, use this timestamp for the event.
665
+ """
666
+ assert _SQLALCHEMY_ENGINE is not None
667
+ cluster_hash = _get_hash_for_existing_cluster(cluster_name)
668
+ if cluster_hash is None:
669
+ logger.debug(f'Hash for cluster {cluster_name} not found. '
670
+ 'Skipping event.')
671
+ return
672
+ if transitioned_at is None:
673
+ transitioned_at = int(time.time())
674
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
675
+ if (_SQLALCHEMY_ENGINE.dialect.name ==
676
+ db_utils.SQLAlchemyDialect.SQLITE.value):
677
+ insert_func = sqlite.insert
678
+ elif (_SQLALCHEMY_ENGINE.dialect.name ==
679
+ db_utils.SQLAlchemyDialect.POSTGRESQL.value):
680
+ insert_func = postgresql.insert
681
+ else:
682
+ session.rollback()
683
+ raise ValueError('Unsupported database dialect')
684
+
685
+ cluster_row = session.query(cluster_table).filter_by(name=cluster_name)
686
+ last_status = cluster_row.first(
687
+ ).status if cluster_row and cluster_row.first() is not None else None
688
+ if nop_if_duplicate:
689
+ last_event = get_last_cluster_event(cluster_hash,
690
+ event_type=event_type)
691
+ if duplicate_regex is not None and last_event is not None:
692
+ if re.search(duplicate_regex, last_event):
693
+ return
694
+ elif last_event == reason:
695
+ return
696
+ try:
697
+ session.execute(
698
+ insert_func(cluster_event_table).values(
699
+ cluster_hash=cluster_hash,
700
+ name=cluster_name,
701
+ starting_status=last_status,
702
+ ending_status=new_status.value if new_status else None,
703
+ reason=reason,
704
+ transitioned_at=transitioned_at,
705
+ type=event_type.value,
706
+ ))
707
+ session.commit()
708
+ except sqlalchemy.exc.IntegrityError as e:
709
+ if 'UNIQUE constraint failed' in str(e):
710
+ # This can happen if the cluster event is added twice.
711
+ # We can ignore this error unless the caller requests
712
+ # to expose the error.
713
+ if expose_duplicate_error:
714
+ raise db_utils.UniqueConstraintViolationError(
715
+ value=reason, message=str(e))
716
+ else:
717
+ pass
718
+ else:
719
+ raise e
720
+
721
+
722
+ def get_last_cluster_event(cluster_hash: str,
723
+ event_type: ClusterEventType) -> Optional[str]:
724
+ assert _SQLALCHEMY_ENGINE is not None
725
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
726
+ row = session.query(cluster_event_table).filter_by(
727
+ cluster_hash=cluster_hash, type=event_type.value).order_by(
728
+ cluster_event_table.c.transitioned_at.desc()).first()
729
+ if row is None:
730
+ return None
731
+ return row.reason
732
+
733
+
734
+ def get_cluster_events(cluster_name: Optional[str], cluster_hash: Optional[str],
735
+ event_type: ClusterEventType) -> List[str]:
736
+ """Returns the cluster events for the cluster.
737
+
738
+ Args:
739
+ cluster_name: Name of the cluster. Cannot be specified if cluster_hash
740
+ is specified.
741
+ cluster_hash: Hash of the cluster. Cannot be specified if cluster_name
742
+ is specified.
743
+ event_type: Type of the event.
744
+ """
745
+ assert _SQLALCHEMY_ENGINE is not None
746
+
747
+ if cluster_name is not None and cluster_hash is not None:
748
+ raise ValueError('Cannot specify both cluster_name and cluster_hash')
749
+ if cluster_name is None and cluster_hash is None:
750
+ raise ValueError('Must specify either cluster_name or cluster_hash')
751
+ if cluster_name is not None:
752
+ cluster_hash = _get_hash_for_existing_cluster(cluster_name)
753
+ if cluster_hash is None:
754
+ raise ValueError(f'Hash for cluster {cluster_name} not found.')
755
+
756
+ with orm.Session(_SQLALCHEMY_ENGINE) as session:
757
+ rows = session.query(cluster_event_table).filter_by(
758
+ cluster_hash=cluster_hash, type=event_type.value).order_by(
759
+ cluster_event_table.c.transitioned_at.asc()).all()
760
+ return [row.reason for row in rows]
761
+
762
+
615
763
  def _get_user_hash_or_current_user(user_hash: Optional[str]) -> str:
616
764
  """Returns the user hash or the current user hash, if user_hash is None.
617
765
 
@@ -662,6 +810,8 @@ def remove_cluster(cluster_name: str, terminate: bool) -> None:
662
810
 
663
811
  if terminate:
664
812
  session.query(cluster_table).filter_by(name=cluster_name).delete()
813
+ session.query(cluster_event_table).filter_by(
814
+ cluster_hash=cluster_hash).delete()
665
815
  else:
666
816
  handle = get_handle_from_cluster_name(cluster_name)
667
817
  if handle is None:
@@ -948,6 +1098,8 @@ def get_cluster_from_name(
948
1098
  user_hash = _get_user_hash_or_current_user(row.user_hash)
949
1099
  user = get_user(user_hash)
950
1100
  user_name = user.name if user is not None else None
1101
+ last_event = get_last_cluster_event(
1102
+ row.cluster_hash, event_type=ClusterEventType.STATUS_CHANGE)
951
1103
  # TODO: use namedtuple instead of dict
952
1104
  record = {
953
1105
  'name': row.name,
@@ -971,6 +1123,7 @@ def get_cluster_from_name(
971
1123
  'last_creation_yaml': row.last_creation_yaml,
972
1124
  'last_creation_command': row.last_creation_command,
973
1125
  'is_managed': bool(row.is_managed),
1126
+ 'last_event': last_event,
974
1127
  }
975
1128
 
976
1129
  return record
@@ -987,6 +1140,8 @@ def get_clusters() -> List[Dict[str, Any]]:
987
1140
  user_hash = _get_user_hash_or_current_user(row.user_hash)
988
1141
  user = get_user(user_hash)
989
1142
  user_name = user.name if user is not None else None
1143
+ last_event = get_last_cluster_event(
1144
+ row.cluster_hash, event_type=ClusterEventType.STATUS_CHANGE)
990
1145
  # TODO: use namedtuple instead of dict
991
1146
  record = {
992
1147
  'name': row.name,
@@ -1010,6 +1165,7 @@ def get_clusters() -> List[Dict[str, Any]]:
1010
1165
  'last_creation_yaml': row.last_creation_yaml,
1011
1166
  'last_creation_command': row.last_creation_command,
1012
1167
  'is_managed': bool(row.is_managed),
1168
+ 'last_event': last_event,
1013
1169
  }
1014
1170
 
1015
1171
  records.append(record)
@@ -1130,6 +1286,8 @@ def get_clusters_from_history(
1130
1286
  'workspace': workspace,
1131
1287
  'last_creation_yaml': row.last_creation_yaml,
1132
1288
  'last_creation_command': row.last_creation_command,
1289
+ 'last_event': get_last_cluster_event(
1290
+ row.cluster_hash, event_type=ClusterEventType.STATUS_CHANGE),
1133
1291
  }
1134
1292
 
1135
1293
  records.append(record)
@@ -1143,9 +1301,9 @@ def get_clusters_from_history(
1143
1301
  def get_cluster_names_start_with(starts_with: str) -> List[str]:
1144
1302
  assert _SQLALCHEMY_ENGINE is not None
1145
1303
  with orm.Session(_SQLALCHEMY_ENGINE) as session:
1146
- rows = session.query(cluster_table).filter(
1304
+ rows = session.query(cluster_table.c.name).filter(
1147
1305
  cluster_table.c.name.like(f'{starts_with}%')).all()
1148
- return [row.name for row in rows]
1306
+ return [row[0] for row in rows]
1149
1307
 
1150
1308
 
1151
1309
  @_init_db
sky/jobs/constants.py CHANGED
@@ -47,7 +47,7 @@ JOBS_CLUSTER_NAME_PREFIX_LENGTH = 25
47
47
  # The version of the lib files that jobs/utils use. Whenever there is an API
48
48
  # change for the jobs/utils, we need to bump this version and update
49
49
  # job.utils.ManagedJobCodeGen to handle the version update.
50
- MANAGED_JOBS_VERSION = 7
50
+ MANAGED_JOBS_VERSION = 8
51
51
 
52
52
  # The command for setting up the jobs dashboard on the controller. It firstly
53
53
  # checks if the systemd services are available, and if not (e.g., Kubernetes
sky/jobs/controller.py CHANGED
@@ -30,7 +30,6 @@ from sky.jobs import recovery_strategy
30
30
  from sky.jobs import scheduler
31
31
  from sky.jobs import state as managed_job_state
32
32
  from sky.jobs import utils as managed_job_utils
33
- from sky.serve import serve_utils
34
33
  from sky.skylet import constants
35
34
  from sky.skylet import job_lib
36
35
  from sky.usage import usage_lib
@@ -10,8 +10,8 @@ import traceback
10
10
  import typing
11
11
  from typing import Optional
12
12
 
13
- import sky
14
13
  from sky import backends
14
+ from sky import dag as dag_lib
15
15
  from sky import exceptions
16
16
  from sky import execution
17
17
  from sky import global_user_state
@@ -61,7 +61,7 @@ class StrategyExecutor:
61
61
  """
62
62
  assert isinstance(backend, backends.CloudVmRayBackend), (
63
63
  'Only CloudVMRayBackend is supported.')
64
- self.dag = sky.Dag()
64
+ self.dag = dag_lib.Dag()
65
65
  self.dag.add(task)
66
66
  # For jobs submitted to a pool, the cluster name might change after each
67
67
  # recovery. Initially this is set to an empty string to indicate that no
@@ -261,6 +261,9 @@ class StrategyExecutor:
261
261
  if self.cluster_name is None:
262
262
  return
263
263
  if self.pool is None:
264
+ global_user_state.add_cluster_event(
265
+ self.cluster_name, None, 'Cluster was cleaned up.',
266
+ global_user_state.ClusterEventType.STATUS_CHANGE)
264
267
  managed_job_utils.terminate_cluster(self.cluster_name)
265
268
 
266
269
  def _launch(self,
@@ -444,7 +447,7 @@ class StrategyExecutor:
444
447
  # We retry immediately for worker pool, since no sky.launch()
445
448
  # is called and the overhead is minimal.
446
449
  gap_seconds = (backoff.current_backoff()
447
- if self.pool is None else 0)
450
+ if self.pool is None else 1)
448
451
  logger.info('Retrying to launch the cluster in '
449
452
  f'{gap_seconds:.1f} seconds.')
450
453
  time.sleep(gap_seconds)