skypilot-nightly 1.0.0.dev20250827__py3-none-any.whl → 1.0.0.dev20250828__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of skypilot-nightly might be problematic. Click here for more details.
- sky/__init__.py +2 -2
- sky/admin_policy.py +11 -10
- sky/authentication.py +1 -1
- sky/backends/backend.py +3 -5
- sky/backends/backend_utils.py +11 -13
- sky/backends/cloud_vm_ray_backend.py +11 -22
- sky/backends/local_docker_backend.py +3 -8
- sky/client/cli/command.py +41 -9
- sky/client/sdk.py +23 -8
- sky/client/sdk_async.py +6 -2
- sky/core.py +1 -4
- sky/dashboard/out/404.html +1 -1
- sky/dashboard/out/_next/static/chunks/{webpack-6e76f636a048e145.js → webpack-6dae1cd599a34def.js} +1 -1
- sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
- sky/dashboard/out/clusters/[cluster].html +1 -1
- sky/dashboard/out/clusters.html +1 -1
- sky/dashboard/out/config.html +1 -1
- sky/dashboard/out/index.html +1 -1
- sky/dashboard/out/infra/[context].html +1 -1
- sky/dashboard/out/infra.html +1 -1
- sky/dashboard/out/jobs/[job].html +1 -1
- sky/dashboard/out/jobs/pools/[pool].html +1 -1
- sky/dashboard/out/jobs.html +1 -1
- sky/dashboard/out/users.html +1 -1
- sky/dashboard/out/volumes.html +1 -1
- sky/dashboard/out/workspace/new.html +1 -1
- sky/dashboard/out/workspaces/[name].html +1 -1
- sky/dashboard/out/workspaces.html +1 -1
- sky/global_user_state.py +24 -12
- sky/jobs/client/sdk.py +5 -2
- sky/jobs/recovery_strategy.py +9 -4
- sky/logs/agent.py +2 -2
- sky/logs/aws.py +6 -3
- sky/provision/do/utils.py +2 -1
- sky/provision/kubernetes/instance.py +55 -11
- sky/provision/kubernetes/utils.py +2 -2
- sky/provision/nebius/utils.py +36 -2
- sky/serve/client/impl.py +5 -4
- sky/serve/replica_managers.py +4 -3
- sky/serve/serve_utils.py +2 -2
- sky/serve/server/impl.py +3 -2
- sky/server/auth/oauth2_proxy.py +10 -4
- sky/server/common.py +3 -2
- sky/server/daemons.py +10 -5
- sky/server/requests/executor.py +2 -1
- sky/server/requests/requests.py +21 -0
- sky/server/server.py +16 -0
- sky/skylet/events.py +2 -3
- sky/skypilot_config.py +10 -10
- sky/task.py +1 -1
- sky/templates/nebius-ray.yml.j2 +4 -8
- sky/usage/usage_lib.py +3 -2
- sky/utils/common_utils.py +0 -72
- sky/utils/controller_utils.py +4 -3
- sky/utils/dag_utils.py +4 -4
- sky/utils/kubernetes/config_map_utils.py +3 -3
- sky/utils/schemas.py +3 -0
- sky/utils/yaml_utils.py +77 -10
- {skypilot_nightly-1.0.0.dev20250827.dist-info → skypilot_nightly-1.0.0.dev20250828.dist-info}/METADATA +1 -1
- {skypilot_nightly-1.0.0.dev20250827.dist-info → skypilot_nightly-1.0.0.dev20250828.dist-info}/RECORD +66 -66
- /sky/dashboard/out/_next/static/{-eL7Ky3bxVivzeLHNB9U6 → 9DW6d9jaP2kZt0NcgIfFa}/_buildManifest.js +0 -0
- /sky/dashboard/out/_next/static/{-eL7Ky3bxVivzeLHNB9U6 → 9DW6d9jaP2kZt0NcgIfFa}/_ssgManifest.js +0 -0
- {skypilot_nightly-1.0.0.dev20250827.dist-info → skypilot_nightly-1.0.0.dev20250828.dist-info}/WHEEL +0 -0
- {skypilot_nightly-1.0.0.dev20250827.dist-info → skypilot_nightly-1.0.0.dev20250828.dist-info}/entry_points.txt +0 -0
- {skypilot_nightly-1.0.0.dev20250827.dist-info → skypilot_nightly-1.0.0.dev20250828.dist-info}/licenses/LICENSE +0 -0
- {skypilot_nightly-1.0.0.dev20250827.dist-info → skypilot_nightly-1.0.0.dev20250828.dist-info}/top_level.txt +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D/%5Bjob%5D-06afb50d25f7c61f.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]/[job]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/4676-9da7fdbde90b5549.js" defer=""></script><script src="/dashboard/_next/static/chunks/754-d0da8ab45f9509e9.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-4a6f1a928fb6d370.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-08b2a1cae076a943.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/1121-8afcf719ea87debc.js" defer=""></script><script src="/dashboard/_next/static/chunks/6856-049014c6d43d127b.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/3015-6c9c09593b1e67b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/9037-89a84fd7fa31362d.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters/%5Bcluster%5D-a0527109c2fab467.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters/[cluster]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/clusters.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/clusters-469814d711d63b1b.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/clusters","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/config.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/config-dfb9bf07b13045f4.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/config","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/index.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/index-444f1804401f04ea.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/infra/%5Bcontext%5D-81351f95f3bec08e.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/infra/[context]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/infra.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/infra-c320641c2bcbbea6.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/infra","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/754-d0da8ab45f9509e9.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-4a6f1a928fb6d370.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/%5Bjob%5D-dd64309c3fe67ed2.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/[job]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/754-d0da8ab45f9509e9.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-4a6f1a928fb6d370.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/pools/%5Bpool%5D-07349868f7905d37.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/pools/[pool]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/jobs.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs-7421e63ac35f8fce.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/users.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/users-018bf31cda52e11b.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/users","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/dashboard/out/volumes.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/volumes-739726d6b823f532.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/volumes","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/7205-88191679e7988c57.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-4a6f1a928fb6d370.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-08b2a1cae076a943.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/1121-8afcf719ea87debc.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/3015-6c9c09593b1e67b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-943efc7aff0f0c06.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-de06e613e20bc977.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
|
@@ -1 +1 @@
|
|
|
1
|
-
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-
|
|
1
|
+
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6dae1cd599a34def.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-be35b22e2046564c.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/9DW6d9jaP2kZt0NcgIfFa/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"9DW6d9jaP2kZt0NcgIfFa","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
|
sky/global_user_state.py
CHANGED
|
@@ -53,6 +53,7 @@ _SQLALCHEMY_ENGINE: Optional[sqlalchemy.engine.Engine] = None
|
|
|
53
53
|
_SQLALCHEMY_ENGINE_LOCK = threading.Lock()
|
|
54
54
|
|
|
55
55
|
DEFAULT_CLUSTER_EVENT_RETENTION_HOURS = 24.0
|
|
56
|
+
DEBUG_CLUSTER_EVENT_RETENTION_HOURS = 30 * 24.0
|
|
56
57
|
MIN_CLUSTER_EVENT_DAEMON_INTERVAL_SECONDS = 3600
|
|
57
58
|
|
|
58
59
|
_UNIQUE_CONSTRAINT_FAILED_ERROR_MSGS = [
|
|
@@ -595,7 +596,7 @@ def add_or_update_cluster(cluster_name: str,
|
|
|
595
596
|
if (is_launch and not cluster_row or
|
|
596
597
|
cluster_row.status != status_lib.ClusterStatus.UP.value):
|
|
597
598
|
conditional_values.update({
|
|
598
|
-
'last_creation_yaml':
|
|
599
|
+
'last_creation_yaml': yaml_utils.dump_yaml_str(task_config)
|
|
599
600
|
if task_config else None,
|
|
600
601
|
'last_creation_command': last_use,
|
|
601
602
|
})
|
|
@@ -807,12 +808,15 @@ def _get_last_cluster_event_multiple(
|
|
|
807
808
|
return {row.cluster_hash: row.reason for row in rows}
|
|
808
809
|
|
|
809
810
|
|
|
810
|
-
def cleanup_cluster_events_with_retention(retention_hours: float
|
|
811
|
+
def cleanup_cluster_events_with_retention(retention_hours: float,
|
|
812
|
+
event_type: ClusterEventType) -> None:
|
|
811
813
|
assert _SQLALCHEMY_ENGINE is not None
|
|
814
|
+
# Once for events with type STATUS_CHANGE.
|
|
812
815
|
with orm.Session(_SQLALCHEMY_ENGINE) as session:
|
|
813
816
|
query = session.query(cluster_event_table).filter(
|
|
814
|
-
cluster_event_table.c.transitioned_at <
|
|
815
|
-
retention_hours * 3600
|
|
817
|
+
cluster_event_table.c.transitioned_at <
|
|
818
|
+
time.time() - retention_hours * 3600,
|
|
819
|
+
cluster_event_table.c.type == event_type.value)
|
|
816
820
|
logger.debug(f'Deleting {query.count()} cluster events.')
|
|
817
821
|
query.delete()
|
|
818
822
|
session.commit()
|
|
@@ -827,9 +831,20 @@ async def cluster_event_retention_daemon():
|
|
|
827
831
|
retention_hours = skypilot_config.get_nested(
|
|
828
832
|
('api_server', 'cluster_event_retention_hours'),
|
|
829
833
|
DEFAULT_CLUSTER_EVENT_RETENTION_HOURS)
|
|
834
|
+
debug_retention_hours = skypilot_config.get_nested(
|
|
835
|
+
('api_server', 'cluster_debug_event_retention_hours'),
|
|
836
|
+
DEBUG_CLUSTER_EVENT_RETENTION_HOURS)
|
|
830
837
|
try:
|
|
831
838
|
if retention_hours >= 0:
|
|
832
|
-
|
|
839
|
+
logger.debug('Cleaning up cluster events with retention '
|
|
840
|
+
f'{retention_hours} hours.')
|
|
841
|
+
cleanup_cluster_events_with_retention(
|
|
842
|
+
retention_hours, ClusterEventType.STATUS_CHANGE)
|
|
843
|
+
if debug_retention_hours >= 0:
|
|
844
|
+
logger.debug('Cleaning up debug cluster events with retention '
|
|
845
|
+
f'{debug_retention_hours} hours.')
|
|
846
|
+
cleanup_cluster_events_with_retention(debug_retention_hours,
|
|
847
|
+
ClusterEventType.DEBUG)
|
|
833
848
|
except asyncio.CancelledError:
|
|
834
849
|
logger.info('Cluster event retention daemon cancelled')
|
|
835
850
|
break
|
|
@@ -837,8 +852,9 @@ async def cluster_event_retention_daemon():
|
|
|
837
852
|
logger.error(f'Error running cluster event retention daemon: {e}')
|
|
838
853
|
|
|
839
854
|
# Run daemon at most once every hour to avoid too frequent cleanup.
|
|
840
|
-
sleep_amount = max(
|
|
841
|
-
|
|
855
|
+
sleep_amount = max(
|
|
856
|
+
min(retention_hours * 3600, debug_retention_hours * 3600),
|
|
857
|
+
MIN_CLUSTER_EVENT_DAEMON_INTERVAL_SECONDS)
|
|
842
858
|
await asyncio.sleep(sleep_amount)
|
|
843
859
|
|
|
844
860
|
|
|
@@ -904,8 +920,7 @@ def update_last_use(cluster_name: str):
|
|
|
904
920
|
|
|
905
921
|
|
|
906
922
|
@_init_db
|
|
907
|
-
def remove_cluster(cluster_name: str, terminate: bool
|
|
908
|
-
remove_events: bool) -> None:
|
|
923
|
+
def remove_cluster(cluster_name: str, terminate: bool) -> None:
|
|
909
924
|
"""Removes cluster_name mapping."""
|
|
910
925
|
assert _SQLALCHEMY_ENGINE is not None
|
|
911
926
|
cluster_hash = _get_hash_for_existing_cluster(cluster_name)
|
|
@@ -933,9 +948,6 @@ def remove_cluster(cluster_name: str, terminate: bool,
|
|
|
933
948
|
|
|
934
949
|
if terminate:
|
|
935
950
|
session.query(cluster_table).filter_by(name=cluster_name).delete()
|
|
936
|
-
if remove_events:
|
|
937
|
-
session.query(cluster_event_table).filter_by(
|
|
938
|
-
cluster_hash=cluster_hash).delete()
|
|
939
951
|
else:
|
|
940
952
|
handle = get_handle_from_cluster_name(cluster_name)
|
|
941
953
|
if handle is None:
|
sky/jobs/client/sdk.py
CHANGED
|
@@ -243,7 +243,7 @@ def tail_logs(name: Optional[str] = None,
|
|
|
243
243
|
controller: bool = False,
|
|
244
244
|
refresh: bool = False,
|
|
245
245
|
tail: Optional[int] = None,
|
|
246
|
-
output_stream: Optional['io.TextIOBase'] = None) -> int:
|
|
246
|
+
output_stream: Optional['io.TextIOBase'] = None) -> Optional[int]:
|
|
247
247
|
"""Tails logs of managed jobs.
|
|
248
248
|
|
|
249
249
|
You can provide either a job name or a job ID to tail logs. If both are not
|
|
@@ -263,6 +263,8 @@ def tail_logs(name: Optional[str] = None,
|
|
|
263
263
|
Exit code based on success or failure of the job. 0 if success,
|
|
264
264
|
100 if the job failed. See exceptions.JobExitCode for possible exit
|
|
265
265
|
codes.
|
|
266
|
+
Will return None if follow is False
|
|
267
|
+
(see note in sky/client/sdk.py::stream_response)
|
|
266
268
|
|
|
267
269
|
Request Raises:
|
|
268
270
|
ValueError: invalid arguments.
|
|
@@ -289,7 +291,8 @@ def tail_logs(name: Optional[str] = None,
|
|
|
289
291
|
return sdk.stream_response(request_id=request_id,
|
|
290
292
|
response=response,
|
|
291
293
|
output_stream=output_stream,
|
|
292
|
-
resumable=(tail == 0)
|
|
294
|
+
resumable=(tail == 0),
|
|
295
|
+
get_result=follow)
|
|
293
296
|
|
|
294
297
|
|
|
295
298
|
@usage_lib.entrypoint
|
sky/jobs/recovery_strategy.py
CHANGED
|
@@ -327,10 +327,15 @@ class StrategyExecutor:
|
|
|
327
327
|
cluster_name=self.cluster_name,
|
|
328
328
|
# We expect to tear down the cluster as soon as
|
|
329
329
|
# the job is finished. However, in case the
|
|
330
|
-
# controller dies,
|
|
331
|
-
#
|
|
332
|
-
|
|
333
|
-
|
|
330
|
+
# controller dies, we may end up with a
|
|
331
|
+
# resource leak.
|
|
332
|
+
# Ideally, we should autodown to be safe,
|
|
333
|
+
# but it's fine to disable it for now, as
|
|
334
|
+
# Nebius doesn't support autodown yet.
|
|
335
|
+
# TODO(kevin): set down=True once Nebius
|
|
336
|
+
# supports autodown.
|
|
337
|
+
# idle_minutes_to_autostop=_AUTODOWN_MINUTES,
|
|
338
|
+
# down=True,
|
|
334
339
|
_is_launched_by_jobs_controller=True)
|
|
335
340
|
else:
|
|
336
341
|
self.cluster_name = (
|
sky/logs/agent.py
CHANGED
|
@@ -5,8 +5,8 @@ import shlex
|
|
|
5
5
|
from typing import Any, Dict
|
|
6
6
|
|
|
7
7
|
from sky.skylet import constants
|
|
8
|
-
from sky.utils import common_utils
|
|
9
8
|
from sky.utils import resources_utils
|
|
9
|
+
from sky.utils import yaml_utils
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
class LoggingAgent(abc.ABC):
|
|
@@ -65,7 +65,7 @@ class FluentbitAgent(LoggingAgent):
|
|
|
65
65
|
'outputs': [self.fluentbit_output_config(cluster_name)],
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
-
return
|
|
68
|
+
return yaml_utils.dump_yaml_str(cfg_dict)
|
|
69
69
|
|
|
70
70
|
@abc.abstractmethod
|
|
71
71
|
def fluentbit_output_config(
|
sky/logs/aws.py
CHANGED
|
@@ -6,8 +6,8 @@ import pydantic
|
|
|
6
6
|
|
|
7
7
|
from sky.logs.agent import FluentbitAgent
|
|
8
8
|
from sky.skylet import constants
|
|
9
|
-
from sky.utils import common_utils
|
|
10
9
|
from sky.utils import resources_utils
|
|
10
|
+
from sky.utils import yaml_utils
|
|
11
11
|
|
|
12
12
|
EC2_MD_URL = '"${AWS_EC2_METADATA_SERVICE_ENDPOINT:-http://169.254.169.254/}"'
|
|
13
13
|
|
|
@@ -130,7 +130,10 @@ class CloudwatchLoggingAgent(FluentbitAgent):
|
|
|
130
130
|
|
|
131
131
|
# If region is specified, set it in the environment
|
|
132
132
|
if self.config.region:
|
|
133
|
-
pre_cmd += f' export AWS_REGION={self.config.region}
|
|
133
|
+
pre_cmd += (f' export AWS_REGION={self.config.region}'
|
|
134
|
+
f' AWS_DEFAULT_REGION={self.config.region};'
|
|
135
|
+
' command -v aws &>/dev/null && '
|
|
136
|
+
f'aws configure set region {self.config.region};')
|
|
134
137
|
else:
|
|
135
138
|
# If region is not specified, check if it's available in
|
|
136
139
|
# the environment or credentials file
|
|
@@ -213,7 +216,7 @@ class CloudwatchLoggingAgent(FluentbitAgent):
|
|
|
213
216
|
}
|
|
214
217
|
}
|
|
215
218
|
|
|
216
|
-
return
|
|
219
|
+
return yaml_utils.dump_yaml_str(cfg_dict)
|
|
217
220
|
|
|
218
221
|
def fluentbit_output_config(
|
|
219
222
|
self, cluster_name: resources_utils.ClusterName) -> Dict[str, Any]:
|
sky/provision/do/utils.py
CHANGED
|
@@ -17,6 +17,7 @@ from sky.provision import constants as provision_constants
|
|
|
17
17
|
from sky.provision.do import constants
|
|
18
18
|
from sky.utils import annotations
|
|
19
19
|
from sky.utils import common_utils
|
|
20
|
+
from sky.utils import yaml_utils
|
|
20
21
|
|
|
21
22
|
logger = sky_logging.init_logger(__name__)
|
|
22
23
|
|
|
@@ -61,7 +62,7 @@ def _init_client():
|
|
|
61
62
|
if get_credentials_path() is None:
|
|
62
63
|
raise DigitalOceanError(
|
|
63
64
|
'No credentials found, please run `doctl auth init`')
|
|
64
|
-
credentials =
|
|
65
|
+
credentials = yaml_utils.read_yaml(get_credentials_path())
|
|
65
66
|
default_token = credentials.get('access-token', None)
|
|
66
67
|
if default_token is not None:
|
|
67
68
|
try:
|