skypilot-nightly 1.0.0.dev20251009__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of skypilot-nightly might be problematic. Click here for more details.

Files changed (231) hide show
  1. sky/__init__.py +6 -2
  2. sky/adaptors/aws.py +25 -7
  3. sky/adaptors/coreweave.py +278 -0
  4. sky/adaptors/kubernetes.py +64 -0
  5. sky/adaptors/shadeform.py +89 -0
  6. sky/admin_policy.py +20 -0
  7. sky/authentication.py +59 -149
  8. sky/backends/backend_utils.py +104 -63
  9. sky/backends/cloud_vm_ray_backend.py +84 -39
  10. sky/catalog/data_fetchers/fetch_runpod.py +698 -0
  11. sky/catalog/data_fetchers/fetch_shadeform.py +142 -0
  12. sky/catalog/kubernetes_catalog.py +24 -28
  13. sky/catalog/runpod_catalog.py +5 -1
  14. sky/catalog/shadeform_catalog.py +165 -0
  15. sky/check.py +25 -13
  16. sky/client/cli/command.py +335 -86
  17. sky/client/cli/flags.py +4 -2
  18. sky/client/cli/table_utils.py +17 -9
  19. sky/client/sdk.py +59 -12
  20. sky/cloud_stores.py +73 -0
  21. sky/clouds/__init__.py +2 -0
  22. sky/clouds/aws.py +71 -16
  23. sky/clouds/azure.py +12 -5
  24. sky/clouds/cloud.py +19 -9
  25. sky/clouds/cudo.py +12 -5
  26. sky/clouds/do.py +4 -1
  27. sky/clouds/fluidstack.py +12 -5
  28. sky/clouds/gcp.py +12 -5
  29. sky/clouds/hyperbolic.py +12 -5
  30. sky/clouds/ibm.py +12 -5
  31. sky/clouds/kubernetes.py +62 -25
  32. sky/clouds/lambda_cloud.py +12 -5
  33. sky/clouds/nebius.py +12 -5
  34. sky/clouds/oci.py +12 -5
  35. sky/clouds/paperspace.py +4 -1
  36. sky/clouds/primeintellect.py +4 -1
  37. sky/clouds/runpod.py +12 -5
  38. sky/clouds/scp.py +12 -5
  39. sky/clouds/seeweb.py +4 -1
  40. sky/clouds/shadeform.py +400 -0
  41. sky/clouds/ssh.py +4 -2
  42. sky/clouds/vast.py +12 -5
  43. sky/clouds/vsphere.py +4 -1
  44. sky/core.py +12 -11
  45. sky/dashboard/out/404.html +1 -1
  46. sky/dashboard/out/_next/static/chunks/1141-e6aa9ab418717c59.js +11 -0
  47. sky/dashboard/out/_next/static/chunks/{1871-49141c317f3a9020.js → 1871-74503c8e80fd253b.js} +1 -1
  48. sky/dashboard/out/_next/static/chunks/2260-7703229c33c5ebd5.js +1 -0
  49. sky/dashboard/out/_next/static/chunks/2755.fff53c4a3fcae910.js +26 -0
  50. sky/dashboard/out/_next/static/chunks/3294.72362fa129305b19.js +1 -0
  51. sky/dashboard/out/_next/static/chunks/{3785.a19328ba41517b8b.js → 3785.ad6adaa2a0fa9768.js} +1 -1
  52. sky/dashboard/out/_next/static/chunks/{4725.10f7a9a5d3ea8208.js → 4725.a830b5c9e7867c92.js} +1 -1
  53. sky/dashboard/out/_next/static/chunks/6856-ef8ba11f96d8c4a3.js +1 -0
  54. sky/dashboard/out/_next/static/chunks/6990-32b6e2d3822301fa.js +1 -0
  55. sky/dashboard/out/_next/static/chunks/7615-3301e838e5f25772.js +1 -0
  56. sky/dashboard/out/_next/static/chunks/8969-1e4613c651bf4051.js +1 -0
  57. sky/dashboard/out/_next/static/chunks/9025.fa408f3242e9028d.js +6 -0
  58. sky/dashboard/out/_next/static/chunks/9353-cff34f7e773b2e2b.js +1 -0
  59. sky/dashboard/out/_next/static/chunks/9360.7310982cf5a0dc79.js +31 -0
  60. sky/dashboard/out/_next/static/chunks/pages/{_app-ce361c6959bc2001.js → _app-bde01e4a2beec258.js} +1 -1
  61. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-c736ead69c2d86ec.js +16 -0
  62. sky/dashboard/out/_next/static/chunks/pages/clusters/{[cluster]-477555ab7c0b13d8.js → [cluster]-a37d2063af475a1c.js} +1 -1
  63. sky/dashboard/out/_next/static/chunks/pages/{clusters-2f61f65487f6d8ff.js → clusters-d44859594e6f8064.js} +1 -1
  64. sky/dashboard/out/_next/static/chunks/pages/infra/{[context]-553b8b5cb65e100b.js → [context]-c0b5935149902e6f.js} +1 -1
  65. sky/dashboard/out/_next/static/chunks/pages/{infra-910a22500c50596f.js → infra-aed0ea19df7cf961.js} +1 -1
  66. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-5796e8d6aea291a0.js +16 -0
  67. sky/dashboard/out/_next/static/chunks/pages/jobs/pools/{[pool]-bc979970c247d8f3.js → [pool]-6edeb7d06032adfc.js} +2 -2
  68. sky/dashboard/out/_next/static/chunks/pages/{jobs-a35a9dc3c5ccd657.js → jobs-479dde13399cf270.js} +1 -1
  69. sky/dashboard/out/_next/static/chunks/pages/{users-98d2ed979084162a.js → users-5ab3b907622cf0fe.js} +1 -1
  70. sky/dashboard/out/_next/static/chunks/pages/{volumes-835d14ba94808f79.js → volumes-b84b948ff357c43e.js} +1 -1
  71. sky/dashboard/out/_next/static/chunks/pages/workspaces/{[name]-e8688c35c06f0ac5.js → [name]-c5a3eeee1c218af1.js} +1 -1
  72. sky/dashboard/out/_next/static/chunks/pages/{workspaces-69c80d677d3c2949.js → workspaces-22b23febb3e89ce1.js} +1 -1
  73. sky/dashboard/out/_next/static/chunks/webpack-2679be77fc08a2f8.js +1 -0
  74. sky/dashboard/out/_next/static/css/0748ce22df867032.css +3 -0
  75. sky/dashboard/out/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js +1 -0
  76. sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
  77. sky/dashboard/out/clusters/[cluster].html +1 -1
  78. sky/dashboard/out/clusters.html +1 -1
  79. sky/dashboard/out/config.html +1 -1
  80. sky/dashboard/out/index.html +1 -1
  81. sky/dashboard/out/infra/[context].html +1 -1
  82. sky/dashboard/out/infra.html +1 -1
  83. sky/dashboard/out/jobs/[job].html +1 -1
  84. sky/dashboard/out/jobs/pools/[pool].html +1 -1
  85. sky/dashboard/out/jobs.html +1 -1
  86. sky/dashboard/out/users.html +1 -1
  87. sky/dashboard/out/volumes.html +1 -1
  88. sky/dashboard/out/workspace/new.html +1 -1
  89. sky/dashboard/out/workspaces/[name].html +1 -1
  90. sky/dashboard/out/workspaces.html +1 -1
  91. sky/data/data_utils.py +92 -1
  92. sky/data/mounting_utils.py +143 -19
  93. sky/data/storage.py +168 -11
  94. sky/exceptions.py +13 -1
  95. sky/execution.py +13 -0
  96. sky/global_user_state.py +189 -113
  97. sky/jobs/client/sdk.py +32 -10
  98. sky/jobs/client/sdk_async.py +9 -3
  99. sky/jobs/constants.py +3 -1
  100. sky/jobs/controller.py +164 -192
  101. sky/jobs/file_content_utils.py +80 -0
  102. sky/jobs/log_gc.py +201 -0
  103. sky/jobs/recovery_strategy.py +59 -82
  104. sky/jobs/scheduler.py +20 -9
  105. sky/jobs/server/core.py +105 -23
  106. sky/jobs/server/server.py +40 -28
  107. sky/jobs/server/utils.py +32 -11
  108. sky/jobs/state.py +588 -110
  109. sky/jobs/utils.py +442 -209
  110. sky/logs/agent.py +1 -1
  111. sky/metrics/utils.py +45 -6
  112. sky/optimizer.py +1 -1
  113. sky/provision/__init__.py +7 -0
  114. sky/provision/aws/instance.py +2 -1
  115. sky/provision/azure/instance.py +2 -1
  116. sky/provision/common.py +2 -0
  117. sky/provision/cudo/instance.py +2 -1
  118. sky/provision/do/instance.py +2 -1
  119. sky/provision/fluidstack/instance.py +4 -3
  120. sky/provision/gcp/instance.py +2 -1
  121. sky/provision/hyperbolic/instance.py +2 -1
  122. sky/provision/instance_setup.py +10 -2
  123. sky/provision/kubernetes/constants.py +0 -1
  124. sky/provision/kubernetes/instance.py +222 -89
  125. sky/provision/kubernetes/network.py +12 -8
  126. sky/provision/kubernetes/utils.py +114 -53
  127. sky/provision/kubernetes/volume.py +5 -4
  128. sky/provision/lambda_cloud/instance.py +2 -1
  129. sky/provision/nebius/instance.py +2 -1
  130. sky/provision/oci/instance.py +2 -1
  131. sky/provision/paperspace/instance.py +2 -1
  132. sky/provision/provisioner.py +11 -2
  133. sky/provision/runpod/instance.py +2 -1
  134. sky/provision/scp/instance.py +2 -1
  135. sky/provision/seeweb/instance.py +3 -3
  136. sky/provision/shadeform/__init__.py +11 -0
  137. sky/provision/shadeform/config.py +12 -0
  138. sky/provision/shadeform/instance.py +351 -0
  139. sky/provision/shadeform/shadeform_utils.py +83 -0
  140. sky/provision/vast/instance.py +2 -1
  141. sky/provision/vsphere/instance.py +2 -1
  142. sky/resources.py +1 -1
  143. sky/schemas/api/responses.py +9 -5
  144. sky/schemas/db/skypilot_config/001_initial_schema.py +30 -0
  145. sky/schemas/db/spot_jobs/004_job_file_contents.py +42 -0
  146. sky/schemas/db/spot_jobs/005_logs_gc.py +38 -0
  147. sky/schemas/generated/jobsv1_pb2.py +52 -52
  148. sky/schemas/generated/jobsv1_pb2.pyi +4 -2
  149. sky/schemas/generated/managed_jobsv1_pb2.py +39 -35
  150. sky/schemas/generated/managed_jobsv1_pb2.pyi +21 -5
  151. sky/serve/client/impl.py +11 -3
  152. sky/serve/replica_managers.py +5 -2
  153. sky/serve/serve_utils.py +9 -2
  154. sky/serve/server/impl.py +7 -2
  155. sky/serve/server/server.py +18 -15
  156. sky/serve/service.py +2 -2
  157. sky/server/auth/oauth2_proxy.py +2 -5
  158. sky/server/common.py +31 -28
  159. sky/server/constants.py +5 -1
  160. sky/server/daemons.py +27 -19
  161. sky/server/requests/executor.py +138 -74
  162. sky/server/requests/payloads.py +9 -1
  163. sky/server/requests/preconditions.py +13 -10
  164. sky/server/requests/request_names.py +120 -0
  165. sky/server/requests/requests.py +485 -153
  166. sky/server/requests/serializers/decoders.py +26 -13
  167. sky/server/requests/serializers/encoders.py +56 -11
  168. sky/server/requests/threads.py +106 -0
  169. sky/server/rest.py +70 -18
  170. sky/server/server.py +283 -104
  171. sky/server/stream_utils.py +233 -59
  172. sky/server/uvicorn.py +18 -17
  173. sky/setup_files/alembic.ini +4 -0
  174. sky/setup_files/dependencies.py +32 -13
  175. sky/sky_logging.py +0 -2
  176. sky/skylet/constants.py +30 -7
  177. sky/skylet/events.py +7 -0
  178. sky/skylet/log_lib.py +8 -2
  179. sky/skylet/log_lib.pyi +1 -1
  180. sky/skylet/services.py +26 -13
  181. sky/skylet/subprocess_daemon.py +103 -29
  182. sky/skypilot_config.py +87 -75
  183. sky/ssh_node_pools/server.py +9 -8
  184. sky/task.py +67 -54
  185. sky/templates/kubernetes-ray.yml.j2 +8 -1
  186. sky/templates/nebius-ray.yml.j2 +1 -0
  187. sky/templates/shadeform-ray.yml.j2 +72 -0
  188. sky/templates/websocket_proxy.py +142 -12
  189. sky/users/permission.py +8 -1
  190. sky/utils/admin_policy_utils.py +16 -3
  191. sky/utils/asyncio_utils.py +78 -0
  192. sky/utils/auth_utils.py +153 -0
  193. sky/utils/cli_utils/status_utils.py +8 -2
  194. sky/utils/command_runner.py +11 -0
  195. sky/utils/common.py +3 -1
  196. sky/utils/common_utils.py +7 -4
  197. sky/utils/context.py +57 -51
  198. sky/utils/context_utils.py +30 -12
  199. sky/utils/controller_utils.py +35 -8
  200. sky/utils/db/db_utils.py +37 -10
  201. sky/utils/db/migration_utils.py +8 -4
  202. sky/utils/locks.py +24 -6
  203. sky/utils/resource_checker.py +4 -1
  204. sky/utils/resources_utils.py +53 -29
  205. sky/utils/schemas.py +23 -4
  206. sky/utils/subprocess_utils.py +17 -4
  207. sky/volumes/server/server.py +7 -6
  208. sky/workspaces/server.py +13 -12
  209. {skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/METADATA +306 -55
  210. {skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/RECORD +215 -195
  211. sky/dashboard/out/_next/static/chunks/1121-d0782b9251f0fcd3.js +0 -1
  212. sky/dashboard/out/_next/static/chunks/1141-3b40c39626f99c89.js +0 -11
  213. sky/dashboard/out/_next/static/chunks/2755.97300e1362fe7c98.js +0 -26
  214. sky/dashboard/out/_next/static/chunks/3015-8d748834fcc60b46.js +0 -1
  215. sky/dashboard/out/_next/static/chunks/3294.1fafbf42b3bcebff.js +0 -1
  216. sky/dashboard/out/_next/static/chunks/6135-4b4d5e824b7f9d3c.js +0 -1
  217. sky/dashboard/out/_next/static/chunks/6856-5fdc9b851a18acdb.js +0 -1
  218. sky/dashboard/out/_next/static/chunks/6990-f6818c84ed8f1c86.js +0 -1
  219. sky/dashboard/out/_next/static/chunks/8969-66237729cdf9749e.js +0 -1
  220. sky/dashboard/out/_next/static/chunks/9025.c12318fb6a1a9093.js +0 -6
  221. sky/dashboard/out/_next/static/chunks/9360.71e83b2ddc844ec2.js +0 -31
  222. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-8f058b0346db2aff.js +0 -16
  223. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-4f7079dcab6ed653.js +0 -16
  224. sky/dashboard/out/_next/static/chunks/webpack-6a5ddd0184bfa22c.js +0 -1
  225. sky/dashboard/out/_next/static/css/4614e06482d7309e.css +0 -3
  226. sky/dashboard/out/_next/static/hIViZcQBkn0HE8SpaSsUU/_buildManifest.js +0 -1
  227. /sky/dashboard/out/_next/static/{hIViZcQBkn0HE8SpaSsUU → zB0ed6ge_W1MDszVHhijS}/_ssgManifest.js +0 -0
  228. {skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/WHEEL +0 -0
  229. {skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/entry_points.txt +0 -0
  230. {skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/licenses/LICENSE +0 -0
  231. {skypilot_nightly-1.0.0.dev20251009.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/top_level.txt +0 -0
sky/dashboard/out/workspace/new.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6a5ddd0184bfa22c.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/hIViZcQBkn0HE8SpaSsUU/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/hIViZcQBkn0HE8SpaSsUU/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"hIViZcQBkn0HE8SpaSsUU","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/0748ce22df867032.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/0748ce22df867032.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-2679be77fc08a2f8.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-bde01e4a2beec258.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/zB0ed6ge_W1MDszVHhijS/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"zB0ed6ge_W1MDszVHhijS","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/workspaces/[name].html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6a5ddd0184bfa22c.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/7359-c8d04e06886000b3.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-66237729cdf9749e.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-f6818c84ed8f1c86.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/1121-d0782b9251f0fcd3.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/3015-8d748834fcc60b46.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-3b40c39626f99c89.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-e8688c35c06f0ac5.js" defer=""></script><script src="/dashboard/_next/static/hIViZcQBkn0HE8SpaSsUU/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/hIViZcQBkn0HE8SpaSsUU/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"hIViZcQBkn0HE8SpaSsUU","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/0748ce22df867032.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/0748ce22df867032.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-2679be77fc08a2f8.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-bde01e4a2beec258.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/7359-c8d04e06886000b3.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-1e4613c651bf4051.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-32b6e2d3822301fa.js" defer=""></script><script src="/dashboard/_next/static/chunks/9353-cff34f7e773b2e2b.js" defer=""></script><script src="/dashboard/_next/static/chunks/2260-7703229c33c5ebd5.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/7615-3301e838e5f25772.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-e6aa9ab418717c59.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-c5a3eeee1c218af1.js" defer=""></script><script src="/dashboard/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/zB0ed6ge_W1MDszVHhijS/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"zB0ed6ge_W1MDszVHhijS","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/workspaces.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-6a5ddd0184bfa22c.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-69c80d677d3c2949.js" defer=""></script><script src="/dashboard/_next/static/hIViZcQBkn0HE8SpaSsUU/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/hIViZcQBkn0HE8SpaSsUU/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"hIViZcQBkn0HE8SpaSsUU","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/0748ce22df867032.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/0748ce22df867032.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-2679be77fc08a2f8.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-bde01e4a2beec258.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-22b23febb3e89ce1.js" defer=""></script><script src="/dashboard/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/zB0ed6ge_W1MDszVHhijS/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"zB0ed6ge_W1MDszVHhijS","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/data/data_utils.py CHANGED
@@ -19,6 +19,7 @@ from sky import sky_logging
19
19
  from sky.adaptors import aws
20
20
  from sky.adaptors import azure
21
21
  from sky.adaptors import cloudflare
22
+ from sky.adaptors import coreweave
22
23
  from sky.adaptors import gcp
23
24
  from sky.adaptors import ibm
24
25
  from sky.adaptors import nebius
@@ -625,6 +626,7 @@ class Rclone:
625
626
  R2 = 'R2'
626
627
  AZURE = 'AZURE'
627
628
  NEBIUS = 'NEBIUS'
629
+ COREWEAVE = 'COREWEAVE'
628
630
 
629
631
  def get_profile_name(self, bucket_name: str) -> str:
630
632
  """Gets the Rclone profile name for a given bucket.
@@ -642,7 +644,8 @@ class Rclone:
642
644
  Rclone.RcloneStores.IBM: 'sky-ibm',
643
645
  Rclone.RcloneStores.R2: 'sky-r2',
644
646
  Rclone.RcloneStores.AZURE: 'sky-azure',
645
- Rclone.RcloneStores.NEBIUS: 'sky-nebius'
647
+ Rclone.RcloneStores.NEBIUS: 'sky-nebius',
648
+ Rclone.RcloneStores.COREWEAVE: 'sky-coreweave'
646
649
  }
647
650
  return f'{profile_prefix[self]}-{bucket_name}'
648
651
 
@@ -748,6 +751,25 @@ class Rclone:
748
751
  endpoint = {endpoint_url}
749
752
  acl = private
750
753
  """)
754
+ elif self is Rclone.RcloneStores.COREWEAVE:
755
+ coreweave_session = coreweave.session()
756
+ coreweave_credentials = coreweave.get_coreweave_credentials(
757
+ coreweave_session)
758
+ # Get endpoint URL from the client
759
+ endpoint_url = coreweave.get_endpoint()
760
+ access_key_id = coreweave_credentials.access_key
761
+ secret_access_key = coreweave_credentials.secret_key
762
+ config = textwrap.dedent(f"""\
763
+ [{rclone_profile_name}]
764
+ type = s3
765
+ provider = Other
766
+ access_key_id = {access_key_id}
767
+ secret_access_key = {secret_access_key}
768
+ endpoint = {endpoint_url}
769
+ region = auto
770
+ acl = private
771
+ force_path_style = false
772
+ """)
751
773
  else:
752
774
  with ux_utils.print_exception_no_traceback():
753
775
  raise NotImplementedError(
@@ -908,3 +930,72 @@ def split_oci_path(oci_path: str) -> Tuple[str, str]:
908
930
  bucket = path_parts.pop(0)
909
931
  key = '/'.join(path_parts)
910
932
  return bucket, key
933
+
934
+
935
+ def create_coreweave_client() -> Client:
936
+ """Create CoreWeave S3 client."""
937
+ return coreweave.client('s3')
938
+
939
+
940
+ def split_coreweave_path(coreweave_path: str) -> Tuple[str, str]:
941
+ """Splits CoreWeave Path into Bucket name and Relative Path to Bucket
942
+
943
+ Args:
944
+ coreweave_path: str; CoreWeave Path, e.g. cw://imagenet/train/
945
+ """
946
+ path_parts = coreweave_path.replace('cw://', '').split('/')
947
+ bucket = path_parts.pop(0)
948
+ key = '/'.join(path_parts)
949
+ return bucket, key
950
+
951
+
952
+ def verify_coreweave_bucket(name: str, retry: int = 0) -> bool:
953
+ """Verify CoreWeave bucket exists and is accessible.
954
+
955
+ Retries head_bucket operation up to retry times with 5 second intervals
956
+ to handle DNS propagation delays or temporary connectivity issues.
957
+ """
958
+ coreweave_client = create_coreweave_client()
959
+ max_retries = retry + 1 # 5s * (retry+1) = total seconds to retry
960
+ retry_count = 0
961
+
962
+ while retry_count < max_retries:
963
+ try:
964
+ coreweave_client.head_bucket(Bucket=name)
965
+ if retry_count > 0:
966
+ logger.debug(
967
+ f'Successfully verified bucket {name} after '
968
+ f'{retry_count} retries ({retry_count * 5} seconds)')
969
+ return True
970
+
971
+ except coreweave.botocore.exceptions.ClientError as e: # type: ignore[union-attr] # pylint: disable=line-too-long:
972
+ error_code = e.response['Error']['Code']
973
+ if error_code == '403':
974
+ logger.error(f'Access denied to bucket {name}')
975
+ return False
976
+ elif error_code == '404':
977
+ logger.debug(f'Bucket {name} does not exist')
978
+ else:
979
+ logger.debug(
980
+ f'Unexpected error checking CoreWeave bucket {name}: {e}')
981
+ except Exception as e: # pylint: disable=broad-except
982
+ logger.debug(
983
+ f'Unexpected error checking CoreWeave bucket {name}: {e}')
984
+
985
+ # Common retry logic for all transient errors
986
+ retry_count += 1
987
+ if retry_count < max_retries:
988
+ logger.debug(f'Error checking CoreWeave bucket {name} '
989
+ f'(attempt {retry_count}/{max_retries}). '
990
+ f'Retrying in 5 seconds...')
991
+ time.sleep(5)
992
+ else:
993
+ attempt_str = 'attempt'
994
+ if max_retries > 1:
995
+ attempt_str += 's'
996
+ logger.error(f'Failed to verify CoreWeave bucket {name} after '
997
+ f'{max_retries} {attempt_str}.')
998
+ return False
999
+
1000
+ # Should not reach here, but just in case
1001
+ return False
sky/data/mounting_utils.py CHANGED
@@ -17,6 +17,16 @@ _TYPE_CACHE_TTL = '5s'
17
17
  _RENAME_DIR_LIMIT = 10000
18
18
  # https://github.com/GoogleCloudPlatform/gcsfuse/releases
19
19
  GCSFUSE_VERSION = '2.2.0'
20
+
21
+ # Some machines do not have fuse/fuse3 installed by default
22
+ # hence rclone will fail on these machines
23
+ FUSE3_INSTALL_CMD = ('(command -v fusermount3 > /dev/null 2>&1 || '
24
+ '((which apt-get > /dev/null 2>&1 && '
25
+ 'sudo apt-get update && sudo apt-get install -y fuse3) || '
26
+ '(which yum > /dev/null 2>&1 && '
27
+ 'sudo yum install -y fuse3) || '
28
+ 'true)) || true')
29
+
20
30
  # Creates a fusermount3 soft link on older (<22) Ubuntu systems to utilize
21
31
  # Rclone's mounting utility.
22
32
  FUSERMOUNT3_SOFT_LINK_CMD = ('[ ! -f /bin/fusermount3 ] && '
@@ -54,10 +64,10 @@ def get_rclone_install_cmd() -> str:
54
64
  f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
55
65
  f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
56
66
  f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb)))'
57
- f' || (which rclone > /dev/null || (cd ~ > /dev/null'
67
+ f' || (which yum > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
58
68
  f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm'
59
69
  f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm -y'
60
- f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm))')
70
+ f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm)))')
61
71
  return install_cmd
62
72
 
63
73
 
@@ -94,6 +104,7 @@ def get_s3_mount_cmd(bucket_name: str,
94
104
  # Use rclone for ARM64 architectures since goofys doesn't support them
95
105
  arch_check = 'ARCH=$(uname -m) && '
96
106
  rclone_mount = (
107
+ f'{FUSE3_INSTALL_CMD} && '
97
108
  f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
98
109
  f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
99
110
  # Have to add --s3-env-auth=true to allow rclone to access private
@@ -128,6 +139,7 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
128
139
  # Use rclone for ARM64 architectures since goofys doesn't support them
129
140
  arch_check = 'ARCH=$(uname -m) && '
130
141
  rclone_mount = (
142
+ f'{FUSE3_INSTALL_CMD} && '
131
143
  f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
132
144
  f'AWS_PROFILE={nebius_profile_name} '
133
145
  f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
@@ -148,6 +160,46 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
148
160
  return mount_cmd
149
161
 
150
162
 
163
+ def get_coreweave_mount_cmd(cw_credentials_path: str,
164
+ coreweave_profile_name: str,
165
+ bucket_name: str,
166
+ endpoint_url: str,
167
+ mount_path: str,
168
+ _bucket_sub_path: Optional[str] = None) -> str:
169
+ """Returns a command to mount CoreWeave bucket"""
170
+ if _bucket_sub_path is None:
171
+ _bucket_sub_path = ''
172
+ else:
173
+ _bucket_sub_path = f':{_bucket_sub_path}'
174
+
175
+ # Use rclone for ARM64 architectures since goofys doesn't support them
176
+ arch_check = 'ARCH=$(uname -m) && '
177
+ rclone_mount = (
178
+ f'{FUSE3_INSTALL_CMD} && '
179
+ f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
180
+ f'AWS_SHARED_CREDENTIALS_FILE={cw_credentials_path} '
181
+ f'AWS_PROFILE={coreweave_profile_name} '
182
+ f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
183
+ f'--s3-force-path-style=false '
184
+ f'--s3-endpoint {endpoint_url} --daemon --allow-other')
185
+ goofys_mount = (f'AWS_SHARED_CREDENTIALS_FILE={cw_credentials_path} '
186
+ f'AWS_PROFILE={coreweave_profile_name} {_GOOFYS_WRAPPER} '
187
+ '-o allow_other '
188
+ f'--stat-cache-ttl {_STAT_CACHE_TTL} '
189
+ f'--type-cache-ttl {_TYPE_CACHE_TTL} '
190
+ f'--subdomain '
191
+ f'--endpoint {endpoint_url} '
192
+ f'{bucket_name}{_bucket_sub_path} {mount_path}')
193
+
194
+ mount_cmd = (f'{arch_check}'
195
+ f'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
196
+ f' {rclone_mount}; '
197
+ f'else '
198
+ f' {goofys_mount}; '
199
+ f'fi')
200
+ return mount_cmd
201
+
202
+
151
203
  def get_gcs_mount_install_cmd() -> str:
152
204
  """Returns a command to install GCS mount utility gcsfuse."""
153
205
  install_cmd = ('ARCH=$(uname -m) && '
@@ -185,27 +237,72 @@ def get_gcs_mount_cmd(bucket_name: str,
185
237
  def get_az_mount_install_cmd() -> str:
186
238
  """Returns a command to install AZ Container mount utility blobfuse2."""
187
239
  install_cmd = (
188
- 'sudo apt-get update; '
189
- 'sudo apt-get install -y '
190
- '-o Dpkg::Options::="--force-confdef" '
191
- 'fuse3 libfuse3-dev || { '
192
- ' echo "fuse3 not available, falling back to fuse"; '
193
- ' sudo apt-get install -y '
194
- ' -o Dpkg::Options::="--force-confdef" '
195
- ' fuse libfuse-dev; '
196
- '} && '
240
+ # Check architecture first - blobfuse2 only supports x86_64
197
241
  'ARCH=$(uname -m) && '
198
242
  'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
199
243
  ' echo "blobfuse2 is not supported on $ARCH" && '
200
244
  f' exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
245
+ 'fi && '
246
+ # Try to install fuse3 from default repos
247
+ 'sudo apt-get update && '
248
+ 'FUSE3_INSTALLED=0 && '
249
+ # On Kubernetes, if FUSERMOUNT_SHARED_DIR is set, it means
250
+ # fusermount and fusermount3 is symlinked to fusermount-shim.
251
+ # If we reinstall fuse3, it may overwrite the symlink, so
252
+ # just install libfuse3, which is needed by blobfuse2.
253
+ 'if [ -n "${FUSERMOUNT_SHARED_DIR:-}" ]; then '
254
+ ' PACKAGES="libfuse3-3 libfuse3-dev"; '
255
+ 'else '
256
+ ' PACKAGES="fuse3 libfuse3-3 libfuse3-dev"; '
257
+ 'fi && '
258
+ 'if sudo apt-get install -y '
259
+ '-o Dpkg::Options::="--force-confdef" '
260
+ '$PACKAGES; then '
261
+ ' FUSE3_INSTALLED=1; '
262
+ ' echo "fuse3 installed from default repos"; '
201
263
  'else '
202
- ' ARCH_SUFFIX="x86_64"; '
264
+ # If fuse3 not available, try focal for Ubuntu <= 20.04
265
+ ' DISTRO=$(grep "^ID=" /etc/os-release | cut -d= -f2 | '
266
+ 'tr -d \'"\' | tr "[:upper:]" "[:lower:]") && '
267
+ ' VERSION=$(grep "^VERSION_ID=" /etc/os-release | cut -d= -f2 | '
268
+ 'tr -d \'"\') && '
269
+ ' if [ "$DISTRO" = "ubuntu" ] && '
270
+ '[ "$(echo "$VERSION 20.04" | '
271
+ 'awk \'{ print ($1 <= $2) }\')" = "1" ]; then '
272
+ ' echo "Trying to install fuse3 from focal for '
273
+ 'Ubuntu $VERSION"; '
274
+ ' echo "deb http://archive.ubuntu.com/ubuntu '
275
+ 'focal main universe" | '
276
+ 'sudo tee /etc/apt/sources.list.d/focal-fuse3.list && '
277
+ ' sudo apt-get update && '
278
+ ' if sudo apt-get install -y '
279
+ '-o Dpkg::Options::="--force-confdef" '
280
+ '-o Dpkg::Options::="--force-confold" '
281
+ '$PACKAGES; then '
282
+ ' FUSE3_INSTALLED=1; '
283
+ ' echo "fuse3 installed from focal"; '
284
+ ' sudo rm /etc/apt/sources.list.d/focal-fuse3.list; '
285
+ ' sudo apt-get update; '
286
+ ' else '
287
+ ' sudo rm -f /etc/apt/sources.list.d/focal-fuse3.list; '
288
+ ' sudo apt-get update; '
289
+ ' fi; '
290
+ ' fi; '
203
291
  'fi && '
204
- 'wget -nc https://github.com/Azure/azure-storage-fuse'
205
- f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
206
- f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
292
+ # Install blobfuse2 only if fuse3 is available
293
+ 'if [ "$FUSE3_INSTALLED" = "1" ]; then '
294
+ ' echo "Installing blobfuse2 with libfuse3 support"; '
295
+ ' wget -nc https://github.com/Azure/azure-storage-fuse'
296
+ f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}/'
297
+ f'blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
207
298
  '-O /tmp/blobfuse2.deb && '
208
- 'sudo dpkg --install /tmp/blobfuse2.deb && '
299
+ ' sudo dpkg --install /tmp/blobfuse2.deb; '
300
+ 'else '
301
+ ' echo "Error: libfuse3 is required for Azure storage '
302
+ 'mounting with fusermount-wrapper."; '
303
+ ' echo "libfuse3 could not be installed on this system."; '
304
+ f' exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
305
+ 'fi && '
209
306
  f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
210
307
 
211
308
  return install_cmd
@@ -277,7 +374,10 @@ def get_az_mount_cmd(container_name: str,
277
374
  f'-- {blobfuse2_cmd} -o nonempty --foreground {{}}')
278
375
  original = f'{blobfuse2_cmd} {blobfuse2_options} {mount_path}'
279
376
  # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
280
- # to avoid requiring root privilege.
377
+ # to avoid requiring privileged containers.
378
+ # fusermount-wrapper requires libfuse3;
379
+ # we install libfuse3 even on older distros like Ubuntu 18.04 by using
380
+ # Ubuntu 20.04 (focal) repositories.
281
381
  # TODO(aylei): feeling hacky, refactor this.
282
382
  get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
283
383
  f'echo "{wrapped}" || echo "{original}"')
@@ -304,6 +404,7 @@ def get_r2_mount_cmd(r2_credentials_path: str,
304
404
  # Use rclone for ARM64 architectures since goofys doesn't support them
305
405
  arch_check = 'ARCH=$(uname -m) && '
306
406
  rclone_mount = (
407
+ f'{FUSE3_INSTALL_CMD} && '
307
408
  f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
308
409
  f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
309
410
  f'AWS_PROFILE={r2_profile_name} '
@@ -333,7 +434,8 @@ def get_cos_mount_cmd(rclone_config: str,
333
434
  _bucket_sub_path: Optional[str] = None) -> str:
334
435
  """Returns a command to mount an IBM COS bucket using rclone."""
335
436
  # stores bucket profile in rclone config file at the cluster's nodes.
336
- configure_rclone_profile = (f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
437
+ configure_rclone_profile = (f'{FUSE3_INSTALL_CMD} && '
438
+ f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
337
439
  f'mkdir -p {constants.RCLONE_CONFIG_DIR} && '
338
440
  f'echo "{rclone_config}" >> '
339
441
  f'{constants.RCLONE_CONFIG_PATH}')
@@ -353,7 +455,8 @@ def get_mount_cached_cmd(rclone_config: str, rclone_profile_name: str,
353
455
  bucket_name: str, mount_path: str) -> str:
354
456
  """Returns a command to mount a bucket using rclone with vfs cache."""
355
457
  # stores bucket profile in rclone config file at the remote nodes.
356
- configure_rclone_profile = (f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
458
+ configure_rclone_profile = (f'{FUSE3_INSTALL_CMD} && '
459
+ f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
357
460
  f'mkdir -p {constants.RCLONE_CONFIG_DIR} && '
358
461
  f'echo {shlex.quote(rclone_config)} >> '
359
462
  f'{constants.RCLONE_CONFIG_PATH}')
@@ -525,7 +628,28 @@ def get_mounting_script(
525
628
  fi
526
629
  fi
527
630
  echo "Mounting $SOURCE_BUCKET to $MOUNT_PATH with $MOUNT_BINARY..."
631
+ set +e
528
632
  {mount_cmd}
633
+ MOUNT_EXIT_CODE=$?
634
+ set -e
635
+ if [ $MOUNT_EXIT_CODE -ne 0 ]; then
636
+ echo "Mount failed with exit code $MOUNT_EXIT_CODE."
637
+ if [ "$MOUNT_BINARY" = "goofys" ]; then
638
+ echo "Looking for goofys log files..."
639
+ # Find goofys log files in /tmp (created by mktemp -t goofys.XXXX.log)
640
+ # Note: if /dev/log exists, goofys logs to syslog instead of a file
641
+ GOOFYS_LOGS=$(ls -t /tmp/goofys.*.log 2>/dev/null | head -1)
642
+ if [ -n "$GOOFYS_LOGS" ]; then
643
+ echo "=== Goofys log file contents ==="
644
+ cat "$GOOFYS_LOGS"
645
+ echo "=== End of goofys log file ==="
646
+ else
647
+ echo "No goofys log file found in /tmp"
648
+ fi
649
+ fi
650
+ # TODO(kevin): Print logs from rclone, etc too for observability.
651
+ exit $MOUNT_EXIT_CODE
652
+ fi
529
653
  echo "Mounting done."
530
654
  """)
531
655