skypilot-nightly 1.0.0.dev20250905__py3-none-any.whl → 1.0.0.dev20251210__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (429) hide show
  1. sky/__init__.py +12 -2
  2. sky/adaptors/aws.py +27 -22
  3. sky/adaptors/common.py +25 -2
  4. sky/adaptors/coreweave.py +278 -0
  5. sky/adaptors/do.py +8 -2
  6. sky/adaptors/gcp.py +11 -0
  7. sky/adaptors/ibm.py +5 -2
  8. sky/adaptors/kubernetes.py +64 -0
  9. sky/adaptors/nebius.py +3 -1
  10. sky/adaptors/primeintellect.py +1 -0
  11. sky/adaptors/seeweb.py +183 -0
  12. sky/adaptors/shadeform.py +89 -0
  13. sky/adaptors/slurm.py +478 -0
  14. sky/admin_policy.py +20 -0
  15. sky/authentication.py +157 -263
  16. sky/backends/__init__.py +3 -2
  17. sky/backends/backend.py +11 -3
  18. sky/backends/backend_utils.py +630 -185
  19. sky/backends/cloud_vm_ray_backend.py +1111 -928
  20. sky/backends/local_docker_backend.py +9 -5
  21. sky/backends/task_codegen.py +971 -0
  22. sky/backends/wheel_utils.py +18 -0
  23. sky/catalog/__init__.py +8 -3
  24. sky/catalog/aws_catalog.py +4 -0
  25. sky/catalog/common.py +19 -1
  26. sky/catalog/data_fetchers/fetch_aws.py +102 -80
  27. sky/catalog/data_fetchers/fetch_gcp.py +30 -3
  28. sky/catalog/data_fetchers/fetch_nebius.py +9 -6
  29. sky/catalog/data_fetchers/fetch_runpod.py +698 -0
  30. sky/catalog/data_fetchers/fetch_seeweb.py +329 -0
  31. sky/catalog/data_fetchers/fetch_shadeform.py +142 -0
  32. sky/catalog/kubernetes_catalog.py +36 -32
  33. sky/catalog/primeintellect_catalog.py +95 -0
  34. sky/catalog/runpod_catalog.py +5 -1
  35. sky/catalog/seeweb_catalog.py +184 -0
  36. sky/catalog/shadeform_catalog.py +165 -0
  37. sky/catalog/slurm_catalog.py +243 -0
  38. sky/check.py +87 -46
  39. sky/client/cli/command.py +1004 -434
  40. sky/client/cli/flags.py +4 -2
  41. sky/{volumes/utils.py → client/cli/table_utils.py} +111 -13
  42. sky/client/cli/utils.py +79 -0
  43. sky/client/common.py +12 -2
  44. sky/client/sdk.py +188 -65
  45. sky/client/sdk_async.py +34 -33
  46. sky/cloud_stores.py +82 -3
  47. sky/clouds/__init__.py +8 -0
  48. sky/clouds/aws.py +337 -129
  49. sky/clouds/azure.py +24 -18
  50. sky/clouds/cloud.py +47 -13
  51. sky/clouds/cudo.py +16 -13
  52. sky/clouds/do.py +9 -7
  53. sky/clouds/fluidstack.py +12 -5
  54. sky/clouds/gcp.py +14 -7
  55. sky/clouds/hyperbolic.py +12 -5
  56. sky/clouds/ibm.py +12 -5
  57. sky/clouds/kubernetes.py +80 -45
  58. sky/clouds/lambda_cloud.py +12 -5
  59. sky/clouds/nebius.py +23 -9
  60. sky/clouds/oci.py +19 -12
  61. sky/clouds/paperspace.py +4 -1
  62. sky/clouds/primeintellect.py +317 -0
  63. sky/clouds/runpod.py +85 -24
  64. sky/clouds/scp.py +12 -8
  65. sky/clouds/seeweb.py +477 -0
  66. sky/clouds/shadeform.py +400 -0
  67. sky/clouds/slurm.py +578 -0
  68. sky/clouds/ssh.py +6 -3
  69. sky/clouds/utils/scp_utils.py +61 -50
  70. sky/clouds/vast.py +43 -27
  71. sky/clouds/vsphere.py +14 -16
  72. sky/core.py +296 -195
  73. sky/dashboard/out/404.html +1 -1
  74. sky/dashboard/out/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js +1 -0
  75. sky/dashboard/out/_next/static/chunks/1141-9c810f01ff4f398a.js +11 -0
  76. sky/dashboard/out/_next/static/chunks/1871-7e202677c42f43fe.js +6 -0
  77. sky/dashboard/out/_next/static/chunks/2260-7703229c33c5ebd5.js +1 -0
  78. sky/dashboard/out/_next/static/chunks/2369.fc20f0c2c8ed9fe7.js +15 -0
  79. sky/dashboard/out/_next/static/chunks/2755.edd818326d489a1d.js +26 -0
  80. sky/dashboard/out/_next/static/chunks/3294.ddda8c6c6f9f24dc.js +1 -0
  81. sky/dashboard/out/_next/static/chunks/3785.7e245f318f9d1121.js +1 -0
  82. sky/dashboard/out/_next/static/chunks/{6601-06114c982db410b6.js → 3800-b589397dc09c5b4e.js} +1 -1
  83. sky/dashboard/out/_next/static/chunks/3850-fd5696f3bbbaddae.js +1 -0
  84. sky/dashboard/out/_next/static/chunks/4725.172ede95d1b21022.js +1 -0
  85. sky/dashboard/out/_next/static/chunks/4937.a2baa2df5572a276.js +15 -0
  86. sky/dashboard/out/_next/static/chunks/6212-7bd06f60ba693125.js +13 -0
  87. sky/dashboard/out/_next/static/chunks/6856-da20c5fd999f319c.js +1 -0
  88. sky/dashboard/out/_next/static/chunks/6990-09cbf02d3cd518c3.js +1 -0
  89. sky/dashboard/out/_next/static/chunks/7359-c8d04e06886000b3.js +30 -0
  90. sky/dashboard/out/_next/static/chunks/7615-019513abc55b3b47.js +1 -0
  91. sky/dashboard/out/_next/static/chunks/8640.5b9475a2d18c5416.js +16 -0
  92. sky/dashboard/out/_next/static/chunks/8969-452f9d5cbdd2dc73.js +1 -0
  93. sky/dashboard/out/_next/static/chunks/9025.fa408f3242e9028d.js +6 -0
  94. sky/dashboard/out/_next/static/chunks/9353-8369df1cf105221c.js +1 -0
  95. sky/dashboard/out/_next/static/chunks/9360.a536cf6b1fa42355.js +31 -0
  96. sky/dashboard/out/_next/static/chunks/9847.3aaca6bb33455140.js +30 -0
  97. sky/dashboard/out/_next/static/chunks/pages/_app-68b647e26f9d2793.js +34 -0
  98. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-33f525539665fdfd.js +16 -0
  99. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-a7565f586ef86467.js +1 -0
  100. sky/dashboard/out/_next/static/chunks/pages/clusters-9e5d47818b9bdadd.js +1 -0
  101. sky/dashboard/out/_next/static/chunks/pages/{config-dfb9bf07b13045f4.js → config-718cdc365de82689.js} +1 -1
  102. sky/dashboard/out/_next/static/chunks/pages/infra/{[context]-6563820e094f68ca.js → [context]-12c559ec4d81fdbd.js} +1 -1
  103. sky/dashboard/out/_next/static/chunks/pages/{infra-aabba60d57826e0f.js → infra-d187cd0413d72475.js} +1 -1
  104. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-895847b6cf200b04.js +16 -0
  105. sky/dashboard/out/_next/static/chunks/pages/jobs/pools/[pool]-8d0f4655400b4eb9.js +21 -0
  106. sky/dashboard/out/_next/static/chunks/pages/jobs-e5a98f17f8513a96.js +1 -0
  107. sky/dashboard/out/_next/static/chunks/pages/plugins/[...slug]-4f46050ca065d8f8.js +1 -0
  108. sky/dashboard/out/_next/static/chunks/pages/users-2f7646eb77785a2c.js +1 -0
  109. sky/dashboard/out/_next/static/chunks/pages/volumes-ef19d49c6d0e8500.js +1 -0
  110. sky/dashboard/out/_next/static/chunks/pages/workspaces/{[name]-af76bb06dbb3954f.js → [name]-96e0f298308da7e2.js} +1 -1
  111. sky/dashboard/out/_next/static/chunks/pages/{workspaces-7598c33a746cdc91.js → workspaces-cb4da3abe08ebf19.js} +1 -1
  112. sky/dashboard/out/_next/static/chunks/webpack-fba3de387ff6bb08.js +1 -0
  113. sky/dashboard/out/_next/static/css/c5a4cfd2600fc715.css +3 -0
  114. sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
  115. sky/dashboard/out/clusters/[cluster].html +1 -1
  116. sky/dashboard/out/clusters.html +1 -1
  117. sky/dashboard/out/config.html +1 -1
  118. sky/dashboard/out/index.html +1 -1
  119. sky/dashboard/out/infra/[context].html +1 -1
  120. sky/dashboard/out/infra.html +1 -1
  121. sky/dashboard/out/jobs/[job].html +1 -1
  122. sky/dashboard/out/jobs/pools/[pool].html +1 -1
  123. sky/dashboard/out/jobs.html +1 -1
  124. sky/dashboard/out/plugins/[...slug].html +1 -0
  125. sky/dashboard/out/users.html +1 -1
  126. sky/dashboard/out/volumes.html +1 -1
  127. sky/dashboard/out/workspace/new.html +1 -1
  128. sky/dashboard/out/workspaces/[name].html +1 -1
  129. sky/dashboard/out/workspaces.html +1 -1
  130. sky/data/data_utils.py +92 -1
  131. sky/data/mounting_utils.py +177 -30
  132. sky/data/storage.py +200 -19
  133. sky/data/storage_utils.py +10 -45
  134. sky/exceptions.py +18 -7
  135. sky/execution.py +74 -31
  136. sky/global_user_state.py +605 -191
  137. sky/jobs/__init__.py +2 -0
  138. sky/jobs/client/sdk.py +101 -4
  139. sky/jobs/client/sdk_async.py +31 -5
  140. sky/jobs/constants.py +15 -8
  141. sky/jobs/controller.py +726 -284
  142. sky/jobs/file_content_utils.py +128 -0
  143. sky/jobs/log_gc.py +193 -0
  144. sky/jobs/recovery_strategy.py +250 -100
  145. sky/jobs/scheduler.py +271 -173
  146. sky/jobs/server/core.py +367 -114
  147. sky/jobs/server/server.py +81 -35
  148. sky/jobs/server/utils.py +89 -35
  149. sky/jobs/state.py +1498 -620
  150. sky/jobs/utils.py +771 -306
  151. sky/logs/agent.py +40 -5
  152. sky/logs/aws.py +9 -19
  153. sky/metrics/utils.py +282 -39
  154. sky/models.py +2 -0
  155. sky/optimizer.py +7 -6
  156. sky/provision/__init__.py +38 -1
  157. sky/provision/aws/config.py +34 -13
  158. sky/provision/aws/instance.py +5 -2
  159. sky/provision/azure/instance.py +5 -3
  160. sky/provision/common.py +22 -0
  161. sky/provision/cudo/instance.py +4 -3
  162. sky/provision/do/instance.py +4 -3
  163. sky/provision/docker_utils.py +112 -28
  164. sky/provision/fluidstack/instance.py +6 -5
  165. sky/provision/gcp/config.py +6 -1
  166. sky/provision/gcp/instance.py +4 -2
  167. sky/provision/hyperbolic/instance.py +4 -2
  168. sky/provision/instance_setup.py +66 -20
  169. sky/provision/kubernetes/__init__.py +2 -0
  170. sky/provision/kubernetes/config.py +7 -44
  171. sky/provision/kubernetes/constants.py +0 -1
  172. sky/provision/kubernetes/instance.py +609 -213
  173. sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml +1 -2
  174. sky/provision/kubernetes/network.py +12 -8
  175. sky/provision/kubernetes/network_utils.py +8 -25
  176. sky/provision/kubernetes/utils.py +422 -422
  177. sky/provision/kubernetes/volume.py +150 -18
  178. sky/provision/lambda_cloud/instance.py +16 -13
  179. sky/provision/nebius/instance.py +6 -2
  180. sky/provision/nebius/utils.py +103 -86
  181. sky/provision/oci/instance.py +4 -2
  182. sky/provision/paperspace/instance.py +4 -3
  183. sky/provision/primeintellect/__init__.py +10 -0
  184. sky/provision/primeintellect/config.py +11 -0
  185. sky/provision/primeintellect/instance.py +454 -0
  186. sky/provision/primeintellect/utils.py +398 -0
  187. sky/provision/provisioner.py +45 -15
  188. sky/provision/runpod/__init__.py +2 -0
  189. sky/provision/runpod/instance.py +4 -3
  190. sky/provision/runpod/volume.py +69 -13
  191. sky/provision/scp/instance.py +307 -130
  192. sky/provision/seeweb/__init__.py +11 -0
  193. sky/provision/seeweb/config.py +13 -0
  194. sky/provision/seeweb/instance.py +812 -0
  195. sky/provision/shadeform/__init__.py +11 -0
  196. sky/provision/shadeform/config.py +12 -0
  197. sky/provision/shadeform/instance.py +351 -0
  198. sky/provision/shadeform/shadeform_utils.py +83 -0
  199. sky/provision/slurm/__init__.py +12 -0
  200. sky/provision/slurm/config.py +13 -0
  201. sky/provision/slurm/instance.py +572 -0
  202. sky/provision/slurm/utils.py +583 -0
  203. sky/provision/vast/instance.py +9 -4
  204. sky/provision/vast/utils.py +10 -6
  205. sky/provision/volume.py +164 -0
  206. sky/provision/vsphere/common/ssl_helper.py +1 -1
  207. sky/provision/vsphere/common/vapiconnect.py +2 -1
  208. sky/provision/vsphere/common/vim_utils.py +3 -2
  209. sky/provision/vsphere/instance.py +8 -6
  210. sky/provision/vsphere/vsphere_utils.py +8 -1
  211. sky/resources.py +11 -3
  212. sky/schemas/api/responses.py +107 -6
  213. sky/schemas/db/global_user_state/008_skylet_ssh_tunnel_metadata.py +34 -0
  214. sky/schemas/db/global_user_state/009_last_activity_and_launched_at.py +89 -0
  215. sky/schemas/db/global_user_state/010_save_ssh_key.py +66 -0
  216. sky/schemas/db/global_user_state/011_is_ephemeral.py +34 -0
  217. sky/schemas/db/kv_cache/001_initial_schema.py +29 -0
  218. sky/schemas/db/serve_state/002_yaml_content.py +34 -0
  219. sky/schemas/db/skypilot_config/001_initial_schema.py +30 -0
  220. sky/schemas/db/spot_jobs/002_cluster_pool.py +3 -3
  221. sky/schemas/db/spot_jobs/004_job_file_contents.py +42 -0
  222. sky/schemas/db/spot_jobs/005_logs_gc.py +38 -0
  223. sky/schemas/db/spot_jobs/006_controller_pid_started_at.py +34 -0
  224. sky/schemas/db/spot_jobs/007_config_file_content.py +34 -0
  225. sky/schemas/generated/jobsv1_pb2.py +86 -0
  226. sky/schemas/generated/jobsv1_pb2.pyi +254 -0
  227. sky/schemas/generated/jobsv1_pb2_grpc.py +542 -0
  228. sky/schemas/generated/managed_jobsv1_pb2.py +76 -0
  229. sky/schemas/generated/managed_jobsv1_pb2.pyi +278 -0
  230. sky/schemas/generated/managed_jobsv1_pb2_grpc.py +278 -0
  231. sky/schemas/generated/servev1_pb2.py +58 -0
  232. sky/schemas/generated/servev1_pb2.pyi +115 -0
  233. sky/schemas/generated/servev1_pb2_grpc.py +322 -0
  234. sky/serve/autoscalers.py +2 -0
  235. sky/serve/client/impl.py +55 -21
  236. sky/serve/constants.py +4 -3
  237. sky/serve/controller.py +17 -11
  238. sky/serve/load_balancing_policies.py +1 -1
  239. sky/serve/replica_managers.py +219 -142
  240. sky/serve/serve_rpc_utils.py +179 -0
  241. sky/serve/serve_state.py +63 -54
  242. sky/serve/serve_utils.py +145 -109
  243. sky/serve/server/core.py +46 -25
  244. sky/serve/server/impl.py +311 -162
  245. sky/serve/server/server.py +21 -19
  246. sky/serve/service.py +84 -68
  247. sky/serve/service_spec.py +45 -7
  248. sky/server/auth/loopback.py +38 -0
  249. sky/server/auth/oauth2_proxy.py +12 -7
  250. sky/server/common.py +47 -24
  251. sky/server/config.py +62 -28
  252. sky/server/constants.py +9 -1
  253. sky/server/daemons.py +109 -38
  254. sky/server/metrics.py +76 -96
  255. sky/server/middleware_utils.py +166 -0
  256. sky/server/plugins.py +222 -0
  257. sky/server/requests/executor.py +384 -145
  258. sky/server/requests/payloads.py +83 -19
  259. sky/server/requests/preconditions.py +15 -13
  260. sky/server/requests/request_names.py +123 -0
  261. sky/server/requests/requests.py +511 -157
  262. sky/server/requests/serializers/decoders.py +48 -17
  263. sky/server/requests/serializers/encoders.py +102 -20
  264. sky/server/requests/serializers/return_value_serializers.py +60 -0
  265. sky/server/requests/threads.py +117 -0
  266. sky/server/rest.py +116 -24
  267. sky/server/server.py +497 -179
  268. sky/server/server_utils.py +30 -0
  269. sky/server/stream_utils.py +219 -45
  270. sky/server/uvicorn.py +30 -19
  271. sky/setup_files/MANIFEST.in +6 -1
  272. sky/setup_files/alembic.ini +8 -0
  273. sky/setup_files/dependencies.py +64 -19
  274. sky/setup_files/setup.py +44 -44
  275. sky/sky_logging.py +13 -5
  276. sky/skylet/attempt_skylet.py +116 -24
  277. sky/skylet/configs.py +3 -1
  278. sky/skylet/constants.py +139 -29
  279. sky/skylet/events.py +74 -14
  280. sky/skylet/executor/__init__.py +1 -0
  281. sky/skylet/executor/slurm.py +189 -0
  282. sky/skylet/job_lib.py +143 -105
  283. sky/skylet/log_lib.py +252 -8
  284. sky/skylet/log_lib.pyi +47 -7
  285. sky/skylet/providers/ibm/node_provider.py +12 -8
  286. sky/skylet/providers/ibm/vpc_provider.py +13 -12
  287. sky/skylet/runtime_utils.py +21 -0
  288. sky/skylet/services.py +524 -0
  289. sky/skylet/skylet.py +27 -2
  290. sky/skylet/subprocess_daemon.py +104 -28
  291. sky/skypilot_config.py +99 -79
  292. sky/ssh_node_pools/constants.py +12 -0
  293. sky/ssh_node_pools/core.py +40 -3
  294. sky/ssh_node_pools/deploy/__init__.py +4 -0
  295. sky/ssh_node_pools/deploy/deploy.py +952 -0
  296. sky/ssh_node_pools/deploy/tunnel_utils.py +199 -0
  297. sky/ssh_node_pools/deploy/utils.py +173 -0
  298. sky/ssh_node_pools/server.py +20 -21
  299. sky/{utils/kubernetes/ssh_utils.py → ssh_node_pools/utils.py} +9 -6
  300. sky/task.py +221 -104
  301. sky/templates/aws-ray.yml.j2 +1 -0
  302. sky/templates/azure-ray.yml.j2 +1 -0
  303. sky/templates/cudo-ray.yml.j2 +1 -0
  304. sky/templates/do-ray.yml.j2 +1 -0
  305. sky/templates/fluidstack-ray.yml.j2 +1 -0
  306. sky/templates/gcp-ray.yml.j2 +1 -0
  307. sky/templates/hyperbolic-ray.yml.j2 +1 -0
  308. sky/templates/ibm-ray.yml.j2 +2 -1
  309. sky/templates/jobs-controller.yaml.j2 +3 -0
  310. sky/templates/kubernetes-ray.yml.j2 +204 -55
  311. sky/templates/lambda-ray.yml.j2 +1 -0
  312. sky/templates/nebius-ray.yml.j2 +3 -0
  313. sky/templates/oci-ray.yml.j2 +1 -0
  314. sky/templates/paperspace-ray.yml.j2 +1 -0
  315. sky/templates/primeintellect-ray.yml.j2 +72 -0
  316. sky/templates/runpod-ray.yml.j2 +1 -0
  317. sky/templates/scp-ray.yml.j2 +1 -0
  318. sky/templates/seeweb-ray.yml.j2 +171 -0
  319. sky/templates/shadeform-ray.yml.j2 +73 -0
  320. sky/templates/slurm-ray.yml.j2 +85 -0
  321. sky/templates/vast-ray.yml.j2 +2 -0
  322. sky/templates/vsphere-ray.yml.j2 +1 -0
  323. sky/templates/websocket_proxy.py +188 -43
  324. sky/usage/usage_lib.py +16 -4
  325. sky/users/model.conf +1 -1
  326. sky/users/permission.py +84 -44
  327. sky/users/rbac.py +31 -3
  328. sky/utils/accelerator_registry.py +6 -3
  329. sky/utils/admin_policy_utils.py +18 -5
  330. sky/utils/annotations.py +128 -6
  331. sky/utils/asyncio_utils.py +78 -0
  332. sky/utils/atomic.py +1 -1
  333. sky/utils/auth_utils.py +153 -0
  334. sky/utils/cli_utils/status_utils.py +12 -7
  335. sky/utils/cluster_utils.py +28 -6
  336. sky/utils/command_runner.py +283 -30
  337. sky/utils/command_runner.pyi +63 -7
  338. sky/utils/common.py +3 -1
  339. sky/utils/common_utils.py +55 -7
  340. sky/utils/config_utils.py +1 -14
  341. sky/utils/context.py +127 -40
  342. sky/utils/context_utils.py +73 -18
  343. sky/utils/controller_utils.py +229 -70
  344. sky/utils/db/db_utils.py +95 -18
  345. sky/utils/db/kv_cache.py +149 -0
  346. sky/utils/db/migration_utils.py +24 -7
  347. sky/utils/env_options.py +4 -0
  348. sky/utils/git.py +559 -1
  349. sky/utils/kubernetes/create_cluster.sh +15 -30
  350. sky/utils/kubernetes/delete_cluster.sh +10 -7
  351. sky/utils/kubernetes/generate_kind_config.py +6 -66
  352. sky/utils/kubernetes/gpu_labeler.py +13 -3
  353. sky/utils/kubernetes/k8s_gpu_labeler_job.yaml +2 -1
  354. sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml +16 -16
  355. sky/utils/kubernetes/kubernetes_deploy_utils.py +187 -260
  356. sky/utils/kubernetes/rsync_helper.sh +11 -3
  357. sky/utils/kubernetes/ssh-tunnel.sh +7 -376
  358. sky/utils/kubernetes_enums.py +7 -15
  359. sky/utils/lock_events.py +4 -4
  360. sky/utils/locks.py +128 -31
  361. sky/utils/log_utils.py +0 -319
  362. sky/utils/resource_checker.py +13 -10
  363. sky/utils/resources_utils.py +53 -29
  364. sky/utils/rich_utils.py +8 -4
  365. sky/utils/schemas.py +138 -52
  366. sky/utils/subprocess_utils.py +17 -4
  367. sky/utils/thread_utils.py +91 -0
  368. sky/utils/timeline.py +2 -1
  369. sky/utils/ux_utils.py +35 -1
  370. sky/utils/volume.py +88 -4
  371. sky/utils/yaml_utils.py +9 -0
  372. sky/volumes/client/sdk.py +48 -10
  373. sky/volumes/server/core.py +59 -22
  374. sky/volumes/server/server.py +46 -17
  375. sky/volumes/volume.py +54 -42
  376. sky/workspaces/core.py +57 -21
  377. sky/workspaces/server.py +13 -12
  378. sky_templates/README.md +3 -0
  379. sky_templates/__init__.py +3 -0
  380. sky_templates/ray/__init__.py +0 -0
  381. sky_templates/ray/start_cluster +183 -0
  382. sky_templates/ray/stop_cluster +75 -0
  383. {skypilot_nightly-1.0.0.dev20250905.dist-info → skypilot_nightly-1.0.0.dev20251210.dist-info}/METADATA +343 -65
  384. skypilot_nightly-1.0.0.dev20251210.dist-info/RECORD +629 -0
  385. skypilot_nightly-1.0.0.dev20251210.dist-info/top_level.txt +2 -0
  386. sky/client/cli/git.py +0 -549
  387. sky/dashboard/out/_next/static/chunks/1121-408ed10b2f9fce17.js +0 -1
  388. sky/dashboard/out/_next/static/chunks/1141-943efc7aff0f0c06.js +0 -1
  389. sky/dashboard/out/_next/static/chunks/1836-37fede578e2da5f8.js +0 -40
  390. sky/dashboard/out/_next/static/chunks/3015-86cabed5d4669ad0.js +0 -1
  391. sky/dashboard/out/_next/static/chunks/3294.c80326aec9bfed40.js +0 -6
  392. sky/dashboard/out/_next/static/chunks/3785.4872a2f3aa489880.js +0 -1
  393. sky/dashboard/out/_next/static/chunks/3850-ff4a9a69d978632b.js +0 -1
  394. sky/dashboard/out/_next/static/chunks/4045.b30465273dc5e468.js +0 -21
  395. sky/dashboard/out/_next/static/chunks/4676-9da7fdbde90b5549.js +0 -10
  396. sky/dashboard/out/_next/static/chunks/4725.10f7a9a5d3ea8208.js +0 -1
  397. sky/dashboard/out/_next/static/chunks/5339.3fda4a4010ff4e06.js +0 -51
  398. sky/dashboard/out/_next/static/chunks/6135-4b4d5e824b7f9d3c.js +0 -1
  399. sky/dashboard/out/_next/static/chunks/649.b9d7f7d10c1b8c53.js +0 -45
  400. sky/dashboard/out/_next/static/chunks/6856-dca7962af4814e1b.js +0 -1
  401. sky/dashboard/out/_next/static/chunks/6990-08b2a1cae076a943.js +0 -1
  402. sky/dashboard/out/_next/static/chunks/7325.b4bc99ce0892dcd5.js +0 -6
  403. sky/dashboard/out/_next/static/chunks/754-d0da8ab45f9509e9.js +0 -18
  404. sky/dashboard/out/_next/static/chunks/7669.1f5d9a402bf5cc42.js +0 -36
  405. sky/dashboard/out/_next/static/chunks/8969-0be3036bf86f8256.js +0 -1
  406. sky/dashboard/out/_next/static/chunks/9025.c12318fb6a1a9093.js +0 -6
  407. sky/dashboard/out/_next/static/chunks/9037-fa1737818d0a0969.js +0 -6
  408. sky/dashboard/out/_next/static/chunks/pages/_app-ce361c6959bc2001.js +0 -34
  409. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-1cbba24bd1bd35f8.js +0 -16
  410. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-0b4b35dc1dfe046c.js +0 -16
  411. sky/dashboard/out/_next/static/chunks/pages/clusters-469814d711d63b1b.js +0 -1
  412. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-dd64309c3fe67ed2.js +0 -11
  413. sky/dashboard/out/_next/static/chunks/pages/jobs/pools/[pool]-07349868f7905d37.js +0 -16
  414. sky/dashboard/out/_next/static/chunks/pages/jobs-1f70d9faa564804f.js +0 -1
  415. sky/dashboard/out/_next/static/chunks/pages/users-018bf31cda52e11b.js +0 -1
  416. sky/dashboard/out/_next/static/chunks/pages/volumes-739726d6b823f532.js +0 -1
  417. sky/dashboard/out/_next/static/chunks/webpack-4fe903277b57b523.js +0 -1
  418. sky/dashboard/out/_next/static/css/4614e06482d7309e.css +0 -3
  419. sky/dashboard/out/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js +0 -1
  420. sky/templates/kubernetes-ssh-jump.yml.j2 +0 -94
  421. sky/utils/kubernetes/cleanup-tunnel.sh +0 -62
  422. sky/utils/kubernetes/deploy_remote_cluster.py +0 -1299
  423. sky/utils/kubernetes/ssh_jump_lifecycle_manager.py +0 -191
  424. skypilot_nightly-1.0.0.dev20250905.dist-info/RECORD +0 -547
  425. skypilot_nightly-1.0.0.dev20250905.dist-info/top_level.txt +0 -1
  426. /sky/dashboard/out/_next/static/{mS-4qZPSkRuA1u-g2wQhg → KYAhEFa3FTfq4JyKVgo-s}/_ssgManifest.js +0 -0
  427. {skypilot_nightly-1.0.0.dev20250905.dist-info → skypilot_nightly-1.0.0.dev20251210.dist-info}/WHEEL +0 -0
  428. {skypilot_nightly-1.0.0.dev20250905.dist-info → skypilot_nightly-1.0.0.dev20251210.dist-info}/entry_points.txt +0 -0
  429. {skypilot_nightly-1.0.0.dev20250905.dist-info → skypilot_nightly-1.0.0.dev20251210.dist-info}/licenses/LICENSE +0 -0
sky/dashboard/out/jobs/pools/[pool].html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/754-d0da8ab45f9509e9.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-0be3036bf86f8256.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/pools/%5Bpool%5D-07349868f7905d37.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/pools/[pool]","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/6212-7bd06f60ba693125.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-fd5696f3bbbaddae.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-452f9d5cbdd2dc73.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs/pools/%5Bpool%5D-8d0f4655400b4eb9.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs/pools/[pool]","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/jobs.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs-1f70d9faa564804f.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/jobs-e5a98f17f8513a96.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/jobs","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/plugins/[...slug].html ADDED
@@ -0,0 +1 @@
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/plugins/%5B...slug%5D-4f46050ca065d8f8.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/plugins/[...slug]","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/users.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/users-018bf31cda52e11b.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/users","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/users-2f7646eb77785a2c.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/users","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/volumes.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/volumes-739726d6b823f532.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/volumes","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/volumes-ef19d49c6d0e8500.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/volumes","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/workspace/new.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspace/new","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/workspaces/[name].html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/1836-37fede578e2da5f8.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-ff4a9a69d978632b.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-0be3036bf86f8256.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-08b2a1cae076a943.js" defer=""></script><script src="/dashboard/_next/static/chunks/6135-4b4d5e824b7f9d3c.js" defer=""></script><script src="/dashboard/_next/static/chunks/1121-408ed10b2f9fce17.js" defer=""></script><script src="/dashboard/_next/static/chunks/6601-06114c982db410b6.js" defer=""></script><script src="/dashboard/_next/static/chunks/3015-86cabed5d4669ad0.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-943efc7aff0f0c06.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-af76bb06dbb3954f.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/616-3d59f75e2ccf9321.js" defer=""></script><script src="/dashboard/_next/static/chunks/5739-d67458fcb1386c92.js" defer=""></script><script src="/dashboard/_next/static/chunks/6130-2be46d70a38f1e82.js" defer=""></script><script src="/dashboard/_next/static/chunks/7411-b15471acd2cba716.js" defer=""></script><script src="/dashboard/_next/static/chunks/1272-1ef0bf0237faccdb.js" defer=""></script><script src="/dashboard/_next/static/chunks/7359-c8d04e06886000b3.js" defer=""></script><script src="/dashboard/_next/static/chunks/6989-01359c57e018caa4.js" defer=""></script><script src="/dashboard/_next/static/chunks/3850-fd5696f3bbbaddae.js" defer=""></script><script src="/dashboard/_next/static/chunks/8969-452f9d5cbdd2dc73.js" defer=""></script><script src="/dashboard/_next/static/chunks/6990-09cbf02d3cd518c3.js" defer=""></script><script src="/dashboard/_next/static/chunks/9353-8369df1cf105221c.js" defer=""></script><script src="/dashboard/_next/static/chunks/2260-7703229c33c5ebd5.js" defer=""></script><script src="/dashboard/_next/static/chunks/3800-b589397dc09c5b4e.js" defer=""></script><script src="/dashboard/_next/static/chunks/7615-019513abc55b3b47.js" defer=""></script><script src="/dashboard/_next/static/chunks/1141-9c810f01ff4f398a.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces/%5Bname%5D-96e0f298308da7e2.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces/[name]","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/dashboard/out/workspaces.html CHANGED
@@ -1 +1 @@
1
- <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/4614e06482d7309e.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/4614e06482d7309e.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-4fe903277b57b523.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-ce361c6959bc2001.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-7598c33a746cdc91.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/mS-4qZPSkRuA1u-g2wQhg/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"mS-4qZPSkRuA1u-g2wQhg","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
1
+ <!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><meta name="next-head-count" content="2"/><link rel="preload" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" as="style"/><link rel="stylesheet" href="/dashboard/_next/static/css/c5a4cfd2600fc715.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/dashboard/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/dashboard/_next/static/chunks/webpack-fba3de387ff6bb08.js" defer=""></script><script src="/dashboard/_next/static/chunks/framework-cf60a09ccd051a10.js" defer=""></script><script src="/dashboard/_next/static/chunks/main-f15ccb73239a3bf1.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/_app-68b647e26f9d2793.js" defer=""></script><script src="/dashboard/_next/static/chunks/pages/workspaces-cb4da3abe08ebf19.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_buildManifest.js" defer=""></script><script src="/dashboard/_next/static/KYAhEFa3FTfq4JyKVgo-s/_ssgManifest.js" defer=""></script></head><body><div id="__next"></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{}},"page":"/workspaces","query":{},"buildId":"KYAhEFa3FTfq4JyKVgo-s","assetPrefix":"/dashboard","nextExport":true,"autoExport":true,"isFallback":false,"scriptLoader":[]}</script></body></html>
sky/data/data_utils.py CHANGED
@@ -19,6 +19,7 @@ from sky import sky_logging
19
19
  from sky.adaptors import aws
20
20
  from sky.adaptors import azure
21
21
  from sky.adaptors import cloudflare
22
+ from sky.adaptors import coreweave
22
23
  from sky.adaptors import gcp
23
24
  from sky.adaptors import ibm
24
25
  from sky.adaptors import nebius
@@ -625,6 +626,7 @@ class Rclone:
625
626
  R2 = 'R2'
626
627
  AZURE = 'AZURE'
627
628
  NEBIUS = 'NEBIUS'
629
+ COREWEAVE = 'COREWEAVE'
628
630
 
629
631
  def get_profile_name(self, bucket_name: str) -> str:
630
632
  """Gets the Rclone profile name for a given bucket.
@@ -642,7 +644,8 @@ class Rclone:
642
644
  Rclone.RcloneStores.IBM: 'sky-ibm',
643
645
  Rclone.RcloneStores.R2: 'sky-r2',
644
646
  Rclone.RcloneStores.AZURE: 'sky-azure',
645
- Rclone.RcloneStores.NEBIUS: 'sky-nebius'
647
+ Rclone.RcloneStores.NEBIUS: 'sky-nebius',
648
+ Rclone.RcloneStores.COREWEAVE: 'sky-coreweave'
646
649
  }
647
650
  return f'{profile_prefix[self]}-{bucket_name}'
648
651
 
@@ -748,6 +751,25 @@ class Rclone:
748
751
  endpoint = {endpoint_url}
749
752
  acl = private
750
753
  """)
754
+ elif self is Rclone.RcloneStores.COREWEAVE:
755
+ coreweave_session = coreweave.session()
756
+ coreweave_credentials = coreweave.get_coreweave_credentials(
757
+ coreweave_session)
758
+ # Get endpoint URL from the client
759
+ endpoint_url = coreweave.get_endpoint()
760
+ access_key_id = coreweave_credentials.access_key
761
+ secret_access_key = coreweave_credentials.secret_key
762
+ config = textwrap.dedent(f"""\
763
+ [{rclone_profile_name}]
764
+ type = s3
765
+ provider = Other
766
+ access_key_id = {access_key_id}
767
+ secret_access_key = {secret_access_key}
768
+ endpoint = {endpoint_url}
769
+ region = auto
770
+ acl = private
771
+ force_path_style = false
772
+ """)
751
773
  else:
752
774
  with ux_utils.print_exception_no_traceback():
753
775
  raise NotImplementedError(
@@ -908,3 +930,72 @@ def split_oci_path(oci_path: str) -> Tuple[str, str]:
908
930
  bucket = path_parts.pop(0)
909
931
  key = '/'.join(path_parts)
910
932
  return bucket, key
933
+
934
+
935
+ def create_coreweave_client() -> Client:
936
+ """Create CoreWeave S3 client."""
937
+ return coreweave.client('s3')
938
+
939
+
940
+ def split_coreweave_path(coreweave_path: str) -> Tuple[str, str]:
941
+ """Splits CoreWeave Path into Bucket name and Relative Path to Bucket
942
+
943
+ Args:
944
+ coreweave_path: str; CoreWeave Path, e.g. cw://imagenet/train/
945
+ """
946
+ path_parts = coreweave_path.replace('cw://', '').split('/')
947
+ bucket = path_parts.pop(0)
948
+ key = '/'.join(path_parts)
949
+ return bucket, key
950
+
951
+
952
+ def verify_coreweave_bucket(name: str, retry: int = 0) -> bool:
953
+ """Verify CoreWeave bucket exists and is accessible.
954
+
955
+ Retries head_bucket operation up to retry times with 5 second intervals
956
+ to handle DNS propagation delays or temporary connectivity issues.
957
+ """
958
+ coreweave_client = create_coreweave_client()
959
+ max_retries = retry + 1 # 5s * (retry+1) = total seconds to retry
960
+ retry_count = 0
961
+
962
+ while retry_count < max_retries:
963
+ try:
964
+ coreweave_client.head_bucket(Bucket=name)
965
+ if retry_count > 0:
966
+ logger.debug(
967
+ f'Successfully verified bucket {name} after '
968
+ f'{retry_count} retries ({retry_count * 5} seconds)')
969
+ return True
970
+
971
+ except coreweave.botocore.exceptions.ClientError as e: # type: ignore[union-attr] # pylint: disable=line-too-long:
972
+ error_code = e.response['Error']['Code']
973
+ if error_code == '403':
974
+ logger.error(f'Access denied to bucket {name}')
975
+ return False
976
+ elif error_code == '404':
977
+ logger.debug(f'Bucket {name} does not exist')
978
+ else:
979
+ logger.debug(
980
+ f'Unexpected error checking CoreWeave bucket {name}: {e}')
981
+ except Exception as e: # pylint: disable=broad-except
982
+ logger.debug(
983
+ f'Unexpected error checking CoreWeave bucket {name}: {e}')
984
+
985
+ # Common retry logic for all transient errors
986
+ retry_count += 1
987
+ if retry_count < max_retries:
988
+ logger.debug(f'Error checking CoreWeave bucket {name} '
989
+ f'(attempt {retry_count}/{max_retries}). '
990
+ f'Retrying in 5 seconds...')
991
+ time.sleep(5)
992
+ else:
993
+ attempt_str = 'attempt'
994
+ if max_retries > 1:
995
+ attempt_str += 's'
996
+ logger.error(f'Failed to verify CoreWeave bucket {name} after '
997
+ f'{max_retries} {attempt_str}.')
998
+ return False
999
+
1000
+ # Should not reach here, but just in case
1001
+ return False
sky/data/mounting_utils.py CHANGED
@@ -17,6 +17,16 @@ _TYPE_CACHE_TTL = '5s'
17
17
  _RENAME_DIR_LIMIT = 10000
18
18
  # https://github.com/GoogleCloudPlatform/gcsfuse/releases
19
19
  GCSFUSE_VERSION = '2.2.0'
20
+
21
+ # Some machines do not have fuse/fuse3 installed by default
22
+ # hence rclone will fail on these machines
23
+ FUSE3_INSTALL_CMD = ('(command -v fusermount3 > /dev/null 2>&1 || '
24
+ '((which apt-get > /dev/null 2>&1 && '
25
+ 'sudo apt-get update && sudo apt-get install -y fuse3) || '
26
+ '(which yum > /dev/null 2>&1 && '
27
+ 'sudo yum install -y fuse3) || '
28
+ 'true)) || true')
29
+
20
30
  # Creates a fusermount3 soft link on older (<22) Ubuntu systems to utilize
21
31
  # Rclone's mounting utility.
22
32
  FUSERMOUNT3_SOFT_LINK_CMD = ('[ ! -f /bin/fusermount3 ] && '
@@ -54,10 +64,10 @@ def get_rclone_install_cmd() -> str:
54
64
  f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
55
65
  f' && sudo dpkg -i rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb'
56
66
  f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.deb)))'
57
- f' || (which rclone > /dev/null || (cd ~ > /dev/null'
67
+ f' || (which yum > /dev/null 2>&1 && (which rclone > /dev/null || (cd ~ > /dev/null'
58
68
  f' && curl -O https://downloads.rclone.org/{RCLONE_VERSION}/rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm'
59
69
  f' && sudo yum --nogpgcheck install rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm -y'
60
- f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm))')
70
+ f' && rm -f rclone-{RCLONE_VERSION}-linux-${{ARCH_SUFFIX}}.rpm)))')
61
71
  return install_cmd
62
72
 
63
73
 
@@ -94,6 +104,7 @@ def get_s3_mount_cmd(bucket_name: str,
94
104
  # Use rclone for ARM64 architectures since goofys doesn't support them
95
105
  arch_check = 'ARCH=$(uname -m) && '
96
106
  rclone_mount = (
107
+ f'{FUSE3_INSTALL_CMD} && '
97
108
  f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
98
109
  f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
99
110
  # Have to add --s3-env-auth=true to allow rclone to access private
@@ -128,6 +139,7 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
128
139
  # Use rclone for ARM64 architectures since goofys doesn't support them
129
140
  arch_check = 'ARCH=$(uname -m) && '
130
141
  rclone_mount = (
142
+ f'{FUSE3_INSTALL_CMD} && '
131
143
  f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
132
144
  f'AWS_PROFILE={nebius_profile_name} '
133
145
  f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
@@ -148,6 +160,46 @@ def get_nebius_mount_cmd(nebius_profile_name: str,
148
160
  return mount_cmd
149
161
 
150
162
 
163
+ def get_coreweave_mount_cmd(cw_credentials_path: str,
164
+ coreweave_profile_name: str,
165
+ bucket_name: str,
166
+ endpoint_url: str,
167
+ mount_path: str,
168
+ _bucket_sub_path: Optional[str] = None) -> str:
169
+ """Returns a command to mount CoreWeave bucket"""
170
+ if _bucket_sub_path is None:
171
+ _bucket_sub_path = ''
172
+ else:
173
+ _bucket_sub_path = f':{_bucket_sub_path}'
174
+
175
+ # Use rclone for ARM64 architectures since goofys doesn't support them
176
+ arch_check = 'ARCH=$(uname -m) && '
177
+ rclone_mount = (
178
+ f'{FUSE3_INSTALL_CMD} && '
179
+ f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
180
+ f'AWS_SHARED_CREDENTIALS_FILE={cw_credentials_path} '
181
+ f'AWS_PROFILE={coreweave_profile_name} '
182
+ f'rclone mount :s3:{bucket_name}{_bucket_sub_path} {mount_path} '
183
+ f'--s3-force-path-style=false '
184
+ f'--s3-endpoint {endpoint_url} --daemon --allow-other')
185
+ goofys_mount = (f'AWS_SHARED_CREDENTIALS_FILE={cw_credentials_path} '
186
+ f'AWS_PROFILE={coreweave_profile_name} {_GOOFYS_WRAPPER} '
187
+ '-o allow_other '
188
+ f'--stat-cache-ttl {_STAT_CACHE_TTL} '
189
+ f'--type-cache-ttl {_TYPE_CACHE_TTL} '
190
+ f'--subdomain '
191
+ f'--endpoint {endpoint_url} '
192
+ f'{bucket_name}{_bucket_sub_path} {mount_path}')
193
+
194
+ mount_cmd = (f'{arch_check}'
195
+ f'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
196
+ f' {rclone_mount}; '
197
+ f'else '
198
+ f' {goofys_mount}; '
199
+ f'fi')
200
+ return mount_cmd
201
+
202
+
151
203
  def get_gcs_mount_install_cmd() -> str:
152
204
  """Returns a command to install GCS mount utility gcsfuse."""
153
205
  install_cmd = ('ARCH=$(uname -m) && '
@@ -171,7 +223,10 @@ def get_gcs_mount_cmd(bucket_name: str,
171
223
  """Returns a command to mount a GCS bucket using gcsfuse."""
172
224
  bucket_sub_path_arg = f'--only-dir {_bucket_sub_path} '\
173
225
  if _bucket_sub_path else ''
174
- mount_cmd = ('gcsfuse -o allow_other '
226
+ log_file = '$(mktemp -t gcsfuse.XXXX.log)'
227
+ mount_cmd = (f'gcsfuse --log-file {log_file} '
228
+ '--debug_fuse_errors '
229
+ '-o allow_other '
175
230
  '--implicit-dirs '
176
231
  f'--stat-cache-capacity {_STAT_CACHE_CAPACITY} '
177
232
  f'--stat-cache-ttl {_STAT_CACHE_TTL} '
@@ -185,27 +240,72 @@ def get_gcs_mount_cmd(bucket_name: str,
185
240
  def get_az_mount_install_cmd() -> str:
186
241
  """Returns a command to install AZ Container mount utility blobfuse2."""
187
242
  install_cmd = (
188
- 'sudo apt-get update; '
189
- 'sudo apt-get install -y '
190
- '-o Dpkg::Options::="--force-confdef" '
191
- 'fuse3 libfuse3-dev || { '
192
- ' echo "fuse3 not available, falling back to fuse"; '
193
- ' sudo apt-get install -y '
194
- ' -o Dpkg::Options::="--force-confdef" '
195
- ' fuse libfuse-dev; '
196
- '} && '
243
+ # Check architecture first - blobfuse2 only supports x86_64
197
244
  'ARCH=$(uname -m) && '
198
245
  'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
199
246
  ' echo "blobfuse2 is not supported on $ARCH" && '
200
247
  f' exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
248
+ 'fi && '
249
+ # Try to install fuse3 from default repos
250
+ 'sudo apt-get update && '
251
+ 'FUSE3_INSTALLED=0 && '
252
+ # On Kubernetes, if FUSERMOUNT_SHARED_DIR is set, it means
253
+ # fusermount and fusermount3 is symlinked to fusermount-shim.
254
+ # If we reinstall fuse3, it may overwrite the symlink, so
255
+ # just install libfuse3, which is needed by blobfuse2.
256
+ 'if [ -n "${FUSERMOUNT_SHARED_DIR:-}" ]; then '
257
+ ' PACKAGES="libfuse3-3 libfuse3-dev"; '
201
258
  'else '
202
- ' ARCH_SUFFIX="x86_64"; '
259
+ ' PACKAGES="fuse3 libfuse3-3 libfuse3-dev"; '
203
260
  'fi && '
204
- 'wget -nc https://github.com/Azure/azure-storage-fuse'
205
- f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
206
- f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
261
+ 'if sudo apt-get install -y '
262
+ '-o Dpkg::Options::="--force-confdef" '
263
+ '$PACKAGES; then '
264
+ ' FUSE3_INSTALLED=1; '
265
+ ' echo "fuse3 installed from default repos"; '
266
+ 'else '
267
+ # If fuse3 not available, try focal for Ubuntu <= 20.04
268
+ ' DISTRO=$(grep "^ID=" /etc/os-release | cut -d= -f2 | '
269
+ 'tr -d \'"\' | tr "[:upper:]" "[:lower:]") && '
270
+ ' VERSION=$(grep "^VERSION_ID=" /etc/os-release | cut -d= -f2 | '
271
+ 'tr -d \'"\') && '
272
+ ' if [ "$DISTRO" = "ubuntu" ] && '
273
+ '[ "$(echo "$VERSION 20.04" | '
274
+ 'awk \'{ print ($1 <= $2) }\')" = "1" ]; then '
275
+ ' echo "Trying to install fuse3 from focal for '
276
+ 'Ubuntu $VERSION"; '
277
+ ' echo "deb http://archive.ubuntu.com/ubuntu '
278
+ 'focal main universe" | '
279
+ 'sudo tee /etc/apt/sources.list.d/focal-fuse3.list && '
280
+ ' sudo apt-get update && '
281
+ ' if sudo apt-get install -y '
282
+ '-o Dpkg::Options::="--force-confdef" '
283
+ '-o Dpkg::Options::="--force-confold" '
284
+ '$PACKAGES; then '
285
+ ' FUSE3_INSTALLED=1; '
286
+ ' echo "fuse3 installed from focal"; '
287
+ ' sudo rm /etc/apt/sources.list.d/focal-fuse3.list; '
288
+ ' sudo apt-get update; '
289
+ ' else '
290
+ ' sudo rm -f /etc/apt/sources.list.d/focal-fuse3.list; '
291
+ ' sudo apt-get update; '
292
+ ' fi; '
293
+ ' fi; '
294
+ 'fi && '
295
+ # Install blobfuse2 only if fuse3 is available
296
+ 'if [ "$FUSE3_INSTALLED" = "1" ]; then '
297
+ ' echo "Installing blobfuse2 with libfuse3 support"; '
298
+ ' wget -nc https://github.com/Azure/azure-storage-fuse'
299
+ f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}/'
300
+ f'blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
207
301
  '-O /tmp/blobfuse2.deb && '
208
- 'sudo dpkg --install /tmp/blobfuse2.deb && '
302
+ ' sudo dpkg --install /tmp/blobfuse2.deb; '
303
+ 'else '
304
+ ' echo "Error: libfuse3 is required for Azure storage '
305
+ 'mounting with fusermount-wrapper."; '
306
+ ' echo "libfuse3 could not be installed on this system."; '
307
+ f' exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
308
+ 'fi && '
209
309
  f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
210
310
 
211
311
  return install_cmd
@@ -277,7 +377,10 @@ def get_az_mount_cmd(container_name: str,
277
377
  f'-- {blobfuse2_cmd} -o nonempty --foreground {{}}')
278
378
  original = f'{blobfuse2_cmd} {blobfuse2_options} {mount_path}'
279
379
  # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
280
- # to avoid requiring root privilege.
380
+ # to avoid requiring privileged containers.
381
+ # fusermount-wrapper requires libfuse3;
382
+ # we install libfuse3 even on older distros like Ubuntu 18.04 by using
383
+ # Ubuntu 20.04 (focal) repositories.
281
384
  # TODO(aylei): feeling hacky, refactor this.
282
385
  get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
283
386
  f'echo "{wrapped}" || echo "{original}"')
@@ -304,6 +407,7 @@ def get_r2_mount_cmd(r2_credentials_path: str,
304
407
  # Use rclone for ARM64 architectures since goofys doesn't support them
305
408
  arch_check = 'ARCH=$(uname -m) && '
306
409
  rclone_mount = (
410
+ f'{FUSE3_INSTALL_CMD} && '
307
411
  f'{FUSERMOUNT3_SOFT_LINK_CMD} && '
308
412
  f'AWS_SHARED_CREDENTIALS_FILE={r2_credentials_path} '
309
413
  f'AWS_PROFILE={r2_profile_name} '
@@ -333,7 +437,8 @@ def get_cos_mount_cmd(rclone_config: str,
333
437
  _bucket_sub_path: Optional[str] = None) -> str:
334
438
  """Returns a command to mount an IBM COS bucket using rclone."""
335
439
  # stores bucket profile in rclone config file at the cluster's nodes.
336
- configure_rclone_profile = (f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
440
+ configure_rclone_profile = (f'{FUSE3_INSTALL_CMD} && '
441
+ f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
337
442
  f'mkdir -p {constants.RCLONE_CONFIG_DIR} && '
338
443
  f'echo "{rclone_config}" >> '
339
444
  f'{constants.RCLONE_CONFIG_PATH}')
@@ -353,7 +458,8 @@ def get_mount_cached_cmd(rclone_config: str, rclone_profile_name: str,
353
458
  bucket_name: str, mount_path: str) -> str:
354
459
  """Returns a command to mount a bucket using rclone with vfs cache."""
355
460
  # stores bucket profile in rclone config file at the remote nodes.
356
- configure_rclone_profile = (f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
461
+ configure_rclone_profile = (f'{FUSE3_INSTALL_CMD} && '
462
+ f'{FUSERMOUNT3_SOFT_LINK_CMD}; '
357
463
  f'mkdir -p {constants.RCLONE_CONFIG_DIR} && '
358
464
  f'echo {shlex.quote(rclone_config)} >> '
359
465
  f'{constants.RCLONE_CONFIG_PATH}')
@@ -485,13 +591,20 @@ def get_mounting_script(
485
591
 
486
592
  {command_runner.ALIAS_SUDO_TO_EMPTY_FOR_ROOT_CMD}
487
593
 
488
- MOUNT_PATH={mount_path}
594
+ MOUNT_PATH=$(eval echo {mount_path})
489
595
  MOUNT_BINARY={mount_binary}
490
596
 
491
597
  # Check if path is already mounted
492
- if grep -q $MOUNT_PATH /proc/mounts ; then
598
+ if findmnt -rn -T "$MOUNT_PATH" >/dev/null 2>&1; then
493
599
  echo "Path already mounted - unmounting..."
494
- fusermount -uz "$MOUNT_PATH"
600
+ (command -v fusermount >/dev/null 2>&1 && fusermount -uz "$MOUNT_PATH") \
601
+ || (command -v fusermount3 >/dev/null 2>&1 && fusermount3 -uz "$MOUNT_PATH") \
602
+ || sudo umount -l "$MOUNT_PATH" || true
603
+ # Ensure it's really gone (avoids races)
604
+ for i in $(seq 1 20); do
605
+ if ! findmnt -rn -T "$MOUNT_PATH" >/dev/null 2>&1; then break; fi
606
+ sleep 0.2
607
+ done
495
608
  echo "Successfully unmounted $MOUNT_PATH."
496
609
  fi
497
610
 
@@ -506,17 +619,51 @@ def get_mounting_script(
506
619
  # Check if mount path exists
507
620
  if [ ! -d "$MOUNT_PATH" ]; then
508
621
  echo "Mount path $MOUNT_PATH does not exist. Creating..."
509
- sudo mkdir -p $MOUNT_PATH
510
- sudo chmod 777 $MOUNT_PATH
622
+ sudo mkdir -p "$MOUNT_PATH"
623
+ sudo chmod 777 "$MOUNT_PATH"
511
624
  else
512
- # Check if mount path contains files
513
- if [ "$(ls -A $MOUNT_PATH)" ]; then
514
- echo "Mount path $MOUNT_PATH is not empty. Please mount to another path or remove it first."
515
- exit {exceptions.MOUNT_PATH_NON_EMPTY_CODE}
516
- fi
625
+ # If not a mountpoint and contains files, clean it to satisfy SkyPilot check
626
+ if ! findmnt -rn -T "$MOUNT_PATH" >/dev/null 2>&1; then
627
+ if [ -n "$(ls -A "$MOUNT_PATH" 2>/dev/null)" ]; then
628
+ echo "Cleaning non-empty mount path before mount..."
629
+ sudo bash -lc 'shopt -s dotglob nullglob; rm -rf --one-file-system -- '"$MOUNT_PATH"'/*' 2>/dev/null || true
630
+ fi
631
+ fi
517
632
  fi
518
633
  echo "Mounting $SOURCE_BUCKET to $MOUNT_PATH with $MOUNT_BINARY..."
634
+ set +e
519
635
  {mount_cmd}
636
+ MOUNT_EXIT_CODE=$?
637
+ set -e
638
+ if [ $MOUNT_EXIT_CODE -ne 0 ]; then
639
+ echo "Mount failed with exit code $MOUNT_EXIT_CODE."
640
+ if [ "$MOUNT_BINARY" = "goofys" ]; then
641
+ echo "Looking for goofys log files..."
642
+ # Find goofys log files in /tmp (created by mktemp -t goofys.XXXX.log)
643
+ # Note: if /dev/log exists, goofys logs to syslog instead of a file
644
+ GOOFYS_LOGS=$(ls -t /tmp/goofys.*.log 2>/dev/null | head -1)
645
+ if [ -n "$GOOFYS_LOGS" ]; then
646
+ echo "=== Goofys log file contents ==="
647
+ cat "$GOOFYS_LOGS"
648
+ echo "=== End of goofys log file ==="
649
+ else
650
+ echo "No goofys log file found in /tmp"
651
+ fi
652
+ elif [ "$MOUNT_BINARY" = "gcsfuse" ]; then
653
+ echo "Looking for gcsfuse log files..."
654
+ # Find gcsfuse log files in /tmp (created by mktemp -t gcsfuse.XXXX.log)
655
+ GCSFUSE_LOGS=$(ls -t /tmp/gcsfuse.*.log 2>/dev/null | head -1)
656
+ if [ -n "$GCSFUSE_LOGS" ]; then
657
+ echo "=== GCSFuse log file contents ==="
658
+ cat "$GCSFUSE_LOGS"
659
+ echo "=== End of gcsfuse log file ==="
660
+ else
661
+ echo "No gcsfuse log file found in /tmp"
662
+ fi
663
+ fi
664
+ # TODO(kevin): Print logs from rclone, blobfuse2, etc too for observability.
665
+ exit $MOUNT_EXIT_CODE
666
+ fi
520
667
  echo "Mounting done."
521
668
  """)
522
669