skypilot-nightly 1.0.0.dev20250509__py3-none-any.whl → 1.0.0.dev20251107__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of skypilot-nightly might be problematic. Click here for more details.

Files changed (512) hide show
  1. sky/__init__.py +22 -6
  2. sky/adaptors/aws.py +25 -7
  3. sky/adaptors/common.py +24 -1
  4. sky/adaptors/coreweave.py +278 -0
  5. sky/adaptors/do.py +8 -2
  6. sky/adaptors/hyperbolic.py +8 -0
  7. sky/adaptors/kubernetes.py +149 -18
  8. sky/adaptors/nebius.py +170 -17
  9. sky/adaptors/primeintellect.py +1 -0
  10. sky/adaptors/runpod.py +68 -0
  11. sky/adaptors/seeweb.py +167 -0
  12. sky/adaptors/shadeform.py +89 -0
  13. sky/admin_policy.py +187 -4
  14. sky/authentication.py +179 -225
  15. sky/backends/__init__.py +4 -2
  16. sky/backends/backend.py +22 -9
  17. sky/backends/backend_utils.py +1299 -380
  18. sky/backends/cloud_vm_ray_backend.py +1715 -518
  19. sky/backends/docker_utils.py +1 -1
  20. sky/backends/local_docker_backend.py +11 -6
  21. sky/backends/wheel_utils.py +37 -9
  22. sky/{clouds/service_catalog → catalog}/__init__.py +21 -19
  23. sky/{clouds/service_catalog → catalog}/aws_catalog.py +27 -8
  24. sky/{clouds/service_catalog → catalog}/azure_catalog.py +10 -7
  25. sky/{clouds/service_catalog → catalog}/common.py +89 -48
  26. sky/{clouds/service_catalog → catalog}/cudo_catalog.py +8 -5
  27. sky/{clouds/service_catalog → catalog}/data_fetchers/analyze.py +1 -1
  28. sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_aws.py +30 -40
  29. sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_cudo.py +38 -38
  30. sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_gcp.py +42 -15
  31. sky/catalog/data_fetchers/fetch_hyperbolic.py +136 -0
  32. sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_lambda_cloud.py +1 -0
  33. sky/catalog/data_fetchers/fetch_nebius.py +335 -0
  34. sky/catalog/data_fetchers/fetch_runpod.py +698 -0
  35. sky/catalog/data_fetchers/fetch_seeweb.py +329 -0
  36. sky/catalog/data_fetchers/fetch_shadeform.py +142 -0
  37. sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_vast.py +1 -1
  38. sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_vsphere.py +1 -1
  39. sky/{clouds/service_catalog → catalog}/do_catalog.py +5 -2
  40. sky/{clouds/service_catalog → catalog}/fluidstack_catalog.py +6 -3
  41. sky/{clouds/service_catalog → catalog}/gcp_catalog.py +41 -15
  42. sky/catalog/hyperbolic_catalog.py +136 -0
  43. sky/{clouds/service_catalog → catalog}/ibm_catalog.py +9 -6
  44. sky/{clouds/service_catalog → catalog}/kubernetes_catalog.py +36 -24
  45. sky/{clouds/service_catalog → catalog}/lambda_catalog.py +9 -6
  46. sky/{clouds/service_catalog → catalog}/nebius_catalog.py +9 -7
  47. sky/{clouds/service_catalog → catalog}/oci_catalog.py +9 -6
  48. sky/{clouds/service_catalog → catalog}/paperspace_catalog.py +5 -2
  49. sky/catalog/primeintellect_catalog.py +95 -0
  50. sky/{clouds/service_catalog → catalog}/runpod_catalog.py +11 -4
  51. sky/{clouds/service_catalog → catalog}/scp_catalog.py +9 -6
  52. sky/catalog/seeweb_catalog.py +184 -0
  53. sky/catalog/shadeform_catalog.py +165 -0
  54. sky/catalog/ssh_catalog.py +167 -0
  55. sky/{clouds/service_catalog → catalog}/vast_catalog.py +6 -3
  56. sky/{clouds/service_catalog → catalog}/vsphere_catalog.py +5 -2
  57. sky/check.py +491 -203
  58. sky/cli.py +5 -6005
  59. sky/client/{cli.py → cli/command.py} +2477 -1885
  60. sky/client/cli/deprecation_utils.py +99 -0
  61. sky/client/cli/flags.py +359 -0
  62. sky/client/cli/table_utils.py +320 -0
  63. sky/client/common.py +70 -32
  64. sky/client/oauth.py +82 -0
  65. sky/client/sdk.py +1203 -297
  66. sky/client/sdk_async.py +833 -0
  67. sky/client/service_account_auth.py +47 -0
  68. sky/cloud_stores.py +73 -0
  69. sky/clouds/__init__.py +13 -0
  70. sky/clouds/aws.py +358 -93
  71. sky/clouds/azure.py +105 -83
  72. sky/clouds/cloud.py +127 -36
  73. sky/clouds/cudo.py +68 -50
  74. sky/clouds/do.py +66 -48
  75. sky/clouds/fluidstack.py +63 -44
  76. sky/clouds/gcp.py +339 -110
  77. sky/clouds/hyperbolic.py +293 -0
  78. sky/clouds/ibm.py +70 -49
  79. sky/clouds/kubernetes.py +563 -162
  80. sky/clouds/lambda_cloud.py +74 -54
  81. sky/clouds/nebius.py +206 -80
  82. sky/clouds/oci.py +88 -66
  83. sky/clouds/paperspace.py +61 -44
  84. sky/clouds/primeintellect.py +317 -0
  85. sky/clouds/runpod.py +164 -74
  86. sky/clouds/scp.py +89 -83
  87. sky/clouds/seeweb.py +466 -0
  88. sky/clouds/shadeform.py +400 -0
  89. sky/clouds/ssh.py +263 -0
  90. sky/clouds/utils/aws_utils.py +10 -4
  91. sky/clouds/utils/gcp_utils.py +87 -11
  92. sky/clouds/utils/oci_utils.py +38 -14
  93. sky/clouds/utils/scp_utils.py +177 -124
  94. sky/clouds/vast.py +99 -77
  95. sky/clouds/vsphere.py +51 -40
  96. sky/core.py +349 -139
  97. sky/dag.py +15 -0
  98. sky/dashboard/out/404.html +1 -1
  99. sky/dashboard/out/_next/static/chunks/1141-e6aa9ab418717c59.js +11 -0
  100. sky/dashboard/out/_next/static/chunks/1272-1ef0bf0237faccdb.js +1 -0
  101. sky/dashboard/out/_next/static/chunks/1871-74503c8e80fd253b.js +6 -0
  102. sky/dashboard/out/_next/static/chunks/2260-7703229c33c5ebd5.js +1 -0
  103. sky/dashboard/out/_next/static/chunks/2350.fab69e61bac57b23.js +1 -0
  104. sky/dashboard/out/_next/static/chunks/2369.fc20f0c2c8ed9fe7.js +15 -0
  105. sky/dashboard/out/_next/static/chunks/2755.fff53c4a3fcae910.js +26 -0
  106. sky/dashboard/out/_next/static/chunks/3294.72362fa129305b19.js +1 -0
  107. sky/dashboard/out/_next/static/chunks/3785.ad6adaa2a0fa9768.js +1 -0
  108. sky/dashboard/out/_next/static/chunks/3850-ff4a9a69d978632b.js +1 -0
  109. sky/dashboard/out/_next/static/chunks/3937.210053269f121201.js +1 -0
  110. sky/dashboard/out/_next/static/chunks/4725.a830b5c9e7867c92.js +1 -0
  111. sky/dashboard/out/_next/static/chunks/4937.a2baa2df5572a276.js +15 -0
  112. sky/dashboard/out/_next/static/chunks/5739-d67458fcb1386c92.js +8 -0
  113. sky/dashboard/out/_next/static/chunks/6130-2be46d70a38f1e82.js +1 -0
  114. sky/dashboard/out/_next/static/chunks/616-3d59f75e2ccf9321.js +39 -0
  115. sky/dashboard/out/_next/static/chunks/6212-7bd06f60ba693125.js +13 -0
  116. sky/dashboard/out/_next/static/chunks/6601-06114c982db410b6.js +1 -0
  117. sky/dashboard/out/_next/static/chunks/6856-ef8ba11f96d8c4a3.js +1 -0
  118. sky/dashboard/out/_next/static/chunks/6989-01359c57e018caa4.js +1 -0
  119. sky/dashboard/out/_next/static/chunks/6990-32b6e2d3822301fa.js +1 -0
  120. sky/dashboard/out/_next/static/chunks/7359-c8d04e06886000b3.js +30 -0
  121. sky/dashboard/out/_next/static/chunks/7411-b15471acd2cba716.js +41 -0
  122. sky/dashboard/out/_next/static/chunks/7615-3301e838e5f25772.js +1 -0
  123. sky/dashboard/out/_next/static/chunks/8640.5b9475a2d18c5416.js +16 -0
  124. sky/dashboard/out/_next/static/chunks/8969-1e4613c651bf4051.js +1 -0
  125. sky/dashboard/out/_next/static/chunks/9025.fa408f3242e9028d.js +6 -0
  126. sky/dashboard/out/_next/static/chunks/9353-cff34f7e773b2e2b.js +1 -0
  127. sky/dashboard/out/_next/static/chunks/9360.7310982cf5a0dc79.js +31 -0
  128. sky/dashboard/out/_next/static/chunks/9847.3aaca6bb33455140.js +30 -0
  129. sky/dashboard/out/_next/static/chunks/fd9d1056-86323a29a8f7e46a.js +1 -0
  130. sky/dashboard/out/_next/static/chunks/framework-cf60a09ccd051a10.js +33 -0
  131. sky/dashboard/out/_next/static/chunks/main-app-587214043926b3cc.js +1 -0
  132. sky/dashboard/out/_next/static/chunks/main-f15ccb73239a3bf1.js +1 -0
  133. sky/dashboard/out/_next/static/chunks/pages/_app-bde01e4a2beec258.js +34 -0
  134. sky/dashboard/out/_next/static/chunks/pages/_error-c66a4e8afc46f17b.js +1 -0
  135. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-c736ead69c2d86ec.js +16 -0
  136. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-a37d2063af475a1c.js +1 -0
  137. sky/dashboard/out/_next/static/chunks/pages/clusters-d44859594e6f8064.js +1 -0
  138. sky/dashboard/out/_next/static/chunks/pages/config-dfb9bf07b13045f4.js +1 -0
  139. sky/dashboard/out/_next/static/chunks/pages/index-444f1804401f04ea.js +1 -0
  140. sky/dashboard/out/_next/static/chunks/pages/infra/[context]-c0b5935149902e6f.js +1 -0
  141. sky/dashboard/out/_next/static/chunks/pages/infra-aed0ea19df7cf961.js +1 -0
  142. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-5796e8d6aea291a0.js +16 -0
  143. sky/dashboard/out/_next/static/chunks/pages/jobs/pools/[pool]-6edeb7d06032adfc.js +21 -0
  144. sky/dashboard/out/_next/static/chunks/pages/jobs-479dde13399cf270.js +1 -0
  145. sky/dashboard/out/_next/static/chunks/pages/users-5ab3b907622cf0fe.js +1 -0
  146. sky/dashboard/out/_next/static/chunks/pages/volumes-b84b948ff357c43e.js +1 -0
  147. sky/dashboard/out/_next/static/chunks/pages/workspace/new-3f88a1c7e86a3f86.js +1 -0
  148. sky/dashboard/out/_next/static/chunks/pages/workspaces/[name]-c5a3eeee1c218af1.js +1 -0
  149. sky/dashboard/out/_next/static/chunks/pages/workspaces-22b23febb3e89ce1.js +1 -0
  150. sky/dashboard/out/_next/static/chunks/webpack-2679be77fc08a2f8.js +1 -0
  151. sky/dashboard/out/_next/static/css/0748ce22df867032.css +3 -0
  152. sky/dashboard/out/_next/static/zB0ed6ge_W1MDszVHhijS/_buildManifest.js +1 -0
  153. sky/dashboard/out/clusters/[cluster]/[job].html +1 -1
  154. sky/dashboard/out/clusters/[cluster].html +1 -1
  155. sky/dashboard/out/clusters.html +1 -1
  156. sky/dashboard/out/config.html +1 -0
  157. sky/dashboard/out/index.html +1 -1
  158. sky/dashboard/out/infra/[context].html +1 -0
  159. sky/dashboard/out/infra.html +1 -0
  160. sky/dashboard/out/jobs/[job].html +1 -1
  161. sky/dashboard/out/jobs/pools/[pool].html +1 -0
  162. sky/dashboard/out/jobs.html +1 -1
  163. sky/dashboard/out/users.html +1 -0
  164. sky/dashboard/out/volumes.html +1 -0
  165. sky/dashboard/out/workspace/new.html +1 -0
  166. sky/dashboard/out/workspaces/[name].html +1 -0
  167. sky/dashboard/out/workspaces.html +1 -0
  168. sky/data/data_utils.py +137 -1
  169. sky/data/mounting_utils.py +269 -84
  170. sky/data/storage.py +1451 -1807
  171. sky/data/storage_utils.py +43 -57
  172. sky/exceptions.py +132 -2
  173. sky/execution.py +206 -63
  174. sky/global_user_state.py +2374 -586
  175. sky/jobs/__init__.py +5 -0
  176. sky/jobs/client/sdk.py +242 -65
  177. sky/jobs/client/sdk_async.py +143 -0
  178. sky/jobs/constants.py +9 -8
  179. sky/jobs/controller.py +839 -277
  180. sky/jobs/file_content_utils.py +80 -0
  181. sky/jobs/log_gc.py +201 -0
  182. sky/jobs/recovery_strategy.py +398 -152
  183. sky/jobs/scheduler.py +315 -189
  184. sky/jobs/server/core.py +829 -255
  185. sky/jobs/server/server.py +156 -115
  186. sky/jobs/server/utils.py +136 -0
  187. sky/jobs/state.py +2092 -701
  188. sky/jobs/utils.py +1242 -160
  189. sky/logs/__init__.py +21 -0
  190. sky/logs/agent.py +108 -0
  191. sky/logs/aws.py +243 -0
  192. sky/logs/gcp.py +91 -0
  193. sky/metrics/__init__.py +0 -0
  194. sky/metrics/utils.py +443 -0
  195. sky/models.py +78 -1
  196. sky/optimizer.py +164 -70
  197. sky/provision/__init__.py +90 -4
  198. sky/provision/aws/config.py +147 -26
  199. sky/provision/aws/instance.py +135 -50
  200. sky/provision/azure/instance.py +10 -5
  201. sky/provision/common.py +13 -1
  202. sky/provision/cudo/cudo_machine_type.py +1 -1
  203. sky/provision/cudo/cudo_utils.py +14 -8
  204. sky/provision/cudo/cudo_wrapper.py +72 -71
  205. sky/provision/cudo/instance.py +10 -6
  206. sky/provision/do/instance.py +10 -6
  207. sky/provision/do/utils.py +4 -3
  208. sky/provision/docker_utils.py +114 -23
  209. sky/provision/fluidstack/instance.py +13 -8
  210. sky/provision/gcp/__init__.py +1 -0
  211. sky/provision/gcp/config.py +301 -19
  212. sky/provision/gcp/constants.py +218 -0
  213. sky/provision/gcp/instance.py +36 -8
  214. sky/provision/gcp/instance_utils.py +18 -4
  215. sky/provision/gcp/volume_utils.py +247 -0
  216. sky/provision/hyperbolic/__init__.py +12 -0
  217. sky/provision/hyperbolic/config.py +10 -0
  218. sky/provision/hyperbolic/instance.py +437 -0
  219. sky/provision/hyperbolic/utils.py +373 -0
  220. sky/provision/instance_setup.py +93 -14
  221. sky/provision/kubernetes/__init__.py +5 -0
  222. sky/provision/kubernetes/config.py +9 -52
  223. sky/provision/kubernetes/constants.py +17 -0
  224. sky/provision/kubernetes/instance.py +789 -247
  225. sky/provision/kubernetes/manifests/fusermount-server-daemonset.yaml +1 -2
  226. sky/provision/kubernetes/network.py +27 -17
  227. sky/provision/kubernetes/network_utils.py +40 -43
  228. sky/provision/kubernetes/utils.py +1192 -531
  229. sky/provision/kubernetes/volume.py +282 -0
  230. sky/provision/lambda_cloud/instance.py +22 -16
  231. sky/provision/nebius/constants.py +50 -0
  232. sky/provision/nebius/instance.py +19 -6
  233. sky/provision/nebius/utils.py +196 -91
  234. sky/provision/oci/instance.py +10 -5
  235. sky/provision/paperspace/instance.py +10 -7
  236. sky/provision/paperspace/utils.py +1 -1
  237. sky/provision/primeintellect/__init__.py +10 -0
  238. sky/provision/primeintellect/config.py +11 -0
  239. sky/provision/primeintellect/instance.py +454 -0
  240. sky/provision/primeintellect/utils.py +398 -0
  241. sky/provision/provisioner.py +110 -36
  242. sky/provision/runpod/__init__.py +5 -0
  243. sky/provision/runpod/instance.py +27 -6
  244. sky/provision/runpod/utils.py +51 -18
  245. sky/provision/runpod/volume.py +180 -0
  246. sky/provision/scp/__init__.py +15 -0
  247. sky/provision/scp/config.py +93 -0
  248. sky/provision/scp/instance.py +531 -0
  249. sky/provision/seeweb/__init__.py +11 -0
  250. sky/provision/seeweb/config.py +13 -0
  251. sky/provision/seeweb/instance.py +807 -0
  252. sky/provision/shadeform/__init__.py +11 -0
  253. sky/provision/shadeform/config.py +12 -0
  254. sky/provision/shadeform/instance.py +351 -0
  255. sky/provision/shadeform/shadeform_utils.py +83 -0
  256. sky/provision/ssh/__init__.py +18 -0
  257. sky/provision/vast/instance.py +13 -8
  258. sky/provision/vast/utils.py +10 -7
  259. sky/provision/vsphere/common/vim_utils.py +1 -2
  260. sky/provision/vsphere/instance.py +15 -10
  261. sky/provision/vsphere/vsphere_utils.py +9 -19
  262. sky/py.typed +0 -0
  263. sky/resources.py +844 -118
  264. sky/schemas/__init__.py +0 -0
  265. sky/schemas/api/__init__.py +0 -0
  266. sky/schemas/api/responses.py +225 -0
  267. sky/schemas/db/README +4 -0
  268. sky/schemas/db/env.py +90 -0
  269. sky/schemas/db/global_user_state/001_initial_schema.py +124 -0
  270. sky/schemas/db/global_user_state/002_add_workspace_to_cluster_history.py +35 -0
  271. sky/schemas/db/global_user_state/003_fix_initial_revision.py +61 -0
  272. sky/schemas/db/global_user_state/004_is_managed.py +34 -0
  273. sky/schemas/db/global_user_state/005_cluster_event.py +32 -0
  274. sky/schemas/db/global_user_state/006_provision_log.py +41 -0
  275. sky/schemas/db/global_user_state/007_cluster_event_request_id.py +34 -0
  276. sky/schemas/db/global_user_state/008_skylet_ssh_tunnel_metadata.py +34 -0
  277. sky/schemas/db/global_user_state/009_last_activity_and_launched_at.py +89 -0
  278. sky/schemas/db/global_user_state/010_save_ssh_key.py +66 -0
  279. sky/schemas/db/script.py.mako +28 -0
  280. sky/schemas/db/serve_state/001_initial_schema.py +67 -0
  281. sky/schemas/db/skypilot_config/001_initial_schema.py +30 -0
  282. sky/schemas/db/spot_jobs/001_initial_schema.py +97 -0
  283. sky/schemas/db/spot_jobs/002_cluster_pool.py +42 -0
  284. sky/schemas/db/spot_jobs/003_pool_hash.py +34 -0
  285. sky/schemas/db/spot_jobs/004_job_file_contents.py +42 -0
  286. sky/schemas/db/spot_jobs/005_logs_gc.py +38 -0
  287. sky/schemas/generated/__init__.py +0 -0
  288. sky/schemas/generated/autostopv1_pb2.py +36 -0
  289. sky/schemas/generated/autostopv1_pb2.pyi +43 -0
  290. sky/schemas/generated/autostopv1_pb2_grpc.py +146 -0
  291. sky/schemas/generated/jobsv1_pb2.py +86 -0
  292. sky/schemas/generated/jobsv1_pb2.pyi +254 -0
  293. sky/schemas/generated/jobsv1_pb2_grpc.py +542 -0
  294. sky/schemas/generated/managed_jobsv1_pb2.py +74 -0
  295. sky/schemas/generated/managed_jobsv1_pb2.pyi +278 -0
  296. sky/schemas/generated/managed_jobsv1_pb2_grpc.py +278 -0
  297. sky/schemas/generated/servev1_pb2.py +58 -0
  298. sky/schemas/generated/servev1_pb2.pyi +115 -0
  299. sky/schemas/generated/servev1_pb2_grpc.py +322 -0
  300. sky/serve/autoscalers.py +357 -5
  301. sky/serve/client/impl.py +310 -0
  302. sky/serve/client/sdk.py +47 -139
  303. sky/serve/client/sdk_async.py +130 -0
  304. sky/serve/constants.py +10 -8
  305. sky/serve/controller.py +64 -19
  306. sky/serve/load_balancer.py +106 -60
  307. sky/serve/load_balancing_policies.py +115 -1
  308. sky/serve/replica_managers.py +273 -162
  309. sky/serve/serve_rpc_utils.py +179 -0
  310. sky/serve/serve_state.py +554 -251
  311. sky/serve/serve_utils.py +733 -220
  312. sky/serve/server/core.py +66 -711
  313. sky/serve/server/impl.py +1093 -0
  314. sky/serve/server/server.py +21 -18
  315. sky/serve/service.py +133 -48
  316. sky/serve/service_spec.py +135 -16
  317. sky/serve/spot_placer.py +3 -0
  318. sky/server/auth/__init__.py +0 -0
  319. sky/server/auth/authn.py +50 -0
  320. sky/server/auth/loopback.py +38 -0
  321. sky/server/auth/oauth2_proxy.py +200 -0
  322. sky/server/common.py +475 -181
  323. sky/server/config.py +81 -23
  324. sky/server/constants.py +44 -6
  325. sky/server/daemons.py +229 -0
  326. sky/server/html/token_page.html +185 -0
  327. sky/server/metrics.py +160 -0
  328. sky/server/requests/executor.py +528 -138
  329. sky/server/requests/payloads.py +351 -17
  330. sky/server/requests/preconditions.py +21 -17
  331. sky/server/requests/process.py +112 -29
  332. sky/server/requests/request_names.py +120 -0
  333. sky/server/requests/requests.py +817 -224
  334. sky/server/requests/serializers/decoders.py +82 -31
  335. sky/server/requests/serializers/encoders.py +140 -22
  336. sky/server/requests/threads.py +106 -0
  337. sky/server/rest.py +417 -0
  338. sky/server/server.py +1290 -284
  339. sky/server/state.py +20 -0
  340. sky/server/stream_utils.py +345 -57
  341. sky/server/uvicorn.py +217 -3
  342. sky/server/versions.py +270 -0
  343. sky/setup_files/MANIFEST.in +5 -0
  344. sky/setup_files/alembic.ini +156 -0
  345. sky/setup_files/dependencies.py +136 -31
  346. sky/setup_files/setup.py +44 -42
  347. sky/sky_logging.py +102 -5
  348. sky/skylet/attempt_skylet.py +1 -0
  349. sky/skylet/autostop_lib.py +129 -8
  350. sky/skylet/configs.py +27 -20
  351. sky/skylet/constants.py +171 -19
  352. sky/skylet/events.py +105 -21
  353. sky/skylet/job_lib.py +335 -104
  354. sky/skylet/log_lib.py +297 -18
  355. sky/skylet/log_lib.pyi +44 -1
  356. sky/skylet/ray_patches/__init__.py +17 -3
  357. sky/skylet/ray_patches/autoscaler.py.diff +18 -0
  358. sky/skylet/ray_patches/cli.py.diff +19 -0
  359. sky/skylet/ray_patches/command_runner.py.diff +17 -0
  360. sky/skylet/ray_patches/log_monitor.py.diff +20 -0
  361. sky/skylet/ray_patches/resource_demand_scheduler.py.diff +32 -0
  362. sky/skylet/ray_patches/updater.py.diff +18 -0
  363. sky/skylet/ray_patches/worker.py.diff +41 -0
  364. sky/skylet/services.py +564 -0
  365. sky/skylet/skylet.py +63 -4
  366. sky/skylet/subprocess_daemon.py +103 -29
  367. sky/skypilot_config.py +506 -99
  368. sky/ssh_node_pools/__init__.py +1 -0
  369. sky/ssh_node_pools/core.py +135 -0
  370. sky/ssh_node_pools/server.py +233 -0
  371. sky/task.py +621 -137
  372. sky/templates/aws-ray.yml.j2 +10 -3
  373. sky/templates/azure-ray.yml.j2 +1 -1
  374. sky/templates/do-ray.yml.j2 +1 -1
  375. sky/templates/gcp-ray.yml.j2 +57 -0
  376. sky/templates/hyperbolic-ray.yml.j2 +67 -0
  377. sky/templates/jobs-controller.yaml.j2 +27 -24
  378. sky/templates/kubernetes-loadbalancer.yml.j2 +2 -0
  379. sky/templates/kubernetes-ray.yml.j2 +607 -51
  380. sky/templates/lambda-ray.yml.j2 +1 -1
  381. sky/templates/nebius-ray.yml.j2 +33 -12
  382. sky/templates/paperspace-ray.yml.j2 +1 -1
  383. sky/templates/primeintellect-ray.yml.j2 +71 -0
  384. sky/templates/runpod-ray.yml.j2 +9 -1
  385. sky/templates/scp-ray.yml.j2 +3 -50
  386. sky/templates/seeweb-ray.yml.j2 +108 -0
  387. sky/templates/shadeform-ray.yml.j2 +72 -0
  388. sky/templates/sky-serve-controller.yaml.j2 +22 -2
  389. sky/templates/websocket_proxy.py +178 -18
  390. sky/usage/usage_lib.py +18 -11
  391. sky/users/__init__.py +0 -0
  392. sky/users/model.conf +15 -0
  393. sky/users/permission.py +387 -0
  394. sky/users/rbac.py +121 -0
  395. sky/users/server.py +720 -0
  396. sky/users/token_service.py +218 -0
  397. sky/utils/accelerator_registry.py +34 -5
  398. sky/utils/admin_policy_utils.py +84 -38
  399. sky/utils/annotations.py +16 -5
  400. sky/utils/asyncio_utils.py +78 -0
  401. sky/utils/auth_utils.py +153 -0
  402. sky/utils/benchmark_utils.py +60 -0
  403. sky/utils/cli_utils/status_utils.py +159 -86
  404. sky/utils/cluster_utils.py +31 -9
  405. sky/utils/command_runner.py +354 -68
  406. sky/utils/command_runner.pyi +93 -3
  407. sky/utils/common.py +35 -8
  408. sky/utils/common_utils.py +310 -87
  409. sky/utils/config_utils.py +87 -5
  410. sky/utils/context.py +402 -0
  411. sky/utils/context_utils.py +222 -0
  412. sky/utils/controller_utils.py +264 -89
  413. sky/utils/dag_utils.py +31 -12
  414. sky/utils/db/__init__.py +0 -0
  415. sky/utils/db/db_utils.py +470 -0
  416. sky/utils/db/migration_utils.py +133 -0
  417. sky/utils/directory_utils.py +12 -0
  418. sky/utils/env_options.py +13 -0
  419. sky/utils/git.py +567 -0
  420. sky/utils/git_clone.sh +460 -0
  421. sky/utils/infra_utils.py +195 -0
  422. sky/utils/kubernetes/cleanup-tunnel.sh +62 -0
  423. sky/utils/kubernetes/config_map_utils.py +133 -0
  424. sky/utils/kubernetes/create_cluster.sh +13 -27
  425. sky/utils/kubernetes/delete_cluster.sh +10 -7
  426. sky/utils/kubernetes/deploy_remote_cluster.py +1299 -0
  427. sky/utils/kubernetes/exec_kubeconfig_converter.py +22 -31
  428. sky/utils/kubernetes/generate_kind_config.py +6 -66
  429. sky/utils/kubernetes/generate_kubeconfig.sh +4 -1
  430. sky/utils/kubernetes/gpu_labeler.py +5 -5
  431. sky/utils/kubernetes/kubernetes_deploy_utils.py +354 -47
  432. sky/utils/kubernetes/ssh-tunnel.sh +379 -0
  433. sky/utils/kubernetes/ssh_utils.py +221 -0
  434. sky/utils/kubernetes_enums.py +8 -15
  435. sky/utils/lock_events.py +94 -0
  436. sky/utils/locks.py +368 -0
  437. sky/utils/log_utils.py +300 -6
  438. sky/utils/perf_utils.py +22 -0
  439. sky/utils/resource_checker.py +298 -0
  440. sky/utils/resources_utils.py +249 -32
  441. sky/utils/rich_utils.py +213 -37
  442. sky/utils/schemas.py +905 -147
  443. sky/utils/serialize_utils.py +16 -0
  444. sky/utils/status_lib.py +10 -0
  445. sky/utils/subprocess_utils.py +38 -15
  446. sky/utils/tempstore.py +70 -0
  447. sky/utils/timeline.py +24 -52
  448. sky/utils/ux_utils.py +84 -15
  449. sky/utils/validator.py +11 -1
  450. sky/utils/volume.py +86 -0
  451. sky/utils/yaml_utils.py +111 -0
  452. sky/volumes/__init__.py +13 -0
  453. sky/volumes/client/__init__.py +0 -0
  454. sky/volumes/client/sdk.py +149 -0
  455. sky/volumes/server/__init__.py +0 -0
  456. sky/volumes/server/core.py +258 -0
  457. sky/volumes/server/server.py +122 -0
  458. sky/volumes/volume.py +212 -0
  459. sky/workspaces/__init__.py +0 -0
  460. sky/workspaces/core.py +655 -0
  461. sky/workspaces/server.py +101 -0
  462. sky/workspaces/utils.py +56 -0
  463. skypilot_nightly-1.0.0.dev20251107.dist-info/METADATA +675 -0
  464. skypilot_nightly-1.0.0.dev20251107.dist-info/RECORD +594 -0
  465. {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/WHEEL +1 -1
  466. sky/benchmark/benchmark_state.py +0 -256
  467. sky/benchmark/benchmark_utils.py +0 -641
  468. sky/clouds/service_catalog/constants.py +0 -7
  469. sky/dashboard/out/_next/static/LksQgChY5izXjokL3LcEu/_buildManifest.js +0 -1
  470. sky/dashboard/out/_next/static/chunks/236-f49500b82ad5392d.js +0 -6
  471. sky/dashboard/out/_next/static/chunks/312-c3c8845990db8ffc.js +0 -15
  472. sky/dashboard/out/_next/static/chunks/37-0a572fe0dbb89c4d.js +0 -6
  473. sky/dashboard/out/_next/static/chunks/678-206dddca808e6d16.js +0 -59
  474. sky/dashboard/out/_next/static/chunks/845-0f8017370869e269.js +0 -1
  475. sky/dashboard/out/_next/static/chunks/979-7bf73a4c7cea0f5c.js +0 -1
  476. sky/dashboard/out/_next/static/chunks/fd9d1056-2821b0f0cabcd8bd.js +0 -1
  477. sky/dashboard/out/_next/static/chunks/framework-87d061ee6ed71b28.js +0 -33
  478. sky/dashboard/out/_next/static/chunks/main-app-241eb28595532291.js +0 -1
  479. sky/dashboard/out/_next/static/chunks/main-e0e2335212e72357.js +0 -1
  480. sky/dashboard/out/_next/static/chunks/pages/_app-e6b013bc3f77ad60.js +0 -1
  481. sky/dashboard/out/_next/static/chunks/pages/_error-1be831200e60c5c0.js +0 -1
  482. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]/[job]-e15db85d0ea1fbe1.js +0 -1
  483. sky/dashboard/out/_next/static/chunks/pages/clusters/[cluster]-f383db7389368ea7.js +0 -1
  484. sky/dashboard/out/_next/static/chunks/pages/clusters-a93b93e10b8b074e.js +0 -1
  485. sky/dashboard/out/_next/static/chunks/pages/index-f9f039532ca8cbc4.js +0 -1
  486. sky/dashboard/out/_next/static/chunks/pages/jobs/[job]-03f279c6741fb48b.js +0 -1
  487. sky/dashboard/out/_next/static/chunks/pages/jobs-a75029b67aab6a2e.js +0 -1
  488. sky/dashboard/out/_next/static/chunks/webpack-830f59b8404e96b8.js +0 -1
  489. sky/dashboard/out/_next/static/css/c6933bbb2ce7f4dd.css +0 -3
  490. sky/jobs/dashboard/dashboard.py +0 -223
  491. sky/jobs/dashboard/static/favicon.ico +0 -0
  492. sky/jobs/dashboard/templates/index.html +0 -831
  493. sky/jobs/server/dashboard_utils.py +0 -69
  494. sky/skylet/providers/scp/__init__.py +0 -2
  495. sky/skylet/providers/scp/config.py +0 -149
  496. sky/skylet/providers/scp/node_provider.py +0 -578
  497. sky/templates/kubernetes-ssh-jump.yml.j2 +0 -94
  498. sky/utils/db_utils.py +0 -100
  499. sky/utils/kubernetes/deploy_remote_cluster.sh +0 -308
  500. sky/utils/kubernetes/ssh_jump_lifecycle_manager.py +0 -191
  501. skypilot_nightly-1.0.0.dev20250509.dist-info/METADATA +0 -361
  502. skypilot_nightly-1.0.0.dev20250509.dist-info/RECORD +0 -396
  503. /sky/{clouds/service_catalog → catalog}/config.py +0 -0
  504. /sky/{benchmark → catalog/data_fetchers}/__init__.py +0 -0
  505. /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_azure.py +0 -0
  506. /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_fluidstack.py +0 -0
  507. /sky/{clouds/service_catalog → catalog}/data_fetchers/fetch_ibm.py +0 -0
  508. /sky/{clouds/service_catalog/data_fetchers → client/cli}/__init__.py +0 -0
  509. /sky/dashboard/out/_next/static/{LksQgChY5izXjokL3LcEu → zB0ed6ge_W1MDszVHhijS}/_ssgManifest.js +0 -0
  510. {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/entry_points.txt +0 -0
  511. {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/licenses/LICENSE +0 -0
  512. {skypilot_nightly-1.0.0.dev20250509.dist-info → skypilot_nightly-1.0.0.dev20251107.dist-info}/top_level.txt +0 -0
sky/utils/git_clone.sh ADDED
@@ -0,0 +1,460 @@
1
+ #!/bin/bash
2
+
3
+ # Git clone script for SkyPilot remote clusters
4
+ # This script handles Git repository cloning with authentication and different ref types
5
+
6
+ set -e # Exit on any error
7
+
8
+ # Function to log messages
9
+ log() {
10
+ echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >&2
11
+ }
12
+
13
+ # Function to check and install Git if not present
14
+ check_and_install_git() {
15
+ # Check if git is already installed
16
+ if command -v git >/dev/null 2>&1; then
17
+ log "Git is already installed: $(git --version)"
18
+ return 0
19
+ fi
20
+
21
+ log "Git not found, attempting to install..."
22
+
23
+ # Function to try installing with error handling
24
+ install_with_error_handling() {
25
+ local cmd_with_sudo="$1"
26
+ local cmd_without_sudo="$2"
27
+ local pkg_manager="$3"
28
+
29
+ log "Trying to install Git using $pkg_manager..."
30
+
31
+ # First, try with sudo
32
+ log "Attempting with sudo..."
33
+ if eval "$cmd_with_sudo" 2>&1 >/tmp/git_install.log; then
34
+ log "Installation with sudo completed successfully"
35
+ rm -f /tmp/git_install.log
36
+ return 0
37
+ else
38
+ log "Installation with sudo failed, trying without sudo..."
39
+ cat /tmp/git_install.log >&2
40
+ rm -f /tmp/git_install.log
41
+ fi
42
+
43
+ # If sudo failed, try without sudo
44
+ log "Attempting without sudo..."
45
+ if eval "$cmd_without_sudo" 2>&1 >/tmp/git_install.log; then
46
+ log "Installation without sudo completed successfully"
47
+ rm -f /tmp/git_install.log
48
+ return 0
49
+ else
50
+ log "Installation failed. Error output:"
51
+ cat /tmp/git_install.log >&2
52
+
53
+ # Check for common error patterns
54
+ if grep -q -i "permission\|denied\|unauthorized" /tmp/git_install.log; then
55
+ log "ERROR: Installation failed due to permission issues"
56
+ log "Current user: $(whoami), UID: $(id -u)"
57
+ log "Try running this script as root or with appropriate permissions"
58
+ elif grep -q -i "command not found\|not found" /tmp/git_install.log; then
59
+ log "ERROR: Package manager command not found"
60
+ log "Please install Git manually using your system's package manager"
61
+ fi
62
+
63
+ rm -f /tmp/git_install.log
64
+ return 1
65
+ fi
66
+ }
67
+
68
+ # Detect the operating system and package manager
69
+ if [ -f /etc/debian_version ]; then
70
+ # Debian/Ubuntu
71
+ log "Detected Debian/Ubuntu system"
72
+ if command -v apt-get >/dev/null 2>&1; then
73
+ install_with_error_handling "sudo apt-get update -y && sudo apt-get install -y git" "apt-get update -y && apt-get install -y git" "apt-get"
74
+ elif command -v apt >/dev/null 2>&1; then
75
+ install_with_error_handling "sudo apt update -y && sudo apt install -y git" "apt update -y && apt install -y git" "apt"
76
+ else
77
+ log "ERROR: Neither apt-get nor apt found on Debian/Ubuntu system"
78
+ return 1
79
+ fi
80
+ elif [ -f /etc/redhat-release ] || [ -f /etc/centos-release ]; then
81
+ # Red Hat/CentOS/Fedora
82
+ log "Detected Red Hat/CentOS/Fedora system"
83
+ if command -v dnf >/dev/null 2>&1; then
84
+ install_with_error_handling "sudo dnf install -y git" "dnf install -y git" "dnf"
85
+ elif command -v yum >/dev/null 2>&1; then
86
+ install_with_error_handling "sudo yum install -y git" "yum install -y git" "yum"
87
+ else
88
+ log "ERROR: Neither dnf nor yum found on Red Hat/CentOS/Fedora system"
89
+ return 1
90
+ fi
91
+ elif [ -f /etc/arch-release ]; then
92
+ # Arch Linux
93
+ log "Detected Arch Linux system"
94
+ if command -v pacman >/dev/null 2>&1; then
95
+ install_with_error_handling "sudo pacman -S --noconfirm git" "pacman -S --noconfirm git" "pacman"
96
+ else
97
+ log "ERROR: pacman not found on Arch Linux system"
98
+ return 1
99
+ fi
100
+ elif [ -f /etc/alpine-release ]; then
101
+ # Alpine Linux
102
+ log "Detected Alpine Linux system"
103
+ if command -v apk >/dev/null 2>&1; then
104
+ install_with_error_handling "sudo apk add --no-cache git" "apk add --no-cache git" "apk"
105
+ else
106
+ log "ERROR: apk not found on Alpine Linux system"
107
+ return 1
108
+ fi
109
+ elif [ "$(uname)" = "Darwin" ]; then
110
+ # macOS
111
+ log "Detected macOS system"
112
+ if command -v brew >/dev/null 2>&1; then
113
+ install_with_error_handling "brew install git" "brew install git" "Homebrew"
114
+ elif command -v port >/dev/null 2>&1; then
115
+ install_with_error_handling "sudo port install git" "port install git" "MacPorts"
116
+ else
117
+ log "ERROR: Neither Homebrew nor MacPorts found on macOS"
118
+ log "Please install Git manually: https://git-scm.com/download/mac"
119
+ return 1
120
+ fi
121
+ else
122
+ log "ERROR: Unsupported operating system: $(uname -s)"
123
+ log "Please install Git manually: https://git-scm.com/downloads"
124
+ return 1
125
+ fi
126
+
127
+ # Verify installation
128
+ if command -v git >/dev/null 2>&1; then
129
+ log "Git successfully installed: $(git --version)"
130
+ return 0
131
+ else
132
+ log "ERROR: Git installation failed or Git is not in PATH"
133
+ log "Please install Git manually and ensure it's in your PATH"
134
+ return 1
135
+ fi
136
+ }
137
+
138
+ # Function to cleanup temporary files
139
+ cleanup() {
140
+ if [ -n "$SSH_KEY_FILE" ] && [ -f "$SSH_KEY_FILE" ]; then
141
+ log "Cleaning up temporary SSH key file"
142
+ rm -f "$SSH_KEY_FILE"
143
+ fi
144
+ }
145
+
146
+ # Function to normalize Git URL for comparison and extract schema
147
+ # Note: URLs from GitRepo are already standardized as HTTPS or SSH format
148
+ # Returns: "normalized_url schema"
149
+ normalize_git_url() {
150
+ local url="$1"
151
+ local schema=""
152
+
153
+ # Remove trailing .git and /
154
+ url=$(echo "$url" | sed 's|\.git/*$||' | sed 's|/*$||')
155
+
156
+ # Convert standardized formats to host/path for comparison and extract schema
157
+ if [[ "$url" =~ ^ssh://([^@]+@)?([^:/]+)(:([0-9]+))?/(.+)$ ]]; then
158
+ # SSH standard format: ssh://user@host:port/path -> host/path
159
+ local host="${BASH_REMATCH[2]}"
160
+ local path="${BASH_REMATCH[5]}"
161
+ schema="ssh"
162
+ echo "$host/$path $schema"
163
+ elif [[ "$url" =~ ^https?://([^@]*@)?([^:/]+)(:([0-9]+))?/(.+)$ ]]; then
164
+ # HTTPS format: https://token@host:port/path -> host/path
165
+ local host="${BASH_REMATCH[2]}"
166
+ local path="${BASH_REMATCH[5]}"
167
+ schema="https"
168
+ echo "$host/$path $schema"
169
+ else
170
+ # Fallback: return as-is (shouldn't happen with standardized URLs)
171
+ schema="unknown"
172
+ echo "$url $schema"
173
+ fi
174
+ }
175
+
176
+ # Set up cleanup trap
177
+ trap cleanup EXIT
178
+
179
+ # Check if target directory is provided
180
+ if [ -z "$1" ]; then
181
+ log "ERROR: Target directory not provided"
182
+ echo "Usage: $0 <target_directory>"
183
+ exit 1
184
+ fi
185
+
186
+ TARGET_DIR="$1"
187
+ log "Target directory: $TARGET_DIR"
188
+
189
+ # Check if GIT_URL is provided
190
+ if [ -z "$GIT_URL" ]; then
191
+ log "GIT_URL environment variable not set, skipping git clone"
192
+ exit 0
193
+ fi
194
+
195
+ log "Git URL: $GIT_URL"
196
+
197
+ # Check and install Git if not present
198
+ if ! check_and_install_git; then
199
+ log "ERROR: Failed to ensure Git is installed"
200
+ exit 1
201
+ fi
202
+
203
+ # Setup SSH key if provided
204
+ SSH_KEY_FILE=""
205
+ if [ -n "$GIT_SSH_KEY" ]; then
206
+ log "Setting up SSH key authentication"
207
+ SSH_KEY_FILE=$(mktemp)
208
+ echo "$GIT_SSH_KEY" > "$SSH_KEY_FILE"
209
+ chmod 600 "$SSH_KEY_FILE"
210
+
211
+ # Setup Git SSH command
212
+ export GIT_SSH_COMMAND="ssh -i $SSH_KEY_FILE -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes"
213
+ log "SSH key written to temporary file: $SSH_KEY_FILE"
214
+ fi
215
+
216
+ # Setup token authentication if provided
217
+ CLONE_URL="$GIT_URL"
218
+ if [ -n "$GIT_TOKEN" ]; then
219
+ log "Setting up token authentication"
220
+ if [[ "$GIT_URL" =~ ^https://(.*)$ ]]; then
221
+ # For container environments, directly embed token in URL for reliability
222
+ # This is simpler and more reliable than credential helper in containers
223
+ CLONE_URL="https://${GIT_TOKEN}@${BASH_REMATCH[1]}"
224
+ log "Token authentication configured via URL embedding"
225
+ else
226
+ log "WARNING: GIT_TOKEN provided but URL is not HTTPS, ignoring token"
227
+ fi
228
+ fi
229
+
230
+ # Determine reference type and clone strategy
231
+ REF_TYPE=""
232
+ REF_VALUE=""
233
+ CLONE_ARGS=""
234
+
235
+ if [ -n "$GIT_BRANCH" ]; then
236
+ REF_TYPE="branch"
237
+ REF_VALUE="$GIT_BRANCH"
238
+ CLONE_ARGS="--depth 1 --branch $GIT_BRANCH"
239
+ log "Using branch: $GIT_BRANCH (shallow clone)"
240
+ elif [ -n "$GIT_TAG" ]; then
241
+ REF_TYPE="tag"
242
+ REF_VALUE="$GIT_TAG"
243
+ CLONE_ARGS="--depth 1 --branch $GIT_TAG"
244
+ log "Using tag: $GIT_TAG (shallow clone)"
245
+ elif [ -n "$GIT_COMMIT_HASH" ]; then
246
+ REF_TYPE="commit"
247
+ REF_VALUE="$GIT_COMMIT_HASH"
248
+ # For commit hash, we need full clone first, then checkout
249
+ CLONE_ARGS=""
250
+ log "Using commit hash: $GIT_COMMIT_HASH (full clone required)"
251
+ else
252
+ # Use git default behavior to clone the default branch
253
+ REF_TYPE="default"
254
+ # Get the remote default branch name before cloning
255
+ log "Determining remote default branch..."
256
+ if REF_VALUE=$(git ls-remote --symref "$CLONE_URL" HEAD 2>/dev/null | grep '^ref:' | sed 's/^ref: refs\/heads\///' | awk '{print $1}' | head -1); then
257
+ if [ -n "$REF_VALUE" ]; then
258
+ CLONE_ARGS="--depth 1 --branch $REF_VALUE"
259
+ log "Remote default branch: $REF_VALUE (shallow clone)"
260
+ else
261
+ REF_VALUE="" # Will be determined after clone
262
+ CLONE_ARGS="--depth 1"
263
+ log "Could not determine remote default branch, using git default behavior"
264
+ fi
265
+ else
266
+ REF_VALUE="" # Will be determined after clone
267
+ CLONE_ARGS="--depth 1"
268
+ log "Could not query remote, using git default behavior"
269
+ fi
270
+ fi
271
+
272
+ # Check if target directory already exists and has git repository
273
+ if [ -d "$TARGET_DIR" ]; then
274
+ if [ -d "$TARGET_DIR/.git" ]; then
275
+ log "Git repository already exists, updating..."
276
+ cd "$TARGET_DIR"
277
+
278
+ # Verify this is the same repository
279
+ CURRENT_REMOTE=$(git remote get-url origin 2>/dev/null || echo "")
280
+ if [ -n "$CURRENT_REMOTE" ]; then
281
+ # Normalize URLs for comparison (handles HTTPS, SSH formats)
282
+ CURRENT_RESULT=$(normalize_git_url "$CURRENT_REMOTE")
283
+ TARGET_RESULT=$(normalize_git_url "$GIT_URL")
284
+
285
+ # Extract normalized URL and schema
286
+ CURRENT_CLEAN=$(echo "$CURRENT_RESULT" | cut -d' ' -f1)
287
+ CURRENT_SCHEMA=$(echo "$CURRENT_RESULT" | cut -d' ' -f2)
288
+ TARGET_CLEAN=$(echo "$TARGET_RESULT" | cut -d' ' -f1)
289
+ TARGET_SCHEMA=$(echo "$TARGET_RESULT" | cut -d' ' -f2)
290
+
291
+ if [ "$CURRENT_CLEAN" != "$TARGET_CLEAN" ]; then
292
+ log "WARNING: Existing repository has different remote URL"
293
+ log "Current: $CURRENT_REMOTE -> $CURRENT_CLEAN ($CURRENT_SCHEMA)"
294
+ log "Target: $GIT_URL -> $TARGET_CLEAN ($TARGET_SCHEMA)"
295
+ log "Please manually remove the directory or use a different target directory."
296
+ exit 1
297
+ elif [ "$CURRENT_SCHEMA" != "$TARGET_SCHEMA" ]; then
298
+ # Same repository but different schema (HTTPS vs SSH)
299
+ log "Repository matches but schema differs"
300
+ log "Current: $CURRENT_REMOTE ($CURRENT_SCHEMA)"
301
+ log "Target: $GIT_URL ($TARGET_SCHEMA)"
302
+ log "Updating remote URL to match authentication method"
303
+ git remote set-url origin "$CLONE_URL"
304
+ log "Remote URL updated successfully"
305
+ NEED_CLONE=false
306
+ else
307
+ log "Repository remote URL matches exactly, fetching updates"
308
+ NEED_CLONE=false
309
+ fi
310
+ else
311
+ log "Cannot determine remote URL, assuming repository needs update"
312
+ NEED_CLONE=false
313
+ fi
314
+ else
315
+ log "Directory exists but is not a git repository"
316
+ cd "$TARGET_DIR"
317
+ NEED_CLONE=true
318
+ fi
319
+ else
320
+ log "Creating target directory: $TARGET_DIR"
321
+ mkdir -p "$TARGET_DIR"
322
+ cd "$TARGET_DIR"
323
+ NEED_CLONE=true
324
+ fi
325
+
326
+ # Clone or update repository
327
+ if [ "$NEED_CLONE" = true ]; then
328
+ log "Cloning repository..."
329
+ if [ "$REF_TYPE" = "commit" ]; then
330
+ # Clone full repository for commit hash
331
+ git clone "$CLONE_URL" .
332
+ log "Successfully cloned repository"
333
+ else
334
+ # Clone with appropriate arguments (branch-specific, tag-specific, or default)
335
+ git clone $CLONE_ARGS "$CLONE_URL" .
336
+ if [ "$REF_TYPE" = "branch" ]; then
337
+ log "Successfully cloned branch: $REF_VALUE"
338
+ elif [ "$REF_TYPE" = "tag" ]; then
339
+ log "Successfully cloned tag: $REF_VALUE"
340
+ else
341
+ # Determine the actual default branch name (if not already known)
342
+ if [ -z "$REF_VALUE" ]; then
343
+ REF_VALUE=$(git branch --show-current)
344
+ fi
345
+ log "Successfully cloned default branch: $REF_VALUE"
346
+ fi
347
+ fi
348
+ else
349
+ # Update existing repository
350
+ log "Fetching updates..."
351
+
352
+ if [ "$REF_TYPE" = "commit" ]; then
353
+ # For commit hash, we need to fetch all refs
354
+ # Check if this is a shallow repository
355
+ if [ -f ".git/shallow" ]; then
356
+ log "Detected shallow repository, converting to full clone for commit access"
357
+ git fetch --unshallow
358
+ else
359
+ git fetch origin
360
+ fi
361
+ elif [ "$REF_TYPE" = "tag" ]; then
362
+ # For specific tag, fetch that tag
363
+ if ! git fetch origin "+refs/tags/$REF_VALUE:refs/tags/$REF_VALUE"; then
364
+ log "ERROR: Failed to fetch tag '$REF_VALUE' from remote"
365
+ log "This could mean:"
366
+ log " 1. Remote tag '$REF_VALUE' does not exist"
367
+ log " 2. Network connectivity issues"
368
+ log " 3. Authentication problems"
369
+ log "Checking available remote tags..."
370
+ git ls-remote --tags origin | sed 's/.*refs\/tags\// - /' || true
371
+ exit 1
372
+ fi
373
+ elif [ "$REF_TYPE" = "default" ]; then
374
+ # For default branch, get current branch and fetch it (if not already known)
375
+ if [ -z "$REF_VALUE" ]; then
376
+ REF_VALUE=$(git branch --show-current)
377
+ fi
378
+ log "Current default branch: $REF_VALUE"
379
+ if ! git fetch origin "+refs/heads/$REF_VALUE:refs/remotes/origin/$REF_VALUE"; then
380
+ log "ERROR: Failed to fetch default branch '$REF_VALUE' from remote"
381
+ log "This could mean the remote branch no longer exists"
382
+ exit 1
383
+ fi
384
+ else
385
+ # For specific branch, fetch that branch and create remote tracking branch
386
+ if ! git fetch origin "+refs/heads/$REF_VALUE:refs/remotes/origin/$REF_VALUE"; then
387
+ log "ERROR: Failed to fetch branch '$REF_VALUE' from remote"
388
+ log "This could mean:"
389
+ log " 1. Remote branch '$REF_VALUE' does not exist"
390
+ log " 2. Network connectivity issues"
391
+ log " 3. Authentication problems"
392
+ log "Checking available remote branches..."
393
+ git ls-remote --heads origin | sed 's/.*refs\/heads\// - /' || true
394
+ exit 1
395
+ fi
396
+ fi
397
+ log "Successfully fetched updates"
398
+ fi
399
+
400
+ # Checkout the desired reference
401
+ if [ "$REF_TYPE" = "commit" ]; then
402
+ # Checkout specific commit
403
+ log "Checking out commit: $REF_VALUE"
404
+ if ! git checkout "$REF_VALUE"; then
405
+ log "ERROR: Failed to checkout commit '$REF_VALUE'"
406
+ log "This could mean:"
407
+ log " 1. The commit hash does not exist in the repository"
408
+ log " 2. The commit hash is incomplete or invalid"
409
+ log " 3. The repository may still be missing some history"
410
+ log "Try providing a longer commit hash or ensure the commit exists"
411
+ exit 1
412
+ fi
413
+ log "Successfully checked out commit: $REF_VALUE"
414
+ elif [ "$REF_TYPE" = "tag" ]; then
415
+ # Checkout specific tag
416
+ log "Checking out tag: $REF_VALUE"
417
+ if ! git checkout "$REF_VALUE"; then
418
+ log "ERROR: Failed to checkout tag '$REF_VALUE'"
419
+ log "This could mean:"
420
+ log " 1. The tag '$REF_VALUE' does not exist in the repository"
421
+ log " 2. The tag name is invalid"
422
+ log " 3. Local repository is in an inconsistent state"
423
+ exit 1
424
+ fi
425
+ log "Successfully checked out tag: $REF_VALUE"
426
+ else
427
+ # Checkout specific branch
428
+ log "Checking out branch: $REF_VALUE"
429
+ if git checkout "$REF_VALUE" 2>/dev/null; then
430
+ log "Switched to existing local branch: $REF_VALUE"
431
+ elif git checkout -b "$REF_VALUE" "origin/$REF_VALUE" 2>/dev/null; then
432
+ log "Created new local branch tracking origin/$REF_VALUE"
433
+ else
434
+ log "ERROR: Failed to checkout branch '$REF_VALUE'"
435
+ log "This could mean:"
436
+ log " 1. Remote branch 'origin/$REF_VALUE' does not exist"
437
+ log " 2. Local repository is in an inconsistent state"
438
+ exit 1
439
+ fi
440
+
441
+ # Ensure we're on the latest commit of the branch
442
+ if [ "$NEED_CLONE" != true ]; then
443
+ git reset --hard "origin/$REF_VALUE"
444
+ fi
445
+ log "Successfully checked out branch: $REF_VALUE"
446
+ fi
447
+
448
+ # Verify final state
449
+ CURRENT_REF=$(git rev-parse HEAD)
450
+ log "Repository is now at commit: $CURRENT_REF"
451
+
452
+ if [ "$REF_TYPE" = "branch" ] || [ "$REF_TYPE" = "default" ]; then
453
+ CURRENT_BRANCH=$(git branch --show-current 2>/dev/null || echo "detached")
454
+ log "Current branch: $CURRENT_BRANCH"
455
+ elif [ "$REF_TYPE" = "tag" ]; then
456
+ CURRENT_TAG=$(git describe --tags --exact-match 2>/dev/null || echo "unknown")
457
+ log "Current tag: $CURRENT_TAG"
458
+ fi
459
+
460
+ log "Git clone/update completed successfully"
@@ -0,0 +1,195 @@
1
+ """Utility functions for handling infrastructure specifications."""
2
+ import dataclasses
3
+ from typing import Optional
4
+
5
+ from sky.utils import common_utils
6
+ from sky.utils import ux_utils
7
+
8
+ _REGION_OR_ZONE_TRUNCATION_LENGTH = 25
9
+
10
+
11
+ @dataclasses.dataclass
12
+ class InfraInfo:
13
+ """Infrastructure information parsed from infra string.
14
+
15
+ When a field is None, it means the field is not specified.
16
+ """
17
+ cloud: Optional[str] = None
18
+ region: Optional[str] = None
19
+ zone: Optional[str] = None
20
+
21
+ def __init__(self,
22
+ cloud: Optional[str] = None,
23
+ region: Optional[str] = None,
24
+ zone: Optional[str] = None):
25
+ assert cloud not in ['none', 'None', 'NONE'], 'cloud must be specified'
26
+ if not cloud or cloud == '*':
27
+ cloud = None
28
+ if not region or region == '*':
29
+ region = None
30
+ if not zone or zone == '*':
31
+ zone = None
32
+
33
+ self.cloud = cloud
34
+ self.region = region
35
+ self.zone = zone
36
+
37
+ @staticmethod
38
+ def from_str(infra: Optional[str]) -> 'InfraInfo':
39
+ """Parse the infra string into cloud, region, and zone components.
40
+
41
+ The format of the infra string is `cloud`, `cloud/region`, or
42
+ `cloud/region/zone`. Examples: `aws`, `aws/us-east-1`,
43
+ `aws/us-east-1/us-east-1a`. For any field, you can use `*` to indicate
44
+ that any value is acceptable.
45
+
46
+ If `*` is used for any field, the InfraInfo will have None for that
47
+ field.
48
+
49
+ Args:
50
+ infra: A string in the format of `cloud`, `cloud/region`, or
51
+ `cloud/region/zone`. Examples: `aws`, `aws/us-east-1`,
52
+ `aws/us-east-1/us-east-1a`.
53
+
54
+ Returns:
55
+ An InfraInfo object containing cloud, region, and zone information.
56
+
57
+ Raises:
58
+ ValueError: If the infra string is malformed.
59
+ """
60
+ if infra is None or not infra.strip():
61
+ return InfraInfo()
62
+
63
+ infra = infra.strip().strip('/')
64
+
65
+ # Split on / to get cloud, region, zone
66
+ parts = [p.strip() for p in infra.strip().split('/')]
67
+
68
+ if '' in parts:
69
+ with ux_utils.print_exception_no_traceback():
70
+ raise ValueError(
71
+ f'Invalid infra format: {infra}. Format should not contain '
72
+ 'empty parts (e.g., double slashes "//").')
73
+
74
+ if not parts or not parts[0]:
75
+ with ux_utils.print_exception_no_traceback():
76
+ raise ValueError(
77
+ f'Invalid infra format: {infra}. Expected format is '
78
+ '"cloud", "cloud/region", or "cloud/region/zone".')
79
+
80
+ cloud_name: Optional[str] = parts[0].lower()
81
+
82
+ # Handle Kubernetes contexts specially, as they can contain slashes
83
+ if cloud_name in ['k8s', 'kubernetes']:
84
+ # For Kubernetes, the entire string after "k8s/" is the
85
+ # context name (region)
86
+ cloud_name = 'kubernetes' # Normalize k8s to kubernetes
87
+ region = '/'.join(parts[1:]) if len(parts) >= 2 else None
88
+ zone = None
89
+ elif cloud_name == 'ssh':
90
+ # For SSH, the entire string after "ssh/" is the
91
+ # node pool name. We prepend 'ssh-' for the internal implementation
92
+ # which reuses the context name.
93
+ # TODO(romilb): This is a workaround while we use the global
94
+ # kubeconfig to store the ssh contexts.
95
+ region = '/'.join(parts[1:]) if len(parts) >= 2 else None
96
+ if region:
97
+ region = f'ssh-{region}'
98
+ zone = None
99
+ else:
100
+ # For non-Kubernetes clouds, continue with regular parsing
101
+ # but be careful to only split into max 3 parts
102
+ region_zone_parts = parts[1:]
103
+ region = None
104
+ zone = None
105
+ if region_zone_parts:
106
+ region = region_zone_parts[0]
107
+ if len(region_zone_parts) > 1:
108
+ zone = region_zone_parts[1]
109
+ if len(region_zone_parts) > 2:
110
+ with ux_utils.print_exception_no_traceback():
111
+ raise ValueError(
112
+ f'Invalid infra format: {infra}. Expected format '
113
+ 'is "cloud", "cloud/region", or '
114
+ '"cloud/region/zone".')
115
+
116
+ if cloud_name == '*':
117
+ cloud_name = None
118
+ if region == '*':
119
+ region = None
120
+ if zone == '*':
121
+ zone = None
122
+ return InfraInfo(cloud=cloud_name, region=region, zone=zone)
123
+
124
+ def to_str(self) -> Optional[str]:
125
+ """Formats cloud, region, and zone into an infra string.
126
+
127
+ Args:
128
+ cloud: The cloud object
129
+ region: The region name
130
+ zone: The zone name
131
+
132
+ Returns:
133
+ A formatted infra string, or None if cloud is None or '*'
134
+ """
135
+ cloud = self.cloud
136
+ region = self.region
137
+ zone = self.zone
138
+
139
+ if cloud is None:
140
+ cloud = '*'
141
+ if region is None:
142
+ region = '*'
143
+ if zone is None:
144
+ zone = '*'
145
+
146
+ # If the cloud is ssh, we remove the ssh- prefix from the region
147
+ # TODO(romilb): This is a workaround while we use the global
148
+ # kubeconfig to store the ssh contexts.
149
+ if region and region.startswith('ssh-'):
150
+ region = region[4:]
151
+
152
+ # Build the parts list and filter out trailing wildcards
153
+ parts = [cloud.lower(), region, zone]
154
+ while parts and parts[-1] == '*':
155
+ parts.pop()
156
+
157
+ if not parts:
158
+ return None
159
+
160
+ # Join the parts with '/'
161
+ return '/'.join(parts)
162
+
163
+ def formatted_str(self, truncate: bool = True) -> str:
164
+ """Formats cloud, region, and zone into an infra string.
165
+
166
+ Args:
167
+ truncate: Whether to truncate the region or zone
168
+
169
+ Returns:
170
+ A formatted infra string, or None if cloud is None or '*'
171
+ """
172
+ if self.cloud is None or self.cloud == '*':
173
+ return '-'
174
+
175
+ region_or_zone = None
176
+ if self.zone is not None and self.zone != '*':
177
+ region_or_zone = self.zone
178
+ elif self.region is not None and self.region != '*':
179
+ # If using region, we remove the ssh- prefix if it exists for SSH
180
+ # Node Pools.
181
+ # TODO(romilb): This is a workaround while we use the global
182
+ # kubeconfig to store the ssh contexts.
183
+ region_or_zone = common_utils.removeprefix(self.region, 'ssh-')
184
+
185
+ if region_or_zone is not None and truncate:
186
+ region_or_zone = common_utils.truncate_long_string(
187
+ region_or_zone,
188
+ _REGION_OR_ZONE_TRUNCATION_LENGTH,
189
+ truncate_middle=True)
190
+
191
+ formatted_str = f'{self.cloud}'
192
+ if region_or_zone is not None:
193
+ formatted_str += f' ({region_or_zone})'
194
+
195
+ return formatted_str