skypilot-nightly 1.0.0.dev20241231__py3-none-any.whl → 1.0.0.dev20250102__py3-none-any.whl

sky/templates/do-ray.yml.j2

@@ -0,0 +1,98 @@
+cluster_name: {{cluster_name_on_cloud}}
+
+# The maximum number of workers nodes to launch in addition to the head node.
+max_workers: {{num_nodes - 1}}
+upscaling_speed: {{num_nodes - 1}}
+idle_timeout_minutes: 60
+
+{%- if docker_image is not none %}
+docker:
+  image: {{docker_image}}
+  container_name: {{docker_container_name}}
+  run_options:
+    - --ulimit nofile=1048576:1048576
+    {%- for run_option in docker_run_options %}
+    - {{run_option}}
+    {%- endfor %}
+  {%- if docker_login_config is not none %}
+  docker_login_config:
+    username: |-
+      {{docker_login_config.username}}
+    password: |-
+      {{docker_login_config.password}}
+    server: |-
+      {{docker_login_config.server}}
+  {%- endif %}
+{%- endif %}
+
+provider:
+  type: external
+  module: sky.provision.do
+  region: "{{region}}"
+
+auth:
+  ssh_user: root
+  ssh_private_key: {{ssh_private_key}}
+  ssh_public_key: |-
+      skypilot:ssh_public_key_content
+
+available_node_types:
+  ray_head_default:
+    resources: {}
+    node_config:
+      InstanceType: {{instance_type}}
+      DiskSize: {{disk_size}}
+      {%- if image_id is not none %}
+      ImageId: {{image_id}}
+      {%- endif %}
+
+head_node_type: ray_head_default
+
+# Format: `REMOTE_PATH : LOCAL_PATH`
+file_mounts: {
+  "{{sky_ray_yaml_remote_path}}": "{{sky_ray_yaml_local_path}}",
+  "{{sky_remote_path}}/{{sky_wheel_hash}}": "{{sky_local_path}}",
+{%- for remote_path, local_path in credentials.items() %}
+  "{{remote_path}}": "{{local_path}}",
+{%- endfor %}
+}
+
+rsync_exclude: []
+
+initialization_commands: []
+
+# List of shell commands to run to set up nodes.
+# NOTE: these are very performance-sensitive. Each new item opens/closes an SSH
+# connection, which is expensive. Try your best to co-locate commands into fewer
+# items!
+#
+# Increment the following for catching performance bugs easier:
+#   current num items (num SSH connections): 1
+setup_commands:
+  # Disable `unattended-upgrades` to prevent apt-get from hanging. It should be called at the beginning before the process started to avoid being blocked. (This is a temporary fix.)
+  # Create ~/.ssh/config file in case the file does not exist in the image.
+  # Line 'rm ..': there is another installation of pip.
+  # Line 'sudo bash ..': set the ulimit as suggested by ray docs for performance. https://docs.ray.io/en/latest/cluster/vms/user-guides/large-cluster-best-practices.html#system-configuration
+  # Line 'sudo grep ..': set the number of threads per process to unlimited to avoid ray job submit stucking issue when the number of running ray jobs increase.
+  # Line 'mkdir -p ..': disable host key check
+  # Line 'python3 -c ..': patch the buggy ray files and enable `-o allow_other` option for `goofys`
+  - {%- for initial_setup_command in initial_setup_commands %}
+    {{ initial_setup_command }}
+    {%- endfor %}
+    sudo systemctl stop unattended-upgrades || true;
+    sudo systemctl disable unattended-upgrades || true;
+    sudo sed -i 's/Unattended-Upgrade "1"/Unattended-Upgrade "0"/g' /etc/apt/apt.conf.d/20auto-upgrades || true;
+    sudo kill -9 `sudo lsof /var/lib/dpkg/lock-frontend | awk '{print $2}' | tail -n 1` || true;
+    sudo pkill -9 apt-get;
+    sudo pkill -9 dpkg;
+    sudo dpkg --configure -a;
+    mkdir -p ~/.ssh; touch ~/.ssh/config;
+    {{ conda_installation_commands }}
+    {{ ray_skypilot_installation_commands }}
+    sudo bash -c 'rm -rf /etc/security/limits.d; echo "* soft nofile 1048576" >> /etc/security/limits.conf; echo "* hard nofile 1048576" >> /etc/security/limits.conf';
+    sudo grep -e '^DefaultTasksMax' /etc/systemd/system.conf || (sudo bash -c 'echo "DefaultTasksMax=infinity" >> /etc/systemd/system.conf'); sudo systemctl set-property user-$(id -u $(whoami)).slice TasksMax=infinity; sudo systemctl daemon-reload;
+    mkdir -p ~/.ssh; (grep -Pzo -q "Host \*\n  StrictHostKeyChecking no" ~/.ssh/config) || printf "Host *\n  StrictHostKeyChecking no\n" >> ~/.ssh/config;
+    [ -f /etc/fuse.conf ] && sudo sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf || (sudo sh -c 'echo "user_allow_other" > /etc/fuse.conf');
+    
+# Command to start ray clusters are now placed in `sky.provision.instance_setup`. 
+# We do not need to list it here anymore.

{skypilot_nightly-1.0.0.dev20241231.dist-info → skypilot_nightly-1.0.0.dev20250102.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20241231
+Version: 1.0.0.dev20250102
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0
@@ -95,6 +95,10 @@ Provides-Extra: fluidstack
 Provides-Extra: cudo
 Requires-Dist: cudo-compute>=0.1.10; extra == "cudo"
 Provides-Extra: paperspace
+Provides-Extra: do
+Requires-Dist: pydo>=0.3.0; extra == "do"
+Requires-Dist: azure-core>=1.24.0; extra == "do"
+Requires-Dist: azure-common; extra == "do"
 Provides-Extra: vsphere
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "vsphere"
 Provides-Extra: all
@@ -138,6 +142,9 @@ Requires-Dist: protobuf!=3.19.5,>=3.15.3; extra == "all"
 Requires-Dist: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3; extra == "all"
 Requires-Dist: runpod>=1.5.1; extra == "all"
 Requires-Dist: cudo-compute>=0.1.10; extra == "all"
+Requires-Dist: pydo>=0.3.0; extra == "all"
+Requires-Dist: azure-core>=1.24.0; extra == "all"
+Requires-Dist: azure-common; extra == "all"
 Requires-Dist: pyvmomi==8.0.1.0.2; extra == "all"
 
 <p align="center">

{skypilot_nightly-1.0.0.dev20241231.dist-info → skypilot_nightly-1.0.0.dev20250102.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=K4pjDvzJHjXsQwePxvouhow-WSenWR6mXdUoTAjrX0o,5944
+sky/__init__.py,sha256=3Skp0G5uYpDz1ila9ybindujnUJkbkcTlZhL0JsFeWU,5944
 sky/admin_policy.py,sha256=hPo02f_A32gCqhUueF0QYy1fMSSKqRwYEg_9FxScN_s,3248
 sky/authentication.py,sha256=kACHmiZgWgRpYd1wx1ofbXRMErfMcFmWrkw4a9NxYrY,20988
 sky/check.py,sha256=s8deMVL-k9y8gd519K7NWZc3DqWsEySwiAr0uH3Vvcc,9459
@@ -21,6 +21,7 @@ sky/adaptors/azure.py,sha256=yjM8nAPW-mlSXfmA8OmJNnSIrZ9lQx2-GxiI-TIVrwE,21910
 sky/adaptors/cloudflare.py,sha256=5NUvRGZ1gNf6xwL4CfsjekQTAsTsQQCH7l3tQIhkFxQ,7587
 sky/adaptors/common.py,sha256=VhgAFPx-GmhXoob8B1YvqGC2BXs2HwtqoA8oFMaqkbc,2551
 sky/adaptors/cudo.py,sha256=WGvIQrlzJkGDe02Ve7pygA56tHwUc4kwS3XHW8kMFAA,239
+sky/adaptors/do.py,sha256=dJ0BYbkQoUWVu6_9Pxq3fOu6PngjZyyCQzgjnODXLCA,777
 sky/adaptors/docker.py,sha256=_kzpZ0fkWHqqQAVVl0llTsCE31KYz3Sjn8psTBQHVkA,468
 sky/adaptors/gcp.py,sha256=OQ9RaqjR0r0iaWYpjvEtIx5vnEhyB4LhUCwbtdxsmVk,3115
 sky/adaptors/ibm.py,sha256=H87vD6izq_wQI8oQC7cx9iVtRgPi_QkAcrfa1Z3PNqU,4906
@@ -30,8 +31,8 @@ sky/adaptors/runpod.py,sha256=4Nt_BfZhJAKQNA3wO8cxvvNI8x4NsDGHu_4EhRDlGYQ,225
 sky/adaptors/vsphere.py,sha256=zJP9SeObEoLrpgHW2VHvZE48EhgVf8GfAEIwBeaDMfM,2129
 sky/backends/__init__.py,sha256=UDjwbUgpTRApbPJnNfR786GadUuwgRk3vsWoVu5RB_c,536
 sky/backends/backend.py,sha256=iBs5gnMaaUoH2OIQ3xhAjWdrJWqj8T61Za9TGsBFpvQ,7515
-sky/backends/backend_utils.py,sha256=s_r37B-uJw92Fz1n3OXBPjHjegOOy2wzG_Cny4joVbY,135338
-sky/backends/cloud_vm_ray_backend.py,sha256=yX55EfXJXjvBNJPJ1uEQ-BjNg0v0FSd33t9xNyA72xA,239215
+sky/backends/backend_utils.py,sha256=J2_-lC-tAnf8XlO9lVbl1ESUce7XOzdipcfIczudylw,135361
+sky/backends/cloud_vm_ray_backend.py,sha256=ou2MfE1aLz-k3ODL2GAaFLw6JUm3kYV99KgRcPrMXr8,239251
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=nSYCjms3HOPjPNOrcCqsUKm1WV3AAovRFjEQ7hcEXW4,17021
 sky/backends/wheel_utils.py,sha256=CUVOwlBtQjOMv-RSDGx2jMQ0M1D0w9ZPm0TDafJwBDI,8180
@@ -39,12 +40,13 @@ sky/backends/monkey_patches/monkey_patch_ray_up.py,sha256=76-y2fCaE3JINj8lEwHT1e
 sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
 sky/benchmark/benchmark_utils.py,sha256=mP8Ox2WiKfthq6LcUAZnHknFQ0n8v9o_rCh1LXLgkqc,26192
-sky/clouds/__init__.py,sha256=WuNIJEnZmBO72tU5awgaaL3rdvFRSkgaYNNeuY68dXo,1356
+sky/clouds/__init__.py,sha256=iORaV6auqMxa6-6FKgt1C9f3UqRk1GASUtakua3tb9A,1395
 sky/clouds/aws.py,sha256=EDhoMe6P20HisrPzi9yGG0y5ePvCFy4GsLeSA8DirBw,49637
 sky/clouds/azure.py,sha256=KtnnNZn4ZEr7xndBHxX91v0YXSI1QWPgIefuM1zDUBA,30784
 sky/clouds/cloud.py,sha256=NIODnzroLwMA3VJq3DOFt1voUoZJB8aielpDkX61iD0,35240
 sky/clouds/cloud_registry.py,sha256=oLoYFjm_SDTgdHokY7b6A5Utq80HXRQNxV0fLjDdVsQ,2361
 sky/clouds/cudo.py,sha256=TjlgTklsNhbzpTaqEZ5TudnH7YW9aJpkt4xyAhyaKj0,13094
+sky/clouds/do.py,sha256=zqibtq1gxNPSNkSkZFPfP5yplfIKCwBss3ry0o4C17c,11198
 sky/clouds/fluidstack.py,sha256=u2I6jXEtTqgqRWi2EafMsKqc8VkUq1cR6CSDUvk72_U,12407
 sky/clouds/gcp.py,sha256=GYlMsM9gAg0tfZF7pGcw4cFxsLPUSwMC8rCL3-Cyw8Q,54674
 sky/clouds/ibm.py,sha256=0ArRTQx1_DpTNGByFhukzFedEDzmVjBsGiiques1bQ0,21447
@@ -60,8 +62,9 @@ sky/clouds/service_catalog/aws_catalog.py,sha256=j33lNC5GXWK6CiGWZORCnumGlRODmCA
 sky/clouds/service_catalog/azure_catalog.py,sha256=5Q51x_WEKvQ2YSgJvZHRH3URlbwIstYuwpjaWW_wJlw,8149
 sky/clouds/service_catalog/common.py,sha256=RInaXJ1_g5vQktJyEhmnvvCC5iI-QBcbgwyZ4AOhFpg,27668
 sky/clouds/service_catalog/config.py,sha256=ylzqewdEBjDg4awvFek6ldYmFrnvD2bVGLZuLPvEVYA,1793
-sky/clouds/service_catalog/constants.py,sha256=u_SjIr3gJI6P-xpRgNAzZG2Z4M5UfE1PL7z4atfQncE,411
+sky/clouds/service_catalog/constants.py,sha256=dJPD7f4wv219mT6fg2FzoCW4GKH9xXEIf93_To3DV-Y,417
 sky/clouds/service_catalog/cudo_catalog.py,sha256=V_takvL6dWTGQaTLCEvjKIotCDPnMujiNUZ87kZKGVI,4673
+sky/clouds/service_catalog/do_catalog.py,sha256=Cug2QaQlSN6nFhba7f1ksyzs6z0ICTj6vSiR-792WnI,3698
 sky/clouds/service_catalog/fluidstack_catalog.py,sha256=21-cvrYEYTIi7n3ZNF2e7_0QX-PF4BkhlVJUWQOvKrY,5059
 sky/clouds/service_catalog/gcp_catalog.py,sha256=jJEfWjZ4ItsE657LjIf9mruJVZERFegCD5Qtu20AFNc,24542
 sky/clouds/service_catalog/ibm_catalog.py,sha256=1iK0KvbI82U7sySb7chr-qm_16x3tTnZ6nIo7o76ouc,4493
@@ -106,11 +109,11 @@ sky/jobs/dashboard/templates/index.html,sha256=su1tqgcsXNl1lGl9hfIR6ig1f531OO57x
 sky/provision/__init__.py,sha256=rnuwL9x3qQGhX3EYW6KyZWfw_yBqGSiFBDz86T5xus4,6339
 sky/provision/common.py,sha256=E8AlSUFcn0FYQq1erNmoVfMAdsF9tP2yxfyk-9PLvQU,10286
 sky/provision/constants.py,sha256=oc_XDUkcoLQ_lwDy5yMeMSWviKS0j0s1c0pjlvpNeWY,800
-sky/provision/docker_utils.py,sha256=l4AMzwXGZd8RyNq8AwOaKV9bFSofLYfSyj2NBhkXYsY,19200
+sky/provision/docker_utils.py,sha256=PKh866psTqix9PM1wBD0puUMqgdY5pvb5PUtQZUUdcM,19477
 sky/provision/instance_setup.py,sha256=8Pudbpke6ah0xufr2UwtsDnNZ64-aAYkz8M44ZA0huI,23218
 sky/provision/logging.py,sha256=yZWgejrFBhhRjAtvFu5N5bRXIMK5TuwNjp1vKQqz2pw,2103
 sky/provision/metadata_utils.py,sha256=LrxeV4wD2QPzNdXV_npj8q-pr35FatxBBjF_jSbpOT0,4013
-sky/provision/provisioner.py,sha256=REh4ajMIzFMJu4vAj81Qo9qAbJ7-3_5_IpzjBjo_MH0,29154
+sky/provision/provisioner.py,sha256=ZOgFOO0NB4QZVPwd4qikRqi615Bq67n0Vcl3cTDVxNE,29153
 sky/provision/aws/__init__.py,sha256=mxq8PeWJqUtalDozTNpbtENErRZ1ktEs8uf2aG9UUgU,731
 sky/provision/aws/config.py,sha256=X0yChaZTBjx0IR7lsJwhzhb0Zcbz5w7nNpe1HMYBOfM,24196
 sky/provision/aws/instance.py,sha256=eCslJ2XfJo_pkQMnKFQqhGnUIRvwKiT12oxBY5-klss,40750
@@ -125,6 +128,11 @@ sky/provision/cudo/cudo_machine_type.py,sha256=_VNXWPELmlFXbtdcnPvkuLuyE9CZ923BU
 sky/provision/cudo/cudo_utils.py,sha256=fIwRpCSOXOyIcCLh-kGMdfl-ACTXdTwyHf6o27rMA6Y,1730
 sky/provision/cudo/cudo_wrapper.py,sha256=WgtA1-ZMytR7tG7UFOH93hSvSAqDWymwWxQREspbZAQ,4678
 sky/provision/cudo/instance.py,sha256=k8TkimcEZMOS6juYOdoLYAwgTa9RdyE-BKXs5FtD8wE,8822
+sky/provision/do/__init__.py,sha256=VAhSO9F9FDAjvEuCDlRPueB-TTA3XHR61vVXyUe85z0,518
+sky/provision/do/config.py,sha256=WWFz6kZLKrUHoJoKRzHGY8ck_ieQiBrMWDb5JP6rWUk,387
+sky/provision/do/constants.py,sha256=NYomY7Zb3Svd33wJbNJTi4NwYMzbD7nGNlHWBX-_EYI,172
+sky/provision/do/instance.py,sha256=7N4ZPYSaFs_kIcpGXMJOSihIsmLfS5TSDY7sokL-nQA,11070
+sky/provision/do/utils.py,sha256=I9YEGHgTVVfhhf_wk4Vi2vosZjnbrvVIEkFH8rafoXQ,10440
 sky/provision/fluidstack/__init__.py,sha256=h2df4JHCHtBMYZwtC05WM0VnLbAsRO-T3GFTf6NpY04,592
 sky/provision/fluidstack/config.py,sha256=hDqesKEVjIhXLTWej3fDdpbHtKBXoybxFGgC6T8U5uo,326
 sky/provision/fluidstack/fluidstack_utils.py,sha256=Y21y2IAiHPLk_b-Lp-ld26SZSfWARhxdDEiu7MtfBmc,5693
@@ -191,7 +199,7 @@ sky/serve/serve_utils.py,sha256=WgPcqEw3WyMOdgRTFg8DSsWyIG1xnRbRkI1-f09tNKg,3974
 sky/serve/service.py,sha256=7bvK9R9D48PZSYcOKSievXQ2mHUMk1d3AAIxtra7WOI,12083
 sky/serve/service_spec.py,sha256=mph-IRhamiFls4RX_8yhC0lTLzbIxGYbnvosy7VZTi4,14999
 sky/setup_files/MANIFEST.in,sha256=WF0T89NLichHxZDDSQzvSpiONtAEFyur2MPmGczgTIo,555
-sky/setup_files/dependencies.py,sha256=-_9ekkrmrU9vF7I9cqz_xP55qaSZ-qz2KfGVgQDQTTY,5903
+sky/setup_files/dependencies.py,sha256=cF_mLf8DTFrK8eCH6EAB6ggkW2hChX0X1jh8GnqePFY,5968
 sky/setup_files/setup.py,sha256=HMqAIxHrhtQUOlm6_Iz5E_bL4dUvsYgXc9YVQIFayPs,7417
 sky/skylet/LICENSE,sha256=BnFrJSvUFpMUoH5mOpWnEvaC5R6Uux8W6WXgrte8iYg,12381
 sky/skylet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -225,6 +233,7 @@ sky/skylet/ray_patches/worker.py.patch,sha256=_OBhibdr3xOy5Qje6Tt8D1eQVm_msi50TJ
 sky/templates/aws-ray.yml.j2,sha256=FK95sqSCrCkoAz9Cdtwq6_g3TqU-3048KE7pUaZfNNg,8814
 sky/templates/azure-ray.yml.j2,sha256=NQKg_f_S7WjsY90ykx0yNDNOGYnnEL3HS4pA3NMIZkM,6112
 sky/templates/cudo-ray.yml.j2,sha256=SEHVY57iBauCOE2HYJtYVFEKlriAkdwQu_p86a1n_bA,3548
+sky/templates/do-ray.yml.j2,sha256=sRKpn0tC-uPYtSZ20OB4fMzE7RbPQUr8kOCIbuJ4b4Q,4191
 sky/templates/fluidstack-ray.yml.j2,sha256=t8TCULgiErCZdtFmBZVsA8ZdcqR7ccwsmQhuDFTBEAU,3541
 sky/templates/gcp-ray.yml.j2,sha256=y95B-Nk6hFxm6vEIaxI1wFzAIcy_GcKC3XMYo9m-ThI,9662
 sky/templates/ibm-ray.yml.j2,sha256=RMBUqPId8i4CnVwcyfK3DbRapF1jFMuGQlY0E0PFbMU,6669
@@ -279,9 +288,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=k0TBoQ4zgf79-sVkixKSGYFHQ7Z
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=h4YwrPFf9727CACnMJvF3EyK_0OeOYKKt4su_daKekw,1256
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=Kq1MDygF2IxFmu9FXpCxqucXLmeUrvs6OtRij6XTQbo,6554
-skypilot_nightly-1.0.0.dev20241231.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20241231.dist-info/METADATA,sha256=sxHdBITmGuD24QkxyiyHPr_ttV-7K5SkZcUjbh7N2F4,20149
-skypilot_nightly-1.0.0.dev20241231.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-skypilot_nightly-1.0.0.dev20241231.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20241231.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20241231.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20250102.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20250102.dist-info/METADATA,sha256=kQYxKaNaYviPfe84I6DK9dHuvKVJsCqECwL7ekSsQCc,20439
+skypilot_nightly-1.0.0.dev20250102.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+skypilot_nightly-1.0.0.dev20250102.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20250102.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20250102.dist-info/RECORD,,