skypilot-nightly 1.0.0.dev20240824__py3-none-any.whl → 1.0.0.dev20240826__py3-none-any.whl

sky/__init__.py

@@ -5,7 +5,7 @@ from typing import Optional
 import urllib.request
 
 # Replaced with the current commit when building the wheels.
-_SKYPILOT_COMMIT_SHA = 'f3defe3c2c14f28af5dc70b10ab6d4ef19d8709b'
+_SKYPILOT_COMMIT_SHA = '6067e2bcd60c9ae48fa4fb883bbede4b98b4d545'
 
 
 def _get_git_commit():
@@ -35,7 +35,7 @@ def _get_git_commit():
 
 
 __commit__ = _get_git_commit()
-__version__ = '1.0.0.dev20240824'
+__version__ = '1.0.0.dev20240826'
 __root_dir__ = os.path.dirname(os.path.abspath(__file__))
 
 

sky/backends/wheel_utils.py

@@ -39,29 +39,30 @@ _WHEEL_PATTERN = (f'{_PACKAGE_WHEEL_NAME}-'
                   f'{version.parse(sky.__version__)}-*.whl')
 
 
-def _get_latest_wheel_and_remove_all_others() -> pathlib.Path:
-    wheel_name = (f'**/{_WHEEL_PATTERN}')
+def _remove_stale_wheels(latest_wheel_dir: pathlib.Path) -> None:
+    """Remove all wheels except the latest one."""
+    for f in WHEEL_DIR.iterdir():
+        if f != latest_wheel_dir:
+            if f.is_dir() and not f.is_symlink():
+                shutil.rmtree(f, ignore_errors=True)
+
+
+def _get_latest_wheel() -> pathlib.Path:
+    wheel_name = f'**/{_WHEEL_PATTERN}'
     try:
         latest_wheel = max(WHEEL_DIR.glob(wheel_name), key=os.path.getctime)
     except ValueError:
         raise FileNotFoundError(
             'Could not find built SkyPilot wheels with glob pattern '
             f'{wheel_name} under {WHEEL_DIR!r}') from None
-
-    latest_wheel_dir_name = latest_wheel.parent
-    # Cleanup older wheels.
-    for f in WHEEL_DIR.iterdir():
-        if f != latest_wheel_dir_name:
-            if f.is_dir() and not f.is_symlink():
-                shutil.rmtree(f, ignore_errors=True)
     return latest_wheel
 
 
-def _build_sky_wheel():
-    """Build a wheel for SkyPilot."""
-    with tempfile.TemporaryDirectory() as tmp_dir:
+def _build_sky_wheel() -> pathlib.Path:
+    """Build a wheel for SkyPilot and return the path to the wheel."""
+    with tempfile.TemporaryDirectory() as tmp_dir_str:
         # prepare files
-        tmp_dir = pathlib.Path(tmp_dir)
+        tmp_dir = pathlib.Path(tmp_dir_str)
         sky_tmp_dir = tmp_dir / 'sky'
         sky_tmp_dir.mkdir()
         for item in SKY_PACKAGE_PATH.iterdir():
@@ -129,6 +130,7 @@ def _build_sky_wheel():
         wheel_dir = WHEEL_DIR / hash_of_latest_wheel
         wheel_dir.mkdir(parents=True, exist_ok=True)
         shutil.move(str(wheel_path), wheel_dir)
+        return wheel_dir / wheel_path.name
 
 
 def build_sky_wheel() -> Tuple[pathlib.Path, str]:
@@ -161,13 +163,22 @@ def build_sky_wheel() -> Tuple[pathlib.Path, str]:
         last_modification_time = _get_latest_modification_time(SKY_PACKAGE_PATH)
         last_wheel_modification_time = _get_latest_modification_time(WHEEL_DIR)
 
-        # only build wheels if the wheel is outdated
-        if last_wheel_modification_time < last_modification_time:
+        # Only build wheels if the wheel is outdated or wheel does not exist
+        # for the requested version.
+        if (last_wheel_modification_time < last_modification_time) or not any(
+                WHEEL_DIR.glob(f'**/{_WHEEL_PATTERN}')):
             if not WHEEL_DIR.exists():
                 WHEEL_DIR.mkdir(parents=True, exist_ok=True)
-            _build_sky_wheel()
-
-        latest_wheel = _get_latest_wheel_and_remove_all_others()
+            latest_wheel = _build_sky_wheel()
+        else:
+            latest_wheel = _get_latest_wheel()
+
+        # We remove all wheels except the latest one for garbage collection.
+        # Otherwise stale wheels will accumulate over time.
+        # TODO(romilb): If the user switches versions every alternate launch,
+        #  the wheel will be rebuilt every time. At the risk of adding
+        #  complexity, we can consider TTL caching wheels by version here.
+        _remove_stale_wheels(latest_wheel.parent)
 
         wheel_hash = latest_wheel.parent.name
 

sky/templates/kubernetes-ray.yml.j2

@@ -317,7 +317,54 @@ available_node_types:
           # Do not change this command - it keeps the pod alive until it is
           # explicitly killed.
           command: ["/bin/bash", "-c", "--"]
-          args: ['trap : TERM INT; sleep infinity & wait;']
+          args: 
+            - |
+              # Tails file and checks every 5 sec for
+              # open file handlers with write access
+              # closes if none exist
+              monitor_file() {
+                tail -f $file &
+                TAIL_PID=$!
+                while kill -0 $TAIL_PID 2> /dev/null; do
+                  # only two PIDs should be accessing the file
+                  # the log appender and log tailer
+                  if [ $(lsof -w $file | wc -l) -lt 3 ]; then
+                    kill $TAIL_PID
+                    break
+                  fi
+                  # Sleep for 5 seconds before checking again. Do not make this
+                  # too short as it will consume CPU, and too long will cause
+                  # the file to be closed too late keeping the pod alive.
+                  sleep 5
+                done
+              }
+
+              log_tail() {
+                FILE_PATTERN="~/sky_logs/*/tasks/*.log"
+                while ! ls $(eval echo $FILE_PATTERN) 1> /dev/null 2>&1; do
+                  sleep 1
+                done
+
+                # Keep track of already monitored files
+                already_monitored=""
+
+                # Infinite loop to continuously check for new files
+                while true; do
+                  for file in $(eval echo $FILE_PATTERN); do
+                    if echo $already_monitored | grep -q $file; then
+                      # File is already being monitored
+                      continue
+                    fi
+                    
+                    # Monitor the new file
+                    monitor_file $file &
+                    already_monitored="${already_monitored} ${file}"
+                  done
+                  sleep 0.1
+                done
+              }
+              trap : TERM INT; log_tail || sleep infinity & wait
+
           ports:
           - containerPort: 22  # Used for SSH
           - containerPort: {{ray_port}}  # Redis port
@@ -365,7 +412,7 @@ setup_commands:
   # Line 'sudo grep ..': set the number of threads per process to unlimited to avoid ray job submit stucking issue when the number of running ray jobs increase.
   # Line 'mkdir -p ..': disable host key check
   # Line 'python3 -c ..': patch the buggy ray files and enable `-o allow_other` option for `goofys`
-  - sudo DEBIAN_FRONTEND=noninteractive apt install gcc patch pciutils rsync fuse curl -y;
+  - sudo DEBIAN_FRONTEND=noninteractive apt install lsof gcc patch pciutils rsync fuse curl -y;
     mkdir -p ~/.ssh; touch ~/.ssh/config;
     {%- for initial_setup_command in initial_setup_commands %}
     {{ initial_setup_command }}

{skypilot_nightly-1.0.0.dev20240824.dist-info → skypilot_nightly-1.0.0.dev20240826.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: skypilot-nightly
-Version: 1.0.0.dev20240824
+Version: 1.0.0.dev20240826
 Summary: SkyPilot: An intercloud broker for the clouds
 Author: SkyPilot Team
 License: Apache 2.0

{skypilot_nightly-1.0.0.dev20240824.dist-info → skypilot_nightly-1.0.0.dev20240826.dist-info}/RECORD

@@ -1,4 +1,4 @@
-sky/__init__.py,sha256=P9pq59ys4031jvg84mXVIEVfAXuAcx355jd3cJvc1g0,5588
+sky/__init__.py,sha256=irQAdlzMvPbSpPhz6DApwrGhWnyaBnXANdTbIfZcdQ0,5588
 sky/authentication.py,sha256=PXKsabUKnvhoTNoNwZzo66qjCLAsuc5pQ8esVu0IYh8,20424
 sky/check.py,sha256=sqUCow5Gqqtp2ouk7tZ3whwDOK23wqyUJvcxMBsich8,9080
 sky/cli.py,sha256=XlgRr617B_yE2HV_nW01D_m5nWntW2GstutNrMp86UA,201504
@@ -33,7 +33,7 @@ sky/backends/backend_utils.py,sha256=k-DQ-xlQElzlb5KhoyoaySw8t36gmsWtgeR24TM23iw
 sky/backends/cloud_vm_ray_backend.py,sha256=EXrJdc2wqKlgPkIrKtF2GYLw-wW_U49HDb0CpaVlIMA,230185
 sky/backends/docker_utils.py,sha256=Hyw1YY20EyghhEbYx6O2FIMDcGkNzBzV9TM7LFynei8,8358
 sky/backends/local_docker_backend.py,sha256=H4GBo0KFUC_EEf-ziv1OUbfAkOI5BrwkYs9fYOxSoNw,16741
-sky/backends/wheel_utils.py,sha256=Kt2WOtVvOdw1DCx-wM5QydQoIMbqzcTcHvbbHa9DYY4,7297
+sky/backends/wheel_utils.py,sha256=3QS4T_Ydvo4DbYhogtyADyNBEf04I6jUCL71M285shQ,7963
 sky/backends/monkey_patches/monkey_patch_ray_up.py,sha256=76-y2fCaE3JINj8lEwHT1eirYzCbpD8O1ySsysuGu8o,3450
 sky/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sky/benchmark/benchmark_state.py,sha256=X8CXmuU9KgsDRhKedhFgjeRMUFWtQsjFs1qECvPG2yg,8723
@@ -226,7 +226,7 @@ sky/templates/jobs-controller.yaml.j2,sha256=QjlIcFBEay48xKT50UWqzgREzili-H7AuUr
 sky/templates/kubernetes-ingress.yml.j2,sha256=73iDklVDWBMbItg0IexCa6_ClXPJOxw7PWz3leku4nE,1340
 sky/templates/kubernetes-loadbalancer.yml.j2,sha256=IxrNYM366N01bbkJEbZ_UPYxUP8wyVEbRNFHRsBuLsw,626
 sky/templates/kubernetes-port-forward-proxy-command.sh,sha256=D5T3OUfTlTyiO4ZPwA8fBDyttfESpyWxlSTYrRJdHHA,2614
-sky/templates/kubernetes-ray.yml.j2,sha256=_qLDrSvIvH2GUIGu7MH6EvRBl1OjT0B8ekNrpPorx0A,16268
+sky/templates/kubernetes-ray.yml.j2,sha256=YjNQNF1AcV_KEnEC-fDSGx5Ix6Ci0y9JOgJpYzwUcgg,18061
 sky/templates/kubernetes-ssh-jump.yml.j2,sha256=k5W5sOIMppU7dDkJMwPlqsUcb92y7L5_TVG3hkgMy8M,2747
 sky/templates/lambda-ray.yml.j2,sha256=SY3-itV1QJ6BoIDQ9Qm8XwfYgphyO7XfAzm7SsQtyc0,5738
 sky/templates/local-ray.yml.j2,sha256=FNHeyHF6nW9nU9QLIZceUWfvrFTTcO51KqhTnYCEFaA,1185
@@ -270,9 +270,9 @@ sky/utils/kubernetes/k8s_gpu_labeler_job.yaml,sha256=KPqp23B-zQ2SZK03jdHeF9fLTog
 sky/utils/kubernetes/k8s_gpu_labeler_setup.yaml,sha256=VLKT2KKimZu1GDg_4AIlIt488oMQvhRZWwsj9vBbPUg,3812
 sky/utils/kubernetes/rsync_helper.sh,sha256=1ad-m4s8QTPxaBQxNIL8AGDfX4un6T0dxZgk-R7OLyE,154
 sky/utils/kubernetes/ssh_jump_lifecycle_manager.py,sha256=RFLJ3k7MR5UN4SKHykQ0lV9SgXumoULpKYIAt1vh-HU,6560
-skypilot_nightly-1.0.0.dev20240824.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
-skypilot_nightly-1.0.0.dev20240824.dist-info/METADATA,sha256=gXiWBPMwBGeLlYfTA9U4QVGpG7M1v6XWuTn1we5EBhY,18870
-skypilot_nightly-1.0.0.dev20240824.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-skypilot_nightly-1.0.0.dev20240824.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
-skypilot_nightly-1.0.0.dev20240824.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
-skypilot_nightly-1.0.0.dev20240824.dist-info/RECORD,,
+skypilot_nightly-1.0.0.dev20240826.dist-info/LICENSE,sha256=emRJAvE7ngL6x0RhQvlns5wJzGI3NEQ_WMjNmd9TZc4,12170
+skypilot_nightly-1.0.0.dev20240826.dist-info/METADATA,sha256=Ys16mAgml6_cs23iKJzpUEWJnF1fR0b6YQUegH-uKSY,18870
+skypilot_nightly-1.0.0.dev20240826.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
+skypilot_nightly-1.0.0.dev20240826.dist-info/entry_points.txt,sha256=StA6HYpuHj-Y61L2Ze-hK2IcLWgLZcML5gJu8cs6nU4,36
+skypilot_nightly-1.0.0.dev20240826.dist-info/top_level.txt,sha256=qA8QuiNNb6Y1OF-pCUtPEr6sLEwy2xJX06Bd_CrtrHY,4
+skypilot_nightly-1.0.0.dev20240826.dist-info/RECORD,,