xmanager-slurm 0.4.15__py3-none-any.whl → 0.4.16__py3-none-any.whl

xm_slurm/contrib/clusters/__init__.py

@@ -26,6 +26,7 @@ def mila(
             "/home/mila/${USER:0:1}/$USER/.local/state/xm-slurm": "/xm-slurm-state",
             "/home/mila/${USER:0:1}/$USER/.ssh": "/home/mila/${USER:0:1}/$USER/.ssh",
         }
+    mounts = dict(mounts) | {"/dev/infiniband": "/dev/infiniband"}
 
     return config.SlurmClusterConfig(
         name="mila",

xm_slurm/contrib/clusters/drac.py

@@ -29,6 +29,7 @@ def _drac_cluster(
             "/home/$USER/.ssh": "/home/$USER/.ssh",
             "/home/$USER/.local/state/xm-slurm": "/xm-slurm-state",
         }
+    mounts = dict(mounts) | {"/dev/infiniband": "/dev/infiniband"}
 
     return config.SlurmClusterConfig(
         name=name,

xm_slurm/templates/slurm/runtimes/apptainer.bash.j2

@@ -3,7 +3,9 @@
 
 # Bundle will be where our built sandbox image is stored
 # container-workdir will be our container's scratch directory
-mkdir -p "$SLURM_TMPDIR"/{container,container-workdir,container-overlay}
+# TODO(jfarebro): We can make this more efficient by doing an srun per node and downloading the container once per node.
+# but this requires apptainer support to have an overlay per procid
+mkdir -p "$SLURM_TMPDIR"/{container-"$SLURM_PROCID",container-workdir-"$SLURM_PROCID",container-overlay-"$SLURM_PROCID"}
 
 retry -c 255 -n 10 -d 1 -b 2 -- \
 {% if job.executable.credentials %}
@@ -14,19 +16,21 @@ retry -c 255 -n 10 -d 1 -b 2 -- \
     --force \
     --sandbox \
     --fix-perms \
-    "$SLURM_TMPDIR"/container \
+    "$SLURM_TMPDIR"/container-"$SLURM_PROCID" \
     docker://{{ job.executable.image }}
 
 {% if runtime == "singularity" and cluster.mounts %}
 {% for source, dest in cluster.mounts.items() %}
-mkdir -p "$SLURM_TMPDIR"/container/{{ dest | trim('/') }}
+mkdir -p "$SLURM_TMPDIR"/container-"$SLURM_PROCID"/{{ dest | trim('/') }}
 {% endfor %}
 {% endif %}
 
-cat << 'ENTRYPOINT_EOF' > "$SLURM_TMPDIR"/container/xm-slurm-entrypoint.sh
+cat << 'ENTRYPOINT_EOF' > "$SLURM_TMPDIR"/container-"$SLURM_PROCID"/xm-slurm-entrypoint.sh
 {{ entrypoint(cluster, job) }}
 ENTRYPOINT_EOF
-chmod +x "$SLURM_TMPDIR"/container/xm-slurm-entrypoint.sh
+chmod +x "$SLURM_TMPDIR"/container-"$SLURM_PROCID"/xm-slurm-entrypoint.sh
+
+for var in "${!SLURM_@}"; do export "{{ runtime | upper }}ENV_${var}=${!var}"; done
 
 exec {{ runtime }} exec \
 {% if job.executor.requirements.accelerator %}
@@ -45,16 +49,16 @@ exec {{ runtime }} exec \
   --bind {{ source }}:{{ dest }} \
 {% endfor %}
 {% endif %}
-  --workdir "$SLURM_TMPDIR"/container-workdir \
+  --workdir "$SLURM_TMPDIR"/container-workdir-"$SLURM_PROCID" \
 {% if (cluster.runtime | string) == "apptainer" %}
-  --overlay "$SLURM_TMPDIR"/container-overlay \
+  --overlay "$SLURM_TMPDIR"/container-overlay-"$SLURM_PROCID" \
 {% else %}
   --writable \
 {% endif %}
 {% if job.executable.workdir %}
   --pwd {{ job.executable.workdir }} \
 {% endif %}
-  "$SLURM_TMPDIR"/container \
+  "$SLURM_TMPDIR"/container-"$SLURM_PROCID" \
   /xm-slurm-entrypoint.sh \
 {% for arg in job.executable.args.to_list() %}
   {{ arg }} \

xm_slurm/templates/slurm/runtimes/podman.bash.j2

@@ -16,6 +16,7 @@ exec podman run \
   --entrypoint /xm-slurm-entrypoint.sh \
   --pull never \
   --restart no \
+  --env "SLURM_*" \
   --rm \
 {% if job.executor.requirements.accelerator %}
   --device nvidia.com/gpu=all \

{xmanager_slurm-0.4.15.dist-info → xmanager_slurm-0.4.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xmanager-slurm
-Version: 0.4.15
+Version: 0.4.16
 Summary: Slurm backend for XManager.
 Project-URL: GitHub, https://github.com/jessefarebro/xm-slurm
 Author-email: Jesse Farebrother <jfarebro@cs.mcgill.ca>

{xmanager_slurm-0.4.15.dist-info → xmanager_slurm-0.4.16.dist-info}/RECORD

@@ -22,8 +22,8 @@ xm_slurm/api/models.py,sha256=_INVh0j-4-rRs0WASyg4fNB6NF1L1nUeGgQ6-XnbwsM,1610
 xm_slurm/api/sqlite/client.py,sha256=jAesCKDuYwnNcAxwJk_1b1TB8cT_QGbSjo1UE3mZjEQ,14037
 xm_slurm/api/web/client.py,sha256=uO67Y7fnQ-w__Vm_A5BEuy7Qi8wQcWk3vIsBGEBkyfk,6261
 xm_slurm/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xm_slurm/contrib/clusters/__init__.py,sha256=XFCVnkThiU3_8uA_tUgDByOBanXNHrxDvfmuptmQ2KE,2214
-xm_slurm/contrib/clusters/drac.py,sha256=_iubsmzz5tK2KTaKqSuykS3IDtsdXqJ0MXep1THSJUM,7020
+xm_slurm/contrib/clusters/__init__.py,sha256=oYkuo4jAA6_VLIJs4k1Rjcp6l3qN0qmJExfkPWfmho4,2281
+xm_slurm/contrib/clusters/drac.py,sha256=-Nawzyx3p471f8Fti_tbweHn8h4ItRN4ec-ktHnUE4Q,7087
 xm_slurm/experimental/parameter_controller.py,sha256=IrFzq104LkZrhzuirit5GUZDXDvv2bBSYNMh3orsiPY,8518
 xm_slurm/packaging/__init__.py,sha256=dh307yLpUT9KN7rJ1e9fYC6hegGKfZcGboUq9nGpDVQ,233
 xm_slurm/packaging/docker.py,sha256=-DWcB9qqbeHmIEqyfF0-v6xOT25ae90u2x-QZ7kluOw,13579
@@ -43,10 +43,10 @@ xm_slurm/templates/slurm/job.bash.j2,sha256=JnK0D8_3tVNpnvPwM5yL_rjLcjqhuHiCtolD
 xm_slurm/templates/slurm/fragments/monitor.bash.j2,sha256=ri5FgoKs6_bQVf5DO8SL4rJf4UsLxV34aOV-OD8VWDU,2526
 xm_slurm/templates/slurm/fragments/proxy.bash.j2,sha256=VJLglZo-Nvx9R-qe3rHTxr07CylTQ6Z9NwBzvIpAZrA,814
 xm_slurm/templates/slurm/library/retry.bash,sha256=bLe59qvfWEk17rE1wZ4EHiHba3RvR2WWZPq-kSe8RAA,2164
-xm_slurm/templates/slurm/runtimes/apptainer.bash.j2,sha256=v0LwHM-kBW8sJqVcVA2jYr1n44imDSZrJqmqlr5uTGc,1980
-xm_slurm/templates/slurm/runtimes/podman.bash.j2,sha256=zWLsFEuVzOMSETOmv4A5ZCV4oQHwCipiR6wi79XVzNI,1188
-xmanager_slurm-0.4.15.dist-info/METADATA,sha256=xgJUFDConlb4R5W0cOK3xb2fuIR7x_tjNpOybMagT_A,1007
-xmanager_slurm-0.4.15.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-xmanager_slurm-0.4.15.dist-info/entry_points.txt,sha256=_HLGmLgxuQLOPmF2gOFYDVq2HqtMVD_SzigHvUh8TCY,49
-xmanager_slurm-0.4.15.dist-info/licenses/LICENSE.md,sha256=IxstXr3MPHwTJ5jMrByHrQsR1ZAGQ2U_uz_4qzI_15Y,11756
-xmanager_slurm-0.4.15.dist-info/RECORD,,
+xm_slurm/templates/slurm/runtimes/apptainer.bash.j2,sha256=XxAQWLxZogL7zjn7tuzKn-DkYUJMx_HjaRzpVkz97lM,2414
+xm_slurm/templates/slurm/runtimes/podman.bash.j2,sha256=8N1ZtwHyXxP-Cjo4HBPsJiZXcTvf7q2GzvW9ao8_aok,1208
+xmanager_slurm-0.4.16.dist-info/METADATA,sha256=j0282EV56cTAG80Q4R419IE-Z74OJVvAkP--IPMCwuo,1007
+xmanager_slurm-0.4.16.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+xmanager_slurm-0.4.16.dist-info/entry_points.txt,sha256=_HLGmLgxuQLOPmF2gOFYDVq2HqtMVD_SzigHvUh8TCY,49
+xmanager_slurm-0.4.16.dist-info/licenses/LICENSE.md,sha256=IxstXr3MPHwTJ5jMrByHrQsR1ZAGQ2U_uz_4qzI_15Y,11756
+xmanager_slurm-0.4.16.dist-info/RECORD,,