xmanager-slurm 0.4.5__py3-none-any.whl → 0.4.7__py3-none-any.whl

xm_slurm/packageables.py

@@ -44,7 +44,6 @@ def docker_container(
     build_args: Mapping[str, str] = immutabledict.immutabledict(),
     cache_from: str | Sequence[str] | None = None,
     labels: Mapping[str, str] = immutabledict.immutabledict(),
-    workdir: pathlib.Path | None = None,
     args: xm.UserArgs | None = None,
     env_vars: Mapping[str, str] = immutabledict.immutabledict(),
 ) -> xm.Packageable:
@@ -59,7 +58,6 @@ def docker_container(
         build_args: Build arguments to docker.
         cache_from: Where to pull the BuildKit cache from. See `--cache-from` in `docker build`.
         labels: The container labels.
-        workdir: The working directory in container.
         args: The user arguments to pass to the executable.
         env_vars: The environment variables to pass to the executable.
 
@@ -93,7 +91,6 @@ def docker_container(
             ssh=ssh,
             build_args=build_args,
             cache_from=cache_from,
-            workdir=workdir,
             labels=labels,
         ),
         args=xm.SequentialArgs.from_collection(args),
@@ -173,8 +170,6 @@ def python_container(
             },
             cache_from=cache_from,
             labels=labels,
-            # We must specify the workdir manually for apptainer support
-            workdir=pathlib.Path("/workspace"),
             args=args,
             env_vars=env_vars,
         )
@@ -241,8 +236,6 @@ def mamba_container(
             },
             cache_from=cache_from,
             labels=labels,
-            # We must specify the workdir manually for apptainer support
-            workdir=pathlib.Path("/workspace"),
             args=args,
             env_vars=env_vars,
         )
@@ -312,8 +305,6 @@ def uv_container(
             },
             cache_from=cache_from,
             labels=labels,
-            # We must specify the workdir manually for apptainer support
-            workdir=pathlib.Path("/workspace"),
             args=args,
             env_vars=env_vars,
         )

xm_slurm/packaging/docker.py

@@ -10,13 +10,13 @@ import os
 import pathlib
 import shlex
 import shutil
-import subprocess
 import tempfile
-from typing import Hashable, Literal, Mapping, Sequence
+import typing as tp
 
 import jinja2 as j2
 from xmanager import xm
 
+from xm_slurm import utils
 from xm_slurm.executables import (
     Dockerfile,
     DockerImage,
@@ -32,7 +32,7 @@ from xm_slurm.packaging.registry import IndexedContainer
 logger = logging.getLogger(__name__)
 
 
-def _hash_digest(obj: Hashable) -> str:
+def _hash_digest(obj: tp.Hashable) -> str:
     return hashlib.sha256(repr(obj).encode()).hexdigest()
 
 
@@ -52,7 +52,7 @@ class DockerClient:
             raise RuntimeError("No Docker client found.")
         self._client_call = client_call
 
-        backend_version = packaging_utils.run_command(
+        backend_version = utils.run_command(
             xm.merge_args(self._client_call, ["buildx", "version"]), return_stdout=True
         )
         if backend_version.stdout.startswith("github.com/docker/buildx"):
@@ -71,14 +71,17 @@ class DockerClient:
             """Parse credentials from a Docker credential helper."""
             if not shutil.which(f"docker-credential-{helper}"):
                 return None
-            returncode, output = subprocess.getstatusoutput(
-                f"echo {hostname} | docker-credential-{helper} get",
+            result = utils.run_command(
+                [f"docker-credential-{helper}", "get"],
+                stdin=hostname,
+                return_stdout=True,
             )
 
-            if returncode == 0:
-                credentials = json.loads(output)
+            if result.returncode == 0:
+                credentials = json.loads(result.stdout)
                 return RemoteRepositoryCredentials(
-                    username=credentials["Username"], password=credentials["Secret"]
+                    username=str.strip(credentials["Username"]),
+                    password=str.strip(credentials["Secret"]),
                 )
             return None
 
@@ -112,7 +115,10 @@ class DockerClient:
                     if registry.domain == hostname:
                         auth = base64.b64decode(metadata["auth"]).decode("utf-8")
                         username, password = auth.split(":")
-                        credentials = RemoteRepositoryCredentials(username, password)
+                        credentials = RemoteRepositoryCredentials(
+                            str.strip(username),
+                            str.strip(password),
+                        )
                         self._credentials_cache[hostname] = credentials
                         return credentials
             return None
@@ -133,6 +139,22 @@ class DockerClient:
                 )
                 return _parse_credentials_from_config(podman_config_path)
 
+    def inspect(
+        self, image: ImageURI, element: str | None = None, type: tp.Literal["image"] = "image"
+    ) -> dict[str, tp.Any]:
+        output = utils.run_command(
+            xm.merge_args(
+                self._client_call,
+                ["inspect"],
+                ["--format", f"{{{{json .{element}}}}}"] if element else [],
+                ["--type", type] if type else [],
+                [str(image)],
+            ),
+            check=True,
+            return_stdout=True,
+        )
+        return json.loads(output.stdout.strip().strip("'"))
+
     @functools.cached_property
     def _bake_template(self) -> j2.Template:
         template_loader = j2.PackageLoader("xm_slurm", "templates/docker")
@@ -143,17 +165,17 @@ class DockerClient:
     def _bake_args(
         self,
         *,
-        targets: str | Sequence[str] | None = None,
+        targets: str | tp.Sequence[str] | None = None,
         builder: str | None = None,
-        files: str | os.PathLike[str] | Sequence[os.PathLike[str] | str] | None = None,
+        files: str | os.PathLike[str] | tp.Sequence[os.PathLike[str] | str] | None = None,
         load: bool = False,
         cache: bool = True,
         print: bool = False,
         pull: bool = False,
         push: bool = False,
         metadata_file: str | os.PathLike[str] | None = None,
-        progress: Literal["auto", "plain", "tty"] = "auto",
-        set: Mapping[str, str] | None = None,
+        progress: tp.Literal["auto", "plain", "tty"] = "auto",
+        set: tp.Mapping[str, str] | None = None,
     ) -> xm.SequentialArgs:
         files = files
         if files is None:
@@ -184,9 +206,7 @@ class DockerClient:
         )
 
     def bake(
-        self,
-        *,
-        targets: Sequence[IndexedContainer[xm.Packageable]],
+        self, *, targets: tp.Sequence[IndexedContainer[xm.Packageable]]
     ) -> list[IndexedContainer[RemoteImage]]:
         executors_by_executables = packaging_utils.collect_executors_by_executable(targets)
         for executable, executors in executors_by_executables.items():
@@ -223,7 +243,7 @@ class DockerClient:
                         push=True,
                     ),
                 )
-                packaging_utils.run_command(bake_command.to_list(), tty=True, check=True)
+                utils.run_command(bake_command.to_list(), tty=True, check=True)
             except Exception as ex:
                 raise RuntimeError(f"Failed to build Dockerfiles: {ex}") from ex
             else:
@@ -239,13 +259,27 @@ class DockerClient:
             uri = ImageURI(target.value.executor_spec.tag).with_digest(
                 executable_metadata["containerimage.digest"]
             )
+            config = self.inspect(uri, "Config")
+            if "WorkingDir" not in config:
+                raise ValueError(
+                    "Docker image does not have a working directory. "
+                    "To support all runtimes, we need to set a working directory. "
+                    "Please set `WORKDIR` in the `Dockerfile`."
+                )
+            if "Entrypoint" not in config:
+                raise ValueError(
+                    "Docker image does not have an entrypoint. "
+                    "To support all runtimes, we need to set an entrypoint. "
+                    "Please set `ENTRYPOINT` in the `Dockerfile`."
+                )
 
             images.append(
                 dataclasses.replace(
                     target,
                     value=RemoteImage(  # type: ignore
                         image=str(uri),
-                        workdir=target.value.executable_spec.workdir,
+                        workdir=config["WorkingDir"],
+                        entrypoint=xm.SequentialArgs.from_collection(config["Entrypoint"]),
                         args=target.value.args,
                         env_vars=target.value.env_vars,
                         credentials=self.credentials(uri.domain),
@@ -263,14 +297,14 @@ def docker_client() -> DockerClient:
 
 @registry.register(Dockerfile)
 def _(
-    targets: Sequence[IndexedContainer[xm.Packageable]],
+    targets: tp.Sequence[IndexedContainer[xm.Packageable]],
 ) -> list[IndexedContainer[RemoteImage]]:
     return docker_client().bake(targets=targets)
 
 
 @registry.register(DockerImage)
 def _(
-    targets: Sequence[IndexedContainer[xm.Packageable]],
+    targets: tp.Sequence[IndexedContainer[xm.Packageable]],
 ) -> list[IndexedContainer[RemoteImage]]:
     """Build Docker images, this is essentially a passthrough."""
     images = []
@@ -285,12 +319,28 @@ def _(
             )
 
         uri = ImageURI(target.value.executable_spec.image)
+
+        config = client.inspect(uri, "Config")
+        if "WorkingDir" not in config:
+            raise ValueError(
+                "Docker image does not have a working directory. "
+                "To support all runtimes, we need to set a working directory. "
+                "Please set `WORKDIR` in the `Dockerfile`."
+            )
+        if "Entrypoint" not in config:
+            raise ValueError(
+                "Docker image does not have an entrypoint. "
+                "To support all runtimes, we need to set an entrypoint. "
+                "Please set `ENTRYPOINT` in the `Dockerfile`."
+            )
+
         images.append(
             dataclasses.replace(
                 target,
                 value=RemoteImage(  # type: ignore
                     image=str(uri),
-                    workdir=target.value.executable_spec.workdir,
+                    workdir=config["WorkingDir"],
+                    entrypoint=xm.SequentialArgs.from_collection(config["Entrypoint"]),
                     args=target.value.args,
                     env_vars=target.value.env_vars,
                     credentials=client.credentials(hostname=uri.domain),

xm_slurm/packaging/utils.py

@@ -1,12 +1,5 @@
 import collections
 import logging
-import os
-import pathlib
-import pty
-import re
-import select
-import shutil
-import subprocess
 from typing import ParamSpec, Sequence, TypeVar
 
 from xmanager import xm
@@ -27,104 +20,3 @@ def collect_executors_by_executable(
     for target in targets:
         executors_by_executable[target.value.executable_spec].add(target.value.executor_spec)
     return executors_by_executable
-
-
-# Cursor commands to filter out from the command data stream
-_CURSOR_ESCAPE_SEQUENCES_REGEX = re.compile(
-    rb"\x1b\[\?25[hl]"  # Matches cursor show/hide commands (CSI ?25h and CSI ?25l)
-    rb"|\x1b\[[0-9;]*[Hf]"  # Matches cursor position commands (CSI n;mH and CSI n;mf)
-    rb"|\x1b\[s"  # Matches cursor save position (CSI s)
-    rb"|\x1b\[u"  # Matches cursor restore position (CSI u)
-    rb"|\x1b\[2J"  # Matches clear screen (CSI 2J)
-    rb"|\x1b\[K"  # Matches clear line (CSI K)
-)
-
-
-def run_command(
-    args: Sequence[str] | xm.SequentialArgs,
-    env: dict[str, str] | None = None,
-    tty: bool = False,
-    cwd: str | os.PathLike[str] | None = None,
-    check: bool = False,
-    return_stdout: bool = False,
-    return_stderr: bool = False,
-) -> subprocess.CompletedProcess[str]:
-    if isinstance(args, xm.SequentialArgs):
-        args = args.to_list()
-    args = list(args)
-
-    executable = shutil.which(args[0])
-    if not executable:
-        raise RuntimeError(f"Couldn't find executable {args[0]}")
-    executable = pathlib.Path(executable)
-
-    subprocess_env = os.environ.copy() | (env if env else {})
-    if executable.name == "docker" and args[1] == "buildx":
-        subprocess_env |= {"DOCKER_CLI_EXPERIMENTAL": "enabled"}
-
-    logger.debug(f"env: {subprocess_env}")
-    logger.debug(f"command: {' '.join(args)}")
-
-    stdout_master, stdout_slave = pty.openpty()
-    stderr_master, stderr_slave = pty.openpty()
-
-    stdout_data, stderr_data = b"", b""
-    with subprocess.Popen(
-        executable=executable,
-        args=args,
-        shell=False,
-        text=True,
-        bufsize=0,
-        stdout=stdout_slave,
-        stderr=stderr_slave,
-        start_new_session=True,
-        close_fds=True,
-        cwd=cwd,
-        env=subprocess_env,
-    ) as process:
-        os.close(stdout_slave)
-        os.close(stderr_slave)
-
-        fds = [stdout_master, stderr_master]
-        while fds:
-            rlist, _, _ = select.select(fds, [], [])
-            for fd in rlist:
-                try:
-                    data = os.read(fd, 1024)
-                except OSError:
-                    data = None
-
-                if not data:
-                    os.close(fd)
-                    fds.remove(fd)
-                    continue
-
-                data = _CURSOR_ESCAPE_SEQUENCES_REGEX.sub(b"", data)
-
-                if fd == stdout_master:
-                    if return_stdout:
-                        stdout_data += data
-                    if tty:
-                        os.write(pty.STDOUT_FILENO, data)
-                elif fd == stderr_master:
-                    if return_stderr:
-                        stderr_data += data
-                    if tty:
-                        os.write(pty.STDERR_FILENO, data)
-                else:
-                    raise RuntimeError("Unexpected file descriptor")
-
-    stdout = stdout_data.decode(errors="replace") if stdout_data else ""
-    stderr = stderr_data.decode(errors="replace") if stderr_data else ""
-
-    retcode = process.poll()
-    assert retcode is not None
-
-    if check and retcode:
-        raise subprocess.CalledProcessError(retcode, process.args)
-    return subprocess.CompletedProcess(
-        process.args,
-        retcode,
-        stdout=stdout,
-        stderr=stderr,
-    )

xm_slurm/scripts/cli.py

@@ -29,8 +29,15 @@ async def logs(
         raise ValueError("Must specify either wid or identity.")
     assert wu is not None
 
-    async for log in wu.logs(num_lines=num_lines, block_size=block_size, wait=True, follow=follow):
-        console.print(log, end="\n")
+    with console.status("Waiting for logs...") as status:
+        waiting = True
+        async for log in wu.logs(
+            num_lines=num_lines, block_size=block_size, wait=True, follow=follow
+        ):
+            if waiting:
+                status.stop()
+                waiting = False
+            console.print(log, end="\n")
 
 
 @xm.run_in_asyncio_loop

xm_slurm/templates/docker/uv.Dockerfile

@@ -17,7 +17,12 @@ RUN apt-get update \
     git $EXTRA_SYSTEM_PACKAGES \
     && rm -rf /var/lib/apt/lists/*
 
-RUN uv pip install --system pysocks $EXTRA_PYTHON_PACKAGES
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system pysocks $EXTRA_PYTHON_PACKAGES
+
+RUN uv venv --system-site-packages
+
+ENV PATH="/workspace/.venv/bin:$PATH"
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
@@ -30,6 +35,4 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=ssh \
     uv sync --frozen --no-dev
 
-ENV PATH="/workspace/.venv/bin:$PATH"
-
 ENTRYPOINT [ "python" ]

xm_slurm/templates/slurm/entrypoint.bash.j2

@@ -0,0 +1,27 @@
+{%- macro entrypoint(cluster, job) -%}
+#!/bin/sh
+set -eux
+
+{% if cluster.container_environment %}
+# Cluster environment variables
+{% for key, value in cluster.container_environment.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endif %}
+
+{% if job.executable.env_vars %}
+# Executable environment variables
+{% for key, value in job.executable.env_vars.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endif %}
+
+{% if job.env_vars %}
+# Job environment variables
+{% for key, value in job.env_vars.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endif %}
+
+exec {{ job.executable.entrypoint.to_list() | join(' ') }} "$@"
+{%- endmacro -%}

xm_slurm/templates/slurm/job-array.bash.j2

@@ -10,18 +10,18 @@ srun \
   --unbuffered \
   --kill-on-bad-exit=0 \
   --overlap \
-  --export={{ export(job, "ALL") }} \
+  --export="ALL" \
   bash <<'SRUN_EOF' &
 set -Eeuxo pipefail
 
-readonly __XM_SLURM_TRIALS=(
+readonly XM_SLURM_TRIAL_ARGS=(
 {% for trial in args %}
   "{{ trial.to_list() | join(" ") }}"
 {% endfor %}
 )
 
-{% call run(job, cluster) %}
-  ${__XM_SLURM_TRIALS[$SLURM_ARRAY_TASK_ID]} \
+{% call run(cluster, job) %}
+  ${XM_SLURM_TRIAL_ARGS[$SLURM_ARRAY_TASK_ID]} \
 {% endcall %}
 
 SRUN_EOF

xm_slurm/templates/slurm/job-group.bash.j2

@@ -31,11 +31,11 @@
 srun \
   --unbuffered \
   --kill-on-bad-exit=0 \
-  --export={{ export(job, "ALL") }} \
+  --export="ALL" \
   --het-group={{ loop.index0 }} \
   bash <<'SRUN_EOF' &
 set -Eeuxo pipefail
-{{ run(job, cluster) }}
+{{ run(cluster, job) }}
 SRUN_EOF
 {% endfor +%}
 {% endblock bootstrap %}

xm_slurm/templates/slurm/job.bash.j2

@@ -33,9 +33,10 @@ set -Eeuxo pipefail
 {% endif %}
 {%- endblock prolog %}
 
-
 {% block environment -%}
-{{ env(cluster.environment) }}
+{% for key, value in cluster.host_environment.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
 {%- endblock environment %}
 
 {% block proxy -%}
@@ -54,10 +55,10 @@ srun \
   --unbuffered \
   --kill-on-bad-exit=0 \
   --overlap \
-  --export={{ export(job, "ALL") }} \
+  --export="ALL" \
   bash <<'SRUN_EOF' &
 set -Eeuxo pipefail
-{{ run(job, cluster) }}
+{{ run(cluster, job) }}
 SRUN_EOF
 {%- endblock bootstrap %}
 

xm_slurm/templates/slurm/runtimes/apptainer.bash.j2

@@ -1,73 +1,33 @@
-{% macro env(env_vars) -%}
-{% for key, value in env_vars.items() %}
-{% if key.startswith("SINGULARITY") or key.startswith("APPTAINER") or key.startswith("_") %}
-{% set key = key.lstrip('_') %}
-export {{ key }}="{{ value }}"
-{% else %}
-export APPTAINERENV_{{ key }}="{{ value }}"
-export SINGULARITYENV_{{ key }}="{{ value }}"
-export {{ key }}="{{ value }}"
-{% endif %}
-{% endfor %}
-{%- endmacro %}
-
-{% macro export(job, mode=None) -%}
-    {%- set combined_envs = operator.or_(job.env_vars, job.executable.env_vars) -%}
-    {%- if job.executable.credentials -%}
-      {%- set combined_envs = operator.or_(combined_envs, {
-          "APPTAINER_DOCKER_USERNAME": job.executable.credentials.username,
-          "APPTAINER_DOCKER_PASSWORD": job.executable.credentials.password,
-          "SINGULARITY_DOCKER_USERNAME": job.executable.credentials.username,
-          "SINGULARITY_DOCKER_PASSWORD": job.executable.credentials.password,
-        })
-      -%}
-    {%- endif %}
-
-    {%- set env_strings = [] -%}
-    {%- for key, value in combined_envs.items() -%}
-        {%- if key.startswith("SINGULARITY") or key.startswith("APPTAINER") -%}
-            {%- set _ = env_strings.append('{0}="{1}"'.format(key, value)) -%}
-        {%- else -%}
-            {%- set _ = env_strings.append('APPTAINERENV_{0}="{1}",SINGULARITYENV_{0}="{1}"'.format(key, value)) -%}
-        {%- endif -%}
-    {%- endfor -%}
-
-     {%- if mode is not none -%}
-      {{- mode -}}{{- "," if combined_envs -}}
-    {%- endif -%}
-
-    {{- env_strings | join(",") -}}
-{% endmacro %}
-
-{% macro run(job, cluster) -%}
-# Determine which binary to use or if an error should be raised
-if [[ $(command -v apptainer) ]]; then
-    readonly CONTAINER_RUNTIME="apptainer"
-elif [[ $(command -v singularity) ]]; then
-    readonly CONTAINER_RUNTIME="singularity"
-else
-    echo "Error: Neither singularity nor apptainer binaries found" >&2
-    exit 1
-fi
+{% macro run(cluster, job) -%}
+{%- set runtime = (cluster.runtime | string | lower) -%}
 
 # Bundle will be where our built sandbox image is stored
 # container-workdir will be our container's scratch directory
 mkdir -p "$SLURM_TMPDIR"/{container,container-workdir,container-overlay}
 
-time ${CONTAINER_RUNTIME} build \
+{% if job.executable.credentials %}
+env {{ runtime | upper }}_DOCKER_USERNAME="{{ job.executable.credentials.username }}" {{ runtime | upper }}_DOCKER_PASSWORD="{{ job.executable.credentials.password }}" time {{ runtime }} build \
+{% else %}
+time {{ runtime }} build \
+{% endif %}
   --force \
   --sandbox \
   --fix-perms \
   "$SLURM_TMPDIR"/container \
   docker://{{ job.executable.image }}
 
-{% if (cluster.runtime | string) == "singularity" and cluster.mounts %}
+{% if runtime == "singularity" and cluster.mounts %}
 {% for source, dest in cluster.mounts.items() %}
 mkdir -p "$SLURM_TMPDIR"/container/{{ dest | trim('/') }}
 {% endfor %}
 {% endif %}
 
-exec ${CONTAINER_RUNTIME} run \
+cat << 'ENTRYPOINT_EOF' > "$SLURM_TMPDIR"/container/xm-slurm-entrypoint.sh
+{{ entrypoint(cluster, job) }}
+ENTRYPOINT_EOF
+chmod +x "$SLURM_TMPDIR"/container/xm-slurm-entrypoint.sh
+
+exec {{ runtime }} exec \
 {% if job.executor.requirements.accelerator %}
   --nv \
 {% endif %}
@@ -75,6 +35,9 @@ exec ${CONTAINER_RUNTIME} run \
   --no-umask \
   --no-home \
   --cleanenv \
+{% if runtime == "apptainer" %}
+  --no-eval \
+{% endif %}
   --containall \
 {% if cluster.mounts %}
 {% for source, dest in cluster.mounts.items() %}
@@ -91,6 +54,7 @@ exec ${CONTAINER_RUNTIME} run \
   --pwd {{ job.executable.workdir }} \
 {% endif %}
   "$SLURM_TMPDIR"/container \
+  /xm-slurm-entrypoint.sh \
 {% for arg in job.executable.args.to_list() %}
   {{ arg }} \
 {% endfor %}

xm_slurm/templates/slurm/runtimes/podman.bash.j2

@@ -1,33 +1,18 @@
-{% macro env(env_vars) -%}
-{% for key, value in env_vars.items() %}
-export PODMANENV_{{ key }}="{{ value }}"
-{% endfor %}
-{%- endmacro %}
-
-{% macro export(job, mode=None) -%}
-    {%- set combined_envs = operator.or_(job.env_vars, job.executable.env_vars) -%}
-
-    {%- set env_strings = [] -%}
-    {%- for key, value in combined_envs.items() -%}
-        {%- set _ = env_strings.append('PODMANENV_{0}="{1}"'.format(key, value)) -%}
-    {%- endfor -%}
-
-    {%- if mode is not none -%}
-        {{- mode -}}{{- "," if combined_envs -}}
-    {%- endif -%}
-
-    {{- env_strings | join(",") -}}
-{% endmacro %}
-
-{% macro run(job, cluster) -%}
-podman pull \
+{% macro run(cluster, job) -%}
+time podman pull \
 {% if job.executable.credentials %}
   --creds {{ job.executable.credentials.username }}:{{ job.executable.credentials.password }} \
 {% endif %}
   {{ job.executable.image }}
 
+cat << 'ENTRYPOINT_EOF' > "$SLURM_TMPDIR"/xm-slurm-entrypoint.sh
+{{ entrypoint(cluster, job) }}
+ENTRYPOINT_EOF
+chmod +x "$SLURM_TMPDIR"/xm-slurm-entrypoint.sh
+
 exec podman run \
-  --env PODMANENV* \
+  --mount type=bind,src="$SLURM_TMPDIR"/xm-slurm-entrypoint.sh,dst=/xm-slurm-entrypoint.sh,ro \
+  --entrypoint /xm-slurm-entrypoint.sh \
   --pull never \
   --restart no \
   --rm \