xmanager-slurm 0.3.0__py3-none-any.whl

xm_slurm/status.py

@@ -0,0 +1,188 @@
+"""Implementation of Slurm work unit statuses."""
+
+import enum
+import re
+from typing import Sequence
+
+from xmanager import xm
+
+
+class SlurmJobState(enum.Enum):
+    BOOT_FAIL = enum.auto()
+    CANCELLED = enum.auto()
+    COMPLETED = enum.auto()
+    DEADLINE = enum.auto()
+    FAILED = enum.auto()
+    NODE_FAIL = enum.auto()
+    OUT_OF_MEMORY = enum.auto()
+    PENDING = enum.auto()
+    PREEMPTED = enum.auto()
+    RUNNING = enum.auto()
+    REQUEUED = enum.auto()
+    RESIZING = enum.auto()
+    REVOKED = enum.auto()
+    SUSPENDED = enum.auto()
+    TIMEOUT = enum.auto()
+
+    @property
+    def message(self) -> str:
+        match self:
+            case SlurmJobState.BOOT_FAIL:
+                return (
+                    "Job terminated due to launch failure, "
+                    "typically due to a hardware failure (e.g. unable to boot "
+                    "the node or block and the job can not be requeued)."
+                )
+            case SlurmJobState.CANCELLED:
+                return (
+                    "Job was explicitly cancelled by the user or "
+                    "system administrator. The job may or may not have been "
+                    "initiated."
+                )
+            case SlurmJobState.COMPLETED:
+                return "Job has terminated all processes on all " "nodes with an exit code of zero."
+            case SlurmJobState.DEADLINE:
+                return "Job terminated on deadline."
+            case SlurmJobState.FAILED:
+                return "Job terminated with non-zero exit code or " "other failure condition."
+            case SlurmJobState.NODE_FAIL:
+                return "Job terminated due to failure of one or " "more allocated nodes."
+            case SlurmJobState.OUT_OF_MEMORY:
+                return "Job experienced out of memory error."
+            case SlurmJobState.PENDING:
+                return "Job is awaiting resource allocation."
+            case SlurmJobState.PREEMPTED:
+                return "Job terminated due to preemption."
+            case SlurmJobState.RUNNING:
+                return "Job currently has an allocation."
+            case SlurmJobState.REQUEUED:
+                return "Job was requeued."
+            case SlurmJobState.RESIZING:
+                return "Job is about to change size."
+            case SlurmJobState.REVOKED:
+                return "Sibling was removed from cluster due to " "other cluster starting the job."
+            case SlurmJobState.SUSPENDED:
+                return "Job has an allocation, but execution has been suspended."
+            case SlurmJobState.TIMEOUT:
+                return "Job terminated upon reaching its time limit."
+            case _:
+                raise ValueError(f"Invalid Slurm job state: {self}")
+
+    def __str__(self) -> str:
+        return f"{self.name}: {self.message}"
+
+    @classmethod
+    def from_str(cls, state: str) -> "SlurmJobState":
+        return cls[state]
+
+    @classmethod
+    def from_slurm_str(cls, state: str) -> "SlurmJobState":
+        _SLURM_JOB_STATE_REGEX = re.compile(f"({'|'.join(entry.name for entry in cls)})\\s?.*")
+        match = _SLURM_JOB_STATE_REGEX.match(state)
+        assert match and len(match.groups()) == 1, f"Failed to parse job state, {state!r}"
+        return cls.from_str(match.group(1))
+
+
+SlurmPendingJobStates = set([
+    SlurmJobState.PENDING,
+    SlurmJobState.REQUEUED,
+    SlurmJobState.RESIZING,
+])
+SlurmRunningJobStates = set([
+    SlurmJobState.RUNNING,
+    SlurmJobState.SUSPENDED,
+])
+SlurmActiveJobStates = SlurmPendingJobStates | SlurmRunningJobStates
+SlurmCompletedJobStates = set([SlurmJobState.COMPLETED])
+SlurmFailedJobStates = set([
+    SlurmJobState.BOOT_FAIL,
+    SlurmJobState.DEADLINE,
+    SlurmJobState.FAILED,
+    SlurmJobState.NODE_FAIL,
+    SlurmJobState.OUT_OF_MEMORY,
+    SlurmJobState.PREEMPTED,
+    SlurmJobState.REVOKED,
+    SlurmJobState.TIMEOUT,
+])
+SlurmCancelledJobStates = set([SlurmJobState.CANCELLED])
+
+
+class SlurmWorkUnitStatusEnum(enum.IntEnum):
+    """Status of a local experiment job."""
+
+    # Work unit was created, but has not started yet.
+    PENDING = 0
+    # Work unit was created, but has not terminated yet.
+    RUNNING = 1
+    # Work unit terminated and was successful.
+    COMPLETED = 2
+    # Work unit terminated and was not succesful.
+    FAILED = 3
+    # Work unit terminated because it was cancelled by the user.
+    CANCELLED = 4
+
+    @classmethod
+    def from_job_state(cls, state: SlurmJobState) -> "SlurmWorkUnitStatusEnum":
+        """Convert a Slurm job state to a SlurmWorkUnitStatusEnum."""
+        if state in SlurmPendingJobStates:
+            return cls.PENDING
+        elif state in SlurmRunningJobStates:
+            return cls.RUNNING
+        elif state in SlurmCompletedJobStates:
+            return cls.COMPLETED
+        elif state in SlurmFailedJobStates:
+            return cls.FAILED
+        elif state in SlurmCancelledJobStates:
+            return cls.CANCELLED
+        else:
+            raise ValueError(f"Invalid Slurm job state: {state}")
+
+
+class SlurmWorkUnitStatus(xm.ExperimentUnitStatus):
+    """Status of a Slurm experiment job."""
+
+    @classmethod
+    def aggregate(cls, states: Sequence[SlurmJobState]) -> "SlurmWorkUnitStatus":
+        """Aggregate a sequence of statuses into a single status."""
+        assert len(states) > 0, "Cannot aggregate empty sequence of statuses."
+        max_error_state: SlurmJobState | None = None
+        for state in states:
+            if not max_error_state:
+                max_error_state = state
+            elif SlurmWorkUnitStatusEnum.from_job_state(
+                state
+            ) > SlurmWorkUnitStatusEnum.from_job_state(max_error_state):
+                max_error_state = state
+        assert max_error_state is not None
+        return cls(max_error_state)
+
+    def __init__(self, state: SlurmJobState) -> None:
+        super().__init__()
+        self._state = state
+        self._status = SlurmWorkUnitStatusEnum.from_job_state(state)
+
+    @property
+    def is_active(self) -> bool:
+        return (
+            self._status == SlurmWorkUnitStatusEnum.RUNNING
+            or self._status == SlurmWorkUnitStatusEnum.PENDING
+        )
+
+    @property
+    def is_completed(self) -> bool:
+        return self._status == SlurmWorkUnitStatusEnum.COMPLETED
+
+    @property
+    def is_failed(self) -> bool:
+        return self._status == SlurmWorkUnitStatusEnum.FAILED
+
+    @property
+    def status(self) -> SlurmWorkUnitStatusEnum:
+        return self._status
+
+    @property
+    def message(self) -> str:
+        return str(self._state)
+
+    def __repr__(self) -> str:
+        return f"<SlurmWorkUnitStatus {self._state!r}>"

xm_slurm/templates/docker/docker-bake.hcl.j2

@@ -0,0 +1,47 @@
+{% for executable, executors in executables.items() %}
+target "{{ hash(executable) }}" {
+    dockerfile-inline = <<EOF
+{{ executable.dockerfile.read_text() }}
+EOF
+    context = "{{ executable.context }}"
+    {% if executable.target %}
+    target = "{{ executable.target }}"
+    {% endif %}
+    pull = true
+    tags = [
+        {% for executor in executors %}
+        "{{ executor.tag }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    output = [
+        "type=registry"
+    ]
+    {% if executable.cache_from %}
+    cache-from = [
+        {% for cache_from in executable.cache_from %}
+        "{{ cache_from }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    {% endif %}
+    cache-to = [
+        "type=inline"
+    ]
+    platforms = [
+        {% for platform in executable.platforms %}
+        "{{ platform }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    labels = {
+        {% for key, value in executable.labels.items() %}
+        "{{ key }}" = "{{ value }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    }
+    {% if executable.build_args %}
+    args = {
+        {% for key, value in executable.build_args.items() %}
+        "{{ key }}" = "{{ value }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    }
+    {% endif %}
+}
+{% endfor %}

xm_slurm/templates/docker/mamba.Dockerfile

@@ -0,0 +1,27 @@
+# syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE=gcr.io/distroless/base-debian10
+
+FROM docker.io/mambaorg/micromamba:jammy as mamba
+ARG CONDA_ENVIRONMENT=environment.yml
+
+USER root
+
+COPY $CONDA_ENVIRONMENT /tmp/
+
+# Setup mamba environment
+RUN --mount=type=cache,target=/opt/conda/pkgs --mount=type=cache,target=/root/.cache/pip \
+    micromamba create --yes --always-copy --no-pyc --prefix /opt/env --file /tmp/environment.yml
+
+RUN find /opt/env/ -follow -type f -name '*.a' -delete && \
+    find /opt/env/ -follow -type f -name '*.js.map' -delete
+
+FROM $BASE_IMAGE
+
+COPY --link --from=mamba /opt/env /opt/env
+
+ENV PATH=$PATH:/opt/env/bin
+
+WORKDIR /workspace
+COPY --link . /workspace
+
+ENTRYPOINT ["/opt/env/bin/python"]

xm_slurm/templates/docker/pdm.Dockerfile

@@ -0,0 +1,31 @@
+# syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE
+
+FROM $BASE_IMAGE AS builder
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip install -U pip setuptools wheel pysocks \
+    && pip install pdm
+
+COPY --link pyproject.toml pdm.lock /workspace/
+WORKDIR /workspace
+
+RUN --mount=type=cache,target=/root/.cache/pdm mkdir __pypackages__ \
+    && PDM_CACHE_DIR=/root/.cache/pdm pdm sync --prod --no-editable
+
+FROM $BASE_IMAGE
+
+ARG PYTHON_MAJOR
+ARG PYTHON_MINOR
+
+ENV PYTHONPATH=/workspace/pkgs:$PYTHONPATH
+COPY --link --from=builder /workspace/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib /workspace/pkgs
+
+WORKDIR /workspace/src
+COPY --link . /workspace/src
+
+ENTRYPOINT ["python"]

xm_slurm/templates/docker/python.Dockerfile

@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE=docker.io/python:3.10-slim
+FROM $BASE_IMAGE as builder
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install and update necesarry global Python packages
+RUN pip install -U pip setuptools wheel pysocks
+
+ARG PIP_REQUIREMENTS=requirements.txt
+
+RUN python -m venv --copies --upgrade --upgrade-deps --system-site-packages /venv
+COPY $PIP_REQUIREMENTS /tmp/requirements.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    PIP_CACHE_DIR=/root/.cache/pip /venv/bin/pip install -r /tmp/requirements.txt \
+    && rm -rf /tmp/requirements.txt
+
+COPY --link . /workspace
+WORKDIR /workspace
+
+ENTRYPOINT [ "/venv/bin/python" ]

xm_slurm/templates/slurm/fragments/monitor.bash.j2

@@ -0,0 +1,32 @@
+{% macro monitor(requeue_max_attempts, requeue_exit_code) -%}
+__xm_slurm_wait_for_children() {
+  if [[ -n "${SLURM_ARRAY_JOB_ID:-}" ]]; then
+    local -r JOB_ID="${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}"
+  else
+    local -r JOB_ID="${SLURM_JOB_ID}"
+  fi
+
+  # If there are no child jobs we should error out
+  if [ -z "$(jobs -p)" ]; then
+    echo "ERROR: no child jobs exist..." >&2
+    exit -1
+  fi
+
+  # Loop through all job IDs in the background job list and wait for them to finish
+  for job in "$(jobs -p)"; do
+    echo "INFO: Waiting for job ${job} to finish..."
+    set +e
+    wait "${job}"
+    local -r JOB_EXIT_CODE="${?}"
+    set -e
+
+    if [ "${JOB_EXIT_CODE}" -eq "{{ requeue_exit_code }}" ] && [ "${SLURM_RESTART_COUNT-0}" -le "{{ requeue_max_attempts }}" ]; then
+      echo "INFO: Received requeue exit code {{ requeue_exit_code }} from job ${job}. Requeing Slurm job ${JOB_ID} after ${SLURM_RESTART_COUNT-0} restarts." >&2
+      scontrol requeue "${JOB_ID}"
+      exit {{ requeue_exit_code }}
+    fi
+  done
+}
+
+__xm_slurm_wait_for_children
+{%- endmacro %}

xm_slurm/templates/slurm/fragments/proxy.bash.j2

@@ -0,0 +1,31 @@
+{% macro proxy(host) -%}
+__xm_slurm_proxy() {
+    local -r GATEWAY="$1"
+    local PORT
+
+    # Find an open port
+    while
+        PORT="$(shuf -n 1 -i 1024-65535)"
+        netstat -atun | grep -q "$PORT"
+    do
+        sleep 0.25
+        continue
+    done
+
+    # Reverse proxy through the gateway
+    ssh -D "$PORT" "$GATEWAY" -N -f
+
+    # Export all env vars for applications to pick up on proxy
+    export ALL_PROXY="socks5://127.0.0.1:$PORT"
+    export all_proxy="socks5://127.0.0.1:$PORT"
+
+    export HTTP_PROXY="socks5://127.0.0.1:$PORT"
+    export http_proxy="socks5://127.0.0.1:$PORT"
+
+    export HTTPS_PROXY="socks5://127.0.0.1:$PORT"
+    export https_proxy="socks5://127.0.0.1:$PORT"
+
+    export JAVA_OPTS="-DsocksProxyHost=127.0.0.1 -DsocksProxyPort=$PORT"
+}
+__xm_slurm_proxy "{{ host }}"
+{%- endmacro %}

xm_slurm/templates/slurm/job-array.bash.j2

@@ -0,0 +1,29 @@
+{% extends "job.bash.j2" %}
+{% block directives %}
+{{ super() -}}
+#SBATCH --array=0-{{ args | length - 1 }}
+#SBATCH --output=xm-%j-%a.stdout
+#SBATCH --error=xm-%j-%a.stderr
+{% endblock directives %}
+
+{% block bootstrap %}
+srun \
+  --unbuffered \
+  --kill-on-bad-exit=0 \
+  --overlap \
+  --export={{ export(job, "ALL") }} \
+  bash <<'SRUN_EOF' &
+set -Eeuxo pipefail
+
+readonly __XM_SLURM_TRIALS=(
+{% for trial in args %}
+  "{{ trial.to_list() | join(" ") }}"
+{% endfor %}
+)
+
+{% call run(job, cluster) %}
+  ${__XM_SLURM_TRIALS[$SLURM_ARRAY_TASK_ID]} \
+{% endcall %}
+
+SRUN_EOF
+{%- endblock bootstrap %}

xm_slurm/templates/slurm/job-group.bash.j2

@@ -0,0 +1,41 @@
+{% extends "job.bash.j2" %}
+{% block directives %}
+#SBATCH --export=NONE
+#SBATCH --comment="{'xid': {{ experiment_id }}}"
+{% if cluster.account %}
+#SBATCH --account={{ cluster.account }}
+{% endif %}
+{% if cluster.partition %}
+#SBATCH --partition={{ cluster.partition }}
+{% endif %}
+{% if cluster.qos %}
+#SBATCH --qos={{ cluster.qos }}
+{% endif %}
+
+{% for job_name, job in job_group.jobs.items() %}
+#SBATCH --output=xm-%j+{{ job_name }}.stdout
+#SBATCH --error=xm-%j+{{ job_name }}.stderr
+{% if identity %}
+#SBATCH --job-name=xm[{{ job_name }}@{{ experiment_id }}.{{ identity }}]
+#SBATCH --dependency=singleton
+{% else %}
+#SBATCH --job-name=xm[{{ job_name }}@{{ experiment_id }}]
+{% endif %}
+{{ job.executor.to_directives() | join("\n") }}
+{{ "\n#SBATCH hetjob\n" if not loop.last }}
+{% endfor %}
+{% endblock directives %}
+
+{% block bootstrap %}
+{% for job in job_group.jobs.values() +%}
+srun \
+  --unbuffered \
+  --kill-on-bad-exit=0 \
+  --export={{ export(job, "ALL") }} \
+  --het-group={{ loop.index0 }} \
+  bash <<'SRUN_EOF' &
+set -Eeuxo pipefail
+{{ run(job, cluster) }}
+SRUN_EOF
+{% endfor +%}
+{% endblock bootstrap %}

xm_slurm/templates/slurm/job.bash.j2

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+{% block directives %}
+#SBATCH --open-mode=append
+#SBATCH --export=NONE
+#SBATCH --output=xm-%j.stdout
+#SBATCH --error=xm-%j.stderr
+#SBATCH --comment="{'xid': {{ experiment_id }}}"
+{% if cluster.account and not job.executor.account %}
+#SBATCH --account={{ cluster.account }}
+{% endif %}
+{% if cluster.partition and not job.executor.partition %}
+#SBATCH --partition={{ cluster.partition }}
+{% endif %}
+{% if cluster.qos and not job.executor.qos %}
+#SBATCH --qos={{ cluster.qos }}
+{% endif %}
+{% if identity %}
+#SBATCH --job-name=xm[{{ experiment_id }}.{{ identity }}]
+#SBATCH --dependency=singleton
+{% else %}
+#SBATCH --job-name=xm[{{ experiment_id }}]
+{% endif %}
+{% for directive in job.executor.to_directives() %}
+#SBATCH {{ directive }}
+{% endfor %}
+{% endblock directives %}
+set -Eeuxo pipefail
+
+{% block prolog %}
+{% if cluster.prolog %}
+{{- cluster.prolog -}}
+{% endif %}
+{%- endblock prolog %}
+
+
+{% block environment -%}
+{{ env(cluster.environment) }}
+{%- endblock environment %}
+
+{% block proxy -%}
+{%- if cluster.proxy %}
+{% from 'fragments/proxy.bash.j2' import proxy %}
+{% if cluster.proxy == "submission-host" %}
+{{ proxy("$SLURM_SUBMIT_HOST") }}
+{% else %}
+{{ proxy(cluster.proxy) }}
+{% endif %}
+{% endif %}
+{%- endblock proxy %}
+
+{% block bootstrap %}
+srun \
+  --unbuffered \
+  --kill-on-bad-exit=0 \
+  --overlap \
+  --export={{ export(job, "ALL") }} \
+  bash <<'SRUN_EOF' &
+set -Eeuxo pipefail
+{{ run(job, cluster) }}
+SRUN_EOF
+{%- endblock bootstrap %}
+
+echo "[INFO] Start timestamp: $(date)"
+
+{%- block epilog -%}
+{% if cluster.epilog %}
+{{ cluster.epilog }}
+{% endif %}
+{%- endblock epilog %}
+
+
+{% block monitor -%}
+{% from 'fragments/monitor.bash.j2' import monitor %}
+{{ monitor(job.executor.requeue_max_attempts, job.executor.requeue_on_exit_code) }}
+{%- endblock monitor %}
+
+
+echo "[INFO] End timestamp: $(date)"

xm_slurm/templates/slurm/runtimes/apptainer.bash.j2

@@ -0,0 +1,103 @@
+{% macro env(env_vars) -%}
+{% for key, value in env_vars.items() %}
+{% if key.startswith("SINGULARITY") or key.startswith("APPTAINER") or key.startswith("_") %}
+{% set key = key.lstrip('_') %}
+export {{ key }}="{{ value }}"
+{% else %}
+export APPTAINERENV_{{ key }}="{{ value }}"
+export SINGULARITYENV_{{ key }}="{{ value }}"
+{% endif %}
+{% endfor %}
+{%- endmacro %}
+
+{% macro export(job, mode=None) -%}
+    {%- set combined_envs = operator.or_(job.env_vars, job.executable.env_vars) -%}
+    {%- if job.executable.credentials -%}
+      {%- set combined_envs = operator.or_(combined_envs, {
+          "APPTAINER_DOCKER_USERNAME": job.executable.credentials.username,
+          "APPTAINER_DOCKER_PASSWORD": job.executable.credentials.password,
+          "SINGULARITY_DOCKER_USERNAME": job.executable.credentials.username,
+          "SINGULARITY_DOCKER_PASSWORD": job.executable.credentials.password,
+        })
+      -%}
+    {%- endif %}
+
+    {%- set env_strings = [] -%}
+    {%- for key, value in combined_envs.items() -%}
+        {%- if key.startswith("SINGULARITY") or key.startswith("APPTAINER") -%}
+            {%- set _ = env_strings.append('{0}="{1}"'.format(key, value)) -%}
+        {%- else -%}
+            {%- set _ = env_strings.append('APPTAINERENV_{0}="{1}",SINGULARITYENV_{0}="{1}"'.format(key, value)) -%}
+        {%- endif -%}
+    {%- endfor -%}
+
+     {%- if mode is not none -%}
+      {{- mode -}}{{- "," if combined_envs -}}
+    {%- endif -%}
+
+    {{- env_strings | join(",") -}}
+{% endmacro %}
+
+{% macro run(job, cluster) -%}
+# Determine which binary to use or if an error should be raised
+if [[ $(command -v apptainer) ]]; then
+    readonly CONTAINER_RUNTIME="apptainer"
+elif [[ $(command -v singularity) ]]; then
+    readonly CONTAINER_RUNTIME="singularity"
+else
+    echo "Error: Neither singularity nor apptainer binaries found" >&2
+    exit 1
+fi
+
+# Bundle will be where our built sandbox image is stored
+# container-workdir will be our container's scratch directory
+mkdir -p "$SLURM_TMPDIR"/{container,container-workdir,container-overlay}
+
+time ${CONTAINER_RUNTIME} build \
+  --force \
+  --sandbox \
+  --fix-perms \
+  "$SLURM_TMPDIR"/container \
+  docker://{{ job.executable.image }}
+
+{% if (cluster.runtime | string) == "singularity" and cluster.mounts %}
+{% for source, dest in cluster.mounts.items() %}
+mkdir -p "$SLURM_TMPDIR"/container/{{ dest | trim('/') }}
+{% endfor %}
+{% endif %}
+
+exec ${CONTAINER_RUNTIME} run \
+{% if job.executor.requirements.accelerator %}
+  --nv \
+{% endif %}
+  --no-init \
+  --no-umask \
+  --no-home \
+  --cleanenv \
+  --containall \
+{% if cluster.mounts %}
+{% for source, dest in cluster.mounts.items() %}
+  --mount type=bind,src={{ source }},dst={{ dest }} \
+{% endfor %}
+{% endif %}
+  --workdir "$SLURM_TMPDIR"/container-workdir \
+{% if (cluster.runtime | string) == "apptainer" %}
+  --overlay "$SLURM_TMPDIR"/container-overlay \
+{% else %}
+  --writable \
+{% endif %}
+{% if job.executable.workdir %}
+  --pwd {{ job.executable.workdir }} \
+{% endif %}
+  "$SLURM_TMPDIR"/container \
+{% for arg in job.executable.args.to_list() %}
+  {{ arg }} \
+{% endfor %}
+{% for arg in job.args.to_list() %}
+  {{ arg }} \
+{% endfor %}
+{% if caller %}
+  {{- caller() -}}
+{% endif %}
+  "$@"
+{%- endmacro %}

xm_slurm/templates/slurm/runtimes/podman.bash.j2

@@ -0,0 +1,56 @@
+{% macro env(env_vars) -%}
+{% for key, value in env_vars.items() %}
+export PODMANENV_{{ key }}="{{ value }}"
+{% endfor %}
+{%- endmacro %}
+
+{% macro export(job, mode=None) -%}
+    {%- set combined_envs = operator.or_(job.env_vars, job.executable.env_vars) -%}
+
+    {%- set env_strings = [] -%}
+    {%- for key, value in combined_envs.items() -%}
+        {%- set _ = env_strings.append('PODMANENV_{0}="{1}"'.format(key, value)) -%}
+    {%- endfor -%}
+
+    {%- if mode is not none -%}
+        {{- mode -}}{{- "," if combined_envs -}}
+    {%- endif -%}
+
+    {{- env_strings | join(",") -}}
+{% endmacro %}
+
+{% macro run(job, cluster) -%}
+podman pull \
+{% if job.executable.credentials %}
+  --creds {{ job.executable.credentials.username }}:{{ job.executable.credentials.password }} \
+{% endif %}
+  {{ job.executable.image }}
+
+exec podman run \
+  --env PODMANENV* \
+  --pull never \
+  --restart no \
+  --rm \
+{% if job.executor.requirements.accelerator %}
+  --device nvidia.com/gpu=all \
+{% endif %}
+{% if cluster.mounts %}
+{% for source, dest in cluster.mounts.items() %}
+  --mount type=bind,src={{ source }},dst={{ dest }} \
+{% endfor %}
+{% endif %}
+{% if job.executable.workdir %}
+  --workdir {{ job.executable.workdir }} \
+{% endif %}
+  {{ job.executable.image }} \
+{% for arg in job.executable.args.to_list() %}
+  {{ arg }} \
+{% endfor %}
+{% for arg in job.args.to_list() %}
+  {{ arg }} \
+{% endfor %}
+{% if caller %}
+  {{- caller() -}}
+{% endif %}
+  "$@"
+{% endmacro %}