xmanager-slurm 0.4.19__py3-none-any.whl

xm_slurm/status.py

@@ -0,0 +1,197 @@
+"""Implementation of Slurm work unit statuses."""
+
+import enum
+import re
+import typing as tp
+
+from xmanager import xm
+
+
+class SlurmJobState(enum.Enum):
+    BOOT_FAIL = enum.auto()
+    CANCELLED = enum.auto()
+    COMPLETED = enum.auto()
+    DEADLINE = enum.auto()
+    FAILED = enum.auto()
+    NODE_FAIL = enum.auto()
+    OUT_OF_MEMORY = enum.auto()
+    PENDING = enum.auto()
+    PREEMPTED = enum.auto()
+    RUNNING = enum.auto()
+    REQUEUED = enum.auto()
+    RESIZING = enum.auto()
+    REVOKED = enum.auto()
+    SUSPENDED = enum.auto()
+    TIMEOUT = enum.auto()
+
+    @property
+    def message(self) -> str:
+        match self:
+            case SlurmJobState.BOOT_FAIL:
+                return (
+                    "Job terminated due to launch failure, "
+                    "typically due to a hardware failure (e.g. unable to boot "
+                    "the node or block and the job can not be requeued)."
+                )
+            case SlurmJobState.CANCELLED:
+                return (
+                    "Job was explicitly cancelled by the user or "
+                    "system administrator. The job may or may not have been "
+                    "initiated."
+                )
+            case SlurmJobState.COMPLETED:
+                return "Job has terminated all processes on all " "nodes with an exit code of zero."
+            case SlurmJobState.DEADLINE:
+                return "Job terminated on deadline."
+            case SlurmJobState.FAILED:
+                return "Job terminated with non-zero exit code or " "other failure condition."
+            case SlurmJobState.NODE_FAIL:
+                return "Job terminated due to failure of one or " "more allocated nodes."
+            case SlurmJobState.OUT_OF_MEMORY:
+                return "Job experienced out of memory error."
+            case SlurmJobState.PENDING:
+                return "Job is awaiting resource allocation."
+            case SlurmJobState.PREEMPTED:
+                return "Job terminated due to preemption."
+            case SlurmJobState.RUNNING:
+                return "Job currently has an allocation."
+            case SlurmJobState.REQUEUED:
+                return "Job was requeued."
+            case SlurmJobState.RESIZING:
+                return "Job is about to change size."
+            case SlurmJobState.REVOKED:
+                return "Sibling was removed from cluster due to " "other cluster starting the job."
+            case SlurmJobState.SUSPENDED:
+                return "Job has an allocation, but execution has been suspended."
+            case SlurmJobState.TIMEOUT:
+                return "Job terminated upon reaching its time limit."
+            case _:
+                raise ValueError(f"Invalid Slurm job state: {self}")
+
+    def __str__(self) -> str:
+        return f"{self.name}: {self.message}"
+
+    @classmethod
+    def from_str(cls, state: str) -> "SlurmJobState":
+        return cls[state]
+
+    @classmethod
+    def from_slurm_str(cls, state: str) -> "SlurmJobState":
+        _SLURM_JOB_STATE_REGEX = re.compile(f"({'|'.join(entry.name for entry in cls)})\\s?.*")
+        match = _SLURM_JOB_STATE_REGEX.match(state)
+        assert match and len(match.groups()) == 1, f"Failed to parse job state, {state!r}"
+        return cls.from_str(match.group(1))
+
+
+SlurmPendingJobStates = set([
+    SlurmJobState.PENDING,
+    SlurmJobState.REQUEUED,
+    SlurmJobState.RESIZING,
+])
+SlurmRunningJobStates = set([
+    SlurmJobState.RUNNING,
+    SlurmJobState.SUSPENDED,
+])
+SlurmActiveJobStates = SlurmPendingJobStates | SlurmRunningJobStates
+SlurmCompletedJobStates = set([SlurmJobState.COMPLETED])
+SlurmFailedJobStates = set([
+    SlurmJobState.BOOT_FAIL,
+    SlurmJobState.DEADLINE,
+    SlurmJobState.FAILED,
+    SlurmJobState.NODE_FAIL,
+    SlurmJobState.OUT_OF_MEMORY,
+    SlurmJobState.PREEMPTED,
+    SlurmJobState.REVOKED,
+    SlurmJobState.TIMEOUT,
+])
+SlurmCancelledJobStates = set([SlurmJobState.CANCELLED])
+
+assert (
+    SlurmPendingJobStates
+    | SlurmRunningJobStates
+    | SlurmActiveJobStates
+    | SlurmCompletedJobStates
+    | SlurmFailedJobStates
+    | SlurmCancelledJobStates
+) == set(SlurmJobState.__members__.values()), "Slurm job states are not exhaustive."
+
+
+class SlurmWorkUnitStatusEnum(enum.IntEnum):
+    """Status of a local experiment job."""
+
+    # Work unit was created, but has not started yet.
+    PENDING = 0
+    # Work unit was created, but has not terminated yet.
+    RUNNING = 1
+    # Work unit terminated and was successful.
+    COMPLETED = 2
+    # Work unit terminated and was not succesful.
+    FAILED = 3
+    # Work unit terminated because it was cancelled by the user.
+    CANCELLED = 4
+
+    @classmethod
+    def from_job_state(cls, state: SlurmJobState) -> "SlurmWorkUnitStatusEnum":
+        """Convert a Slurm job state to a SlurmWorkUnitStatusEnum."""
+        if state in SlurmPendingJobStates:
+            return cls.PENDING
+        elif state in SlurmRunningJobStates:
+            return cls.RUNNING
+        elif state in SlurmCompletedJobStates:
+            return cls.COMPLETED
+        elif state in SlurmFailedJobStates:
+            return cls.FAILED
+        elif state in SlurmCancelledJobStates:
+            return cls.CANCELLED
+        else:
+            raise ValueError(f"Invalid Slurm job state: {state}")
+
+
+class SlurmWorkUnitStatus(xm.ExperimentUnitStatus):
+    """Status of a Slurm experiment job."""
+
+    @classmethod
+    def aggregate(cls, states: tp.Sequence[SlurmJobState]) -> "SlurmWorkUnitStatus":
+        """Aggregate a sequence of statuses into a single status."""
+        assert len(states) > 0, "Cannot aggregate empty sequence of statuses."
+        max_error_state: SlurmJobState | None = None
+        for state in states:
+            if not max_error_state:
+                max_error_state = state
+            elif SlurmWorkUnitStatusEnum.from_job_state(
+                state
+            ) > SlurmWorkUnitStatusEnum.from_job_state(max_error_state):
+                max_error_state = state
+        assert max_error_state is not None
+        return cls(max_error_state)
+
+    def __init__(self, state: SlurmJobState) -> None:
+        super().__init__()
+        self._state = state
+        self._status = SlurmWorkUnitStatusEnum.from_job_state(state)
+
+    @property
+    def is_active(self) -> bool:
+        return (
+            self._status == SlurmWorkUnitStatusEnum.RUNNING
+            or self._status == SlurmWorkUnitStatusEnum.PENDING
+        )
+
+    @property
+    def is_completed(self) -> bool:
+        return self._status == SlurmWorkUnitStatusEnum.COMPLETED
+
+    @property
+    def is_failed(self) -> bool:
+        return self._status == SlurmWorkUnitStatusEnum.FAILED
+
+    @property
+    def status(self) -> SlurmWorkUnitStatusEnum:
+        return self._status
+
+    @property
+    def message(self) -> str:
+        return str(self._state)
+
+    def __repr__(self) -> str:
+        return f"<SlurmWorkUnitStatus {self._state!r}>"

xm_slurm/templates/docker/docker-bake.hcl.j2

@@ -0,0 +1,54 @@
+{% for executable, executors in executables.items() %}
+target "{{ hash(executable) }}" {
+    dockerfile-inline = <<EOF
+{{ executable.dockerfile.read_text() }}
+EOF
+    context = "{{ executable.context }}"
+    {% if executable.ssh %}
+    ssh = [
+        {% for ssh_val in executable.ssh %}
+        "{{ ssh_val }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    {% endif %}
+    {% if executable.target %}
+    target = "{{ executable.target }}"
+    {% endif %}
+    pull = false
+    tags = [
+        {% for executor in executors %}
+        "{{ executor.tag }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    output = [
+        "type=registry"
+    ]
+    {% if executable.cache_from %}
+    cache-from = [
+        {% for cache_from in executable.cache_from %}
+        "{{ cache_from }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    {% endif %}
+    cache-to = [
+        "type=inline"
+    ]
+    platforms = [
+        {% for platform in executable.platforms %}
+        "{{ platform }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    ]
+    labels = {
+        {% for key, value in executable.labels.items() %}
+        "{{ key }}" = "{{ value }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    }
+    {% if executable.build_args %}
+    args = {
+        {% for key, value in executable.build_args.items() %}
+        "{{ key }}" = "{{ value }}"{% if not loop.last %},{% endif %}
+        {% endfor %}
+    }
+    {% endif %}
+}
+{% endfor %}

xm_slurm/templates/docker/mamba.Dockerfile

@@ -0,0 +1,29 @@
+# syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE=gcr.io/distroless/base-debian10
+
+FROM docker.io/mambaorg/micromamba:bookworm-slim as mamba
+ARG CONDA_ENVIRONMENT=environment.yml
+
+USER root
+
+COPY $CONDA_ENVIRONMENT /tmp/
+
+# Setup mamba environment
+RUN --mount=type=cache,target=/opt/conda/pkgs \
+    --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=ssh \
+    micromamba create --yes --always-copy --no-pyc --prefix /opt/env --file /tmp/environment.yml
+
+RUN find /opt/env/ -follow -type f -name '*.a' -delete && \
+    find /opt/env/ -follow -type f -name '*.js.map' -delete
+
+FROM $BASE_IMAGE
+
+COPY --link --from=mamba /opt/env /opt/env
+
+ENV PATH=$PATH:/opt/env/bin
+
+WORKDIR /workspace
+COPY --link . /workspace
+
+ENTRYPOINT ["/opt/env/bin/python"]

xm_slurm/templates/docker/python.Dockerfile

@@ -0,0 +1,32 @@
+# syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE=docker.io/python:3.10-slim-bookworm
+FROM $BASE_IMAGE as builder
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+
+ARG EXTRA_SYSTEM_PACKAGES=""
+ARG EXTRA_PYTHON_PACKAGES=""
+
+ENV UV_PYTHON_DOWNLOADS=0
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
+
+WORKDIR /workspace
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    git $EXTRA_SYSTEM_PACKAGES \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install and update necesarry global Python packages
+RUN uv pip install --system pysocks $EXTRA_PYTHON_PACKAGES
+
+ARG PIP_REQUIREMENTS=requirements.txt
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=$PIP_REQUIREMENTS,target=requirements.txt \
+    --mount=type=ssh \
+    uv pip install --system --requirement requirements.txt
+
+COPY --link . /workspace
+
+ENTRYPOINT [ "python" ]

xm_slurm/templates/docker/uv.Dockerfile

@@ -0,0 +1,38 @@
+# syntax=docker/dockerfile:1.4
+ARG BASE_IMAGE=docker.io/python:3.10-slim-bookworm
+FROM $BASE_IMAGE
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
+
+ARG EXTRA_SYSTEM_PACKAGES=""
+ARG EXTRA_PYTHON_PACKAGES=""
+
+WORKDIR /workspace
+
+ENV UV_PYTHON_DOWNLOADS=0
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    git $EXTRA_SYSTEM_PACKAGES \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system pysocks $EXTRA_PYTHON_PACKAGES
+
+RUN uv venv --system-site-packages
+
+ENV PATH="/workspace/.venv/bin:$PATH"
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=ssh \
+    uv sync --frozen --no-install-project --no-dev --no-editable
+
+COPY --link . /workspace
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=ssh \
+    uv sync --frozen --no-dev
+
+ENTRYPOINT [ "python" ]

xm_slurm/templates/slurm/entrypoint.bash.j2

@@ -0,0 +1,27 @@
+{%- macro entrypoint(cluster, job) -%}
+#!/bin/sh
+set -eux
+
+{% if cluster.container_environment %}
+# Cluster environment variables
+{% for key, value in cluster.container_environment.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endif %}
+
+{% if job.executable.env_vars %}
+# Executable environment variables
+{% for key, value in job.executable.env_vars.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endif %}
+
+{% if job.env_vars %}
+# Job environment variables
+{% for key, value in job.env_vars.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endif %}
+
+exec {{ job.executable.entrypoint.to_list() | join(' ') }} "$@"
+{%- endmacro -%}

xm_slurm/templates/slurm/fragments/monitor.bash.j2

@@ -0,0 +1,78 @@
+{% macro monitor(requeue_max_attempts, requeue_exit_code, requeue_on_timeout, requeue_timeout) -%}
+__xm_slurm_wait_for_children() {
+  if [[ -n "${SLURM_ARRAY_JOB_ID:-}" ]]; then
+    local -r JOB_ID="${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}"
+  else
+    local -r JOB_ID="${SLURM_JOB_ID}"
+  fi
+
+  # If there are no child jobs we should error out
+  children=( $(jobs -p) )
+{% raw %}
+  if [ ${#children[@]} -eq 0 ]; then
+{% endraw %}
+    echo "ERROR: no child jobs exist..." >&2
+    exit 1
+  fi
+
+{% if requeue_on_timeout %}
+  # Start a watchdog process to signal timeout.
+  sleep {{ requeue_timeout }} &
+  timeout_pid=$!
+{% endif %}
+
+{% raw %}
+  while [ ${#children[@]} -gt 0 ]; do
+{% endraw %}
+    echo "INFO: Waiting for child processes to finish..."
+    set +e
+{% if requeue_on_timeout %}
+    # Wait on either one of the child processes or the timeout process.
+    wait -n -p child_pid "${children[@]}" "${timeout_pid}"
+{% else %}
+    wait -n -p child_pid "${children[@]}"
+{% endif %}
+    local child_exit_code=$?
+    set -e
+
+{% if requeue_on_timeout %}
+    # If the finished process is the watchdog, trigger the timeout handling.
+    if [ "${child_pid}" = "${timeout_pid}" ]; then
+      echo "INFO: Timeout of {{ requeue_timeout }} seconds reached. Killing remaining processes: ${children[*]}" >&2
+      kill "${children[@]}" 2>/dev/null || true
+      scontrol requeue "${JOB_ID}"
+      exit {{ requeue_exit_code }}
+    fi
+{% endif %}
+
+    echo "INFO: Process ${child_pid} finished with exit code ${child_exit_code}."
+
+    # Handle the exit code of the finished process.
+    if [ "${child_exit_code}" -eq "{{ requeue_exit_code }}" ] && [ "${SLURM_RESTART_COUNT:-0}" -le "{{ requeue_max_attempts }}" ]; then
+      echo "INFO: Received requeue exit code {{ requeue_exit_code }} from process ${child_pid}. Requeuing Slurm job ${JOB_ID} after ${SLURM_RESTART_COUNT-0} restarts." >&2
+      scontrol requeue "${JOB_ID}"
+      exit {{ requeue_exit_code }}
+    elif [ "${child_exit_code}" -ne 0 ]; then
+      echo "ERROR: Process ${child_pid} exited with code ${child_exit_code}." >&2
+      exit "${child_exit_code}"
+    fi
+
+    # Remove the finished PID from the array in a concise way.
+    for i in "${!children[@]}"; do
+      if [ "${children[i]}" = "$child_pid" ]; then
+        unset 'children[i]'
+        break
+      fi
+    done
+
+    # Reindex the array.
+    children=( "${children[@]}" )
+  done
+
+{% if requeue_on_timeout %}
+  kill "$timeout_pid" 2>/dev/null || true
+{% endif %}
+}
+
+__xm_slurm_wait_for_children
+{%- endmacro %}

xm_slurm/templates/slurm/fragments/proxy.bash.j2

@@ -0,0 +1,31 @@
+{% macro proxy(host) -%}
+__xm_slurm_proxy() {
+    local -r GATEWAY="$1"
+    local PORT
+
+    # Find an open port
+    while
+        PORT="$(shuf -n 1 -i 1024-65535)"
+        netstat -atun | grep -q "$PORT"
+    do
+        sleep 0.25
+        continue
+    done
+
+    # Reverse proxy through the gateway
+    ssh -D "$PORT" "$GATEWAY" -N -f
+
+    # Export all env vars for applications to pick up on proxy
+    export ALL_PROXY="socks5://127.0.0.1:$PORT"
+    export all_proxy="socks5://127.0.0.1:$PORT"
+
+    export HTTP_PROXY="socks5://127.0.0.1:$PORT"
+    export http_proxy="socks5://127.0.0.1:$PORT"
+
+    export HTTPS_PROXY="socks5://127.0.0.1:$PORT"
+    export https_proxy="socks5://127.0.0.1:$PORT"
+
+    export JAVA_OPTS="-DsocksProxyHost=127.0.0.1 -DsocksProxyPort=$PORT"
+}
+__xm_slurm_proxy "{{ host }}"
+{%- endmacro %}

xm_slurm/templates/slurm/job-array.bash.j2

@@ -0,0 +1,31 @@
+{% extends "job.bash.j2" %}
+{% block directives %}
+{{ super() -}}
+#SBATCH --array=0-{{ args | length - 1 }}
+#SBATCH --output=slurm-%A_%a.out
+{% endblock directives %}
+
+{% block bootstrap %}
+srun \
+  --label \
+  --unbuffered \
+  --kill-on-bad-exit=1 \
+  --export="ALL" \
+{% for directive in job.executor.step_directives() %}
+  {{ directive }} \
+{% endfor %}
+  bash <<'SRUN_EOF' &
+set -Eeuxo pipefail
+
+readonly XM_SLURM_TRIAL_ARGS=(
+{% for trial in args %}
+  "{{ trial.to_list() | join(" ") }}"
+{% endfor %}
+)
+
+{% call run(cluster, job) %}
+  ${XM_SLURM_TRIAL_ARGS[$SLURM_ARRAY_TASK_ID]} \
+{% endcall %}
+
+SRUN_EOF
+{%- endblock bootstrap %}

xm_slurm/templates/slurm/job-group.bash.j2

@@ -0,0 +1,47 @@
+{% extends "job.bash.j2" %}
+{% block directives %}
+#SBATCH --export=NONE
+#SBATCH --comment="{'xid': {{ experiment_id }}}"
+{% if cluster.account %}
+#SBATCH --account={{ cluster.account }}
+{% endif %}
+{% if cluster.partition %}
+#SBATCH --partition={{ cluster.partition }}
+{% endif %}
+{% if cluster.qos %}
+#SBATCH --qos={{ cluster.qos }}
+{% endif %}
+
+{% for job_name, job in job_group.jobs.items() %}
+#SBATCH --output=xm-%j+{{ job_name }}.stdout
+#SBATCH --error=xm-%j+{{ job_name }}.stderr
+{% if identity %}
+#SBATCH --job-name=xm[{{ job_name }}@{{ experiment_id }}.{{ identity }}]
+#SBATCH --dependency=singleton
+{% else %}
+#SBATCH --job-name=xm[{{ job_name }}@{{ experiment_id }}]
+{% endif %}
+{% for directive in job.executor.batch_directives() %}
+#SBATCH {{ directive }}
+{% endfor %}
+{{ "\n#SBATCH hetjob\n" if not loop.last }}
+{% endfor %}
+{% endblock directives %}
+
+{% block bootstrap %}
+{% for job in job_group.jobs.values() +%}
+srun \
+  --label \
+  --unbuffered \
+  --kill-on-bad-exit=1 \
+  --export="ALL" \
+{% for directive in job.executor.step_directives() %}
+  {{ directive }} \
+{% endfor %}
+  --het-group={{ loop.index0 }} \
+  bash <<'SRUN_EOF' &
+set -Eeuxo pipefail
+{{ run(cluster, job) }}
+SRUN_EOF
+{% endfor +%}
+{% endblock bootstrap %}

xm_slurm/templates/slurm/job.bash.j2

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+{% block directives %}
+#SBATCH --open-mode=append
+#SBATCH --export=NONE
+#SBATCH --output=slurm-%j.out
+#SBATCH --comment="{'xid': {{ experiment_id }}}"
+{% if cluster.account and not job.executor.account %}
+#SBATCH --account={{ cluster.account }}
+{% endif %}
+{% if cluster.partition and not job.executor.partition %}
+#SBATCH --partition={{ cluster.partition }}
+{% endif %}
+{% if cluster.qos and not job.executor.qos %}
+#SBATCH --qos={{ cluster.qos }}
+{% endif %}
+{% if identity %}
+#SBATCH --job-name=xm[{{ experiment_id }}.{{ identity }}]
+{% else %}
+{% if dependency %}
+#SBATCH {{ dependency.to_directive() }}
+{% endif %}
+#SBATCH --job-name=xm[{{ experiment_id }}]
+{% endif %}
+{% for directive in job.executor.batch_directives() %}
+#SBATCH {{ directive }}
+{% endfor %}
+{% endblock directives %}
+set -Eeuxo pipefail
+
+{% if stdlib %}
+# --- Helper functions ---
+{% for fn in stdlib %}
+{{ fn }}
+{% endfor %}
+{% endif %}
+
+{% block prolog %}
+{% if cluster.prolog %}
+{{- cluster.prolog -}}
+{% endif %}
+{%- endblock prolog %}
+
+{% block environment -%}
+{% for key, value in cluster.host_environment.items() %}
+export {{ key }}="{{ value }}"
+{% endfor %}
+{%- endblock environment %}
+
+{% block proxy -%}
+{%- if cluster.proxy %}
+{% from 'fragments/proxy.bash.j2' import proxy %}
+{% if cluster.proxy == "submission-host" %}
+{{ proxy("$SLURM_SUBMIT_HOST") }}
+{% else %}
+{{ proxy(cluster.proxy) }}
+{% endif %}
+{% endif %}
+{%- endblock proxy %}
+
+{% block bootstrap %}
+srun \
+  --label \
+  --unbuffered \
+  --kill-on-bad-exit=1 \
+  --export="ALL" \
+{% for directive in job.executor.step_directives() %}
+  {{ directive }} \
+{% endfor %}
+  bash <<'SRUN_EOF' &
+set -Eeuxo pipefail
+{{ run(cluster, job) }}
+SRUN_EOF
+{%- endblock bootstrap %}
+
+echo "[INFO] Start timestamp: $(date)"
+
+{%- block epilog -%}
+{% if cluster.epilog %}
+{{ cluster.epilog }}
+{% endif %}
+{%- endblock epilog %}
+
+
+{% block monitor -%}
+{% from 'fragments/monitor.bash.j2' import monitor %}
+{{ monitor(job.executor.requeue_max_attempts, job.executor.requeue_on_exit_code, job.executor.requeue_on_timeout, job.executor.requeue_timeout.seconds) }}
+{%- endblock monitor %}
+
+
+echo "[INFO] End timestamp: $(date)"