idmtools-platform-slurm 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

dockerized_slurm/Dockerfile

@@ -0,0 +1,107 @@
+FROM centos:7
+
+LABEL org.opencontainers.image.source="https://github.com/giovtorres/slurm-docker-cluster" \
+      org.opencontainers.image.title="slurm-docker-cluster" \
+      org.opencontainers.image.description="Slurm Docker cluster on CentOS 7" \
+      org.label-schema.docker.cmd="docker-compose up -d" \
+      maintainer="Giovanni Torres"
+
+ARG SLURM_TAG=slurm-19-05-1-2
+ARG GOSU_VERSION=1.11
+
+RUN set -ex \
+    && yum makecache fast \
+    && yum -y update \
+    && yum -y install epel-release \
+    && yum -y install \
+       wget \
+       bzip2 \
+       perl \
+       gcc \
+       gcc-c++\
+       git \
+       gnupg \
+       make \
+       munge \
+       munge-devel \
+       openssh-server \
+       python-devel \
+       python-pip \
+       python36 \
+       python36-devel \
+       python36-pip \
+       mariadb-server \
+       mariadb-devel \
+       psmisc \
+       bash-completion \
+       vim-enhanced \
+    && yum clean all \
+    && rm -rf /var/cache/yum
+
+RUN pip install Cython nose && pip3.6 install Cython nose
+
+RUN set -ex \
+    && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" \
+    && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64.asc" \
+    && export GNUPGHOME="$(mktemp -d)" \
+    && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+    && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+    && rm -rf "${GNUPGHOME}" /usr/local/bin/gosu.asc \
+    && chmod +x /usr/local/bin/gosu \
+    && gosu nobody true
+
+RUN set -x \
+    && git clone https://github.com/SchedMD/slurm.git \
+    && pushd slurm \
+    && git checkout tags/$SLURM_TAG \
+    && ./configure --enable-debug --prefix=/usr --sysconfdir=/etc/slurm \
+        --with-mysql_config=/usr/bin  --libdir=/usr/lib64 \
+    && make install \
+    && install -D -m644 etc/cgroup.conf.example /etc/slurm/cgroup.conf.example \
+    && install -D -m644 etc/slurm.conf.example /etc/slurm/slurm.conf.example \
+    && install -D -m644 etc/slurmdbd.conf.example /etc/slurm/slurmdbd.conf.example \
+    && install -D -m644 contribs/slurm_completion_help/slurm_completion.sh /etc/profile.d/slurm_completion.sh \
+    && popd \
+    && rm -rf slurm
+RUN /usr/bin/ssh-keygen -A \
+    && groupadd -r --gid=996 slurm-data \
+    && groupadd -r --gid=995 slurm \
+    && useradd -r -g slurm -G slurm-data --uid=995 slurm \
+    && groupadd -r --gid=1005 test \
+    && useradd -r -g test -G slurm-data --uid=1005 test \
+    && echo "test:test" | chpasswd \
+    && mkdir -p /home/test/.ssh \
+    && chown -R test:test /home/test \
+    && mkdir /etc/sysconfig/slurm \
+        /var/spool/slurmd \
+        /var/run/slurmd \
+        /var/run/slurmdbd \
+        /var/lib/slurmd \
+        /var/log/slurm \
+        /data \
+    && touch /var/lib/slurmd/node_state \
+        /var/lib/slurmd/front_end_state \
+        /var/lib/slurmd/job_state \
+        /var/lib/slurmd/resv_state \
+        /var/lib/slurmd/trigger_state \
+        /var/lib/slurmd/assoc_mgr_state \
+        /var/lib/slurmd/assoc_usage \
+        /var/lib/slurmd/qos_usage \
+        /var/lib/slurmd/fed_mgr_state \
+    && chown -R slurm:slurm /var/*/slurm* \
+    && /sbin/create-munge-key
+
+COPY id_rsa id_rsa.pub /home/test/.ssh/
+COPY slurm.conf /etc/slurm/slurm.conf
+COPY slurmdbd.conf /etc/slurm/slurmdbd.conf
+
+
+RUN cp /home/test/.ssh/id_rsa.pub /home/test/.ssh/authorized_keys \
+    && chmod 400 /home/test/.ssh/id_rsa.pub \
+    && chmod 644 /home/test/.ssh/authorized_keys \
+    && chown -R 1005:1005 /home/test/.ssh/
+
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+
+CMD ["slurmdbd"]

dockerized_slurm/README.md

@@ -0,0 +1,17 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+To use the slurm docker test platform, you need to follow these steps
+
+1. Within this directory, run docker-compose up -d
+2. Wait one minute are check the docker logs. Once the slurmctld is ready, then go to the next step
+3. Run register_cluster.sh . On windows, you can run the docker-compose exec commands
+4. Grab the IP Address of slurmctld
+   `docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' slurmctld`
+5. Update the remote_host to the ip address from previous output in idmtools_platform_slurm/tests/idmtools.ini
+6. Update the key_file path in idmtools_platform_slurm/tests/idmtools.ini to point to the absolute path of the 
+   idmtools_platform_slurm/dockerized_slurm/id_rsa on your machine

dockerized_slurm/docker-compose.yml

@@ -0,0 +1,89 @@
+version: "2.2"
+
+services:
+  mysql:
+    image: mysql:5.7
+    hostname: mysql
+    container_name: mysql
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+      MYSQL_DATABASE: slurm_acct_db
+      MYSQL_USER: slurm
+      MYSQL_PASSWORD: password
+    volumes:
+      - var_lib_mysql:/var/lib/mysql
+
+  slurmdbd:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmdbd"]
+    container_name: slurmdbd
+    hostname: slurmdbd
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6819"
+    depends_on:
+      - mysql
+
+  slurmctld:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmctld"]
+    container_name: slurmctld
+    hostname: slurmctld
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - ./test_slurm_data:/data
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6817"
+    ports:
+      - "2222:22"
+    depends_on:
+      - "slurmdbd"
+
+  c1:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmd"]
+    hostname: c1
+    container_name: c1
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - ./test_slurm_data:/data
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6818"
+    depends_on:
+      - "slurmctld"
+
+  c2:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmd"]
+    hostname: c2
+    container_name: c2
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - ./test_slurm_data:/data
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6818"
+    depends_on:
+      - "slurmctld"
+
+volumes:
+  etc_munge:
+  etc_slurm:
+  var_lib_mysql:
+  var_log_slurm:

dockerized_slurm/docker-entrypoint.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+chown -R slurm:slurm-data /data
+chmod 0775 /data
+
+if [ "$1" = "slurmdbd" ]
+then
+    echo "---> Starting the MUNGE Authentication service (munged) ..."
+    gosu munge /usr/sbin/munged
+
+    echo "---> Starting the Slurm Database Daemon (slurmdbd) ..."
+
+    {
+        . /etc/slurm/slurmdbd.conf
+        until echo "SELECT 1" | mysql -h $StorageHost -u$StorageUser -p$StoragePass 2>&1 > /dev/null
+        do
+            echo "-- Waiting for database to become active ..."
+            sleep 2
+        done
+    }
+    echo "-- Database is now active ..."
+
+    exec gosu slurm /usr/sbin/slurmdbd -Dvvv
+fi
+
+if [ "$1" = "slurmctld" ]
+then
+    echo "---> Starting the MUNGE Authentication service (munged) ..."
+    gosu munge /usr/sbin/munged
+
+    echo "---> Waiting for slurmdbd to become active before starting slurmctld ..."
+
+    until 2>/dev/null >/dev/tcp/slurmdbd/6819
+    do
+        echo "-- slurmdbd is not available.  Sleeping ..."
+        sleep 2
+    done
+    echo "-- slurmdbd is now active ..."
+
+    echo "---> Starting the Slurm Controller Daemon (slurmctld) ..."
+    exec /usr/sbin/sshd -D &
+    exec gosu slurm /usr/sbin/slurmctld -Dvvv
+fi
+
+if [ "$1" = "slurmd" ]
+then
+    echo "---> Starting the MUNGE Authentication service (munged) ..."
+    gosu munge /usr/sbin/munged
+
+    echo "---> Waiting for slurmctld to become active before starting slurmd..."
+
+    until 2>/dev/null >/dev/tcp/slurmctld/6817
+    do
+        echo "-- slurmctld is not available.  Sleeping ..."
+        sleep 2
+    done
+    echo "-- slurmctld is now active ..."
+
+    echo "---> Starting the Slurm Node Daemon (slurmd) ..."
+    exec /usr/sbin/slurmd -Dvvv
+fi
+
+exec "$@"

dockerized_slurm/id_rsa

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEApPDLAzN1H0re2oKCHRnqBOlv07MPL13CpQAD7eqzoQajhP+t
+qps0zcNAKsiC1fRbn/+oX4jLlGXApnPvzo2X7PDUw8slIwS7odFNle3vQVf24r3L
+ZGlBrWaomX3cYRO5gFFqfZwS6PDa22/YuoEVSHanml3u2+hBoqAFrGuR0RzF4GUG
+1bZATBH2/YC3wEDR3kCQ98Z+mOS/tenf4/0s1rz1x/7ljQVTbGssc8Xis4S5JBEc
+vg3VKsbz4mb0iuKcojvUJqAWDrpZVqeH6fUXuOHrz4vNFbJh+YBmES9/MxwRxd9U
+JJ8oF2RO2udr6JFFYkorOr3bJiWYoLC28QsVwQIDAQABAoIBAA8ZF4w4dp0hrlqU
+HbLqP1ipwZnAR0CPtZSC9tkdZcn0oJ05Bj2arW+0UrhX2FobXxO7RD9Sd0gjNEpI
+TIg8v85pkSBHBSQ6d65tSUvTFtaFZc0FkIulcuSbhA1gzv533sAXM8dBtR1rhq0V
+hOI1lKwoaFkiBg5NKUzolvxccGGSugsHCER1itoHblZBTxvVnJ/dXhFzQ2yAhErQ
+4vYjjXbyM3cIWv3edoB7oI8Kc4w0cKdvSQ12GPIYSrq3k5RDMB36+GhokNlez0t2
+VhcxeMBJAdIwif3tFXluwBYBmQsn9B7ovnpNm594Nn833vD56dFeH0a9soFm4Q/x
+BotVxFECgYEA0/Ne5LVzGcftzxr5VY014+oPNKeQwnx9OIUWHdVgoFocZFtfCGa7
+jC9cVj27GRw1WUEMKaTcUhreXA3dMzymFDNau89YX8BR3s5DZRtaM48aJCBDoK7W
+YC+pHTXgfhd66+Q5LWtACRRz+lx6O1ZuMF3G4/CJXsvhH5FHEjobw10CgYEAxzhM
+GxBaZPdZLwZY5XUy8BwixN8yEfsJOlIwwPJL/2LS59Qood25LwYA1oYj8JwMIG+s
+uNwJFSBT7NYExV3Sg9we09VW74jX1C3Ejhvn63iGGbzsE2AHQtgKrzLjgcelyaBo
+FkrtnK61L4UcUOyRqJ0+rJpuz28pRRdYo3YvebUCgYAUUP3Fmob/76Qh+AnyY2jJ
+AgSXHYDIw7oVEty921g5xql2DTQc0AwmMdv+AEjQ7V1Hwu5xh3X/AMhTtph/cs7Q
+nOOToRptgzfVzZu0M10AGDV/RQB2hIvUCH2DaUitjX6g4e2BJwiqViWP2BF3Yp+J
+T1PjQYlJZu2bYweW/a7vmQKBgQCcPwMxlrnOxlAkGY5PKIQy3V5HmeXjREgQfbXP
+HjmMqy1OtY7IpOVAhCzUE5DMfRblubB1q91TvG8WKSTExrj8wf9LlN8CLwlXWC34
+ZtqWS4ihVxKwf3gybM60ae0VNEhKwovgMBP79uoTwwpaTbBP1kP5i4WtGzn6/jx+
+t4q0oQKBgAy9CCCO8lnToFcso4KSh8+z2FLRFBjNmFDfb6UHkaI///PVG/posSl9
+pAghYIVCRtlapc7D9SYB5dQdmJ38RDGkkrvz87J3PQZeCGVWou0rYhHyKlVyUVor
+JlwyOKbIXih8ew81IzbYAJKni981nj58dwJViA9m3Zhyb+1p2tKg
+-----END RSA PRIVATE KEY-----

dockerized_slurm/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCk8MsDM3UfSt7agoIdGeoE6W/Tsw8vXcKlAAPt6rOhBqOE/62qmzTNw0AqyILV9Fuf/6hfiMuUZcCmc+/OjZfs8NTDyyUjBLuh0U2V7e9BV/bivctkaUGtZqiZfdxhE7mAUWp9nBLo8Nrbb9i6gRVIdqeaXe7b6EGioAWsa5HRHMXgZQbVtkBMEfb9gLfAQNHeQJD3xn6Y5L+16d/j/SzWvPXH/uWNBVNsayxzxeKzhLkkERy+DdUqxvPiZvSK4pyiO9QmoBYOullWp4fp9Re44evPi80VsmH5gGYRL38zHBHF31QknygXZE7a52vokUViSis6vdsmJZigsLbxCxXB clinton@sahara

dockerized_slurm/register_cluster.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+# https://github.com/giovtorres/docker-centos7-slurm/issues/3
+# sacctmgr add cluster linux
+#
+
+docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add cluster name=linux" && \
+docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add account test Cluster=linux Description='none' Organization='none'" && \
+docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add user test DefaultAccount=test" && \
+docker-compose restart slurmdbd slurmctld c1 c2
+#ssh-copy-id -i id

dockerized_slurm/slurm.conf

@@ -0,0 +1,94 @@
+# slurm.conf
+#
+# See the slurm.conf man page for more information.
+#
+ClusterName=linux
+ControlMachine=slurmctld
+ControlAddr=slurmctld
+#BackupController=
+#BackupAddr=
+#
+SlurmUser=slurm
+#SlurmdUser=root
+SlurmctldPort=6817
+SlurmdPort=6818
+AuthType=auth/munge
+#JobCredentialPrivateKey=
+#JobCredentialPublicCertificate=
+StateSaveLocation=/var/lib/slurmd
+SlurmdSpoolDir=/var/spool/slurmd
+SwitchType=switch/none
+MpiDefault=none
+SlurmctldPidFile=/var/run/slurmd/slurmctld.pid
+SlurmdPidFile=/var/run/slurmd/slurmd.pid
+ProctrackType=proctrack/linuxproc
+#PluginDir=
+CacheGroups=0
+#FirstJobId=
+ReturnToService=0
+#MaxJobCount=
+#PlugStackConfig=
+#PropagatePrioProcess=
+#PropagateResourceLimits=
+#PropagateResourceLimitsExcept=
+#Prolog=
+#Epilog=
+#SrunProlog=
+#SrunEpilog=
+#TaskProlog=
+#TaskEpilog=
+#TaskPlugin=
+#TrackWCKey=no
+#TreeWidth=50
+#TmpFS=
+#UsePAM=1
+#
+# TIMERS
+SlurmctldTimeout=300
+SlurmdTimeout=300
+InactiveLimit=0
+MinJobAge=300
+KillWait=30
+Waittime=0
+#
+# SCHEDULING
+SchedulerType=sched/backfill
+#SchedulerAuth=
+#SchedulerPort=
+#SchedulerRootFilter=
+SelectType=select/cons_res
+SelectTypeParameters=CR_CPU_Memory
+FastSchedule=1
+#PriorityType=priority/multifactor
+#PriorityDecayHalfLife=14-0
+#PriorityUsageResetPeriod=14-0
+#PriorityWeightFairshare=100000
+#PriorityWeightAge=1000
+#PriorityWeightPartition=10000
+#PriorityWeightJobSize=1000
+#PriorityMaxAge=1-0
+#
+# LOGGING
+SlurmctldDebug=3
+SlurmctldLogFile=/var/log/slurm/slurmctld.log
+SlurmdDebug=3
+SlurmdLogFile=/var/log/slurm/slurmd.log
+JobCompType=jobcomp/filetxt
+JobCompLoc=/var/log/slurm/jobcomp.log
+#
+# ACCOUNTING
+JobAcctGatherType=jobacct_gather/linux
+JobAcctGatherFrequency=30
+#
+AccountingStorageType=accounting_storage/slurmdbd
+AccountingStorageHost=slurmdbd
+AccountingStoragePort=6819
+AccountingStorageLoc=slurm_acct_db
+#AccountingStoragePass=
+#AccountingStorageUser=
+#
+# COMPUTE NODES
+NodeName=c[1-2] RealMemory=1000 State=UNKNOWN
+#
+# PARTITIONS
+PartitionName=normal Default=yes Nodes=c[1-2] Priority=50 DefMemPerCPU=500 Shared=NO MaxNodes=1 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP

dockerized_slurm/slurmdbd.conf

@@ -0,0 +1,37 @@
+#
+# Example slurmdbd.conf file.
+#
+# See the slurmdbd.conf man page for more information.
+#
+# Archive info
+#ArchiveJobs=yes
+#ArchiveDir="/tmp"
+#ArchiveSteps=yes
+#ArchiveScript=
+#JobPurge=12
+#StepPurge=1
+#
+# Authentication info
+AuthType=auth/munge
+#AuthInfo=/var/run/munge/munge.socket.2
+#
+# slurmDBD info
+DbdAddr=slurmdbd
+DbdHost=slurmdbd
+#DbdPort=6819
+SlurmUser=slurm
+#MessageTimeout=300
+DebugLevel=4
+#DefaultQOS=normal,standby
+LogFile=/var/log/slurm/slurmdbd.log
+PidFile=/var/run/slurmdbd/slurmdbd.pid
+#PluginDir=/usr/lib/slurm
+#PrivateData=accounts,users,usage,jobs
+#TrackWCKey=yes
+#
+# Database info
+StorageType=accounting_storage/mysql
+StorageHost=mysql
+StorageUser=slurm
+StoragePass=password
+StorageLoc=slurm_acct_db

idmtools_platform_slurm/__init__.py

@@ -1,8 +1,12 @@
-"""
-idmtools-platform-slurm - Placeholder Package
-
-This is a placeholder package to reserve the name on PyPI.
-The actual package will be published later.
-"""
-
-__version__ = "0.0.0.dev0"
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    # Python < 3.8
+    from importlib_metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("idmtools-platform-slurm")  # Use your actual package name
+except PackageNotFoundError:
+    # Package not installed, use fallback
+    __version__ = "0.0.0+unknown"
+

idmtools_platform_slurm/assets/__init__.py

@@ -0,0 +1,157 @@
+"""
+SlurmPlatform utilities.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from pathlib import Path
+from jinja2 import Template
+from typing import TYPE_CHECKING, Optional, Union
+from idmtools.entities.experiment import Experiment
+from idmtools_platform_slurm.platform_operations.utils import check_home
+
+if TYPE_CHECKING:
+    from idmtools_platform_slurm.slurm_platform import SlurmPlatform, CONFIG_PARAMETERS
+
+DEFAULT_TEMPLATE_FILE = Path(__file__).parent.joinpath("sbatch.sh.jinja2")
+BATCH_TEMPLATE_FILE = Path(__file__).parent.joinpath("batch.sh.jinja2")
+
+
+def generate_batch(platform: 'SlurmPlatform', experiment: Experiment,
+                   max_running_jobs: Optional[int] = None, array_batch_size: Optional[int] = None,
+                   dependency: Optional[bool] = None,
+                   template: Union[Path, str] = BATCH_TEMPLATE_FILE, **kwargs) -> None:
+    """
+    Generate bash script file batch.sh
+    Args:
+        platform: Slurm Platform
+        experiment: idmtools Experiment
+        max_running_jobs: int, how many allowed to run
+        array_size: INT, array size for slurm job
+        dependency: bool, determine if Slurm jobs depend on each other
+        template: template to be used to build batch file
+        kwargs: keyword arguments used to expand functionality
+    Returns:
+        None
+    """
+    template_vars = dict(njobs=experiment.simulation_count)
+
+    # Set max_running_jobs
+    if max_running_jobs is not None:
+        if platform.max_running_jobs is not None:
+            template_vars['max_running_jobs'] = min(max_running_jobs, platform.max_running_jobs)
+        else:
+            template_vars['max_running_jobs'] = max_running_jobs
+    else:
+        if platform.max_running_jobs is not None:
+            template_vars['max_running_jobs'] = platform.max_running_jobs
+        else:
+            template_vars['max_running_jobs'] = 1
+
+    # Set array_size
+    if array_batch_size is not None:
+        platform.array_batch_size = array_batch_size
+
+    if platform._max_array_size is not None:
+        if platform.array_batch_size is not None:
+            template_vars['array_batch_size'] = min(platform._max_array_size, platform.array_batch_size,
+                                                    experiment.simulation_count)
+        else:
+            template_vars['array_batch_size'] = min(platform._max_array_size, experiment.simulation_count)
+    elif platform.array_batch_size is not None:
+        template_vars['array_batch_size'] = min(platform.array_batch_size, experiment.simulation_count)
+    else:
+        template_vars['array_batch_size'] = experiment.simulation_count
+
+    # Consider dependency
+    if dependency is None:
+        dependency = True
+    template_vars['dependency'] = dependency
+
+    # Update with possible override values
+    template_vars.update(kwargs)
+
+    # Build batch based on the given template
+    with open(template) as file_:
+        t = Template(file_.read())
+
+    # Write out file
+    output_target = platform.get_directory(experiment).joinpath("batch.sh")
+    with open(output_target, "w") as tout:
+        tout.write(t.render(template_vars))
+
+    # Make executable
+    platform.update_script_mode(output_target)
+
+
+def generate_script(platform: 'SlurmPlatform', experiment: Experiment, max_running_jobs: Optional[int] = None,
+                    template: Union[Path, str] = DEFAULT_TEMPLATE_FILE, **kwargs) -> None:
+    """
+    Generate batch file sbatch.sh
+    Args:
+        platform: Slurm Platform
+        experiment: idmtools Experiment
+        max_running_jobs: int, how many allowed to run at the same time
+        template: template to be used to build batch file
+        kwargs: keyword arguments used to expand functionality
+    Returns:
+        None
+    """
+    from idmtools_platform_slurm.slurm_platform import CONFIG_PARAMETERS
+    template_vars = dict(njobs=experiment.simulation_count)
+    # populate from our platform config vars
+    for p in CONFIG_PARAMETERS:
+        if getattr(platform, p) is not None:
+            template_vars[p] = getattr(platform, p)
+
+    # Set default here
+    if max_running_jobs is not None:
+        template_vars['max_running_jobs'] = max_running_jobs
+    if max_running_jobs is None and platform.max_running_jobs is None:
+        template_vars['max_running_jobs'] = 1
+
+    # Add any overides. We need some validation here later
+    # TODO add validation for valid config options
+    template_vars.update(kwargs)
+
+    if platform.modules:
+        template_vars['modules'] = platform.modules
+
+    with open(template) as file_:
+        t = Template(file_.read())
+
+    # Write out file
+    output_target = platform.get_directory(experiment).joinpath("sbatch.sh")
+    with open(output_target, "w") as tout:
+        tout.write(t.render(template_vars))
+    # Make executable
+    platform.update_script_mode(output_target)
+
+
+def generate_simulation_script(platform: 'SlurmPlatform', simulation, retries: Optional[int] = None) -> None:
+    """
+    Generate batch file _run.sh
+    Args:
+        platform: Slurm Platform
+        simulation: idmtools Simulation
+        retries: int
+    Returns:
+        None
+    """
+    experiment_dir = platform.get_directory(simulation.parent).absolute()
+    experiment_dir = str(experiment_dir).replace('\\', '/')
+    check = check_home(experiment_dir)
+    sim_script = platform.get_directory(simulation).joinpath("_run.sh")
+    with open(sim_script, "w") as tout:
+        with open(Path(__file__).parent.parent.joinpath("assets/_run.sh.jinja2")) as tin:
+            tvars = dict(
+                platform=platform,
+                simulation=simulation,
+                retries=retries if retries else platform.retries
+            )
+            if not check:
+                tvars['experiment_dir'] = str(experiment_dir)
+
+            t = Template(tin.read())
+            tout.write(t.render(tvars))
+    # Make executable
+    platform.update_script_mode(sim_script)

idmtools_platform_slurm/assets/_run.sh.jinja2

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+#SBATCH --signal=B:SIGTERM@30
+
+# define the handler function
+term_handler()
+{
+    # do whatever cleanup you want here
+    echo "-1" > job_status.txt
+    exit -1
+}
+
+# associate the function "term_handler" with the TERM signal
+trap 'term_handler' TERM
+
+echo ${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID} > job_id.txt
+
+n=0
+until [ "$n" -ge {{retries}} ]
+do
+    echo "100" > job_status.txt
+    {% if simulation.task.sif_path is defined and simulation.task.sif_path %}
+        {% if simulation.task.command.cmd.startswith('singularity') %}
+            {{simulation.task.command.cmd}}
+        {% else %}
+                {% if experiment_dir is defined and experiment_dir %}
+                    singularity exec --bind {{experiment_dir}} {{simulation.task.sif_path}} {{simulation.task.command.cmd}}
+                {% else %}
+                    singularity exec {{simulation.task.sif_path}} {{simulation.task.command.cmd}}
+                {% endif %}
+        {% endif %}
+    {% else %}
+        {{simulation.task.command.cmd}}
+    {% endif %}
+   RESULT=$?
+   if [ $RESULT -eq 0 ]; then
+      echo "0" > job_status.txt
+      exit $RESULT
+   fi
+   n=$((n+1))
+   sleep 15
+done
+echo "-1" > job_status.txt
+exit $RESULT