idmtools-platform-slurm 0.0.0.dev0__tar.gz → 0.0.2__tar.gz

idmtools_platform_slurm-0.0.2/.bumpversion.cfg

@@ -0,0 +1,28 @@
+[bumpversion]
+current_version = 3.0.0+nightly
+commit = False
+tag = False
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<release>\+\w+)?(\.(?P<build>\d+))?
+serialize = 
+	{major}.{minor}.{patch}{release}.{build}
+	{major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:release]
+optional_value = prod
+first_value = prod
+values = 
+	prod
+	+nightly
+
+[bumpversion:file:./idmtools_platform_slurm/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"
+serialize = {major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}
+
+[bumpversion:file:./pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"
+serialize = {major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}

idmtools_platform_slurm-0.0.2/LICENSE.TXT

@@ -0,0 +1,3 @@
+idmtools is licensed under the Creative Commons Attribution-Noncommercial-ShareAlike 4.0 License.
+
+To view a copy of this license, visit https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode

idmtools_platform_slurm-0.0.2/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include idmtools_platform_slurm/assets/*
+include LICENSE.TXT

idmtools_platform_slurm-0.0.2/Makefile

@@ -0,0 +1,5 @@
+PACKAGE_NAME=idmtools_slurm
+include $(abspath ../dev_scripts/package_general.mk)
+
+help:
+	help-from-makefile -f ../dev_scripts/package_general.mk -f ./Makefile

idmtools_platform_slurm-0.0.2/PKG-INFO

@@ -0,0 +1,185 @@
+Metadata-Version: 2.4
+Name: idmtools_platform_slurm
+Version: 0.0.2
+Summary: Provides ability to run against Slurm
+Author-email: Sharon Chen <schen@idmod.org>, Clinton Collins <ccollins@idmod.org>, Zhaowei Du <zdu@idmod.org>, Clark Kirkman IV <ckirkman@idmod.org>, Benoit Raybaud <braybaud@idmod.org>
+Project-URL: Homepage, https://github.com/InstituteforDiseaseModeling/idmtools
+Keywords: modeling,IDM
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.TXT
+Requires-Dist: idmtools_platform_general<1.0.0,>=0.0.0
+Requires-Dist: dataclasses-json
+Provides-Extra: test
+Requires-Dist: idmtools[test]; extra == "test"
+Requires-Dist: idmtools_models; extra == "test"
+Requires-Dist: idmtools_test; extra == "test"
+Dynamic: license-file
+
+![Staging: idmtools-platform-slurm](https://github.com/InstituteforDiseaseModeling/idmtools/workflows/Staging:%20idmtools-platform-slurm/badge.svg?branch=dev)
+
+# idmtools-platform-slurm
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+- [Introduction](#introduction)
+- [Setting Up Virtual Environment](#setting-up-virtual-environment)
+- [Development Tips](#development-tips)
+- [Manually run a script as a Slurm job](#manually-run-a-script-as-a-slurm-job)
+- [Use SlurmJob to run a script as a Slurm job](#use-slurmjob-to-run-a-script-as-a-slurm-job)
+- [With SlurmPlatform to run a script as a Slurm job](#with-slurmplatform-to-run-a-script-as-a-slurm-job)
+- [Folder structure:](#folder-structure)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Introduction
+
+**SlurmPlatform** is a platform designed to facilitate the execution of experiments and simulations in slurm cluster. 
+
+## Setting Up Virtual Environment
+
+To set up a virtual environment for **SlurmPlatform**, follow these steps:
+
+1. **Install Python**
+
+   Ensure you have Python 3.8+ installed on your system.
+
+2. **Create Virtual Environment**
+   
+   There are multiple ways to create a virtual environment. Below is an example using `venv`:
+
+    ```bash
+    python -m venv slurm_env
+    ```
+
+3. **Activate Virtual Environment**
+    - On Windows:
+        ```bash
+        slurm_env\Scripts\activate
+        ```
+    - On Linux:
+        ```bash
+        source slurm_env/bin/activate
+        ```
+
+4. **Install SlurmPlatform**
+    ```bash
+    pip install idmtools-platform-slurm --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+    ```
+
+5. **Install Dependencies**
+    ```bash
+    pip install -r requirements.txt
+    ```
+6. **Optional(No need step #4 and #5), Install all slurm platform related packages**
+    ```bash
+    pip install idmtools[slurm] --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+    ```
+
+## Development Tips
+
+There is a Makefile file available for most common development tasks. Here is a list of commands
+
+```bash
+clean       -   Clean up temproary files
+lint        -   Lint package and tests
+test        -   Run All tests
+coverage    -   Run tests and generate coverage report that is shown in browser
+```
+
+On Windows, you can use `pymake` instead of `make`
+
+
+## Manually run a script as a Slurm job
+
+Preparation
+
+(1).Have target script ready, say my_script.py, suppose you have folder structure like::
+
+```bash
+script_folder
+    my_script.py
+    ......
+```
+
+(2). Created a virtual environment and activated it.
+
+Steps
+
+1. within the target script folder, create a batch file 'sbatch.sh' (without quote) with content:
+
+```bash
+#!/bin/bash
+
+#SBATCH --partition=b1139
+#SBATCH --time=10:00:00
+#SBATCH --account=b1139
+
+#SBATCH --output=stdout.txt
+#SBATCH --error=stderr.txt
+
+# replace with your script file
+python3 my_script.py
+    
+exit $RESULT
+```
+
+Note: the content here is based on Northwestern University QUEST Slurm system. For general case, above content (required #SBATCH parameters) may be a little bit different.
+
+2. run your target script as SLURM job
+   execute the following commands from console (under virtual environment):
+
+   cd path_to_script_folder
+
+   `sbatch sbatch.sh`
+
+Note: any output information from my_script.py is stored in file stdout.txt under the current folder. For example, if my_script.py kicks out another Slurm job, then its Slurm id information can be found in file stdout.txt.
+
+
+## Use SlurmJob to run a script as a Slurm job
+
+The example can be simple as the following:
+
+--script.py--
+
+```python
+
+   from idmtools.core.platform_factory import Platform
+   from idmtools_platform_slurm.utils.slurm_job.slurm_job import SlurmJob
+
+   script = '<user script path>'
+   # script = 'example_path/python_sim_slurm.py'   # example
+   platform = Platform('SLURM_LOCAL', job_directory='<job_directory>')
+   sj = SlurmJob(script_path=script, platform=platform)
+   sj.run()
+```
+
+## With SlurmPlatform to run a script as a Slurm job
+
+We have SlurmJob integrated into SlurmPlatform and any Python script can run as a Slurm job simply doing:
+
+--script.py--
+```python
+
+   from idmtools.entities.command_task import CommandTask
+   from idmtools.entities.experiment import Experiment
+   from idmtools.core.platform_factory import Platform
+   
+   platform = Platform('SLURM_LOCAL', job_directory='<job_directory>')
+   # Define task
+   command = "echo 'Hello, World!'"
+   task = CommandTask(command=command)
+   # Run an experiment
+   experiment = Experiment.from_task(task, name="example")
+   experiment.run(platform=platform)
+```
+
+## Folder structure:
+[See Folder Structure](../idmtools_platform_container/README.md#folder-structure)

idmtools_platform_slurm-0.0.2/README.md

@@ -0,0 +1,162 @@
+![Staging: idmtools-platform-slurm](https://github.com/InstituteforDiseaseModeling/idmtools/workflows/Staging:%20idmtools-platform-slurm/badge.svg?branch=dev)
+
+# idmtools-platform-slurm
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+- [Introduction](#introduction)
+- [Setting Up Virtual Environment](#setting-up-virtual-environment)
+- [Development Tips](#development-tips)
+- [Manually run a script as a Slurm job](#manually-run-a-script-as-a-slurm-job)
+- [Use SlurmJob to run a script as a Slurm job](#use-slurmjob-to-run-a-script-as-a-slurm-job)
+- [With SlurmPlatform to run a script as a Slurm job](#with-slurmplatform-to-run-a-script-as-a-slurm-job)
+- [Folder structure:](#folder-structure)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Introduction
+
+**SlurmPlatform** is a platform designed to facilitate the execution of experiments and simulations in slurm cluster. 
+
+## Setting Up Virtual Environment
+
+To set up a virtual environment for **SlurmPlatform**, follow these steps:
+
+1. **Install Python**
+
+   Ensure you have Python 3.8+ installed on your system.
+
+2. **Create Virtual Environment**
+   
+   There are multiple ways to create a virtual environment. Below is an example using `venv`:
+
+    ```bash
+    python -m venv slurm_env
+    ```
+
+3. **Activate Virtual Environment**
+    - On Windows:
+        ```bash
+        slurm_env\Scripts\activate
+        ```
+    - On Linux:
+        ```bash
+        source slurm_env/bin/activate
+        ```
+
+4. **Install SlurmPlatform**
+    ```bash
+    pip install idmtools-platform-slurm --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+    ```
+
+5. **Install Dependencies**
+    ```bash
+    pip install -r requirements.txt
+    ```
+6. **Optional(No need step #4 and #5), Install all slurm platform related packages**
+    ```bash
+    pip install idmtools[slurm] --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+    ```
+
+## Development Tips
+
+There is a Makefile file available for most common development tasks. Here is a list of commands
+
+```bash
+clean       -   Clean up temproary files
+lint        -   Lint package and tests
+test        -   Run All tests
+coverage    -   Run tests and generate coverage report that is shown in browser
+```
+
+On Windows, you can use `pymake` instead of `make`
+
+
+## Manually run a script as a Slurm job
+
+Preparation
+
+(1).Have target script ready, say my_script.py, suppose you have folder structure like::
+
+```bash
+script_folder
+    my_script.py
+    ......
+```
+
+(2). Created a virtual environment and activated it.
+
+Steps
+
+1. within the target script folder, create a batch file 'sbatch.sh' (without quote) with content:
+
+```bash
+#!/bin/bash
+
+#SBATCH --partition=b1139
+#SBATCH --time=10:00:00
+#SBATCH --account=b1139
+
+#SBATCH --output=stdout.txt
+#SBATCH --error=stderr.txt
+
+# replace with your script file
+python3 my_script.py
+    
+exit $RESULT
+```
+
+Note: the content here is based on Northwestern University QUEST Slurm system. For general case, above content (required #SBATCH parameters) may be a little bit different.
+
+2. run your target script as SLURM job
+   execute the following commands from console (under virtual environment):
+
+   cd path_to_script_folder
+
+   `sbatch sbatch.sh`
+
+Note: any output information from my_script.py is stored in file stdout.txt under the current folder. For example, if my_script.py kicks out another Slurm job, then its Slurm id information can be found in file stdout.txt.
+
+
+## Use SlurmJob to run a script as a Slurm job
+
+The example can be simple as the following:
+
+--script.py--
+
+```python
+
+   from idmtools.core.platform_factory import Platform
+   from idmtools_platform_slurm.utils.slurm_job.slurm_job import SlurmJob
+
+   script = '<user script path>'
+   # script = 'example_path/python_sim_slurm.py'   # example
+   platform = Platform('SLURM_LOCAL', job_directory='<job_directory>')
+   sj = SlurmJob(script_path=script, platform=platform)
+   sj.run()
+```
+
+## With SlurmPlatform to run a script as a Slurm job
+
+We have SlurmJob integrated into SlurmPlatform and any Python script can run as a Slurm job simply doing:
+
+--script.py--
+```python
+
+   from idmtools.entities.command_task import CommandTask
+   from idmtools.entities.experiment import Experiment
+   from idmtools.core.platform_factory import Platform
+   
+   platform = Platform('SLURM_LOCAL', job_directory='<job_directory>')
+   # Define task
+   command = "echo 'Hello, World!'"
+   task = CommandTask(command=command)
+   # Run an experiment
+   experiment = Experiment.from_task(task, name="example")
+   experiment.run(platform=platform)
+```
+
+## Folder structure:
+[See Folder Structure](../idmtools_platform_container/README.md#folder-structure)

idmtools_platform_slurm-0.0.2/dockerized_slurm/Dockerfile

@@ -0,0 +1,107 @@
+FROM centos:7
+
+LABEL org.opencontainers.image.source="https://github.com/giovtorres/slurm-docker-cluster" \
+      org.opencontainers.image.title="slurm-docker-cluster" \
+      org.opencontainers.image.description="Slurm Docker cluster on CentOS 7" \
+      org.label-schema.docker.cmd="docker-compose up -d" \
+      maintainer="Giovanni Torres"
+
+ARG SLURM_TAG=slurm-19-05-1-2
+ARG GOSU_VERSION=1.11
+
+RUN set -ex \
+    && yum makecache fast \
+    && yum -y update \
+    && yum -y install epel-release \
+    && yum -y install \
+       wget \
+       bzip2 \
+       perl \
+       gcc \
+       gcc-c++\
+       git \
+       gnupg \
+       make \
+       munge \
+       munge-devel \
+       openssh-server \
+       python-devel \
+       python-pip \
+       python36 \
+       python36-devel \
+       python36-pip \
+       mariadb-server \
+       mariadb-devel \
+       psmisc \
+       bash-completion \
+       vim-enhanced \
+    && yum clean all \
+    && rm -rf /var/cache/yum
+
+RUN pip install Cython nose && pip3.6 install Cython nose
+
+RUN set -ex \
+    && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" \
+    && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64.asc" \
+    && export GNUPGHOME="$(mktemp -d)" \
+    && gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+    && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+    && rm -rf "${GNUPGHOME}" /usr/local/bin/gosu.asc \
+    && chmod +x /usr/local/bin/gosu \
+    && gosu nobody true
+
+RUN set -x \
+    && git clone https://github.com/SchedMD/slurm.git \
+    && pushd slurm \
+    && git checkout tags/$SLURM_TAG \
+    && ./configure --enable-debug --prefix=/usr --sysconfdir=/etc/slurm \
+        --with-mysql_config=/usr/bin  --libdir=/usr/lib64 \
+    && make install \
+    && install -D -m644 etc/cgroup.conf.example /etc/slurm/cgroup.conf.example \
+    && install -D -m644 etc/slurm.conf.example /etc/slurm/slurm.conf.example \
+    && install -D -m644 etc/slurmdbd.conf.example /etc/slurm/slurmdbd.conf.example \
+    && install -D -m644 contribs/slurm_completion_help/slurm_completion.sh /etc/profile.d/slurm_completion.sh \
+    && popd \
+    && rm -rf slurm
+RUN /usr/bin/ssh-keygen -A \
+    && groupadd -r --gid=996 slurm-data \
+    && groupadd -r --gid=995 slurm \
+    && useradd -r -g slurm -G slurm-data --uid=995 slurm \
+    && groupadd -r --gid=1005 test \
+    && useradd -r -g test -G slurm-data --uid=1005 test \
+    && echo "test:test" | chpasswd \
+    && mkdir -p /home/test/.ssh \
+    && chown -R test:test /home/test \
+    && mkdir /etc/sysconfig/slurm \
+        /var/spool/slurmd \
+        /var/run/slurmd \
+        /var/run/slurmdbd \
+        /var/lib/slurmd \
+        /var/log/slurm \
+        /data \
+    && touch /var/lib/slurmd/node_state \
+        /var/lib/slurmd/front_end_state \
+        /var/lib/slurmd/job_state \
+        /var/lib/slurmd/resv_state \
+        /var/lib/slurmd/trigger_state \
+        /var/lib/slurmd/assoc_mgr_state \
+        /var/lib/slurmd/assoc_usage \
+        /var/lib/slurmd/qos_usage \
+        /var/lib/slurmd/fed_mgr_state \
+    && chown -R slurm:slurm /var/*/slurm* \
+    && /sbin/create-munge-key
+
+COPY id_rsa id_rsa.pub /home/test/.ssh/
+COPY slurm.conf /etc/slurm/slurm.conf
+COPY slurmdbd.conf /etc/slurm/slurmdbd.conf
+
+
+RUN cp /home/test/.ssh/id_rsa.pub /home/test/.ssh/authorized_keys \
+    && chmod 400 /home/test/.ssh/id_rsa.pub \
+    && chmod 644 /home/test/.ssh/authorized_keys \
+    && chown -R 1005:1005 /home/test/.ssh/
+
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+
+CMD ["slurmdbd"]

idmtools_platform_slurm-0.0.2/dockerized_slurm/README.md

@@ -0,0 +1,17 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+To use the slurm docker test platform, you need to follow these steps
+
+1. Within this directory, run docker-compose up -d
+2. Wait one minute are check the docker logs. Once the slurmctld is ready, then go to the next step
+3. Run register_cluster.sh . On windows, you can run the docker-compose exec commands
+4. Grab the IP Address of slurmctld
+   `docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' slurmctld`
+5. Update the remote_host to the ip address from previous output in idmtools_platform_slurm/tests/idmtools.ini
+6. Update the key_file path in idmtools_platform_slurm/tests/idmtools.ini to point to the absolute path of the 
+   idmtools_platform_slurm/dockerized_slurm/id_rsa on your machine

idmtools_platform_slurm-0.0.2/dockerized_slurm/docker-compose.yml

@@ -0,0 +1,89 @@
+version: "2.2"
+
+services:
+  mysql:
+    image: mysql:5.7
+    hostname: mysql
+    container_name: mysql
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+      MYSQL_DATABASE: slurm_acct_db
+      MYSQL_USER: slurm
+      MYSQL_PASSWORD: password
+    volumes:
+      - var_lib_mysql:/var/lib/mysql
+
+  slurmdbd:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmdbd"]
+    container_name: slurmdbd
+    hostname: slurmdbd
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6819"
+    depends_on:
+      - mysql
+
+  slurmctld:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmctld"]
+    container_name: slurmctld
+    hostname: slurmctld
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - ./test_slurm_data:/data
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6817"
+    ports:
+      - "2222:22"
+    depends_on:
+      - "slurmdbd"
+
+  c1:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmd"]
+    hostname: c1
+    container_name: c1
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - ./test_slurm_data:/data
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6818"
+    depends_on:
+      - "slurmctld"
+
+  c2:
+    image: slurm-docker-cluster:19.05.1
+    build:
+      context: .
+    command: ["slurmd"]
+    hostname: c2
+    container_name: c2
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - ./test_slurm_data:/data
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6818"
+    depends_on:
+      - "slurmctld"
+
+volumes:
+  etc_munge:
+  etc_slurm:
+  var_lib_mysql:
+  var_log_slurm:

idmtools_platform_slurm-0.0.2/dockerized_slurm/docker-entrypoint.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+chown -R slurm:slurm-data /data
+chmod 0775 /data
+
+if [ "$1" = "slurmdbd" ]
+then
+    echo "---> Starting the MUNGE Authentication service (munged) ..."
+    gosu munge /usr/sbin/munged
+
+    echo "---> Starting the Slurm Database Daemon (slurmdbd) ..."
+
+    {
+        . /etc/slurm/slurmdbd.conf
+        until echo "SELECT 1" | mysql -h $StorageHost -u$StorageUser -p$StoragePass 2>&1 > /dev/null
+        do
+            echo "-- Waiting for database to become active ..."
+            sleep 2
+        done
+    }
+    echo "-- Database is now active ..."
+
+    exec gosu slurm /usr/sbin/slurmdbd -Dvvv
+fi
+
+if [ "$1" = "slurmctld" ]
+then
+    echo "---> Starting the MUNGE Authentication service (munged) ..."
+    gosu munge /usr/sbin/munged
+
+    echo "---> Waiting for slurmdbd to become active before starting slurmctld ..."
+
+    until 2>/dev/null >/dev/tcp/slurmdbd/6819
+    do
+        echo "-- slurmdbd is not available.  Sleeping ..."
+        sleep 2
+    done
+    echo "-- slurmdbd is now active ..."
+
+    echo "---> Starting the Slurm Controller Daemon (slurmctld) ..."
+    exec /usr/sbin/sshd -D &
+    exec gosu slurm /usr/sbin/slurmctld -Dvvv
+fi
+
+if [ "$1" = "slurmd" ]
+then
+    echo "---> Starting the MUNGE Authentication service (munged) ..."
+    gosu munge /usr/sbin/munged
+
+    echo "---> Waiting for slurmctld to become active before starting slurmd..."
+
+    until 2>/dev/null >/dev/tcp/slurmctld/6817
+    do
+        echo "-- slurmctld is not available.  Sleeping ..."
+        sleep 2
+    done
+    echo "-- slurmctld is now active ..."
+
+    echo "---> Starting the Slurm Node Daemon (slurmd) ..."
+    exec /usr/sbin/slurmd -Dvvv
+fi
+
+exec "$@"