idmtools-platform-container 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

docker_image/BASE_VERSION

@@ -0,0 +1 @@
+0.0

docker_image/Dockerfile

@@ -0,0 +1,48 @@
+# FROM docker-production.packages.idmod.org/dtk-rocky-buildenv:0.2-dev
+# Replace the following lines with above line after the offical image is available
+#############################################################
+FROM rockylinux:9.2
+# Set the timezone and the frontend
+ENV TZ=UTC DEBIAN_FRONTEND=noninteractive
+# To suppress warning without altering the installation when pip install with root user
+ENV PIP_ROOT_USER_ACTION=ignore
+# Install the necessary packages
+RUN yum -y install rpm dnf-plugins-core \
+ && yum -y update \
+ && dnf -y install \
+    python3 \
+    python3-devel \
+    mpich \
+    snappy \
+    glibc-devel \
+    epel-release \
+    wget \
+    nano
+
+# Update the system
+RUN yum clean all
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # This is for setting up the timezone
+
+# Install pip
+RUN python3 -m pip install pip --upgrade
+# Create a symbolic link to python3
+RUN ln -s /usr/bin/python3 /usr/bin/python
+# Create a directory for pip configurations
+RUN mkdir /root/.pip
+# Copy the pip configuration file
+COPY pip.conf /root/.pip
+
+# Set the path and library path
+ENV PATH ${PATH}:/usr/lib64/mpich/bin
+ENV LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/usr/lib64/mpich/lib
+
+# Add the requirements file
+ADD requirements.txt /tmp/
+
+# make the PIP index configurable so we can build against staging, production, or a local PyPI server
+ARG CONTAINER_VERSION
+
+# Install the packages
+RUN bash -c "pip3 install -r /tmp/requirements.txt"
+

docker_image/Dockerfile_buildenv

@@ -0,0 +1,46 @@
+#FROM docker-production.packages.idmod.org/dtk-rocky-buildenv:0.2-dev
+# Replace the following lines with above line after the offical image is available
+#############################################################
+FROM rockylinux:9.2
+ENV TZ=UTC DEBIAN_FRONTEND=noninteractive
+ENV GNUMAKEFLAGS=-j4
+
+RUN yum -y install rpm dnf-plugins-core \
+ && yum -y update \
+ && dnf -y install \
+    python3 \
+    python3-devel \
+    snappy \
+    mpich \
+    sudo \
+    epel-release \
+    git \
+    wget \
+    nano
+
+RUN yum -y update
+RUN yum -y group install "Development Tools" \
+ && yum clean all
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # This is for setting up the timezone
+
+RUN python3 -m pip install pip --upgrade
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN mkdir /root/.pip
+COPY pip.conf /root/.pip
+# RUN dnf install -y R
+ENV PATH ${PATH}:/usr/lib64/mpich/bin
+ENV LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:/usr/lib64/mpich/lib
+
+#############################################################
+RUN dnf -y install dos2unix
+
+# Add the requirements file
+ADD requirements.txt /tmp/
+
+# make the PIP index configurable so we can build against staging, production, or a local PyPI server
+ARG CONTAINER_VERSION
+
+# Install the packages
+RUN bash -c "pip3 install -r /tmp/requirements.txt"
+

docker_image/ImageName

@@ -0,0 +1 @@
+container-rocky-runtime

docker_image/README.md

@@ -0,0 +1,78 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+- [idmtools_platform_container Docker image](#idmtools_platform_container-docker-image)
+  - [Introduction](#introduction)
+  - [Prerequisites](#prerequisites)
+  - [Build Docker image](#build-docker-image)
+  - [Docker image versioning](#docker-image-versioning)
+  - [Docker image usage](#docker-image-usage)
+  - [Publish Docker image](#publish-docker-image)
+  - [Extend the Docker image](#extend-the-docker-image)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# idmtools_platform_container Docker image
+
+**Note**: This document is primarily for developers. It is not necessary for users to build and push the Docker image. However, if you wish to use your own Docker image, please follow the instructions below at [Extend the Docker Image](#extend-the-docker-image).
+
+## Introduction
+This Docker image is designed to facilitate idmtools_platform_container. It serves as local running platform with all necessary tools and dependencies installed. The image is based on rockylinux 9.2 and includes the following tools:
+- python 3.9
+- mipch 4.1.1
+- emod-api 1.33.3
+- emod-api's dependencies like numpy, pandas, scipy, matplotlib etc.
+
+## Prerequisites
+- Docker
+  On Windows or Mac, please use Docker Desktop 2.1.0.5 or 2.2.0.1
+
+## Build Docker image
+Note, you do not need to build the image locally. The image is auto built in github action and pushed to idmod artifactory. 
+
+To build Docker image locally, run:
+```bash
+python build_docker_image.py --username <username> --password <password>
+```
+where `<username>` and `<password>` are the username and password for the idmod artifactory account.
+
+You can also build the image with different docker file and image name by specifying `--dockerfile` and `--image_name` arguments.
+```bash
+python build_docker_image.py --username <username> --password <password> --dockerfile Dockerfile_buildenv --image_name container-rocky-buildenv
+```
+which will build image name as 'idm-docker-staging.packages.idmod.org/idmtools/container-rocky-buildenv:x.x.x'
+
+## Docker image versioning
+The Docker image version is determined by the version in idm docker-staging artifactory. It will advance the version number by 0.0.1 for each new build.
+
+## Docker image usage
+By default, you DO NOT need to worry about the image build and how to use image. The image is auto built in github action and pushed to idmod artifactory. The image is used in idmtools_platform_container Platform object. For example, 
+```python
+from idmtools.core.platform_factory import Platform
+platform = Platform('CONTAINER', job_directory='any_dir', docker_image='idm-docker-public.packages.idmod.org/idmtools/container-rocky-runtime:x.x.x')
+```
+where docker_image can be your local built image or the image in idmod artifactory.
+If you do not provide docker_image, the default image will be used.
+
+## Publish Docker image
+Note, you do not need to push docker image to artifactory. The image is auto built in github action and pushed to idmod artifactory.
+If you want to push the image to artifactory, run:
+```bash
+python push_docker_image.py --username <username> --password <password>
+```
+where `<username>` and `<password>` are the username and password for the idmod artifactory account.
+
+## Extend the Docker image
+If you want to build you own Docker image, please use our ``container-rocky-runtime`` as your baseline image and add the following line to the top of your Dockerfile:
+
+```bash
+   FROM docker-production-public/idmtools/container-rocky-runtime/0.0.3
+```
+You can use general Docker build [command](https://docs.docker.com/reference/cli/docker/buildx/build/) to build your own Docker image.
+Then use that image in the Platform object. For example:
+
+```bash
+   from idmtools_platform_container import Platform
+   platform = Platform('CONTAINER', job_directory='any_dir', docker_image='your_own_image_name:x.x.x')
+```

docker_image/__init__.py

@@ -0,0 +1,6 @@
+"""idmtools comps platform.
+
+Build docker image.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

docker_image/build_docker_image.py

@@ -0,0 +1,145 @@
+"""This script is used to build a docker image for the idmtools_platform_container.
+
+Copyright 2024, Bill Gates Foundation. All rights reserved.
+"""
+import argparse
+import os
+import subprocess
+import sys
+import requests
+import keyring
+from logging import getLogger, basicConfig, DEBUG, INFO
+from getpass import getpass
+from requests.auth import HTTPBasicAuth
+from natsort import natsorted
+
+
+logger = getLogger(__name__)
+# Global Configurations
+KEYRING_NAME = "idmtools_container_docker_builder"
+BASE_REPO = 'packages.idmod.org'
+REPO_KEY = 'idm-docker-staging'
+DOCKER_REPO = f'{REPO_KEY}.{BASE_REPO}'
+BASE_IMAGE_NAME = f'{DOCKER_REPO}/idmtools'
+current_working_directory = os.getcwd()
+BASE_VERSION = open(os.path.join(current_working_directory, 'BASE_VERSION')).read().strip()
+
+logger.info("Please be sure you are logged into the docker-production.packages.idmod.org Docker Repo")
+
+
+def get_username_and_password(disable_keyring_load=False, disable_keyring_save=False):
+    """
+    Try to get username.
+
+    It first attempts loading from environment vars, then keyring if not disabled, then lastly prompts.
+
+    Args:
+        disable_keyring_load: Disable loading credentials from keyring
+        disable_keyring_save: Disable keyring save
+
+    Returns:
+        Username password
+    """
+    if 'PYPI_STAGING_USERNAME' in os.environ:
+        logger.info("Loading Credentials from environment")
+        if 'PYPI_STAGING_PASSWORD' not in os.environ:
+            logger.error("When specifying username from environment variable, you must also specify password")
+            sys.exit(-1)
+        username = os.environ['PYPI_STAGING_USERNAME']
+        password = os.environ['PYPI_STAGING_PASSWORD']
+    elif not disable_keyring_load and keyring.get_credential(KEYRING_NAME, "username"):
+        username = keyring.get_password(KEYRING_NAME, "username")
+        password = keyring.get_password(KEYRING_NAME, "password")
+    else:
+        username = input('Username:')
+        password = getpass(prompt='Password:')
+        if not disable_keyring_save:
+            logger.info("Saving Credentials")
+            keyring.set_password(KEYRING_NAME, "username", username)
+            keyring.set_password(KEYRING_NAME, "password", password)
+    return username, password
+
+
+def get_latest_image_version_from_registry(username, password, image_name):
+    """
+    Fetch the latest image version from repo.
+    Args:
+        username: Username to use with registry
+        password: Password to use with registry
+        image_name: Docker image name to use for building image
+    Returns:
+        Latest version published in the registry
+    """
+    url = f'https://{BASE_REPO}/artifactory/api/docker/{REPO_KEY}/v2/idmtools/{image_name}/tags/list'
+    auth = HTTPBasicAuth(username=username, password=password)
+    logger.info(f"Loading Credentials from {url}")
+    response = requests.get(url, auth=auth)
+    logger.debug(f"Return Code: {response.status_code}")
+    if response.status_code != 200 and response.status_code != 404:
+        print(response.status_code)
+        print(response.content)
+        raise Exception('Could not load images')
+    elif response.status_code == 404:
+        logger.info(f"First Version {url}")
+        return f'{BASE_VERSION}.0'
+    else:
+        images = natsorted(response.json()['tags'], reverse=True)
+        images = [i for i in images if len(i) >= 5]
+        logger.debug(f"Images: {images}")
+        last_version = images[0]
+        logger.info(f"Last Version {url}")
+        version_parts = last_version.split('.')
+        base_part = '.'.join(version_parts[:-1])
+        if BASE_VERSION in base_part:
+            version_parts[-1] = str(int(version_parts[-1]) + 1)
+            version = '.'.join(version_parts)
+        else:
+            version = f'{BASE_VERSION}.0'
+        logger.info(f"Next Version: {version}")
+        return version
+
+
+def build_image(username, password, dockerfile, image_name, disable_keyring_load, disable_keyring_save):
+    """
+    Run the docker build command.
+
+    Args:
+        username: Username to use with registry
+        password: Password to use with registry
+        dockerfile: Dockerfile to use for building image
+        image_name: Docker image name to use for building image
+        disable_keyring_load: Disable keyring which caches passwords
+        disable_keyring_save: Disable caching password to the keyring
+
+    Returns:
+        None
+    """
+    if username is None or password is None:
+        username, password = get_username_and_password(disable_keyring_load, disable_keyring_save)
+    version = get_latest_image_version_from_registry(username, password, image_name)
+    cmd = ['docker', 'build', '--network=host', '--build-arg', f'CONTAINER_VERSION={version}', '--tag',
+           f'{BASE_IMAGE_NAME}/{image_name}:{version}', '-f', dockerfile, '.']
+    logger.info(f'Running: {" ".join(cmd)}')
+    p = subprocess.Popen(" ".join(cmd), cwd=current_working_directory, shell=True)
+    p.wait()
+
+    if p.returncode == 0:
+        logger.info("Tagging image")
+        os.system(f'docker tag {BASE_IMAGE_NAME}/{image_name}:{version} {BASE_IMAGE_NAME}/{image_name}:{version[:-2]}')
+    sys.exit(p.returncode)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Build Container Image")
+    parser.add_argument("--username", default=None, help="Docker Production Username")
+    parser.add_argument("--password", default=None, help="Docker Production Password")
+    parser.add_argument("--dockerfile", default="Dockerfile", help="Dockerfile to use for building image")
+    parser.add_argument("--image_name", default="container-rocky-runtime", help="image name to use for building image")
+    parser.add_argument("--disable-keyring-load", default=False, help="Disable loading password from keyring")
+    parser.add_argument("--disable-keyring-save", default=False, help="Disable saving password to keyring after user prompts")
+    parser.add_argument("--verbose", default=False, help="Enable Debug logging")
+    parser.add_argument("--debug", default=False, help="Enable Debug logging")
+    args = parser.parse_args()
+
+    basicConfig(filename="build.log", level=DEBUG if any([args.verbose, args.debug]) else INFO)
+    build_image(args.username, args.password, args.dockerfile, args.image_name, args.disable_keyring_load, args.disable_keyring_save)

docker_image/debian/BASE_VERSION

@@ -0,0 +1 @@
+0.0

docker_image/debian/Dockerfile

@@ -0,0 +1,40 @@
+FROM debian:12.6
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+
+RUN set -x && \
+    apt-get update --fix-missing && \
+    apt-get install -y --no-install-recommends \
+        python3 \
+        python3-pip \
+        python3-dev \
+        python3-venv \
+        mpich \
+        wget \
+        nano \
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*
+
+# Update the system
+RUN apt-get clean
+
+RUN mkdir /root/.pip
+# Copy the pip configuration file
+COPY pip.conf /root/.pip
+
+# Set up a virtual environment
+RUN python3 -m venv /pyenv
+ENV PATH="/pyenv/bin:$PATH"
+
+RUN pip3 install --upgrade pip setuptools
+# Add the requirements file
+ADD requirements.txt /tmp/
+# Install the packages
+RUN pip3 install -r /tmp/requirements.txt
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+ARG CONTAINER_VERSION
+
+ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}
+
+

docker_image/debian/ImageName

@@ -0,0 +1 @@
+container-debian-runtime

docker_image/debian/README.md

@@ -0,0 +1,48 @@
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+- [Debian12 Docker Image](#debian12-docker-image)
+  - [Introduction](#introduction)
+  - [Prerequisites](#prerequisites)
+  - [Build Docker Image locally](#build-docker-image-locally)
+  - [Use docker image in idmtools](#use-docker-image-in-idmtools)
+  - [Build and Publish Debian Docker Image in GitHub Actions](#build-and-publish-debian-docker-image-in-github-actions)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# Debian12 Docker Image
+
+## Introduction
+This Docker image is designed to facilitate idmtools_platform_container. It serves as local running platform with all necessary tools and dependencies installed. 
+
+The image built from this Dockerfile is based on debian:12 docker image and includes the following packages and utilities:
+- python 3.11
+- mipch 4.0.2
+- emod-api 2.0.1
+
+## Prerequisites
+- Docker
+
+## Build Docker Image locally
+```bash
+docker build -t debian12-env:0.0.1 .
+```
+
+## Use docker image in idmtools
+```python
+from idmtools.core.platform_factory import Platform
+platform = Platform('CONTAINER', docker_image='debian12-env:0.0.1')
+```
+Note, here `debian12-env:0.0.1` is the image name you built in the previous step. 
+
+If your script needs extra python packages, you can install with extra_packages argument in Platform at runtime:
+```python
+from idmtools.core.platform_factory import Platform
+platform = Platform('CONTAINER', docker_image='debian12-env', extra_packages=['emodpy~=2.0.0', 'pytest'])
+
+# or use idmtools cli command to install before run script: idmtools container install <package>
+```
+
+## Build and Publish Debian Docker Image in GitHub Actions
+You can build and publish the docker image to idmod artifactory using GitHub Actions. The script is located at `.github/workflows/build_debian_image.yml`.

docker_image/debian/pip.conf

@@ -0,0 +1,3 @@
+[global]
+index-url = https://packages.idmod.org/api/pypi/pypi-production/simple
+extra-index-url =  https://packages.idmod.org/api/pypi/pypi-production/simple

docker_image/debian/requirements.txt

@@ -0,0 +1 @@
+emod-api~=2.0.1

docker_image/docker_image_history.py

@@ -0,0 +1,101 @@
+"""This script fetches the history of a Docker image and prints it in a tabular format."""
+import argparse
+import re
+import shutil
+import subprocess
+import pandas as pd
+from tabulate import tabulate
+
+
+def parse_size(size_str):
+    """
+    Parse the size string from Docker and convert to bytes.
+    Args:
+        size_str:
+
+    Returns:
+        Size in bytes
+    """
+    """Parse the size string from Docker and convert to bytes."""
+    units = {'B': 1, 'KB': 1024, 'MB': 1024**2, 'GB': 1024**3}
+    size_str = size_str.upper().replace(" ", "")
+    num, unit = re.findall(r'([0-9.]+)([A-Z]+)', size_str)[0]
+    return float(num) * units.get(unit, 1)
+
+
+def format_size(size_in_bytes):
+    """
+    Convert a size in bytes to a human-readable string of the most appropriate unit.
+    Args:
+        size_in_bytes:
+
+    Returns:
+        Size in human-readable format
+    """
+    if size_in_bytes < 1024:
+        return f"{size_in_bytes} B"  # Bytes
+    elif size_in_bytes < 1024**2:
+        return f"{size_in_bytes / 1024:.2f} KB"  # Kilobytes
+    elif size_in_bytes < 1024**3:
+        return f"{size_in_bytes / 1024**2:.2f} MB"  # Megabytes
+    elif size_in_bytes < 1024**4:
+        return f"{size_in_bytes / 1024**3:.2f} GB"  # Gigabytes
+    else:
+        return f"{size_in_bytes / 1024**4:.2f} TB"  # Terabytes
+
+
+def get_docker_image_history(image_id_or_name):
+    """
+    Fetch the history of a Docker image.
+    Args:
+        image_id_or_name:
+
+    Returns:
+        df
+    """
+    result = subprocess.run(
+        ['docker', 'history', '--no-trunc', image_id_or_name, '--format', '{{.ID}}|{{.CreatedBy}}|{{.Size}}'],
+        capture_output=True,
+        text=True,
+        encoding='utf-8'
+    )
+    if result.returncode != 0:
+        print("Error fetching Docker image history")
+        return None
+
+    history_lines = result.stdout.strip().split('\n')
+    data = [line.split('|') for line in history_lines]
+    for i, line in enumerate(data):
+        if len(line) == 4:
+            data[i] = [line[0], line[1] + " " + line[2], line[3]]
+    df = pd.DataFrame(data, columns=['LAYER ID', 'COMMAND', 'SIZE'])
+    df = df.iloc[:, -2:]
+    return df
+
+
+def main(image_id):
+    """
+    This script fetches the history of a Docker image and prints it in a tabular format.
+    Returns:
+        None
+    """
+    df = get_docker_image_history(image_id)
+    total_size = sum(parse_size(size) for size in df['SIZE'])
+    size = format_size(total_size)
+    if df is not None:
+        terminal_width = shutil.get_terminal_size().columns
+        max_command_width = terminal_width - 10
+        table = tabulate(df, headers='keys', tablefmt='fancy_grid', showindex=False,
+                         maxcolwidths=[None, max_command_width, None])
+        print(table)
+        with open('rocky_meta_runtime.txt', 'w', encoding='utf-8') as outputfile:
+            outputfile.write(f"Total size: {size}\n\n")
+            outputfile.write(table)
+
+
+if __name__ == "__main__":
+    if __name__ == "__main__":
+        parser = argparse.ArgumentParser("Get Image Build History")
+        parser.add_argument("--image_id", default="10bed3221522", help="Docker image id")
+        args = parser.parse_args()
+        main(args.image_id)

docker_image/pip.conf

@@ -0,0 +1,3 @@
+[global]
+index-url = https://packages.idmod.org/api/pypi/pypi-production/simple
+extra-index-url =  https://pypi.org/simple

docker_image/push_docker_image.py

@@ -0,0 +1,62 @@
+"""This is the script to push docker image to idm staging artifactory.
+
+Copyright 2024, Bill Gates Foundation. All rights reserved.
+"""
+import argparse
+import os
+import subprocess
+import sys
+from logging import getLogger, basicConfig, DEBUG, INFO
+current_directory = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_directory))
+from build_docker_image import get_username_and_password, get_latest_image_version_from_registry  # noqa: E402
+
+current_working_directory = os.getcwd()
+BASE_VERSION = open(os.path.join(current_working_directory, 'BASE_VERSION')).read().strip()
+BASE_REPO = 'packages.idmod.org'
+REPO_KEY = 'idm-docker-staging'
+DOCKER_REPO = f'{REPO_KEY}.{BASE_REPO}'
+BASE_IMAGE_NAME = f'{DOCKER_REPO}/idmtools'
+
+logger = getLogger(__name__)
+
+
+def push_image(username, password, dockerfile, image_name, disable_keyring_load, disable_keyring_save):
+    """
+    Push docker image to idm staging artifactory.
+
+    Args:
+        username: Username to use with registry
+        password: Password to use with registry
+        dockerfile: Dockerfile to use for building image
+        image_name: Docker image name to use for building image
+        disable_keyring_load: Disable keyring which caches passwords
+        disable_keyring_save: Disable caching password to the keyring
+
+    Returns:
+        None
+    """
+    if username is None or password is None:
+        username, password = get_username_and_password(disable_keyring_load, disable_keyring_save)
+    version = get_latest_image_version_from_registry(username, password, image_name)
+    cmd = ['docker', 'push', f'{BASE_IMAGE_NAME}/{image_name}:{version}']
+    print(f'Running: {" ".join(cmd)}')
+    p = subprocess.Popen(" ".join(cmd), cwd=current_working_directory, shell=True)
+    p.wait()
+    sys.exit(p.returncode)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Publish Container Image to Artifactory")
+    parser.add_argument("--username", default=None, help="Docker Staging Username")
+    parser.add_argument("--password", default=None, help="Docker Staging Password")
+    parser.add_argument("--dockerfile", default="Dockerfile", help="Dockerfile to use for building image")
+    parser.add_argument("--image_name", default="container-rocky-runtime", help="image name to use for building image")
+    parser.add_argument("--disable-keyring-load", default=False, help="Disable loading password from keyring")
+    parser.add_argument("--disable-keyring-save", default=False, help="Disable saving password to keyring after user prompts")
+    parser.add_argument("--verbose", default=False, help="Enable Debug logging")
+    parser.add_argument("--debug", default=False, help="Enable Debug logging")
+    args = parser.parse_args()
+
+    basicConfig(filename="build.log", level=DEBUG if any([args.verbose, args.debug]) else INFO)
+    push_image(args.username, args.password, args.dockerfile, args.image_name, args.disable_keyring_load, args.disable_keyring_save)

docker_image/requirements.txt

@@ -0,0 +1 @@
+emod-api~=1.30