idmtools-platform-comps 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools_platform_comps/utils/sweeping.py

@@ -0,0 +1,162 @@
+"""
+idmtools utility.
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""
+import numpy as np
+from typing import Dict, Any, List
+from idmtools.entities.simulation import Simulation
+from logging import getLogger, DEBUG
+
+logger = getLogger()
+
+
+##################################################
+# Sweeping utility functions
+##################################################
+def set_param(simulation: Simulation, param: str, value: Any) -> Dict[str, Any]:
+    """
+    Set a specific parameter value on the simulation task config.
+
+    Args:
+        simulation (Simulation): idmtools Simulation object.
+        param (str): Name of the parameter to modify.
+        value (Any): New value to set.
+
+    Returns:
+        Dict[str, Any]: A dictionary containing the parameter name and value.
+    """
+    try:
+        return simulation.task.set_parameter(param, value)
+    except ValueError:
+        if "parameters" in simulation.task.config:
+            config = simulation.task.config.parameters
+        else:
+            config = simulation.task.config
+
+        config[param] = value
+        return {param: value}
+
+
+def sweep_functions(simulation: Simulation, func_list: List) -> Dict[str, Any]:
+    """
+    Apply a list of sweep functions to a simulation.
+
+    Args:
+        simulation (Simulation): The simulation to update.
+        func_list (List[Callable]): List of functions that apply sweeps to the simulation.
+
+    Returns:
+        Dict[str, Any]: A dictionary of aggregated metadata from each sweep function.
+    """
+    tags_updated = {}
+    for func in func_list:
+        tags = func(simulation)
+        if tags:
+            tags_updated.update(tags)
+    return tags_updated
+
+
+class ItvFn:
+    """
+    Sweeping utility for modifying interventions (campaign layer) during sweeps.
+
+    Requirements:
+        - func must accept an emod-api campaign object as its first argument.
+        - func must return a dictionary of metadata (i.e: tags).
+
+    Returns:
+        Dict[str, Any]: Metadata returned by the intervention function, with numpy types cast to Python types.
+    """
+    def __init__(self, func, *args, **kwargs):  # noqa: D107
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self, simulation: Simulation):  # noqa: D102
+        import emod_api.campaign as campaign
+        campaign.reset()
+
+        matadata = self.func(campaign, *self.args, **self.kwargs)
+
+        # Add new campaign events
+        events = campaign.campaign_dict["Events"]
+        simulation.task.campaign.add_events(events)
+
+        # Handle adhoc (custom) individual events
+        adhoc_events = campaign.get_adhocs()
+        if len(adhoc_events) > 0:
+            if logger.isEnabledFor(DEBUG):
+                logger.debug("Found adhoc events in campaign. Needs some special processing behind the scenes.")
+            if "Custom_Individual_Events" in simulation.task.config.parameters:
+                ev_exist = set(simulation.task.config.parameters.Custom_Individual_Events)
+                ev_addhoc = set(adhoc_events.keys())
+                simulation.task.config.parameters.Custom_Individual_Events = list(ev_exist.union(ev_addhoc))
+            else:
+                simulation.task.config.parameters.Report_Event_Recorder_Events.extend(list(set(adhoc_events.keys())))
+
+        # Convert numpy types
+        if matadata:
+            for k, v in matadata.items():
+                if isinstance(v, (np.int64, np.float64, np.float32, np.uint32, np.int16, np.int32)):
+                    matadata[k] = v.item()
+
+        return matadata
+
+
+class CfgFn:
+    """
+    Sweeping utility for modifying simulation configuration parameters.
+
+    Requirements:
+        - func must accept simulation.task.config as the first parameter.
+        - func must return a dictionary for tagging.
+
+    Returns:
+        Dict[str, Any]: Metadata dictionary with Python-native types.
+    """
+
+    def __init__(self, func, *args, **kwargs):  # noqa: D107
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self, simulation: Simulation):  # noqa: D102
+        matadata = self.func(simulation.task.config, *self.args, **self.kwargs)
+
+        # Make sure we cast numpy types into normal system types
+        if matadata:
+            for k, v in matadata.items():
+                if isinstance(v, (np.int64, np.float64, np.float32, np.uint32, np.int16, np.int32)):
+                    matadata[k] = v.item()
+
+        return matadata
+
+
+class SwpFn:
+    """
+    Sweeping utility for modifying task-level elements (e.g., reports, demographics, climate).
+
+    Requirements:
+        - func must accept simulation.task as the first parameter.
+        - func must return a metadata dictionary (tags).
+
+    Returns:
+        Dict[str, Any]: Metadata with numpy types cast to Python-native types.
+    """
+
+    def __init__(self, func, *args, **kwargs):  # noqa: D107
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(self, simulation: Simulation):  # noqa: D102
+        matadata = self.func(simulation.task, *self.args, **self.kwargs)
+
+        # Make sure we cast numpy types into normal system types
+        if matadata:
+            for k, v in matadata.items():
+                if isinstance(v, (np.int64, np.float64, np.float32, np.uint32, np.int16, np.int32)):
+                    matadata[k] = v.item()
+
+        return matadata

idmtools_platform_comps-0.0.2.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: idmtools_platform_comps
+Version: 0.0.2
+Summary: Comps platform for IDM-Tools
+Author-email: Zhaowei Du <zdu@idmod.org>, Sharon Chen <shchen@idmod.org>, Clinton Collins <ccollins@idmod.org>, Benoit Raybaud <braybaud@idmod.org>, Clark Kirkman IV <ckirkman@idmod.org>, Ye Chen <yechen@idmod.org>, Mary Fisher <mafisher@idmod.org>, Mandy Izzo <mizzo@idmod.org>, Jen Schripsema <jschripsema@idmod.org>, Ross Carter <rcarter@idmod.org>
+Project-URL: Homepage, https://github.com/InstituteforDiseaseModeling/idmtools
+Keywords: modeling,IDM
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.TXT
+Requires-Dist: idmtools<1.0.0,>=0.0.0
+Requires-Dist: pyCOMPS~=2.11
+Requires-Dist: natsort~=8.4.0
+Provides-Extra: test
+Requires-Dist: idmtools[test]; extra == "test"
+Requires-Dist: idmtools_test; extra == "test"
+Requires-Dist: idmtools_models; extra == "test"
+Provides-Extra: packaging
+Requires-Dist: flake8; extra == "packaging"
+Requires-Dist: coverage; extra == "packaging"
+Requires-Dist: bump2version; extra == "packaging"
+Requires-Dist: twine; extra == "packaging"
+Requires-Dist: natsort; extra == "packaging"
+Dynamic: license-file
+
+![Staging: idmtools-platform-comps](https://github.com/InstituteforDiseaseModeling/idmtools/workflows/Staging:%20idmtools-platform-comps/badge.svg?branch=dev)
+
+# idmtools-platform-comps
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+  - [Installing](#installing)
+- [Development Tips](#development-tips)
+- [Building SSMT Docker Image](#building-ssmt-docker-image)
+- [Choose SSMT Docker Image to use in test/script](#choose-ssmt-docker-image-to-use-in-testscript)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Installing
+
+```bash
+pip install idmtools-platform-comps --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+```
+
+# Development Tips
+
+There is a Makefile file available for most common development tasks. Here is a list of commands
+```bash
+clean       -   Clean up temproary files
+lint        -   Lint package and tests
+test        -   Run All tests
+coverage    -   Run tests and generate coverage report that is shown in browser
+```
+On Windows, you can use `pymake` instead of `make`
+
+# Building SSMT Docker Image
+
+To build the SSMT Docker image, follow these steps
+
+1. ```bash
+   docker login docker-production.packages.idmod.org
+   ```
+2. ```bash
+   make ssmt-image
+   ```
+3. When prompted, enter your idm username and password
+
+# Choose SSMT Docker Image to use in test/script
+
+There are three ways to choose which ssmt docker image to use in your script:
+
+1. specify docker_image in SSMTWorkItem creation, for example,
+```bash
+   wi = SSMTWorkItem(name=wi_name, command=command, docker_image='my_test_ssmt_docker_image')
+```   
+2. define docker_image in your idmtools.ini, for example:
+```bash
+    [COMPS2]
+    type = COMPS
+    endpoint = https://comps.idmod.org
+    environment = Calculon
+    ......
+    docker_image = my_test_ssmt_docker_image
+```
+
+3. if not above two cases, idomtools system will determine the default ssmt docker image from platform for you:
+
+   if endpoint = https://comps.idmod.org, it will use production docker image
+   
+   for all other cases, it will use the staging docker image
+   
+Note: if user overrode docker image in wi (case #1) and also defined docker image in idmtools.ini (case #2), 
+      it will take #1 as higher priority

idmtools_platform_comps-0.0.2.dist-info/RECORD

@@ -0,0 +1,62 @@
+idmtools_platform_comps/__init__.py,sha256=q761fnOdJ5dHMHpppTworPXdxYXPL8F05yBSCYEbtn0,829
+idmtools_platform_comps/comps_cli.py,sha256=WdeC7Z1-J1-l5FKu0g5SYlYjQQhOTtrPIP-UDGYRrag,1688
+idmtools_platform_comps/comps_platform.py,sha256=ZMyEzDXnGi4-8Tewam0a0jyDL46P4C0DLZ_oLQ8uMQ4,14762
+idmtools_platform_comps/plugin_info.py,sha256=NBKl8cPNzxy185x5XgVbm92Yl8DZl7pL_OWWHutCQds,5751
+idmtools_platform_comps/ssmt_platform.py,sha256=Yp8FTT8bKjmnRC7Cdz5PmOBvGvy5Oif27hSSlAtlxdc,1808
+idmtools_platform_comps/cli/__init__.py,sha256=YSMg0oRXQjJME5sfJqZ4AK2LTtcK4vng8IIQfztNNhQ,105
+idmtools_platform_comps/cli/cli_functions.py,sha256=ajLR0OmRFvSImAn7tBXy63ul5nt4w5VzztQ8mmIC95k,1561
+idmtools_platform_comps/cli/comps.py,sha256=GFIhCfq_IskHEP8VDKAbiCglRvW-J1mkYqw6KBZX1tM,25676
+idmtools_platform_comps/comps_operations/__init__.py,sha256=nbpywJN29X4gFDmU__MbBAiSra03h0xMmL-HbSt0RIs,177
+idmtools_platform_comps/comps_operations/asset_collection_operations.py,sha256=tx_O_CJoW8S04AWL9D6sY3KDd8Bo1aLWt5E2o943pXg,11555
+idmtools_platform_comps/comps_operations/experiment_operations.py,sha256=kpM7bSKvBNnhmHNwcHpz2X5XuMFJVT45aMx4MIV0cms,23756
+idmtools_platform_comps/comps_operations/simulation_operations.py,sha256=uBagWIBSqbWkxyJlkuxRPfPsQGy_PpdFnIPn-iLv2wk,29951
+idmtools_platform_comps/comps_operations/suite_operations.py,sha256=KcjoK1rL6y3wPBU1PHfYBU20Y1Y1Iar5fzjknDh8sf8,7573
+idmtools_platform_comps/comps_operations/workflow_item_operations.py,sha256=zMXjTYc3oNWDvFHv2v4UdIRfzpPTa56qWEjrjzkjgzE,9989
+idmtools_platform_comps/ssmt_operations/__init__.py,sha256=cBNIFW7ml6eFxEMoagrNUAg5-zBpsYgI8O_WcRenITw,218
+idmtools_platform_comps/ssmt_operations/simulation_operations.py,sha256=447SRVUiQMatCwSd5XLwb12ZOylkXU8-ZU-Mi00VJGg,2991
+idmtools_platform_comps/ssmt_operations/workflow_item_operations.py,sha256=T-V__FTFFlGzrEvYVa5XSemI2dhECNx16SCdTXg1WjY,2801
+idmtools_platform_comps/ssmt_work_items/__init__.py,sha256=0OknVYmNvKuidiudz3I8z8Kf8vtL5QF_vcoccqAODQM,104
+idmtools_platform_comps/ssmt_work_items/comps_work_order_task.py,sha256=caGz5oFZkiDhSHBdn4XYkmFHWAPVK6BRybQ-9TrrYUM,873
+idmtools_platform_comps/ssmt_work_items/comps_workitems.py,sha256=7nMsv-L4_A-ugSFM6qcM82fy3pYP8P95j-33BFupFwg,3717
+idmtools_platform_comps/ssmt_work_items/icomps_workflowitem.py,sha256=yUpETmG4geXJAIrtVHh7mUFqSpdfYsjKnmtH-2c6qYs,1897
+idmtools_platform_comps/ssmt_work_items/work_order.py,sha256=G6kcqZb-OLzykbNLtIDDFyS4fhiU-OPbgnmOnfmMImo,1653
+idmtools_platform_comps/utils/__init__.py,sha256=eerSmYyuqPdwpt6aHUEsPjhNcYGaSfQOQdLJ-8rhBDo,100
+idmtools_platform_comps/utils/base_singularity_work_order.json,sha256=YCrJSkhiX5igP_VYjMNlunUs0Y9jueWCPj5L5eGZYYA,87
+idmtools_platform_comps/utils/download_experiment.py,sha256=5hkrYD90tMDjM9Vgw3YjNyCMe4qUt9LVUJCbRH0qMFw,3333
+idmtools_platform_comps/utils/file_filter_workitem.py,sha256=0d1ApTZQGVP-9EaJhYB8h88yjZRZ1nACvz6rN5foxY4,21074
+idmtools_platform_comps/utils/general.py,sha256=Xp8au0JSK_E5cIOMkQSJ0ppGMWhirxODoZVOWYlSe_o,12366
+idmtools_platform_comps/utils/linux_mounts.py,sha256=xai-RUIm6dPnjYamgEjc4lYiJBpZqyupbSVNinB_XmQ,2605
+idmtools_platform_comps/utils/lookups.py,sha256=PBMuRaENuJEYxYkd1PA7aFDBDIOM7IX2Xqi5sEc3OHY,4378
+idmtools_platform_comps/utils/package_version.py,sha256=r1UyATqvMCOMKCbwHCUMniGHv8RJGrd68jc28Yt7UMo,16149
+idmtools_platform_comps/utils/python_version.py,sha256=15DxlGjo4lrKm2w43dD2_Vf7p5oKfOoF9g8W7qp9zlI,1290
+idmtools_platform_comps/utils/scheduling.py,sha256=-hkNSXtHzd1BT_InbGq5CUB-ChN0L1K6Bd9oai91N3g,6173
+idmtools_platform_comps/utils/singularity_build.py,sha256=TzbabKEbsP6rlADF7hZ6C4TyOEVMP7bNKIi9FSTeoh8,20569
+idmtools_platform_comps/utils/spatial_output.py,sha256=bNSwjb957D0L3BKo-wOPmTAg5lGMEVTIz6ypA12C6fQ,2334
+idmtools_platform_comps/utils/sweeping.py,sha256=8GofrbofLrJu9POQNl09lNoZk1d6nWlPaIjgEaajcxw,5434
+idmtools_platform_comps/utils/assetize_output/__init__.py,sha256=hRdtgRQ4EQNTXZQXJ4_Ym9dG7Bn49cCsum3JEdHa3pA,104
+idmtools_platform_comps/utils/assetize_output/assetize_output.py,sha256=HhZCbdb29UFwhMYofupzsLPtt2bqE_oMHWZs17fztZM,5314
+idmtools_platform_comps/utils/assetize_output/assetize_ssmt_script.py,sha256=nUSvFEFeuLLaXttM9NgguiLKtdjULCzYoAhfcd7IuN0,4844
+idmtools_platform_comps/utils/download/__init__.py,sha256=y9lZ4H9k0MbHS70b02BUnpF4lSoTDKrjDnWM67nrpQU,106
+idmtools_platform_comps/utils/download/download.py,sha256=VFJALnMJtUz39xRaqyLcXOz1gimwcn3iZE6vsB6oJPo,6871
+idmtools_platform_comps/utils/download/download_ssmt.py,sha256=Iu2ggZlLbq5nBQi2Fh4dKjscqXK_tE_d-oLET3_FADU,3194
+idmtools_platform_comps/utils/python_requirements_ac/__init__.py,sha256=5SB84oji6LJ9TD_c_WT8OXgvonDuGfSW6joe3PHQJec,117
+idmtools_platform_comps/utils/python_requirements_ac/create_asset_collection.py,sha256=14xYbrfc5pew4hI0iTN_wb5wq0T3SUlbLbxri8WkQJY,5042
+idmtools_platform_comps/utils/python_requirements_ac/install_requirements.py,sha256=0Oo1SEbBoC8EDvNwmWz1s2xp3w62zExGd0oEmJp6RmU,3515
+idmtools_platform_comps/utils/python_requirements_ac/requirements_to_asset_collection.py,sha256=aMgmlgqcB0_eYe3bdhKyjgXhe2b2LWedOYowB6xZOOk,13632
+idmtools_platform_comps/utils/ssmt_utils/__init__.py,sha256=9p8SmJ5g21Tk2dOAkfc3Kh8MZ3uWEMWo-ysdk4Wllak,158
+idmtools_platform_comps/utils/ssmt_utils/common.py,sha256=67-sGWOBnJaR31BE6SvtZ2ajFF57lc_jsmvH3eJ_wDY,2272
+idmtools_platform_comps/utils/ssmt_utils/file_filter.py,sha256=Nx2_OF4kXDMgO48BhbIcglNJgUim-tdcZ7gsTJMCbfI,26810
+idmtools_platform_comps-0.0.2.dist-info/licenses/LICENSE.TXT,sha256=l9S8Ydr_LcejxKoqK8191ZAOsmVX-nJLSPoLKZDUgcg,197
+ssmt_image/Dockerfile,sha256=5cENyIiv8hnmmGHxEW1vCGgsJsQ2T-sq4Nvdbjz3k_Q,1787
+ssmt_image/Makefile,sha256=kq4BufedyACnD9Z96ypNXsuNI__UEhH73IILuSoOaOk,620
+ssmt_image/__init__.py,sha256=tKN1Oxn8Y7VjaLoKeOviyb_MNrdyTru1NRkxaKWRyM8,130
+ssmt_image/bootstrap.sh,sha256=V0psR_4rJ-sWcmsy-XDnNrbqkgpAXu1rrwVUlRpHgNQ,755
+ssmt_image/build_docker_image.py,sha256=v1zlLTn8TU0HlClUGvmAdcUUMHa0vZcI2uvqK22cq4E,6671
+ssmt_image/pip.conf,sha256=hKl-m_qBXwyQ3QU_lq5tber-7MVizUomo9Kxw5Snb-I,157
+ssmt_image/push_docker_image.py,sha256=GshSewoKZAibWh32YKFWTv9nu78HSXJD6FOY1HcIVvE,1806
+ssmt_image/requirements.txt,sha256=1jO8Nl0sU0fqOdVwIf0bXHK8DiVjRezbTKZUug4Bmn4,135
+idmtools_platform_comps-0.0.2.dist-info/METADATA,sha256=dSWkDTi2ERgazHwdW9iNb2pXnMIlMS_8ozlJBPb1HTA,3793
+idmtools_platform_comps-0.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+idmtools_platform_comps-0.0.2.dist-info/entry_points.txt,sha256=y9ye9nY5E2wjFygMgGnvW9pLyZG4a8FW_0vVjgyH06Q,354
+idmtools_platform_comps-0.0.2.dist-info/top_level.txt,sha256=JGCUUIm6XvUKwv2sqP1HMROi7fMhZxD5q5uhiILd0C0,35
+idmtools_platform_comps-0.0.2.dist-info/RECORD,,

idmtools_platform_comps-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,9 @@
+[console_scripts]
+comps-cli = idmtools_platform_comps.cli.comps:comps
+
+[idmtools_cli.cli_plugins]
+comps_subcommand = idmtools_platform_comps.cli.comps:comps
+
+[idmtools_platform]
+idmtools_platform_comps = idmtools_platform_comps.plugin_info:COMPSPlatformSpecification
+idmtools_platform_ssmt = idmtools_platform_comps.plugin_info:SSMTPlatformSpecification

idmtools_platform_comps-0.0.2.dist-info/licenses/LICENSE.TXT

@@ -0,0 +1,3 @@
+idmtools is licensed under the Creative Commons Attribution-Noncommercial-ShareAlike 4.0 License.
+
+To view a copy of this license, visit https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode

{idmtools_platform_comps-0.0.0.dev0.dist-info → idmtools_platform_comps-0.0.2.dist-info}/top_level.txt

@@ -1 +1,2 @@
 idmtools_platform_comps
+ssmt_image

ssmt_image/Dockerfile

@@ -0,0 +1,52 @@
+FROM ubuntu:20.04
+
+# Install dependencies
+RUN apt-get update
+
+ENV PYTHONPATH=./Assets/:./Assets/site-packages/:.
+# set the timezone - do this before installing python
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y tzdata
+RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
+RUN dpkg-reconfigure --frontend noninteractive tzdata
+
+# install Python3
+RUN apt-get install -y python3-pip
+RUN apt-get install -y python3-dev
+RUN pip install --upgrade pip setuptools
+
+# r packages to support analysis in R
+RUN apt-get install -y xz-utils llvm r-base \
+    # clean up temporary data
+    && mkdir -p /root/.pip
+
+#    && apt-get clean && \     <--- We do this at the end of this file
+#    mkdir -p /root/.pip && \
+#    rm -rf /var/lib/apt/lists/*    <--- We do this at the end of this file
+
+# add requirements file
+ADD requirements.txt /tmp/
+# set default index to IDM Pip Prod(Caching and internal packages IDMTools may depend on like pyComps
+ADD pip.conf /etc/pip.conf
+
+COPY .depends/* /tmp/
+
+RUN pip3 install -U pip && \
+    pip install pygit2
+
+RUN bash -c "pip3 install /tmp/*.gz --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple"
+
+# make the PIP index configurable so we can build against staging, production, or a local PyPI server
+ARG SSMT_VERSION
+
+# Install the packages
+RUN pip3 install --upgrade pip --index-url=https://pypi.python.org/simple && \
+    pip3 install dataclasses && \
+    pip3 install -r /tmp/requirements.txt && \
+    rm -rf /root/.cache
+    # To make this effective, we need to collapse this to one layer....... so don't do it for now
+    # remove stuff we needed for pip but don't want in our final image
+    # apt-get remove build-essential
+
+RUN apt-get clean && rm -rf /var/lib/apt/lists/*

ssmt_image/Makefile

@@ -0,0 +1,21 @@
+.PHONY: help clean clean-all docker docker-release-staging
+IPY=python -c
+PY?=python
+PDS=$(PY) ../../dev_scripts/
+PDR=$(PDS)run.py
+
+help:
+	help-from-makefile -f ./Makefile
+
+clean-all: clean ## Deleting package info hides plugins so we only want to do that for packaging
+	$(CLDIR) --dir-patterns "**/*.egg-info/"
+	docker rmi idmtools_comps_ssmt_worker
+
+docker: ## Build our docker image using the local pypi
+	$(MAKE) -C ../../idmtools_core dist
+	$(MAKE) -C ../../idmtools_models dist
+	$(MAKE) -C ../../idmtools_platform_comps dist
+	python build_docker_image.py
+
+docker-release-staging: docker
+	python push_docker_image.py

ssmt_image/__init__.py

@@ -0,0 +1,6 @@
+"""idmtools comps platform.
+
+Build docker image.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

ssmt_image/bootstrap.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+echo "---> bootstrap.sh: Starting"
+cd $COMPS_WORKING_DIR
+echo "---> bootstrap.sh: Environment variables:"
+env
+echo "---> bootstrap.sh: End of environment variables."
+
+if [[ $COMPS_WORKING_DIR != $PWD ]]; then
+	exit 254
+fi
+EXITCODE=255
+if [[ -z "${COMPS_STARTUP_CMD// }" ]]; then
+	echo "---> bootstrap.sh: COMPS_STARTUP_CMD environment variable not specified."
+else
+   echo Executing system command: $COMPS_STARTUP_CMD
+   eval "$COMPS_STARTUP_CMD"
+	EXITCODE=$?
+fi
+
+if [[ -z "${COMPS_USER_CMD// }" ]]; then
+	echo "---> bootstrap.sh: COMPS_USER_CMD environment variable not specified."
+else
+   echo Executing user command: $COMPS_USER_CMD
+   eval "$COMPS_USER_CMD"
+	EXITCODE=$?
+fi
+
+echo "---> bootstrap.sh: Finished ($EXITCODE)"
+exit $EXITCODE

ssmt_image/build_docker_image.py

@@ -0,0 +1,161 @@
+"""This script is currently a workaround so that we can use bump2version with docker since the nightly versions doesn't work with docker registry.
+
+Notes:
+    If you are using this script locally, you need to set the environment variables *PYPI_STAGING_USERNAME* and *PYPI_STAGING_PASSWORD*.
+    These can be set to your idm email/password
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+from logging import getLogger, basicConfig, DEBUG, INFO
+import sys
+from getpass import getpass
+import requests
+from requests.auth import HTTPBasicAuth
+import keyring
+from natsort import natsorted
+
+
+logger = getLogger(__name__)
+# Global Configurations
+KEYRING_NAME = "idmtools_ssmt_builder"
+BASE_REPO = 'packages.idmod.org'
+REPO_KEY = 'idm-docker-staging'
+DOCKER_REPO = f'{REPO_KEY}.{BASE_REPO}'
+IMAGE_NAME = 'idmtools/comps_ssmt_worker'
+BASE_IMAGE_NAME = f'{DOCKER_REPO}/{IMAGE_NAME}'
+CURRENT_DIRECTORY = os.path.dirname(__file__)
+BASE_VERSION = open(os.path.join(CURRENT_DIRECTORY, '..', 'VERSION')).read().strip()
+
+logger.info("Please be sure you are logged into the docker-production.packages.idmod.org Docker Repo")
+BASE_DIR = os.path.abspath(os.path.join(CURRENT_DIRECTORY, '..', '..'))
+LOCAL_PACKAGE_DIR = os.path.join(BASE_DIR, 'idmtools_platform_comps/ssmt_image')
+
+
+def get_dependency_packages():
+    """
+    Get python packages required to build image.
+
+    Returns:
+        None
+    """
+    os.makedirs(os.path.abspath('.depends'), exist_ok=True)
+    for root, _dirs, files in os.walk(os.path.join(LOCAL_PACKAGE_DIR, '.depends')):
+        for file in files:
+            os.remove(os.path.join(root, file))
+    for package in ['idmtools_core', 'idmtools_models', 'idmtools_platform_comps']:
+        for file in glob.glob(os.path.join(BASE_DIR, package, 'dist', '**.gz')):
+            shutil.copy(file, os.path.join(LOCAL_PACKAGE_DIR, '.depends', os.path.basename(file)))
+
+
+def get_username_and_password(disable_keyring_load=False, disable_keyring_save=False):
+    """
+    Try to get username.
+
+    It first attempts loading from environment vars, then keyring if not disabled, then lastly prompts.
+
+    Args:
+        disable_keyring_load: Disable loading credentials from keyring
+        disable_keyring_save: Disable keyring save
+
+    Returns:
+        Username password
+    """
+    if 'PYPI_STAGING_USERNAME' in os.environ:
+        logger.info("Loading Credentials from environment")
+        if 'PYPI_STAGING_PASSWORD' not in os.environ:
+            logger.error("When specifying username from environment variable, you must also specify password")
+            sys.exit(-1)
+        username = os.environ['PYPI_STAGING_USERNAME']
+        password = os.environ['PYPI_STAGING_PASSWORD']
+    elif not disable_keyring_load and keyring.get_credential(KEYRING_NAME, "username"):
+        username = keyring.get_password(KEYRING_NAME, "username")
+        password = keyring.get_password(KEYRING_NAME, "password")
+    else:
+        username = input('Username:')
+        password = getpass(prompt='Password:')
+        if not disable_keyring_save:
+            logger.info("Saving Credentials")
+            keyring.set_password(KEYRING_NAME, "username", username)
+            keyring.set_password(KEYRING_NAME, "password", password)
+    return username, password
+
+
+def get_latest_image_version_from_registry(username, password):
+    """
+    Fetch the latest image version from repo.
+
+    Returns:
+        Latest version published in the registry
+    """
+    url = f'https://{BASE_REPO}/artifactory/api/docker/{REPO_KEY}/v2/{IMAGE_NAME}/tags/list'
+    auth = HTTPBasicAuth(username=username, password=password)
+    logger.info(f"Loading Credentials from {url}")
+    response = requests.get(url, auth=auth)
+    logger.debug(f"Return Code: {response.status_code}")
+    if response.status_code != 200:
+        print(response.status_code)
+        print(response.content)
+        raise Exception('Could not load images')
+    else:
+        images = natsorted(response.json()['tags'], reverse=True)
+        images = [i for i in images if len(i) > 6]
+        logger.debug(f"Images: {images}")
+        last_version = images[0]
+        logger.info(f"Last Version {url}")
+        version_parts = last_version.split('.')
+        base_part = '.'.join(version_parts[:-1])
+        if BASE_VERSION in base_part:
+            version_parts[-1] = str(int(version_parts[-1]) + 1)
+            version = '.'.join(version_parts)
+        else:
+            version = f'{BASE_VERSION}.0'
+        logger.info(f"Next Version: {version}")
+        return version
+
+
+def build_image(username, password, disable_keyring_load, disable_keyring_save):
+    """
+    Run the docker build command.
+
+    Args:
+        username: Username to use with registry
+        password: Password to use with registry
+        disable_keyring_load: Disable keyring which caches passwords
+        disable_keyring_save: Disable caching password to the keyring
+
+    Returns:
+        None
+    """
+    if username is None or password is None:
+        username, password = get_username_and_password(disable_keyring_load, disable_keyring_save)
+    get_dependency_packages()
+    version = get_latest_image_version_from_registry(username, password)
+    cmd = ['docker', 'build', '--network=host', '--build-arg', f'SSMT_VERSION={version}', '--tag',
+           f'{DOCKER_REPO}/{IMAGE_NAME}:{version}', '.']
+    logger.info(f'Running: {" ".join(cmd)}')
+    p = subprocess.Popen(" ".join(cmd), cwd=os.path.abspath(os.path.dirname(__file__)), shell=True)
+    p.wait()
+
+    if p.returncode == 0:
+        logger.info("Tagging image")
+        os.system(f'docker tag {DOCKER_REPO}/{IMAGE_NAME}:{version} {DOCKER_REPO}/{IMAGE_NAME}:{version[:-2]}')
+    sys.exit(p.returncode)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Build SSMT Image")
+    parser.add_argument("--username", default=None, help="Docker Production Username")
+    parser.add_argument("--password", default=None, help="Docker Production Password")
+    parser.add_argument("--disable-keyring-load", default=False, help="Disable loading password from keyring")
+    parser.add_argument("--disable-keyring-save", default=False, help="Disable saving password to keyring after user prompts")
+    parser.add_argument("--verbose", default=False, help="Enable Debug logging")
+    parser.add_argument("--debug", default=False, help="Enable Debug logging")
+    args = parser.parse_args()
+
+    basicConfig(filename="build.log", level=DEBUG if any([args.verbose, args.debug]) else INFO)
+    build_image(args.username, args.password, args.disable_keyring_load, args.disable_keyring_save)

ssmt_image/pip.conf

@@ -0,0 +1,3 @@
+[global]
+index-url = https://packages.idmod.org/api/pypi/pypi-production/simple
+extra-index-url =  https://packages.idmod.org/api/pypi/pypi-production/simple