idmtools-platform-general 0.0.0.dev0__tar.gz → 0.0.3__tar.gz

idmtools_platform_general-0.0.3/.bumpversion.cfg

@@ -0,0 +1,39 @@
+[bumpversion]
+current_version = 3.0.0+nightly
+commit = False
+tag = False
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<release>\+\w+)?(\.(?P<build>\d+))?
+serialize = 
+	{major}.{minor}.{patch}{release}.{build}
+	{major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:release]
+optional_value = prod
+first_value = prod
+values = 
+	prod
+	+nightly
+
+[bumpversion:part:build]
+
+[bumpversion:file:VERSION]
+serialize = {major}.{minor}.{patch}
+
+[bumpversion:file:./idmtools_platform_file/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"
+serialize = {major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}
+
+[bumpversion:file:./idmtools_platform_process/__init__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"
+serialize = {major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}
+
+[bumpversion:file:./pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"
+serialize = {major}.{minor}.{patch}{release}
+	{major}.{minor}.{patch}

idmtools_platform_general-0.0.3/LICENSE.TXT

@@ -0,0 +1,3 @@
+idmtools is licensed under the Creative Commons Attribution-Noncommercial-ShareAlike 4.0 License.
+
+To view a copy of this license, visit https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode

idmtools_platform_general-0.0.3/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include idmtools_platform_file/assets/*
+include LICENSE.TXT

idmtools_platform_general-0.0.3/Makefile

@@ -0,0 +1,13 @@
+PACKAGE_NAME=idmtools_platform_file
+include $(abspath ../dev_scripts/package_general.mk)
+
+
+help: 
+	help-from-makefile -f ../dev_scripts/package_general.mk -f ./Makefile
+
+# Release related rules
+ssmt-image: ## Build SSMT Docker image
+	$(MAKE) -C ssmt_image docker
+
+ssmt-image-release: ## Push image to staging
+	$(MAKE) -C ssmt_image  docker-release-staging

idmtools_platform_general-0.0.3/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.4
+Name: idmtools_platform_general
+Version: 0.0.3
+Summary: General platform for IDM-Tools
+Author-email: Zhaowei Du <zdu@idmod.org>, Sharon Chen <shchen@idmod.org>, Clinton Collins <ccollins@idmod.org>, Jen Schripsema <jschripsema@idmod.org>, Ross Carter <rcarter@idmod.org>
+Project-URL: Homepage, https://github.com/InstituteforDiseaseModeling/idmtools
+Keywords: modeling,IDM
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.TXT
+Requires-Dist: idmtools<1.0.0,>=0.0.0
+Requires-Dist: idmtools_cli<1.0.0,>=0.0.0
+Provides-Extra: test
+Requires-Dist: idmtools[test]; extra == "test"
+Requires-Dist: idmtools_test; extra == "test"
+Provides-Extra: packaging
+Requires-Dist: flake8; extra == "packaging"
+Requires-Dist: coverage; extra == "packaging"
+Requires-Dist: bump2version; extra == "packaging"
+Requires-Dist: twine; extra == "packaging"
+Requires-Dist: natsort; extra == "packaging"
+Dynamic: license-file
+
+![Staging: idmtools-platform-file](https://github.com/InstituteforDiseaseModeling/idmtools/workflows/Staging:%20idmtools-platform-file/badge.svg?branch=dev)
+
+# idmtools-platform-general
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+  - [Installing](#installing)
+- [Development Tips](#development-tips)
+- [Use Cases](#use-cases)
+- [Feature Roadmap](#feature-roadmap)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Installing
+
+```bash
+pip install idmtools-platform-general --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+```
+
+# Development Tips
+
+There is a Makefile file available for most common development tasks. Here is a list of commands
+
+```bash
+clean       -   Clean up temproary files
+lint        -   Lint package and tests
+test        -   Run All tests
+coverage    -   Run tests and generate coverage report that is shown in browser
+```
+
+On Windows, you can use `pymake` instead of `make`
+
+# Use Cases
+
+* Testing
+  * Test core functionality
+  * Performance Testing
+* Integration with other systems
+  * Other HPC Systems
+  * Local Executions Systems
+  * Jupyter notebooks
+* Basis for future local platforms
+  * Process
+  * Thread
+  * Dask
+  * Asyncio
+
+# Feature Roadmap
+
+* First Version
+  * Support for basic provisioning on a linux filesystem

idmtools_platform_general-0.0.3/README.md

@@ -0,0 +1,53 @@
+![Staging: idmtools-platform-file](https://github.com/InstituteforDiseaseModeling/idmtools/workflows/Staging:%20idmtools-platform-file/badge.svg?branch=dev)
+
+# idmtools-platform-general
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**
+
+  - [Installing](#installing)
+- [Development Tips](#development-tips)
+- [Use Cases](#use-cases)
+- [Feature Roadmap](#feature-roadmap)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Installing
+
+```bash
+pip install idmtools-platform-general --index-url=https://packages.idmod.org/api/pypi/pypi-production/simple
+```
+
+# Development Tips
+
+There is a Makefile file available for most common development tasks. Here is a list of commands
+
+```bash
+clean       -   Clean up temproary files
+lint        -   Lint package and tests
+test        -   Run All tests
+coverage    -   Run tests and generate coverage report that is shown in browser
+```
+
+On Windows, you can use `pymake` instead of `make`
+
+# Use Cases
+
+* Testing
+  * Test core functionality
+  * Performance Testing
+* Integration with other systems
+  * Other HPC Systems
+  * Local Executions Systems
+  * Jupyter notebooks
+* Basis for future local platforms
+  * Process
+  * Thread
+  * Dask
+  * Asyncio
+
+# Feature Roadmap
+
+* First Version
+  * Support for basic provisioning on a linux filesystem

idmtools_platform_general-0.0.3/VERSION

@@ -0,0 +1 @@
+3.0.0

idmtools_platform_general-0.0.3/idmtools_platform_file/__init__.py

@@ -0,0 +1,18 @@
+"""
+idmtools file platform.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# flake8: noqa F821
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    # Python < 3.8
+    from importlib_metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("idmtools-platform-general")  # Use your actual package name
+except PackageNotFoundError:
+    # Package not installed, use fallback
+    __version__ = "0.0.0+unknown"
+

idmtools_platform_general-0.0.3/idmtools_platform_file/assets/__init__.py

@@ -0,0 +1,77 @@
+"""
+Utility functions to generate batch scripts.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from pathlib import Path
+from jinja2 import Template
+from typing import TYPE_CHECKING, Optional
+from idmtools.entities.experiment import Experiment
+from idmtools.entities.simulation import Simulation
+
+if TYPE_CHECKING:
+    from idmtools_platform_file.file_platform import FilePlatform
+
+DEFAULT_TEMPLATE_FILE = Path(__file__).parent.joinpath("batch.sh.jinja2")
+DEFAULT_SIMULATION_TEMPLATE = Path(__file__).parent.parent.joinpath("assets/_run.sh.jinja2")
+
+
+def generate_script(platform: 'FilePlatform', experiment: Experiment, max_job: int = None, run_sequence: bool = None,
+                    **kwargs) -> None:
+    """
+    Generate batch file batch.sh.
+    Args:
+        platform: File Platform
+        experiment: idmtools Experiment
+        max_job: int
+        run_sequence: bool
+        kwargs: keyword arguments used to expand functionality
+    Returns:
+        None
+    """
+    output_target = platform.get_directory(experiment).joinpath("batch.sh")
+    with open(output_target, "w") as tout:
+        with open(DEFAULT_TEMPLATE_FILE) as tin:
+            t = Template(tin.read())
+            tvars = dict(
+                platform=platform,
+                max_job=max_job if max_job is not None else platform.max_job,
+                run_sequence=run_sequence if run_sequence is not None else platform.run_sequence
+            )
+            if platform.modules:
+                tvars['modules'] = platform.modules
+            if platform.extra_packages:
+                tvars['packages'] = platform.extra_packages
+            tout.write(t.render(tvars))
+
+    # Make executable
+    platform.update_script_mode(output_target)
+
+
+def generate_simulation_script(platform: 'FilePlatform', simulation: Simulation, retries: Optional[int] = None,
+                               **kwargs) -> None:
+    """
+    Generate batch file _run.sh.
+    Args:
+        platform: File Platform
+        simulation: idmtools Simulation
+        retries: int
+        extra_packages: List of extra packages to install
+        kwargs: keyword arguments used to expand functionality
+    Returns:
+        None
+    """
+    sim_script = platform.get_directory(simulation).joinpath("_run.sh")
+    with open(sim_script, "w") as tout:
+        with open(DEFAULT_SIMULATION_TEMPLATE) as tin:
+            t = Template(tin.read())
+            tvars = dict(
+                platform=platform,
+                simulation=simulation,
+                retries=retries if retries else platform.retries,
+                ntasks=platform.ntasks
+            )
+            tout.write(t.render(tvars))
+
+    # Make executable
+    platform.update_script_mode(sim_script)

idmtools_platform_general-0.0.3/idmtools_platform_file/assets/_run.sh.jinja2

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# define the handler function
+term_handler()
+{
+    # do whatever cleanup you want here
+    echo "-1" > job_status.txt
+    exit -1
+}
+
+# Register cleanup function to handle SIGINT and SIGTERM signals
+trap 'term_handler' SIGINT SIGTERM
+
+n=0
+{% set mpi_command = "mpirun -n " + ntasks|string if ntasks > 1 else "" %}
+
+until [ "$n" -ge {{retries}} ]
+do
+    echo "100" > job_status.txt
+    {% if simulation.task.sif_path is defined and simulation.task.sif_path %}
+        {% if simulation.task.command.cmd.startswith('singularity') %}
+            {{ mpi_command }} {{simulation.task.command.cmd}} &
+        {% else %}
+           singularity exec {{simulation.task.sif_path}} {{ mpi_command }} {{simulation.task.command.cmd}} &
+        {% endif %}
+    {% else %}
+        exec -a "SIMULATION:{{simulation.id}}" {{ mpi_command }} {{simulation.task.command.cmd}} &
+    {% endif %}
+
+   child_pid=$!
+   echo "Running simulation with PID: $child_pid"
+   # Wait for the child process to complete
+   wait $child_pid
+
+   RESULT=$?
+   if [ $RESULT -eq 0 ]; then
+      echo "0" > job_status.txt
+      exit 0
+   elif [ $RESULT -eq 255 ] || [ $RESULT -eq -1 ]; then   # Normalize -1 or 255 to 1 to avoid process abort
+      echo "-1" > job_status.txt
+      echo "_run.sh exiting with code: $RESULT" >> exit_code.log
+      exit 1
+   fi
+   n=$((n+1))
+done
+echo "-1" > job_status.txt
+exit $RESULT

idmtools_platform_general-0.0.3/idmtools_platform_file/assets/batch.sh.jinja2

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+chmod +x run_simulation.sh
+
+{% if packages is defined and packages is not none and packages|length > 0 %}
+echo "Installing custom packages"
+
+{% for package in packages %}
+    echo "Installing {{ package }}" 1>> stdout.txt 2>> stderr.txt
+    pip install {{ package }} --extra-index-url=https://packages.idmod.org/api/pypi/pypi-production/simple --upgrade 1>> stdout.txt 2>> stderr.txt
+{% endfor %}
+{% endif %}
+
+{% if modules is defined and modules is not none and modules|length > 0 %}
+{% for m in modules %}
+module load {{m}}
+{% endfor %}
+{% endif %}
+
+{% if run_sequence is defined and run_sequence %}
+    find $(pwd) -maxdepth 2 -name "_run.sh" -print0 | xargs -0 -I% dirname % | xargs -d "\n" -I% bash -c 'cd $(pwd) && $(pwd)/run_simulation.sh %  1>> stdout.txt 2>> stderr.txt'
+{% else %}
+    find $(pwd) -maxdepth 2 -name "_run.sh" -print0 | xargs -0 -I% dirname % | xargs -d "\n" -P {{ max_job }} -I% bash -c 'cd $(pwd) && $(pwd)/run_simulation.sh %  1>> stdout.txt 2>> stderr.txt'
+{% endif %}

idmtools_platform_general-0.0.3/idmtools_platform_file/assets/run_simulation.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+JOB_DIRECTORY=$1
+echo "enter directory: '$JOB_DIRECTORY'"
+cd $JOB_DIRECTORY
+sed -i 's/\r//g' _run.sh
+chmod +x _run.sh
+bash _run.sh 1> stdout.txt 2> stderr.txt

idmtools_platform_general-0.0.3/idmtools_platform_file/cli/__init__.py

@@ -0,0 +1,5 @@
+"""
+idmtools file cli module.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools_platform_general-0.0.3/idmtools_platform_file/cli/file.py

@@ -0,0 +1,185 @@
+"""
+idmtools FilePlatform CLI commands.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import json
+import click
+from idmtools.core import ItemType
+from idmtools.core.platform_factory import Platform
+from idmtools_platform_file.tools.status_report.status_report import generate_status_report
+from idmtools_platform_file.tools.status_report.utils import get_latest_experiment, check_status, clear_history
+from logging import getLogger
+
+user_logger = getLogger('user')
+
+
+@click.group(short_help="File platform related commands.")
+@click.argument('job-directory')
+@click.pass_context
+def file(ctx: click.Context, job_directory):
+    """
+    Commands related to managing the File Platform.
+
+    job_directory: FilePlatform Working Directory
+    """
+    ctx.obj = dict(job_directory=job_directory)
+
+
+@file.command(help="Get simulation's report")
+@click.option('--suite-id', default=None, help="Idmtools Suite id")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--status-filter', type=click.Choice(['0', '-1', '100']), multiple=True, help="list of status")
+@click.option('--sim-filter', multiple=True, help="list of simulations")
+@click.option('--verbose/--no-verbose', default=True, help="Enable verbose output in results")
+@click.option('--display/--no-display', default=True, help="Display with working directory or not")
+@click.option('--display-count', default=20, help="Display Count")
+@click.pass_context
+def status_report(ctx: click.Context, suite_id, exp_id, status_filter, sim_filter, verbose, display,
+                  display_count):
+    """
+    Build status report.
+    Args:
+        ctx: click.Context
+        suite_id: suite id
+        exp_id: experiment id
+        status_filter: status filter
+        sim_filter: simulation filter
+        verbose: bool True/False
+        display: bool True/False
+        display_count: how many to display
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    if suite_id is not None:
+        scope = (suite_id, ItemType.SUITE)
+    elif exp_id is not None:
+        scope = (exp_id, ItemType.EXPERIMENT)
+    else:
+        scope = None
+
+    generate_status_report(platform=platform, scope=scope,
+                           status_filter=status_filter if len(status_filter) > 0 else None,
+                           sim_filter=sim_filter if len(sim_filter) > 0 else None,
+                           verbose=verbose, display=display, display_count=display_count)
+
+
+@file.command(help="Get the latest experiment info")
+@click.pass_context
+def get_latest(ctx: click.Context):
+    """
+    Get the latest experiment directory.
+    Args:
+        ctx: click.Context
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    result = get_latest_experiment(platform)
+    user_logger.info(json.dumps(result, indent=3))
+
+
+@file.command(help="Get Suite/Experiment/Simulation directory")
+@click.option('--sim-id', default=None, help="Idmtools Simulation id")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--suite-id', default=None, help="Idmtools Suite id")
+@click.pass_context
+def get_path(ctx: click.Context, sim_id, exp_id, suite_id):
+    """
+    Get entity directory.
+    Args:
+        ctx: click.Context
+        sim_id: simulation id
+        exp_id: experiment id
+        suite_id: suite id
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    if sim_id is not None:
+        item_dir = platform.get_directory_by_id(sim_id, ItemType.SIMULATION)
+    elif exp_id is not None:
+        item_dir = platform.get_directory_by_id(exp_id, ItemType.EXPERIMENT)
+    elif suite_id is not None:
+        item_dir = platform.get_directory_by_id(suite_id, ItemType.SUITE)
+    else:
+        raise Exception('Must provide at least one: suite-id, exp-id or sim-id!')
+
+    user_logger.info(item_dir)
+
+
+@file.command(help="Get status of Experiment/Simulation")
+@click.option('--sim-id', default=None, help="Idmtools Simulation id")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.pass_context
+def get_status(ctx: click.Context, sim_id, exp_id):
+    """
+    Retrieve status.
+    Args:
+        ctx: click.Context
+        sim_id: simulation id
+        exp_id: experiment id
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    if sim_id is not None:
+        status = platform.get_simulation_status(sim_id)
+    elif exp_id is not None:
+        exp = platform.get_item(exp_id, ItemType.EXPERIMENT)
+        status = exp.status
+    else:
+        raise Exception('Must provide at least one: exp-id or sim-id!')
+
+    user_logger.info(status.name if status else None)
+
+
+@file.command(help="Get simulation's status")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--display/--no-display', default=False, help="Display with working directory or not")
+@click.pass_context
+def status(ctx: click.Context, exp_id, display):
+    """
+    Get job status.
+    Args:
+        ctx: click.Context
+        exp_id: experiment id
+        display: bool True/False
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    check_status(platform=platform, exp_id=exp_id, display=display)
+
+
+@file.command(help="Clear generated files/folders")
+@click.option('--exp-id', default=None, help="Idmtools Experiment id")
+@click.option('--sim-id', multiple=True, help="Idmtools Simulation id")
+@click.option('--remove', multiple=True, help="list of files/folders to be removed from simulation")
+@click.pass_context
+def clear_files(ctx: click.Context, exp_id, sim_id, remove):
+    """
+    Clear running history.
+    Args:
+        ctx: click.Context
+        exp_id: experiment id
+        sim_id: simulation id
+        remove: list of files/folders
+    Returns:
+        None
+    """
+    job_dir = ctx.obj['job_directory']
+    platform = Platform('FILE', job_directory=job_dir)
+
+    clear_history(platform=platform, exp_id=exp_id, sim_id=sim_id, remove_list=remove)

idmtools_platform_general-0.0.3/idmtools_platform_file/file_operations/__init__.py

@@ -0,0 +1,4 @@
+"""idmtools comps utils.
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""