vivarium-psimulate 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vivarium_psimulate-1.0.0/.readthedocs.yaml +42 -0
- vivarium_psimulate-1.0.0/CHANGELOG.rst +3 -0
- vivarium_psimulate-1.0.0/Jenkinsfile +40 -0
- vivarium_psimulate-1.0.0/LICENSE +27 -0
- vivarium_psimulate-1.0.0/Makefile +129 -0
- vivarium_psimulate-1.0.0/PKG-INFO +157 -0
- vivarium_psimulate-1.0.0/README.rst +105 -0
- vivarium_psimulate-1.0.0/docs/nitpick-exceptions +31 -0
- vivarium_psimulate-1.0.0/docs/source/_static/style.css +3 -0
- vivarium_psimulate-1.0.0/docs/source/_templates/layout.html +4 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/branches.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/cli.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/cli_options.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/environment.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/index.rst +11 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/jobs.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/model_specification.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/paths.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/performance_logger.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/programming_environment.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/results/cli_options.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/results/index.rst +7 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/results/writing.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/runner.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/simulation_tasks.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/vipin/cli.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/vipin/index.rst +10 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/vipin/perf_counters.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/vipin/perf_report.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/worker/index.rst +7 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/worker/load_test_work_horse.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/worker/task_runner.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/api_reference/worker/vivarium_work_horse.rst +1 -0
- vivarium_psimulate-1.0.0/docs/source/branch.rst +329 -0
- vivarium_psimulate-1.0.0/docs/source/conf.py +235 -0
- vivarium_psimulate-1.0.0/docs/source/distributed_runner.rst +104 -0
- vivarium_psimulate-1.0.0/docs/source/glossary.rst +55 -0
- vivarium_psimulate-1.0.0/docs/source/index.rst +15 -0
- vivarium_psimulate-1.0.0/docs/source/installation.rst +19 -0
- vivarium_psimulate-1.0.0/docs/source/logging.rst +42 -0
- vivarium_psimulate-1.0.0/docs/source/yaml_basics.rst +130 -0
- vivarium_psimulate-1.0.0/pyproject.toml +130 -0
- vivarium_psimulate-1.0.0/python_versions.json +1 -0
- vivarium_psimulate-1.0.0/setup.cfg +4 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/__init__.py +33 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/branches.py +400 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/cli.py +447 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/cli_options.py +202 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/environment.py +58 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/jobmon_workflow.py +181 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/jobs.py +218 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/model_specification.py +86 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/paths.py +337 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/performance_logger.py +289 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/pip_env.py +104 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/py.typed +0 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/results/__init__.py +8 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/results/cli_options.py +20 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/results/writing.py +145 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/runner.py +263 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/simulation_tasks.py +326 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/testing/__init__.py +1 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/testing/fail_once_component.py +66 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/vipin/__init__.py +8 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/vipin/cli.py +49 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/vipin/perf_counters.py +82 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/vipin/perf_report.py +206 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/worker/__init__.py +20 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/worker/load_test_work_horse.py +78 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/worker/task_runner.py +163 -0
- vivarium_psimulate-1.0.0/src/vivarium/psimulate/worker/vivarium_work_horse.py +264 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/PKG-INFO +157 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/SOURCES.txt +103 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/dependency_links.txt +1 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/entry_points.txt +3 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/requires.txt +31 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/scm_file_list.json +99 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/scm_version.json +8 -0
- vivarium_psimulate-1.0.0/src/vivarium_psimulate.egg-info/top_level.txt +1 -0
- vivarium_psimulate-1.0.0/tests/__init__.py +0 -0
- vivarium_psimulate-1.0.0/tests/conftest.py +66 -0
- vivarium_psimulate-1.0.0/tests/data/artifact_perf_df.csv +13 -0
- vivarium_psimulate-1.0.0/tests/data/artifactless_perf_df.csv +13 -0
- vivarium_psimulate-1.0.0/tests/data/e2e_branches.yaml +4 -0
- vivarium_psimulate-1.0.0/tests/data/e2e_model_spec.yaml +27 -0
- vivarium_psimulate-1.0.0/tests/data/e2e_model_spec_fail_once.yaml +32 -0
- vivarium_psimulate-1.0.0/tests/results/__init__.py +0 -0
- vivarium_psimulate-1.0.0/tests/results/test_writing.py +126 -0
- vivarium_psimulate-1.0.0/tests/test_appending_perf_logs.py +542 -0
- vivarium_psimulate-1.0.0/tests/test_branches.py +78 -0
- vivarium_psimulate-1.0.0/tests/test_cli.py +768 -0
- vivarium_psimulate-1.0.0/tests/test_cli_options.py +67 -0
- vivarium_psimulate-1.0.0/tests/test_e2e.py +638 -0
- vivarium_psimulate-1.0.0/tests/test_jobmon_workflow.py +343 -0
- vivarium_psimulate-1.0.0/tests/test_jobs.py +84 -0
- vivarium_psimulate-1.0.0/tests/test_paths.py +167 -0
- vivarium_psimulate-1.0.0/tests/test_runner.py +288 -0
- vivarium_psimulate-1.0.0/tests/test_simulation_tasks.py +401 -0
- vivarium_psimulate-1.0.0/tests/vipin/__init__.py +0 -0
- vivarium_psimulate-1.0.0/tests/vipin/test_perf_report.py +80 -0
- vivarium_psimulate-1.0.0/tests/worker/__init__.py +0 -0
- vivarium_psimulate-1.0.0/tests/worker/test_load_test_work_horse.py +29 -0
- vivarium_psimulate-1.0.0/tests/worker/test_logging.py +42 -0
- vivarium_psimulate-1.0.0/tests/worker/test_task_runner.py +575 -0
- vivarium_psimulate-1.0.0/tests/worker/test_vivarium_work_horse.py +156 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Paths are repo-root-relative even when this file lives in a subdirectory,
|
|
2
|
+
# so the package path must be qualified with libs/psimulate/.
|
|
3
|
+
#
|
|
4
|
+
# This file is read by RTD only when the vivarium-psimulate RTD
|
|
5
|
+
# project's Admin > Advanced Settings > "Path for .readthedocs.yaml" points
|
|
6
|
+
# here. The root-level /.readthedocs.yaml is a placeholder used by the
|
|
7
|
+
# import wizard and ignored for any project that has overridden the path.
|
|
8
|
+
version: 2
|
|
9
|
+
|
|
10
|
+
sphinx:
|
|
11
|
+
configuration: libs/psimulate/docs/source/conf.py
|
|
12
|
+
fail_on_warning: true
|
|
13
|
+
|
|
14
|
+
build:
|
|
15
|
+
os: ubuntu-22.04
|
|
16
|
+
tools:
|
|
17
|
+
python: "3.13"
|
|
18
|
+
jobs:
|
|
19
|
+
post_checkout:
|
|
20
|
+
- git fetch --unshallow origin 2>/dev/null || git fetch origin main
|
|
21
|
+
# Cancel PR (external) builds that don't touch this lib (Exit 183 is RTD's
|
|
22
|
+
# signal to cancel the build immediately).
|
|
23
|
+
- |
|
|
24
|
+
if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet "origin/main...HEAD" -- libs/psimulate/; then
|
|
25
|
+
echo "Skipping docs build: this PR changes nothing under libs/psimulate/."
|
|
26
|
+
exit 183
|
|
27
|
+
fi
|
|
28
|
+
pre_install:
|
|
29
|
+
# RTD installs dependencies from PyPI. Install cluster-tools from the tree first, at
|
|
30
|
+
# the version its CHANGELOG is about to release, so the docs build before that
|
|
31
|
+
# release reaches PyPI (setuptools_scm alone would give it the previous tag's version).
|
|
32
|
+
- SETUPTOOLS_SCM_PRETEND_VERSION_FOR_VIVARIUM_CLUSTER_TOOLS=$(grep -m1 -oE '[0-9]+\.[0-9]+\.[0-9]+' libs/cluster-tools/CHANGELOG.rst) "$READTHEDOCS_VIRTUALENV_PATH/bin/python" -m pip install ./libs/cluster-tools
|
|
33
|
+
|
|
34
|
+
python:
|
|
35
|
+
install:
|
|
36
|
+
- method: pip
|
|
37
|
+
path: libs/psimulate
|
|
38
|
+
extra_requirements:
|
|
39
|
+
- docs
|
|
40
|
+
|
|
41
|
+
formats:
|
|
42
|
+
- pdf
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/* This Jenkinsfile simply loads the `reusable_pipeline` pipeline from the
|
|
2
|
+
vivarium-build-utils package, which lives at ``libs/build-utils/`` inside the
|
|
3
|
+
``ihmeuw/vivarium-suite`` monorepo (https://github.com/ihmeuw/vivarium-suite).
|
|
4
|
+
|
|
5
|
+
vivarium-build-utils is loaded as a Jenkins shared library
|
|
6
|
+
(https://www.jenkins.io/doc/book/pipeline/shared-libraries/).
|
|
7
|
+
Jenkins shared library convention dictates that importable modules must be stored
|
|
8
|
+
in the 'vars' folder.
|
|
9
|
+
|
|
10
|
+
Jenkins shared libraries can be configured in the Jenkins UI:
|
|
11
|
+
* Manage Jenkins
|
|
12
|
+
* Configure System
|
|
13
|
+
* Global Pipeline Libraries section
|
|
14
|
+
* Library subsection
|
|
15
|
+
* Name: The Name for the lib
|
|
16
|
+
* Version: The branch you want to use. Throws an error
|
|
17
|
+
for nonexistent branches.
|
|
18
|
+
* Project Repository: Url to the shared lib
|
|
19
|
+
* Credentials: SSH key to access the repo
|
|
20
|
+
|
|
21
|
+
Note that updating the shared repo will take effect on the next pipeline invocation.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
// Load the get_vbu_version function from build-utils/bootstrap/
|
|
25
|
+
// (the directory to load from is defined in the Jenkins shared library configuration)
|
|
26
|
+
@Library("get_vbu_version@main") _
|
|
27
|
+
|
|
28
|
+
// Load the full vivarium_build_utils library at the expected version
|
|
29
|
+
library("vivarium_build_utils@${get_vbu_version()}")
|
|
30
|
+
|
|
31
|
+
reusable_pipeline(
|
|
32
|
+
scheduled_branches: [
|
|
33
|
+
// Add additional branches for cron jobs below.
|
|
34
|
+
// Note: Newly added branches will require a manual trigger of the first build
|
|
35
|
+
// for subsequent scheduled builds to run.
|
|
36
|
+
"main"
|
|
37
|
+
],
|
|
38
|
+
requires_slurm: "weekly",
|
|
39
|
+
env_reqs: "ci_jenkins",
|
|
40
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright 2018 Institute for Health Metrics and Evaluation
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without modification,
|
|
6
|
+
are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this list
|
|
9
|
+
of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this
|
|
12
|
+
list of conditions and the following disclaimer in the documentation and/or other
|
|
13
|
+
materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its contributors may
|
|
16
|
+
be used to endorse or promote products derived from this software without specific
|
|
17
|
+
prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY
|
|
20
|
+
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
21
|
+
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
|
|
22
|
+
SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
23
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
|
|
24
|
+
OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
25
|
+
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
26
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
27
|
+
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# Check if we're running in Jenkins
|
|
2
|
+
ifdef JENKINS_URL
|
|
3
|
+
# Files are already in workspace from shared library
|
|
4
|
+
MAKE_INCLUDES := .
|
|
5
|
+
else
|
|
6
|
+
# For local dev, use the installed vivarium.build_utils package if it exists
|
|
7
|
+
# First, check if we can import vivarium.build_utils and assign 'yes' or 'no'.
|
|
8
|
+
# We do this by importing the package in python and redirecting stderr to the null device.
|
|
9
|
+
# If the import is successful (&&), it will print 'yes', otherwise (||) it will print 'no'.
|
|
10
|
+
VIVARIUM_BUILD_UTILS_AVAILABLE := $(shell python -c "import vivarium.build_utils" 2>/dev/null && echo "yes" || echo "no")
|
|
11
|
+
# If vivarium.build_utils is available, get the makefiles path or else set it to empty
|
|
12
|
+
ifeq ($(VIVARIUM_BUILD_UTILS_AVAILABLE),yes)
|
|
13
|
+
MAKE_INCLUDES := $(shell python -c "from vivarium.build_utils.resources import get_makefiles_path; print(get_makefiles_path())")
|
|
14
|
+
else
|
|
15
|
+
MAKE_INCLUDES :=
|
|
16
|
+
endif
|
|
17
|
+
endif
|
|
18
|
+
|
|
19
|
+
# Set the package name as the last part of this file's parent directory path
|
|
20
|
+
PACKAGE_NAME = $(notdir $(CURDIR))
|
|
21
|
+
|
|
22
|
+
ifneq ($(MAKE_INCLUDES),) # not empty
|
|
23
|
+
# Include makefiles from vivarium.build_utils
|
|
24
|
+
include $(MAKE_INCLUDES)/base.mk
|
|
25
|
+
include $(MAKE_INCLUDES)/test.mk
|
|
26
|
+
else # empty
|
|
27
|
+
# Use this help message (since the vivarium.build_utils version is not available)
|
|
28
|
+
help:
|
|
29
|
+
@echo
|
|
30
|
+
@echo "For Make's standard help, run 'make --help'."
|
|
31
|
+
@echo
|
|
32
|
+
@echo "Most of our Makefile targets are provided by the vivarium.build_utils"
|
|
33
|
+
@echo "package. To access them, you need to create a development environment first."
|
|
34
|
+
@echo
|
|
35
|
+
@echo "make build-env"
|
|
36
|
+
@echo
|
|
37
|
+
@echo "USAGE:"
|
|
38
|
+
@echo " make build-env [name=<environment name>] [py=<python version>] [artifactory_access=<yes|no>] [keep_env=<yes|no>]"
|
|
39
|
+
@echo
|
|
40
|
+
@echo "ARGUMENTS:"
|
|
41
|
+
@echo " name [optional]"
|
|
42
|
+
@echo " Name of the conda environment to create (defaults to <PACKAGE_NAME>)"
|
|
43
|
+
@echo " py [optional]"
|
|
44
|
+
@echo " Python version (defaults to latest supported)"
|
|
45
|
+
@echo " artifactory_access [optional]"
|
|
46
|
+
@echo " Whether the build host can reach the IHME artifactory. Either 'yes'"
|
|
47
|
+
@echo " (default) or 'no'; use 'no' to build off-VPN without artifactory."
|
|
48
|
+
@echo " keep_env [optional]"
|
|
49
|
+
@echo " Whether to keep a failed build's partial environment for inspection."
|
|
50
|
+
@echo " Either 'yes' or 'no' (default, i.e. tear it down)."
|
|
51
|
+
@echo
|
|
52
|
+
@echo "After creating the environment:"
|
|
53
|
+
@echo " 1. Activate it: 'conda activate <environment_name>'"
|
|
54
|
+
@echo " 2. Run 'make help' again to see all newly available targets"
|
|
55
|
+
@echo
|
|
56
|
+
endif
|
|
57
|
+
|
|
58
|
+
build-env: # Create a new environment with installed packages
|
|
59
|
+
# Validate arguments - exit if unsupported arguments are passed
|
|
60
|
+
@allowed="name py artifactory_access keep_env"; \
|
|
61
|
+
for arg in $(filter-out build-env,$(MAKECMDGOALS)) $(MAKEFLAGS); do \
|
|
62
|
+
case $$arg in \
|
|
63
|
+
*=*) \
|
|
64
|
+
arg_name=$${arg%%=*}; \
|
|
65
|
+
if ! echo " $$allowed " | grep -q " $$arg_name "; then \
|
|
66
|
+
allowed_list=$$(echo $$allowed | sed 's/ /, /g'); \
|
|
67
|
+
echo "Error: Invalid argument '$$arg_name'. Allowed arguments are: $$allowed_list" >&2; \
|
|
68
|
+
exit 1; \
|
|
69
|
+
fi \
|
|
70
|
+
;; \
|
|
71
|
+
esac; \
|
|
72
|
+
done
|
|
73
|
+
|
|
74
|
+
# Handle arguments and set defaults
|
|
75
|
+
# name
|
|
76
|
+
@$(eval name ?= $(PACKAGE_NAME))
|
|
77
|
+
# python version - validate if specified, else get default from json file
|
|
78
|
+
@supported_versions=$$(python -c "import json; print(' '.join(json.load(open('python_versions.json'))))" 2>/dev/null || echo ""); \
|
|
79
|
+
if [ -n "$(py)" ] && ! echo "$$supported_versions" | grep -q "$(py)"; then \
|
|
80
|
+
echo "Error: Python version '$(py)' is not supported. Available: $$(echo $$supported_versions | sed 's/ /, /g')" >&2; \
|
|
81
|
+
exit 1; \
|
|
82
|
+
fi
|
|
83
|
+
@$(eval py ?= $(shell python -c "import json; print(max(json.load(open('python_versions.json')), key=lambda x: tuple(map(int, x.split('.')))))"))
|
|
84
|
+
# artifactory_access
|
|
85
|
+
@$(eval artifactory_access ?= yes)
|
|
86
|
+
@case "$(artifactory_access)" in \
|
|
87
|
+
yes|no) ;; \
|
|
88
|
+
*) echo "Error: 'artifactory_access' must be 'yes' or 'no', got '$(artifactory_access)'" >&2; exit 1 ;; \
|
|
89
|
+
esac
|
|
90
|
+
# keep_env
|
|
91
|
+
@$(eval keep_env ?= no)
|
|
92
|
+
@case "$(keep_env)" in \
|
|
93
|
+
yes|no) ;; \
|
|
94
|
+
*) echo "Error: 'keep_env' must be 'yes' or 'no', got '$(keep_env)'" >&2; exit 1 ;; \
|
|
95
|
+
esac
|
|
96
|
+
|
|
97
|
+
# Tear down on failure rather than leave an env that activates with nothing installed.
|
|
98
|
+
@if ! $(MAKE) --no-print-directory _build-env \
|
|
99
|
+
name=$(name) py=$(py) artifactory_access=$(artifactory_access); then \
|
|
100
|
+
echo >&2; \
|
|
101
|
+
if [ "$(keep_env)" = "yes" ]; then \
|
|
102
|
+
echo "Error: failed to build environment '$(name)'. Keeping it (keep_env=yes)." >&2; \
|
|
103
|
+
else \
|
|
104
|
+
echo "Error: failed to build environment '$(name)'. Removing it; pass keep_env=yes to keep it for inspection." >&2; \
|
|
105
|
+
conda env remove -n $(name) --yes; \
|
|
106
|
+
fi; \
|
|
107
|
+
exit 1; \
|
|
108
|
+
fi
|
|
109
|
+
|
|
110
|
+
@echo
|
|
111
|
+
@echo "Finished building environment"
|
|
112
|
+
@echo " name: $(name)"
|
|
113
|
+
@echo " python version: $(py)"
|
|
114
|
+
@echo
|
|
115
|
+
@echo "Don't forget to activate it with:"
|
|
116
|
+
@echo "conda activate $(name)"
|
|
117
|
+
@echo
|
|
118
|
+
|
|
119
|
+
# Private: the build steps live here so `build-env` can tear down the env if any fails.
|
|
120
|
+
.PHONY: _build-env
|
|
121
|
+
_build-env:
|
|
122
|
+
conda create -n $(name) python=$(py) --yes
|
|
123
|
+
# --no-capture-output streams output, so a failure isn't hidden behind a long silence.
|
|
124
|
+
conda run --no-capture-output -n $(name) pip install vivarium-build-utils
|
|
125
|
+
# artifactory_access=no passes an empty IHME_PYPI, which clears base.mk's EXTRA_INDEX_FLAGS
|
|
126
|
+
conda run --no-capture-output -n $(name) make install $(if $(filter no,$(artifactory_access)),IHME_PYPI=)
|
|
127
|
+
# A fresh env's only editable install is this checkout, so an empty list means it never installed.
|
|
128
|
+
@conda run --no-capture-output -n $(name) pip list --editable --format=freeze | grep -q . \
|
|
129
|
+
|| { echo "Error: nothing was installed into '$(name)' from $(CURDIR)." >&2; exit 1; }
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vivarium-psimulate
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Run many vivarium simulations in parallel on a Slurm cluster.
|
|
5
|
+
Author-email: The vivarium developers <vivarium.dev@gmail.com>
|
|
6
|
+
License: BSD-3-Clause
|
|
7
|
+
Project-URL: Homepage, https://github.com/ihmeuw/vivarium-suite
|
|
8
|
+
Project-URL: Source, https://github.com/ihmeuw/vivarium-suite/tree/main/libs/psimulate
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Education
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
13
|
+
Classifier: Natural Language :: English
|
|
14
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
15
|
+
Classifier: Operating System :: POSIX
|
|
16
|
+
Classifier: Operating System :: POSIX :: BSD
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
19
|
+
Classifier: Programming Language :: Python
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Classifier: Topic :: Education
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Description-Content-Type: text/x-rst
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: vivarium-dependencies[click,loguru,numpy,pandas,pyarrow,pyyaml,tables]
|
|
28
|
+
Requires-Dist: vivarium-build-utils<5.0.0,>=4.0.0
|
|
29
|
+
Requires-Dist: vivarium-cluster-tools>=5.0.0
|
|
30
|
+
Requires-Dist: vivarium-engine>=6.0.0
|
|
31
|
+
Requires-Dist: vivarium-config-tree>=5.0.0
|
|
32
|
+
Requires-Dist: dill
|
|
33
|
+
Requires-Dist: psutil
|
|
34
|
+
Provides-Extra: cluster
|
|
35
|
+
Requires-Dist: jobmon_installer_ihme; extra == "cluster"
|
|
36
|
+
Provides-Extra: test
|
|
37
|
+
Requires-Dist: vivarium-dependencies[pytest]; extra == "test"
|
|
38
|
+
Requires-Dist: pytest-vivarium; extra == "test"
|
|
39
|
+
Provides-Extra: docs
|
|
40
|
+
Requires-Dist: vivarium-dependencies[ipython,matplotlib,sphinx,sphinx-click]; extra == "docs"
|
|
41
|
+
Provides-Extra: lint
|
|
42
|
+
Requires-Dist: vivarium-dependencies[lint]; extra == "lint"
|
|
43
|
+
Requires-Dist: types-psutil; extra == "lint"
|
|
44
|
+
Requires-Dist: pyarrow-stubs; extra == "lint"
|
|
45
|
+
Provides-Extra: dev
|
|
46
|
+
Requires-Dist: vivarium-psimulate[cluster,docs,lint,test]; extra == "dev"
|
|
47
|
+
Provides-Extra: ci-github
|
|
48
|
+
Requires-Dist: vivarium-psimulate[docs,lint,test]; extra == "ci-github"
|
|
49
|
+
Provides-Extra: ci-jenkins
|
|
50
|
+
Requires-Dist: vivarium-psimulate[ci_github,cluster]; extra == "ci-jenkins"
|
|
51
|
+
Dynamic: license-file
|
|
52
|
+
|
|
53
|
+
==================
|
|
54
|
+
Vivarium psimulate
|
|
55
|
+
==================
|
|
56
|
+
|
|
57
|
+
.. image:: https://badge.fury.io/py/vivarium-psimulate.svg
|
|
58
|
+
:target: https://badge.fury.io/py/vivarium-psimulate
|
|
59
|
+
|
|
60
|
+
.. image:: https://readthedocs.org/projects/vivarium-psimulate/badge/?version=latest
|
|
61
|
+
:target: https://vivarium-psimulate.readthedocs.io/en/latest/?badge=latest
|
|
62
|
+
:alt: Documentation Status
|
|
63
|
+
|
|
64
|
+
This library provides ``psimulate``, a command that runs many ``vivarium``
|
|
65
|
+
simulations in parallel on a Slurm cluster, and ``vipin``, a command that
|
|
66
|
+
summarizes the performance logs of a ``psimulate`` run.
|
|
67
|
+
|
|
68
|
+
**Supported Python versions: 3.11, 3.12, 3.13**
|
|
69
|
+
|
|
70
|
+
You can install ``vivarium-psimulate`` from PyPI with pip:
|
|
71
|
+
|
|
72
|
+
.. code-block:: bash
|
|
73
|
+
|
|
74
|
+
pip install vivarium-psimulate
|
|
75
|
+
|
|
76
|
+
or build it from source by cloning the monorepo and installing this package:
|
|
77
|
+
|
|
78
|
+
.. code-block:: bash
|
|
79
|
+
|
|
80
|
+
git clone https://github.com/ihmeuw/vivarium-suite.git
|
|
81
|
+
cd vivarium-suite
|
|
82
|
+
pip install libs/psimulate
|
|
83
|
+
|
|
84
|
+
.. note::
|
|
85
|
+
|
|
86
|
+
The ``[cluster]`` extra (``pip install vivarium-psimulate[cluster]``) pulls in
|
|
87
|
+
``jobmon_installer_ihme`` from IHME's internal PyPI mirror. ``psimulate run``
|
|
88
|
+
and the other commands that submit jobs need it. The base install resolves on
|
|
89
|
+
public PyPI and is enough to inspect results or build the docs.
|
|
90
|
+
|
|
91
|
+
A simple example
|
|
92
|
+
----------------
|
|
93
|
+
|
|
94
|
+
If you have a ``vivarium`` model specification file that defines a model, you
|
|
95
|
+
can use it with a **branches file** to launch many simulations at once, with
|
|
96
|
+
variations in the input data, the random seed or the parameter settings.
|
|
97
|
+
|
|
98
|
+
.. code-block:: console
|
|
99
|
+
|
|
100
|
+
psimulate run -M /path/to/model_specification.yaml -B /path/to/branches_file.yaml \
|
|
101
|
+
-o /path/to/results -P proj_simscience
|
|
102
|
+
|
|
103
|
+
The simplest branches file gives a count of input data draws and random seeds
|
|
104
|
+
to launch.
|
|
105
|
+
|
|
106
|
+
.. code-block:: yaml
|
|
107
|
+
|
|
108
|
+
input_draw_count: 25
|
|
109
|
+
random_seed_count: 10
|
|
110
|
+
|
|
111
|
+
This branches file defines one simulation for each combination of 25 input
|
|
112
|
+
draws and 10 random seeds, so it runs 250 simulations in total.
|
|
113
|
+
|
|
114
|
+
A branches file can also vary the model's configuration. The model
|
|
115
|
+
specification below defines a model of lower respiratory infections and a
|
|
116
|
+
vaccine intervention.
|
|
117
|
+
|
|
118
|
+
.. code-block:: yaml
|
|
119
|
+
|
|
120
|
+
components:
|
|
121
|
+
vivarium.public_health:
|
|
122
|
+
population:
|
|
123
|
+
- BasePopulation()
|
|
124
|
+
disease.models:
|
|
125
|
+
- SIS('lower_respiratory_infections')
|
|
126
|
+
my_lri_intervention:
|
|
127
|
+
components:
|
|
128
|
+
- GiveKidsVaccines()
|
|
129
|
+
|
|
130
|
+
configuration:
|
|
131
|
+
population:
|
|
132
|
+
population_size: 1000
|
|
133
|
+
age_start: 0
|
|
134
|
+
age_end: 5
|
|
135
|
+
lri_vaccine:
|
|
136
|
+
coverage: 0.2
|
|
137
|
+
efficacy: 0.8
|
|
138
|
+
|
|
139
|
+
This branches file varies the vaccine's coverage and efficacy as well as the
|
|
140
|
+
input draws and random seeds.
|
|
141
|
+
|
|
142
|
+
.. code-block:: yaml
|
|
143
|
+
|
|
144
|
+
input_draw_count: 25
|
|
145
|
+
random_seed_count: 10
|
|
146
|
+
|
|
147
|
+
branches:
|
|
148
|
+
- lri_vaccine:
|
|
149
|
+
coverage: [0.0, 0.2, 0.4, 0.8, 1.0]
|
|
150
|
+
efficacy: [0.4, 0.6, 0.8]
|
|
151
|
+
|
|
152
|
+
Each combination of coverage and efficacy replaces the ``lri_vaccine``
|
|
153
|
+
configuration in the model specification, and each runs for every combination
|
|
154
|
+
of input draw and random seed. That makes 25 * 10 * 5 * 3 = 3750 simulations.
|
|
155
|
+
|
|
156
|
+
`Check out the docs! <https://vivarium-psimulate.readthedocs.io/en/latest/>`_
|
|
157
|
+
-----------------------------------------------------------------------------
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
==================
|
|
2
|
+
Vivarium psimulate
|
|
3
|
+
==================
|
|
4
|
+
|
|
5
|
+
.. image:: https://badge.fury.io/py/vivarium-psimulate.svg
|
|
6
|
+
:target: https://badge.fury.io/py/vivarium-psimulate
|
|
7
|
+
|
|
8
|
+
.. image:: https://readthedocs.org/projects/vivarium-psimulate/badge/?version=latest
|
|
9
|
+
:target: https://vivarium-psimulate.readthedocs.io/en/latest/?badge=latest
|
|
10
|
+
:alt: Documentation Status
|
|
11
|
+
|
|
12
|
+
This library provides ``psimulate``, a command that runs many ``vivarium``
|
|
13
|
+
simulations in parallel on a Slurm cluster, and ``vipin``, a command that
|
|
14
|
+
summarizes the performance logs of a ``psimulate`` run.
|
|
15
|
+
|
|
16
|
+
**Supported Python versions: 3.11, 3.12, 3.13**
|
|
17
|
+
|
|
18
|
+
You can install ``vivarium-psimulate`` from PyPI with pip:
|
|
19
|
+
|
|
20
|
+
.. code-block:: bash
|
|
21
|
+
|
|
22
|
+
pip install vivarium-psimulate
|
|
23
|
+
|
|
24
|
+
or build it from source by cloning the monorepo and installing this package:
|
|
25
|
+
|
|
26
|
+
.. code-block:: bash
|
|
27
|
+
|
|
28
|
+
git clone https://github.com/ihmeuw/vivarium-suite.git
|
|
29
|
+
cd vivarium-suite
|
|
30
|
+
pip install libs/psimulate
|
|
31
|
+
|
|
32
|
+
.. note::
|
|
33
|
+
|
|
34
|
+
The ``[cluster]`` extra (``pip install vivarium-psimulate[cluster]``) pulls in
|
|
35
|
+
``jobmon_installer_ihme`` from IHME's internal PyPI mirror. ``psimulate run``
|
|
36
|
+
and the other commands that submit jobs need it. The base install resolves on
|
|
37
|
+
public PyPI and is enough to inspect results or build the docs.
|
|
38
|
+
|
|
39
|
+
A simple example
|
|
40
|
+
----------------
|
|
41
|
+
|
|
42
|
+
If you have a ``vivarium`` model specification file that defines a model, you
|
|
43
|
+
can use it with a **branches file** to launch many simulations at once, with
|
|
44
|
+
variations in the input data, the random seed or the parameter settings.
|
|
45
|
+
|
|
46
|
+
.. code-block:: console
|
|
47
|
+
|
|
48
|
+
psimulate run -M /path/to/model_specification.yaml -B /path/to/branches_file.yaml \
|
|
49
|
+
-o /path/to/results -P proj_simscience
|
|
50
|
+
|
|
51
|
+
The simplest branches file gives a count of input data draws and random seeds
|
|
52
|
+
to launch.
|
|
53
|
+
|
|
54
|
+
.. code-block:: yaml
|
|
55
|
+
|
|
56
|
+
input_draw_count: 25
|
|
57
|
+
random_seed_count: 10
|
|
58
|
+
|
|
59
|
+
This branches file defines one simulation for each combination of 25 input
|
|
60
|
+
draws and 10 random seeds, so it runs 250 simulations in total.
|
|
61
|
+
|
|
62
|
+
A branches file can also vary the model's configuration. The model
|
|
63
|
+
specification below defines a model of lower respiratory infections and a
|
|
64
|
+
vaccine intervention.
|
|
65
|
+
|
|
66
|
+
.. code-block:: yaml
|
|
67
|
+
|
|
68
|
+
components:
|
|
69
|
+
vivarium.public_health:
|
|
70
|
+
population:
|
|
71
|
+
- BasePopulation()
|
|
72
|
+
disease.models:
|
|
73
|
+
- SIS('lower_respiratory_infections')
|
|
74
|
+
my_lri_intervention:
|
|
75
|
+
components:
|
|
76
|
+
- GiveKidsVaccines()
|
|
77
|
+
|
|
78
|
+
configuration:
|
|
79
|
+
population:
|
|
80
|
+
population_size: 1000
|
|
81
|
+
age_start: 0
|
|
82
|
+
age_end: 5
|
|
83
|
+
lri_vaccine:
|
|
84
|
+
coverage: 0.2
|
|
85
|
+
efficacy: 0.8
|
|
86
|
+
|
|
87
|
+
This branches file varies the vaccine's coverage and efficacy as well as the
|
|
88
|
+
input draws and random seeds.
|
|
89
|
+
|
|
90
|
+
.. code-block:: yaml
|
|
91
|
+
|
|
92
|
+
input_draw_count: 25
|
|
93
|
+
random_seed_count: 10
|
|
94
|
+
|
|
95
|
+
branches:
|
|
96
|
+
- lri_vaccine:
|
|
97
|
+
coverage: [0.0, 0.2, 0.4, 0.8, 1.0]
|
|
98
|
+
efficacy: [0.4, 0.6, 0.8]
|
|
99
|
+
|
|
100
|
+
Each combination of coverage and efficacy replaces the ``lri_vaccine``
|
|
101
|
+
configuration in the model specification, and each runs for every combination
|
|
102
|
+
of input draw and random seed. That makes 25 * 10 * 5 * 3 = 3750 simulations.
|
|
103
|
+
|
|
104
|
+
`Check out the docs! <https://vivarium-psimulate.readthedocs.io/en/latest/>`_
|
|
105
|
+
-----------------------------------------------------------------------------
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# numpy
|
|
2
|
+
py:class numpy.random.mtrand.RandomState
|
|
3
|
+
# pandas
|
|
4
|
+
py:class pandas.core.indexes.base.Index
|
|
5
|
+
py:class pandas.core.indexes.multi.MultiIndex
|
|
6
|
+
py:class pandas._libs.tslibs.timestamps.Timestamp
|
|
7
|
+
py:class pandas._libs.tslibs.timedeltas.Timedelta
|
|
8
|
+
py:class pandas._libs.interval.Interval
|
|
9
|
+
py:class pandas.core.frame.DataFrame
|
|
10
|
+
py:class pandas.core.series.Series
|
|
11
|
+
py:class pandas.core.generic.PandasObject
|
|
12
|
+
# pathlib
|
|
13
|
+
py:class Path
|
|
14
|
+
py:class pathlib._local.Path
|
|
15
|
+
# typing
|
|
16
|
+
py:class vivarium.cluster_tools.cli_tools.R
|
|
17
|
+
# psutil
|
|
18
|
+
py:class scpustats
|
|
19
|
+
py:class sdiskio
|
|
20
|
+
py:class scpufreq
|
|
21
|
+
py:class snetio
|
|
22
|
+
# config_tree
|
|
23
|
+
py:class vivarium.config_tree.main.ConfigTree
|
|
24
|
+
py:class ConfigTree
|
|
25
|
+
py:class Component
|
|
26
|
+
# jobmon
|
|
27
|
+
py:class jobmon.client.api.Tool
|
|
28
|
+
py:class jobmon.client.task.Task
|
|
29
|
+
py:class jobmon.client.workflow.Workflow
|
|
30
|
+
py:class Tool
|
|
31
|
+
py:class Task
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.branches
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.cli
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.cli_options
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.environment
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.jobs
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.model_specification
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.paths
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.performance_logger
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.pip_env
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.results.cli_options
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.results.writing
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.runner
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.simulation_tasks
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.vipin.cli
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.vipin.perf_counters
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.vipin.perf_report
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.worker.load_test_work_horse
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.worker.task_runner
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
.. automodule:: vivarium.psimulate.worker.vivarium_work_horse
|