vivarium-dagger 0.1.0__tar.gz

vivarium_dagger-0.1.0/.readthedocs.yaml

@@ -0,0 +1,42 @@
+# Paths are repo-root-relative even when this file lives in a subdirectory,
+# so the package path must be qualified with libs/dagger/.
+#
+# This file is read by RTD only when the vivarium-dagger RTD
+# project's Admin > Advanced Settings > "Path for .readthedocs.yaml" points
+# here. The root-level /.readthedocs.yaml is a placeholder used by the
+# import wizard and ignored for any project that has overridden the path.
+version: 2
+
+sphinx:
+  configuration: libs/dagger/docs/source/conf.py
+  fail_on_warning: true
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.13"
+  jobs:
+    post_checkout:
+      - git fetch --unshallow origin 2>/dev/null || git fetch origin main
+      # Cancel PR (external) builds that don't touch this lib (Exit 183 is RTD's
+      # signal to cancel the build immediately).
+      - |
+        if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet "origin/main...HEAD" -- libs/dagger/; then
+          echo "Skipping docs build: this PR changes nothing under libs/dagger/."
+          exit 183
+        fi
+    pre_install:
+      # RTD installs dependencies from PyPI. Install cluster-tools from the tree first, at
+      # the version its CHANGELOG is about to release, so the docs build before that
+      # release reaches PyPI (setuptools_scm alone would give it the previous tag's version).
+      - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_VIVARIUM_CLUSTER_TOOLS=$(grep -m1 -oE '[0-9]+\.[0-9]+\.[0-9]+' libs/cluster-tools/CHANGELOG.rst) "$READTHEDOCS_VIRTUALENV_PATH/bin/python" -m pip install ./libs/cluster-tools
+
+python:
+  install:
+    - method: pip
+      path: libs/dagger
+      extra_requirements:
+        - docs
+
+formats:
+  - pdf

vivarium_dagger-0.1.0/CHANGELOG.rst

@@ -0,0 +1,8 @@
+**0.1.0 - 11/04/26**
+
+- Initial release: Dagger moved from vivarium-cluster-tools
+- Fix ``dagger restart -P/-q`` not reaching steps that took the workflow's project or queue
+- Refuse a workflow whose ``simulation`` steps would write their runs to the same directory
+- Finish Dagger runs that have a ``simulation`` step and were started under
+  ``vivarium-cluster-tools`` 4.x before upgrading. psimulate's worker module path changed, so
+  restarting such a run after the upgrade is not supported

vivarium_dagger-0.1.0/Jenkinsfile

@@ -0,0 +1,41 @@
+/* This Jenkinsfile simply loads the `reusable_pipeline` pipeline from the
+vivarium-build-utils package, which lives at ``libs/build-utils/`` inside the
+``ihmeuw/vivarium-suite`` monorepo (https://github.com/ihmeuw/vivarium-suite).
+
+vivarium-build-utils is loaded as a Jenkins shared library
+(https://www.jenkins.io/doc/book/pipeline/shared-libraries/).
+Jenkins shared library convention dictates that importable modules must be stored
+in the 'vars' folder.
+
+Jenkins shared libraries can be configured in the Jenkins UI:
+* Manage Jenkins
+  * Configure System
+    * Global Pipeline Libraries section
+      * Library subsection
+        * Name: The Name for the lib
+        * Version: The branch you want to use. Throws an error
+                   for nonexistent branches.
+        * Project Repository: Url to the shared lib
+        * Credentials: SSH key to access the repo
+
+Note that updating the shared repo will take effect on the next pipeline invocation.
+*/
+
+// Load the get_vbu_version function from build-utils/bootstrap/
+// (the directory to load from is defined in the Jenkins shared library configuration)
+@Library("get_vbu_version@main") _
+
+// Load the full vivarium_build_utils library at the expected version
+library("vivarium_build_utils@${get_vbu_version()}")
+
+reusable_pipeline(
+    scheduled_branches: [
+        // Add additional branches for cron jobs below.
+        // Note: Newly added branches will require a manual trigger of the first build
+        // for subsequent scheduled builds to run.
+        "main",
+    ],
+    requires_slurm: "weekly",
+    skip_doc_build: true,
+    env_reqs: "ci_jenkins",
+)

vivarium_dagger-0.1.0/LICENSE

@@ -0,0 +1,27 @@
+BSD 3-Clause License
+
+Copyright 2026 Institute for Health Metrics and Evaluation
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list
+   of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this
+   list of conditions and the following disclaimer in the documentation and/or other
+   materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may
+   be used to endorse or promote products derived from this software without specific
+   prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vivarium_dagger-0.1.0/Makefile

@@ -0,0 +1,129 @@
+# Check if we're running in Jenkins
+ifdef JENKINS_URL
+# 	Files are already in workspace from shared library
+	MAKE_INCLUDES := .
+else
+# 	For local dev, use the installed vivarium.build_utils package if it exists
+# 	First, check if we can import vivarium.build_utils and assign 'yes' or 'no'.
+# 	We do this by importing the package in python and redirecting stderr to the null device.
+# 	If the import is successful (&&), it will print 'yes', otherwise (||) it will print 'no'.
+	VIVARIUM_BUILD_UTILS_AVAILABLE := $(shell python -c "import vivarium.build_utils" 2>/dev/null && echo "yes" || echo "no")
+# 	If vivarium.build_utils is available, get the makefiles path or else set it to empty
+	ifeq ($(VIVARIUM_BUILD_UTILS_AVAILABLE),yes)
+		MAKE_INCLUDES := $(shell python -c "from vivarium.build_utils.resources import get_makefiles_path; print(get_makefiles_path())")
+	else
+		MAKE_INCLUDES :=
+	endif
+endif
+
+# Set the package name as the last part of this file's parent directory path
+PACKAGE_NAME = $(notdir $(CURDIR))
+
+ifneq ($(MAKE_INCLUDES),) # not empty
+# Include makefiles from vivarium.build_utils
+include $(MAKE_INCLUDES)/base.mk
+include $(MAKE_INCLUDES)/test.mk
+else # empty
+# Use this help message (since the vivarium.build_utils version is not available)
+help:
+	@echo
+	@echo "For Make's standard help, run 'make --help'."
+	@echo
+	@echo "Most of our Makefile targets are provided by the vivarium.build_utils"
+	@echo "package. To access them, you need to create a development environment first."
+	@echo
+	@echo "make build-env"
+	@echo
+	@echo "USAGE:"
+	@echo "  make build-env [name=<environment name>] [py=<python version>] [artifactory_access=<yes|no>] [keep_env=<yes|no>]"
+	@echo
+	@echo "ARGUMENTS:"
+	@echo "  name [optional]"
+	@echo "      Name of the conda environment to create (defaults to <PACKAGE_NAME>)"
+	@echo "  py [optional]"
+	@echo "      Python version (defaults to latest supported)"
+	@echo "  artifactory_access [optional]"
+	@echo "      Whether the build host can reach the IHME artifactory. Either 'yes'"
+	@echo "      (default) or 'no'; use 'no' to build off-VPN without artifactory."
+	@echo "  keep_env [optional]"
+	@echo "      Whether to keep a failed build's partial environment for inspection."
+	@echo "      Either 'yes' or 'no' (default, i.e. tear it down)."
+	@echo
+	@echo "After creating the environment:"
+	@echo "  1. Activate it: 'conda activate <environment_name>'"
+	@echo "  2. Run 'make help' again to see all newly available targets"
+	@echo
+endif
+
+build-env: # Create a new environment with installed packages
+#	Validate arguments - exit if unsupported arguments are passed
+	@allowed="name py artifactory_access keep_env"; \
+	for arg in $(filter-out build-env,$(MAKECMDGOALS)) $(MAKEFLAGS); do \
+		case $$arg in \
+			*=*) \
+				arg_name=$${arg%%=*}; \
+				if ! echo " $$allowed " | grep -q " $$arg_name "; then \
+					allowed_list=$$(echo $$allowed | sed 's/ /, /g'); \
+					echo "Error: Invalid argument '$$arg_name'. Allowed arguments are: $$allowed_list" >&2; \
+					exit 1; \
+				fi \
+				;; \
+		esac; \
+	done
+	
+#   Handle arguments and set defaults
+#	name
+	@$(eval name ?= $(PACKAGE_NAME))
+#	python version - validate if specified, else get default from json file
+	@supported_versions=$$(python -c "import json; print(' '.join(json.load(open('python_versions.json'))))" 2>/dev/null || echo ""); \
+	if [ -n "$(py)" ] && ! echo "$$supported_versions" | grep -q "$(py)"; then \
+		echo "Error: Python version '$(py)' is not supported. Available: $$(echo $$supported_versions | sed 's/ /, /g')" >&2; \
+		exit 1; \
+	fi
+	@$(eval py ?= $(shell python -c "import json; print(max(json.load(open('python_versions.json')), key=lambda x: tuple(map(int, x.split('.')))))"))
+#	artifactory_access
+	@$(eval artifactory_access ?= yes)
+	@case "$(artifactory_access)" in \
+		yes|no) ;; \
+		*) echo "Error: 'artifactory_access' must be 'yes' or 'no', got '$(artifactory_access)'" >&2; exit 1 ;; \
+	esac
+#	keep_env
+	@$(eval keep_env ?= no)
+	@case "$(keep_env)" in \
+		yes|no) ;; \
+		*) echo "Error: 'keep_env' must be 'yes' or 'no', got '$(keep_env)'" >&2; exit 1 ;; \
+	esac
+
+#	Tear down on failure rather than leave an env that activates with nothing installed.
+	@if ! $(MAKE) --no-print-directory _build-env \
+			name=$(name) py=$(py) artifactory_access=$(artifactory_access); then \
+		echo >&2; \
+		if [ "$(keep_env)" = "yes" ]; then \
+			echo "Error: failed to build environment '$(name)'. Keeping it (keep_env=yes)." >&2; \
+		else \
+			echo "Error: failed to build environment '$(name)'. Removing it; pass keep_env=yes to keep it for inspection." >&2; \
+			conda env remove -n $(name) --yes; \
+		fi; \
+		exit 1; \
+	fi
+
+	@echo
+	@echo "Finished building environment"
+	@echo "  name: $(name)"
+	@echo "  python version: $(py)"
+	@echo
+	@echo "Don't forget to activate it with:"
+	@echo "conda activate $(name)"
+	@echo
+
+# Private: the build steps live here so `build-env` can tear down the env if any fails.
+.PHONY: _build-env
+_build-env:
+	conda create -n $(name) python=$(py) --yes
+# 	--no-capture-output streams output, so a failure isn't hidden behind a long silence.
+	conda run --no-capture-output -n $(name) pip install vivarium-build-utils
+# 	artifactory_access=no passes an empty IHME_PYPI, which clears base.mk's EXTRA_INDEX_FLAGS
+	conda run --no-capture-output -n $(name) make install $(if $(filter no,$(artifactory_access)),IHME_PYPI=)
+# 	A fresh env's only editable install is this checkout, so an empty list means it never installed.
+	@conda run --no-capture-output -n $(name) pip list --editable --format=freeze | grep -q . \
+		|| { echo "Error: nothing was installed into '$(name)' from $(CURDIR)." >&2; exit 1; }

vivarium_dagger-0.1.0/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: vivarium-dagger
+Version: 0.1.0
+Summary: Build and run multi-step data pipelines.
+Author-email: The vivarium developers <vivarium.dev@gmail.com>
+License: BSD-3-Clause
+Project-URL: Homepage, https://github.com/ihmeuw/vivarium-suite
+Project-URL: Source, https://github.com/ihmeuw/vivarium-suite/tree/main/libs/dagger
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Natural Language :: English
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: POSIX :: BSD
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Education
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.11
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: vivarium-dependencies[click,loguru,pyyaml]
+Requires-Dist: vivarium-build-utils<5.0.0,>=4.0.0
+Requires-Dist: vivarium-cluster-tools>=5.0.0
+Provides-Extra: cluster
+Requires-Dist: jobmon_installer_ihme; extra == "cluster"
+Provides-Extra: simulation
+Requires-Dist: vivarium-psimulate>=1.0.0; extra == "simulation"
+Provides-Extra: test
+Requires-Dist: vivarium-dependencies[pytest]; extra == "test"
+Requires-Dist: pytest-vivarium; extra == "test"
+Provides-Extra: docs
+Requires-Dist: vivarium-dependencies[docutils,sphinx,sphinx-click]; extra == "docs"
+Provides-Extra: interactive
+Requires-Dist: vivarium-dependencies[interactive]; extra == "interactive"
+Provides-Extra: lint
+Requires-Dist: vivarium-dependencies[lint]; extra == "lint"
+Provides-Extra: dev
+Requires-Dist: vivarium-dagger[cluster,docs,interactive,lint,simulation,test]; extra == "dev"
+Provides-Extra: ci-github
+Requires-Dist: vivarium-dagger[docs,lint,simulation,test]; extra == "ci-github"
+Provides-Extra: ci-jenkins
+Requires-Dist: vivarium-dagger[ci_github,cluster]; extra == "ci-jenkins"
+Dynamic: license-file
+
+===============
+Vivarium Dagger
+===============
+
+.. image:: https://badge.fury.io/py/vivarium-dagger.svg
+    :target: https://badge.fury.io/py/vivarium-dagger
+
+.. image:: https://readthedocs.org/projects/vivarium-dagger/badge/?version=latest
+    :target: https://vivarium-dagger.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
+This library builds and runs multi-step data pipelines. It provides ``dagger``,
+a command that runs a pipeline described in a YAML file as one Jobmon workflow
+on a Slurm cluster.
+
+**Supported Python versions: 3.11, 3.12, 3.13**
+
+You can install ``vivarium-dagger`` from PyPI with pip:
+
+.. code-block:: bash
+
+   pip install vivarium-dagger
+
+or build it from source by cloning the monorepo and installing this package:
+
+.. code-block:: bash
+
+   git clone https://github.com/ihmeuw/vivarium-suite.git
+   cd vivarium-suite
+   pip install libs/dagger
+
+.. note::
+
+    The ``[cluster]`` extra (``pip install vivarium-dagger[cluster]``) pulls in
+    ``jobmon_installer_ihme`` from IHME's internal PyPI mirror. ``dagger`` needs
+    it to submit jobs. The ``[simulation]`` extra installs ``vivarium-psimulate``,
+    which a workflow with a ``simulation`` step needs.
+
+Multi-step workflows with ``dagger``
+------------------------------------
+
+A ``dagger`` workflow file lists steps that run in order. Each step gives its
+command and its compute resources:
+
+.. code-block:: yaml
+
+    workflow:
+      name: my_pipeline
+      project: proj_simscience
+      queue: all.q
+      output_directory: /path/to/output
+      steps:
+        - name: run_sims
+          type: simulation
+          resources:
+            memory_gb: 3
+            runtime: "24:00:00"
+          args:
+            model_specification: /path/to/model_specification.yaml
+            branch_configuration: /path/to/branches.yaml
+        - name: post_process
+          command: my_post_processing_script /path/to/output
+          resources:
+            memory_gb: 8
+            runtime: "00:30:00"
+
+Launch the workflow with
+
+.. code-block:: console
+
+    dagger run -c /path/to/workflow.yaml
+
+If a run fails partway through, resume it from its output directory, skipping
+steps that already completed, with
+
+.. code-block:: console
+
+    dagger restart /path/to/output
+
+`Check out the docs! <https://vivarium-dagger.readthedocs.io/en/latest/>`_
+--------------------------------------------------------------------------

vivarium_dagger-0.1.0/README.rst

@@ -0,0 +1,81 @@
+===============
+Vivarium Dagger
+===============
+
+.. image:: https://badge.fury.io/py/vivarium-dagger.svg
+    :target: https://badge.fury.io/py/vivarium-dagger
+
+.. image:: https://readthedocs.org/projects/vivarium-dagger/badge/?version=latest
+    :target: https://vivarium-dagger.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
+
+This library builds and runs multi-step data pipelines. It provides ``dagger``,
+a command that runs a pipeline described in a YAML file as one Jobmon workflow
+on a Slurm cluster.
+
+**Supported Python versions: 3.11, 3.12, 3.13**
+
+You can install ``vivarium-dagger`` from PyPI with pip:
+
+.. code-block:: bash
+
+   pip install vivarium-dagger
+
+or build it from source by cloning the monorepo and installing this package:
+
+.. code-block:: bash
+
+   git clone https://github.com/ihmeuw/vivarium-suite.git
+   cd vivarium-suite
+   pip install libs/dagger
+
+.. note::
+
+    The ``[cluster]`` extra (``pip install vivarium-dagger[cluster]``) pulls in
+    ``jobmon_installer_ihme`` from IHME's internal PyPI mirror. ``dagger`` needs
+    it to submit jobs. The ``[simulation]`` extra installs ``vivarium-psimulate``,
+    which a workflow with a ``simulation`` step needs.
+
+Multi-step workflows with ``dagger``
+------------------------------------
+
+A ``dagger`` workflow file lists steps that run in order. Each step gives its
+command and its compute resources:
+
+.. code-block:: yaml
+
+    workflow:
+      name: my_pipeline
+      project: proj_simscience
+      queue: all.q
+      output_directory: /path/to/output
+      steps:
+        - name: run_sims
+          type: simulation
+          resources:
+            memory_gb: 3
+            runtime: "24:00:00"
+          args:
+            model_specification: /path/to/model_specification.yaml
+            branch_configuration: /path/to/branches.yaml
+        - name: post_process
+          command: my_post_processing_script /path/to/output
+          resources:
+            memory_gb: 8
+            runtime: "00:30:00"
+
+Launch the workflow with
+
+.. code-block:: console
+
+    dagger run -c /path/to/workflow.yaml
+
+If a run fails partway through, resume it from its output directory, skipping
+steps that already completed, with
+
+.. code-block:: console
+
+    dagger restart /path/to/output
+
+`Check out the docs! <https://vivarium-dagger.readthedocs.io/en/latest/>`_
+--------------------------------------------------------------------------

vivarium_dagger-0.1.0/docs/nitpick-exceptions

@@ -0,0 +1,6 @@
+# pathlib
+py:class pathlib._local.Path
+# jobmon
+py:class jobmon.client.api.Tool
+py:class jobmon.client.task.Task
+py:class jobmon.client.workflow.Workflow

vivarium_dagger-0.1.0/docs/source/api_reference/cli.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.cli

vivarium_dagger-0.1.0/docs/source/api_reference/config/builder.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.config.builder

vivarium_dagger-0.1.0/docs/source/api_reference/config/config.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.config.config

vivarium_dagger-0.1.0/docs/source/api_reference/config/index.rst

@@ -0,0 +1,7 @@
+.. automodule:: vivarium.dagger.config
+
+.. toctree::
+   :maxdepth: 2
+   :glob:
+
+   *

vivarium_dagger-0.1.0/docs/source/api_reference/config/interface.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.config.interface

vivarium_dagger-0.1.0/docs/source/api_reference/config/parsing.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.config.parsing

vivarium_dagger-0.1.0/docs/source/api_reference/config/serialization.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.config.serialization

vivarium_dagger-0.1.0/docs/source/api_reference/config/validation.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.config.validation

vivarium_dagger-0.1.0/docs/source/api_reference/index.rst

@@ -0,0 +1,11 @@
+API Reference
+=============
+
+.. automodule:: vivarium.dagger
+
+.. toctree::
+   :maxdepth: 2
+   :glob:
+
+   *
+   */index

vivarium_dagger-0.1.0/docs/source/api_reference/runner.rst

@@ -0,0 +1 @@
+.. automodule:: vivarium.dagger.runner