toil 8.2.0__tar.gz → 9.0.0__tar.gz

{toil-8.2.0 → toil-9.0.0}/.gitlab-ci.yml

@@ -208,8 +208,8 @@ py39_main:
     python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py310_appliance_build:
@@ -253,8 +253,8 @@ py310_main:
     python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py311_appliance_build:
@@ -298,8 +298,8 @@ py311_main:
     python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py312_appliance_build:
@@ -343,8 +343,8 @@ py312_main:
     python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py313_appliance_build:
@@ -379,8 +379,8 @@ py313_main:
     python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 && pip install -U pip wheel && make prepare && make develop extras=[all]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 
@@ -420,7 +420,7 @@ cwl_v1.2:
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     # Run CWL integration tests excluded from cwl_misc
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests excluded from cwl_misc"
-    time make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k '(TestCWLWorkflow or cwl_small) and integrative'"
+    time make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k 'integrative and not conformance'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   artifacts:
     reports:
@@ -545,7 +545,7 @@ cwl_misc:
     python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the miscellaneous CWL tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k '(TestCWLWorkflow or cwl_small) and not integrative'"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k 'not integrative and not conformance'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 #cwl_v1.2_kubernetes:
@@ -642,6 +642,12 @@ server_integration:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
     - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH
+      changes:
+        compare_to: 'refs/heads/master'
+        paths:
+          - 'src/toil/server/*'
+          - 'src/toil/test/server/*'
   stage: integration
   cache:
     key: cache-$MAIN_PYTHON_PKG
@@ -657,7 +663,7 @@ server_integration:
     - python setup_gitlab_ssh.py
     - chmod 400 /root/.ssh/id_rsa
     # Test server and its integration with AWS
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/server"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/server -k 'integrative'"
 
 provisioner_integration:
   rules:
@@ -732,7 +738,7 @@ mesos:
     - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
     - export TOIL_GOOGLE_KEYNAME=id_rsa
     - export TOIL_GOOGLE_PROJECTID=toil-dev
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest src/toil/test/sort/sortTest.py::SortTest::testAwsMesos src/toil/test/sort/sortTest.py::SortTest::testFileMesos src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos src/toil/test/cwl/cwlTest.py::TestCWLv10::test_mesos_cwl_conformance src/toil/test/cwl/cwlTest.py::TestCWLv10::test_mesos_cwl_conformance_with_caching src/toil/test/src/promisedRequirementTest.py::TestMesosPromisedRequirements src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest src/toil/test/sort/sortTest.py::SortTest::testAwsMesos src/toil/test/sort/sortTest.py::SortTest::testFileMesos src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos src/toil/test/cwl/cwlTest.py::TestCWLv10Conformance::test_mesos_cwl_conformance src/toil/test/cwl/cwlTest.py::TestCWLv10Conformance::test_mesos_cwl_conformance_with_caching src/toil/test/src/promisedRequirementTest.py::TestMesosPromisedRequirements src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"
 
 batchsystem:
   rules:
@@ -780,15 +786,4 @@ cactus_integration:
     - chmod 400 /root/.ssh/id_rsa
     - make test threads="${TEST_THREADS}" tests=src/toil/test/cactus/test_cactus_integration.py
 
-# Plugin tests
-plugin:
-  rules:
-    - if: $CI_PIPELINE_SOURCE != "schedule"
-  stage: integration
-  cache:
-    key: cache-$MAIN_PYTHON_PKG
-    paths:
-      - .cache/pip
-  script:
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
-    - make test threads="${TEST_THREADS}" tests=src/toil/test/batchSystems/batch_system_plugin_test.py
+

{toil-8.2.0 → toil-9.0.0}/Makefile

@@ -65,7 +65,7 @@ the appliance images to, for example:
 You might also want to build just for one architecture and load into your
 Docker daemon. We have a 'load_docker' target for this.
 
-    make load_docker arch=amd64
+    make load_docker arch=linux/amd64
 
 If Docker is not installed, Docker-related targets tasks and tests will be skipped. The
 same can be achieved by setting TOIL_DOCKER_REGISTRY to an empty string.
@@ -129,7 +129,7 @@ develop: check_venv
 
 clean_develop: check_venv
 	- rm -rf src/*.egg-info
-	- rm src/toil/version.py
+	- rm -f src/toil/version.py
 
 uninstall:
 	- pip uninstall -y toil
@@ -149,7 +149,7 @@ dist/$(sdist_name):
 
 clean_sdist:
 	- rm -rf dist
-	- rm src/toil/version.py
+	- rm -f src/toil/version.py
 
 download_cwl_spec:
 	git clone https://github.com/common-workflow-language/cwl-v1.2.git src/toil/test/cwl/spec_v12 || true && cd src/toil/test/cwl/spec_v12 && git checkout 0d538a0dbc5518f3c6083ce4571926f65cb84f76
@@ -184,11 +184,12 @@ test_offline: check_venv check_build_reqs
 	TOIL_HISTORY=0 \
 	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(cov) $(tests) $(pytest_args)
 
-# This target will run about 1 minute of tests, and stop at the first failure
+# This target will run about 1 minute of tests, and stop at the first failure.
+# There can be more than one failure because of parallel execution.
 test_1min: check_venv check_build_reqs
-	TOIL_SKIP_DOCKER=True \
+	TOIL_SKIP_DOCKER=False \
 	TOIL_HISTORY=0 \
-	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=10  --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
+	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=30  --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
 
 ifdef TOIL_DOCKER_REGISTRY
 
@@ -206,7 +207,7 @@ endef
 
 docker: toil_docker prometheus_docker grafana_docker mtail_docker
 
-toil_docker: docker/Dockerfile
+toil_docker: docker/Dockerfile src/toil/version.py
 	mkdir -p .docker_cache
 	@set -ex \
 	; cd docker \

{toil-8.2.0 → toil-9.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toil
-Version: 8.2.0
+Version: 9.0.0
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community
@@ -53,11 +53,11 @@ Requires-Dist: boto3-stubs[autoscaling,boto3,ec2,iam,s3,sdb,sts]<2,>=1.28.3.post
 Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: moto<6,>=5.0.3; extra == "aws"
-Requires-Dist: ec2_metadata<3; extra == "aws"
+Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "aws"
 Provides-Extra: cwl
 Requires-Dist: cwltool==3.1.20250110105449; extra == "cwl"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
-Requires-Dist: galaxy-tool-util<25; extra == "cwl"
+Requires-Dist: galaxy-tool-util<26; extra == "cwl"
 Requires-Dist: galaxy-util<25; extra == "cwl"
 Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
@@ -77,18 +77,18 @@ Requires-Dist: types-urllib3; extra == "kubernetes"
 Requires-Dist: types-PyYAML; extra == "kubernetes"
 Requires-Dist: idna>=2; extra == "kubernetes"
 Provides-Extra: wdl
-Requires-Dist: miniwdl==1.12.1; extra == "wdl"
+Requires-Dist: miniwdl==1.13.0; extra == "wdl"
 Requires-Dist: wdlparse==0.1.0; extra == "wdl"
 Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
 Provides-Extra: server
-Requires-Dist: connexion[swagger-ui]<4,>=2.10.0; extra == "server"
-Requires-Dist: flask<2.3,>=2.0; extra == "server"
-Requires-Dist: werkzeug<4,>=2.0; extra == "server"
-Requires-Dist: flask-cors==5.0.1; extra == "server"
-Requires-Dist: gunicorn==23.0.0; extra == "server"
-Requires-Dist: celery<6,>=5.1.0; extra == "server"
-Requires-Dist: wes-service<5,>=4.0.0; extra == "server"
-Requires-Dist: ruamel.yaml<0.19,>=0.15; extra == "server"
+Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "server"
+Requires-Dist: flask<4,>=3; extra == "server"
+Requires-Dist: werkzeug<4,>=3; extra == "server"
+Requires-Dist: flask-cors<6,>=5.0.1; extra == "server"
+Requires-Dist: gunicorn<24,>=23.0.0; extra == "server"
+Requires-Dist: celery<6,>=5.4.0; extra == "server"
+Requires-Dist: wes-service<6,>=5.0.0; extra == "server"
+Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "server"
 Provides-Extra: htcondor
 Requires-Dist: htcondor<25,>=23.6.0; sys_platform != "darwin" and extra == "htcondor"
 Provides-Extra: mesos
@@ -98,10 +98,10 @@ Requires-Dist: boto3-stubs[autoscaling,boto3,ec2,iam,s3,sdb,sts]<2,>=1.28.3.post
 Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: moto<6,>=5.0.3; extra == "all"
-Requires-Dist: ec2_metadata<3; extra == "all"
+Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "all"
 Requires-Dist: cwltool==3.1.20250110105449; extra == "all"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
-Requires-Dist: galaxy-tool-util<25; extra == "all"
+Requires-Dist: galaxy-tool-util<26; extra == "all"
 Requires-Dist: galaxy-util<25; extra == "all"
 Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
@@ -117,17 +117,17 @@ Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
 Requires-Dist: types-urllib3; extra == "all"
 Requires-Dist: types-PyYAML; extra == "all"
 Requires-Dist: idna>=2; extra == "all"
-Requires-Dist: miniwdl==1.12.1; extra == "all"
+Requires-Dist: miniwdl==1.13.0; extra == "all"
 Requires-Dist: wdlparse==0.1.0; extra == "all"
 Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
-Requires-Dist: connexion[swagger-ui]<4,>=2.10.0; extra == "all"
-Requires-Dist: flask<2.3,>=2.0; extra == "all"
-Requires-Dist: werkzeug<4,>=2.0; extra == "all"
-Requires-Dist: flask-cors==5.0.1; extra == "all"
-Requires-Dist: gunicorn==23.0.0; extra == "all"
-Requires-Dist: celery<6,>=5.1.0; extra == "all"
-Requires-Dist: wes-service<5,>=4.0.0; extra == "all"
-Requires-Dist: ruamel.yaml<0.19,>=0.15; extra == "all"
+Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "all"
+Requires-Dist: flask<4,>=3; extra == "all"
+Requires-Dist: werkzeug<4,>=3; extra == "all"
+Requires-Dist: flask-cors<6,>=5.0.1; extra == "all"
+Requires-Dist: gunicorn<24,>=23.0.0; extra == "all"
+Requires-Dist: celery<6,>=5.4.0; extra == "all"
+Requires-Dist: wes-service<6,>=5.0.0; extra == "all"
+Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "all"
 Requires-Dist: pymesos<0.4,>=0.3.15; python_version < "3.11" and extra == "all"
 Dynamic: author
 Dynamic: author-email

{toil-8.2.0 → toil-9.0.0}/docker/Dockerfile.py

@@ -72,6 +72,21 @@ dependencies = ' '.join(python_packages[python] +
                          # Dependencies for singularity on kubernetes
                          'tzdata'])
 
+# pymesos's http-parser dependency can't build on Python later than 3.10, as
+# released in 0.9.0. The upstream pymesos can, but we write it out of Toil's
+# dependencies on later Python versions, since a working http-parser is not
+# available in PyPI. So we need to manually inject a working http-parser, and
+# pymesos, into the Docker images.
+extra_mesos_python_modules = {
+    'python3.9': [],
+    'python3.10': [],
+    'python3.11': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
+    'python3.12': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
+    'python3.13': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15']
+}
+
+extra_python_modules = " ".join(extra_mesos_python_modules[python])
+
 
 def heredoc(s):
     s = textwrap.dedent(s).format(**globals())
@@ -182,18 +197,22 @@ print(heredoc('''
     # The stock pip is too old and can't install from sdist with extras
     RUN curl -sS https://bootstrap.pypa.io/get-pip.py | {python}
 
-    # Include virtualenv, as it is still the recommended way to deploy pipelines
-    RUN {pip} install --upgrade virtualenv==20.25.1
+    # Include virtualenv, as it is still the recommended way to deploy
+    # pipelines.
+    #
+    # We need to --ignore-installed here to allow shadowing system packages
+    # from apt in /usr/lib/python3/dist-packages when the installed package
+    # needs newer versions. We just hope that doesn't break the Ubuntu system
+    # too badly when we're actually on the system Python, or if Toil needs to
+    # upgrade a distutils or setuptools dependency. On the deadsnakes Pythons,
+    # installations into the version-specific package directory won't be seen
+    # by the system Python which is a different version.
+    #
+    # TODO: Change to nested virtual environments and .pth files and teach Toil
+    # to just ship the user-level one for hot deploy.
+    RUN {pip} install --ignore-installed --upgrade 'virtualenv>=20.25.1,<21'
 
-    # Install s3am (--never-download prevents silent upgrades to pip, wheel and setuptools)
-    # Install setuptools within the virtual environment to properly access distutils due to PEP 632 and gh-95299 in Python 3.12 release notes
-    # https://docs.python.org/3/whatsnew/3.12.html#summary-release-highlights
-    RUN virtualenv --python {python} --never-download /home/s3am \
-        && /home/s3am/bin/pip install setuptools \
-        && /home/s3am/bin/pip install s3am==2.0 \
-        && ln -s /home/s3am/bin/s3am /usr/local/bin/
-    
-    RUN {pip} install --upgrade setuptools==69.2.0
+    RUN {pip} install --ignore-installed --upgrade 'setuptools>=80,<81'
 
     # Fix for https://issues.apache.org/jira/browse/MESOS-3793
     ENV MESOS_LAUNCHER=posix
@@ -222,7 +241,7 @@ print(heredoc('''
 
     # This component changes most frequently and keeping it last maximizes Docker cache hits.
     COPY {sdistName} .
-    RUN {pip} install {sdistName}[all]
+    RUN {pip} install --ignore-installed --upgrade {sdistName}[all] {extra_python_modules}
     RUN rm {sdistName}
 
     # We intentionally inherit the default ENTRYPOINT and CMD from the base image, to the effect

{toil-8.2.0 → toil-9.0.0}/docs/cwl/conformance.rst

@@ -64,6 +64,8 @@ CWL 1.2 Optional Features
 
 .. |docker| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/docker.json%3Fjob=cwl_badge
 
+.. |env_var| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/env_var.json%3Fjob=cwl_badge
+
 .. |expression_tool| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/expression_tool.json%3Fjob=cwl_badge
 
 .. |format_checking| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/format_checking.json%3Fjob=cwl_badge
@@ -82,16 +84,26 @@ CWL 1.2 Optional Features
 
 .. |multiple_input| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/multiple_input.json%3Fjob=cwl_badge
 
+.. |multiple| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/multiple.json%3Fjob=cwl_badge
+
 .. |networkaccess| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/networkaccess.json%3Fjob=cwl_badge
 
+.. |require| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/require.json%3Fjob=cwl_badge
+
 .. |resource| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/resource.json%3Fjob=cwl_badge
 
 .. |scatter| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/scatter.json%3Fjob=cwl_badge
 
 .. |schema_def| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/schema_def.json%3Fjob=cwl_badge
 
+.. |secondary_files| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/secondary_files.json%3Fjob=cwl_badge
+
+.. |shell_command| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/shell_command.json%3Fjob=cwl_badge
+
 .. |step_input| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/step_input.json%3Fjob=cwl_badge
 
+.. |subworkflow| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/subworkflow.json%3Fjob=cwl_badge
+
 .. |timelimit| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/timelimit.json%3Fjob=cwl_badge
 
 .. |workflow| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/workflow.json%3Fjob=cwl_badge

{toil-8.2.0 → toil-9.0.0}/docs/cwl/running.rst

@@ -100,8 +100,8 @@ Besides the normal Toil options and the options supported by cwltool, toil-cwl-r
   --reference-inputs    Do not copy remote inputs into Toil's file store and assume they are accessible in place from all nodes.
   --disable-streaming   Do not allow streaming of job input files. By default, files marked with ``streamable`` True are streamed from remote job stores.
   --cwl-default-ram     Apply CWL specification default ramMin.
-  --no-cwl-default-ram  Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies.
-
+  --no-cwl-default-ram  Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies. This can help jobs get to Slurm with no memory limit assigned.
+  --cwl-min-ram BYTES   Specify a minimum memory allocation for all tasks. If ``--no-cwl-default-ram`` is passed, this does not apply to tools that do not specify a memory requirement; ``--defaultMemory`` is used for those tools in that case.
 
 Running CWL in the Cloud
 ------------------------

{toil-8.2.0 → toil-9.0.0}/docs/gettingStarted/quickStart.rst

@@ -150,10 +150,73 @@ For something beyond a "Hello, world!" example, refer to :ref:`runningDetail`.
 
 Toil's customization options are available in Python workflows. Run ``python3 helloWorld.py --help`` to see a complete list of available options.
 
+
+.. _mitochondriaExample:
+
+Example: mitochondrial variant calling
+--------------------------------------
+
+Let's run a more realistic workflow with Toil. This workflow is mitochondrial variant calling: It will take the human reference genome and sequenced reads from an individual and determine how that person's mitochondrial DNA differs from the reference genome.
+
+We will run an example `workflow from Dockstore <https://dockstore.org/workflows/github.com/broadinstitute/gatk/MitochondriaPipeline:master?tab=info>`_.
+First, grab an example workflow input::
+
+    (venv) $ wget https://toil-datasets.s3.us-west-2.amazonaws.com/MitochondriaInputs.zip && unzip MitochondriaInputs.zip
+
+Then, change your current working directory::
+
+    (venv) $ cd MitochondriaInputs
+
+This workflow will take approximately 30 minutes to run.
+
+Since Toil supports Dockstore TRS IDs that allows the WDL runner to run any workflow on Dockstore, we will run it directly::
+
+      (venv) $ toil-wdl-runner '#workflow/github.com/broadinstitute/gatk/MitochondriaPipeline:master' -i ExampleInputsMitochondriaPipeline.json --logInfo --container docker --quantCheck false --outputFile mitochondria.json
+
+
+.. note::
+        * ``--logInfo`` runs the workflow with INFO level logging. For different levels of logging, see ``--logLevel``, ``--logCritical``, ``--logError``, ``--logWarning``, ``--logDebug``, and ``--logTrace``.
+        * ``--container docker`` uses Docker as the container backend. By default, Toil will run with Singularity. To set explicitly, use ``--container singularity``.
+        * ``--outputFile`` will put the workflow JSON outputs into a file. If omitted, Toil will put the workflow outputs onto the commandline.
+        * ``--quantCheck false`` disables certain type checks. This is useful for Cromwell compatibility.
+
+Unless fakeroot support is set up for Singularity, this particular workflow must be run with Docker because it assumes commands in the container will run as root.
+Additionally, WDL workflows sometimes depend on non-spec compliant behaviors. To see if Toil has an workaround option, see :ref:`wdlOptions`.
+
+Once the workflow is done running, you can look at your JSON output with ``jq . mitochondria.json``. For example, if we want the ``out_vcf`` output from the workflow, we can run ``jq -r '.["MitochondriaPipeline.out_vcf"]' mitochondria.json`` to get its path::
+
+    /private/groups/patenlab/toil-dev/mitochondria/wdl-out-c6o9mjop/MitochondriaPipeline.AlignAndCall.FilterContamination/HG02571.GRCh38.chrM.vcf
+
+To open the VCF file, we can run ``less $(jq -r '.["MitochondriaPipeline.out_vcf"]' mitochondria.json)``
+
+.. note::
+        For outputs that aren't files, their values are directly in the JSON. For example, with ``jq '.["MitochondriaPipeline.median_coverage"]' mitochondria.json`` you can fetch the ``median_coverage`` output's value::
+
+            4183.5
+
+
+Toil uses a jobstore to store all of a workflow's files and to communicate between workers. If not specified, Toil will use an ephemeral directory that is deleted after Toil is done running.
+To control where those files are placed or allow a workflow to be restarted, you can use the ``--jobStore`` option. If you specify a jobstore explicitly, the jobstore will stick around if the workflow fails. To keep the jobstore after a successful completion, use ``--clean never``. To remove the jobstore even after a failing run, use ``--clean always``.
+
+On a cluster, the jobstore must be somewhere accessible to all worker nodes. Here's an example of running the workflow with a specified jobstore::
+
+      (venv) $ toil-wdl-runner MitochondriaPipeline.wdl -i ExampleInputsMitochondriaPipeline.json --logInfo --container docker --outputFile mitochondria.json --jobstore mitochondriaJobstore
+
+Toil supports several batch systems. By default, Toil will use ``single_machine``, which will run everything on the local machine. Other batch systems are available. For example, you can use `--batchSystem slurm` to run on a Slurm cluster::
+
+      (venv) $ toil-wdl-runner MitochondriaPipeline.wdl -i ExampleInputsMitochondriaPipeline.json --logInfo --container docker --outputFile mitochondria.json --jobStore mitochondriaJobstore --batchSystem slurm
+
+See :ref:`runningSlurm` for more information, including how to specify time limits and partitions.
+
+Sometimes, a workflow may fail. If this is the case, the workflow can be restarted from the point of failure with ``--restart``, as long as you still have the jobstore::
+
+    (venv) $ toil-wdl-runner MitochondriaPipeline.wdl -i ExampleInputsMitochondriaPipeline.json --logInfo --container docker --outputFile mitochondria.json --jobStore mitochondriaJobstore --batchSystem slurm
+
+
 .. _runningDetail:
 
-A (more) real-world example
----------------------------
+Example: sorting
+----------------
 
 For a more detailed example and explanation, we've developed a sample pipeline
 that merge-sorts a temporary file. This is not supposed to be an efficient

{toil-8.2.0 → toil-9.0.0}/docs/running/cliOptions.rst

@@ -527,7 +527,8 @@ systems have issues!).
 
 **Miscellaneous Options**
 
-  --disableChaining     Disables chaining of jobs (chaining uses one job's
+  --disableChaining BOOL
+                        Disables chaining of jobs (chaining uses one job's
                         resource allocation for its successor job if
                         possible).
   --disableJobStoreChecksumVerification

{toil-8.2.0 → toil-9.0.0}/docs/wdl/running.rst

@@ -48,6 +48,8 @@ workflow, you can do::
 .. _`5.12.0`: https://github.com/DataBiosphere/toil/releases/tag/releases%2F5.12.0
 .. _`Cromwell`: https://github.com/broadinstitute/cromwell#readme
 
+.. _wdlOptions:
+
 Toil WDL Runner Options
 -----------------------
 
@@ -84,6 +86,23 @@ the `Cromwell Output Organizer (croo)`_, will default to ``True``.
 
 .. _`Cromwell Output Organizer (croo)`: https://github.com/ENCODE-DCC/croo
 
+``--strict``: Specifies whether Toil should immediately exit on a lint warning. By default, this is false.
+
+``--quantCheck``: Specifies whether quantifier validation type checking should be enabled.
+Disabling this is useful for workflows which Cromwell can run but don't fully comply with the WDL spec.
+By default, this is true.
+
+``--runImportsOnWorkers``: Run file imports on workers. This is useful if the leader is not network optimized
+and lots of downloads are necessary. By default, this is false.
+
+``--importWorkersBatchSize``: Requires ``--runImportsOnWorkers`` to be true. Specify the target batch size in bytes for batched imports.
+As many files as can fit will go into each batch import job. This also accepts abbreviations, such as ``G`` or ``Gi``.
+
+``--importWorkersDisk``: Requires ``--runImportsOnWorkers`` to be true. Specify the disk size each import worker will get.
+This usually will not need to be set as Toil will attempt to use file streaming when downloading files.
+If not possible, for example, when downloading from AWS to a GCE job store,
+this should be set to the largest file size of all files to import. By default, this is 1 MiB.
+
 Any number of other Toil options may also be specified. For defined Toil options,
 see :ref:`commandRef`.
 
@@ -121,6 +140,21 @@ The final number is a sequential counter: if a step has to be retried, or if
 you run the workflow multiple times without clearing out the logs directory, it
 will increment.
 
+Enabling WDL Call Cache
+-----------------------
+
+Toil can cache the task and workflow outputs to use outputs of already ran tasks and workflows.
+This can save time when debugging long running workflows where a later task fails. However, this is only guaranteed for
+running locally and can use up a considerable amount of disk space.
+
+To use, set the following environment variables before running the workflow::
+
+    export MINIWDL__CALL_CACHE__PUT=True
+    export MINIWDL__CALL_CACHE__GET=True
+    export MINIWDL__CALL_CACHE__DIR=/absolute_path/to/cache
 
+The path to the cache directory must be an absolute path.
 
+For setting up call cache permanently, see the `MiniWDL call cache`_ documentation.
 
+.. _MiniWDL call cache: https://miniwdl.readthedocs.io/en/latest/runner_reference.html#call-cache

{toil-8.2.0 → toil-9.0.0}/requirements-aws.txt

@@ -2,4 +2,4 @@ boto3-stubs[s3,sdb,iam,sts,boto3,ec2,autoscaling]>=1.28.3.post2, <2
 mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced
 mypy-boto3-s3>=1.28.3.post2, <2
 moto>=5.0.3, <6
-ec2_metadata<3
+ec2_metadata>=2.14.0,<3

{toil-8.2.0 → toil-9.0.0}/requirements-cwl.txt

@@ -1,6 +1,6 @@
 cwltool==3.1.20250110105449
 schema-salad>=8.4.20230128170514,<9
-galaxy-tool-util<25
+galaxy-tool-util<26
 galaxy-util<25
 ruamel.yaml>=0.15,<=0.19
 ruamel.yaml.clib>=0.2.6

{toil-8.2.0 → toil-9.0.0}/requirements-dev.txt

@@ -1,6 +1,7 @@
 pytest>=6.2.1,<9
 pytest-cov>=2.12.1,<7
 pytest-timeout>=1.4.2,<3
+pytest_httpserver>=1.1.3,<2
 pytest-subtests
 stubserver>=1.1,<2
 setuptools>=65.5.1,<81
@@ -10,7 +11,7 @@ astroid>=3,<4
 sphinx-autodoc-typehints>=1.24.0,<4
 sphinxcontrib-autoprogram==0.1.9
 cwltest>=2.6.20250314152537
-mypy==1.15.0
+mypy==1.16.1
 types-aws-xray-sdk
 types-boto<2.49.18.20241020
 types-Flask-Cors

toil-9.0.0/requirements-server.txt

@@ -0,0 +1,8 @@
+connexion[swagger-ui]>=3,<4
+flask>=3,<4
+werkzeug>=3,<4
+flask-cors>=5.0.1,<6
+gunicorn>=23.0.0,<24
+celery>=5.4.0,<6
+wes-service>=5.0.0,<6
+ruamel.yaml>=0.18.7,<1

{toil-8.2.0 → toil-9.0.0}/requirements-wdl.txt

@@ -1,3 +1,3 @@
-miniwdl==1.12.1
+miniwdl==1.13.0
 wdlparse==0.1.0
 graphlib-backport==1.0 ; python_version < '3.9'

{toil-8.2.0 → toil-9.0.0}/setup.cfg

@@ -31,6 +31,7 @@ markers =
 	singularity
 	torque
 	wes_server
+	conformance
 	cwl_small_log_dir
 	cwl_small
 	wdl