toil 8.2.0__tar.gz → 9.1.0__tar.gz

{toil-8.2.0 → toil-9.1.0}/.github/workflows/codeql-analysis.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

{toil-8.2.0 → toil-9.1.0}/.gitlab-ci.yml

@@ -164,7 +164,8 @@ quick_test_offline:
     make develop extras=[aws,google,wdl]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the offline tests"
-    TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"
+    # Make sure not to run the doctests here because we didn't install all possible extras
+    TOIL_TEST_QUICK=True make test_offline tests=src/toil/test threads="${TEST_THREADS}"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py39_appliance_build:
@@ -208,8 +209,9 @@ py39_main:
     python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make doctest threads="${TEST_THREADS}"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py310_appliance_build:
@@ -253,8 +255,9 @@ py310_main:
     python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make doctest threads="${TEST_THREADS}"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py311_appliance_build:
@@ -298,8 +301,9 @@ py311_main:
     python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make doctest threads="${TEST_THREADS}"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py312_appliance_build:
@@ -343,8 +347,9 @@ py312_main:
     python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make doctest threads="${TEST_THREADS}"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py313_appliance_build:
@@ -379,11 +384,11 @@ py313_main:
     python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 && pip install -U pip wheel && make prepare && make develop extras=[all]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests=src/toil/test/lib
+    make doctest threads="${TEST_THREADS}"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
-
 slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -420,7 +425,7 @@ cwl_v1.2:
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     # Run CWL integration tests excluded from cwl_misc
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests excluded from cwl_misc"
-    time make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k '(TestCWLWorkflow or cwl_small) and integrative'"
+    time make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k 'integrative and not conformance'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   artifacts:
     reports:
@@ -458,7 +463,7 @@ cwl_badge:
     rm -rf badges1.2
     echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
-    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
+    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   allow_failure: true
   artifacts:
@@ -491,7 +496,7 @@ cwl_badge_release:
     rm -rf badges1.2
     echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
-    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
+    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   allow_failure: true
   artifacts:
@@ -545,7 +550,7 @@ cwl_misc:
     python setup_gitlab_docker.py  # login to increase the docker.io rate limit
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the miscellaneous CWL tests"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k '(TestCWLWorkflow or cwl_small) and not integrative'"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/cwl/cwlTest.py -k 'not integrative and not conformance'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 #cwl_v1.2_kubernetes:
@@ -642,6 +647,12 @@ server_integration:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
     - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH
+      changes:
+        compare_to: 'refs/heads/master'
+        paths:
+          - 'src/toil/server/*'
+          - 'src/toil/test/server/*'
   stage: integration
   cache:
     key: cache-$MAIN_PYTHON_PKG
@@ -657,7 +668,7 @@ server_integration:
     - python setup_gitlab_ssh.py
     - chmod 400 /root/.ssh/id_rsa
     # Test server and its integration with AWS
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/server"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/server -k 'integrative'"
 
 provisioner_integration:
   rules:
@@ -679,7 +690,7 @@ provisioner_integration:
     - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
     # This reads GITLAB_SECRET_FILE_SSH_KEYS
     - python setup_gitlab_ssh.py
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::TestUtils::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest src/toil/test/wdl/wdltoil_test_kubernetes.py::WDLKubernetesClusterTest"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::TestUtils::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::TestAWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/wdl/wdltoil_test_kubernetes.py::WDLKubernetesClusterTest"
 #    - make test tests=src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run
 
 google_jobstore:
@@ -732,7 +743,7 @@ mesos:
     - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
     - export TOIL_GOOGLE_KEYNAME=id_rsa
     - export TOIL_GOOGLE_PROJECTID=toil-dev
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest src/toil/test/sort/sortTest.py::SortTest::testAwsMesos src/toil/test/sort/sortTest.py::SortTest::testFileMesos src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos src/toil/test/cwl/cwlTest.py::TestCWLv10::test_mesos_cwl_conformance src/toil/test/cwl/cwlTest.py::TestCWLv10::test_mesos_cwl_conformance_with_caching src/toil/test/src/promisedRequirementTest.py::TestMesosPromisedRequirements src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest src/toil/test/sort/sortTest.py::SortTest::testAwsMesos src/toil/test/sort/sortTest.py::SortTest::testFileMesos src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos src/toil/test/cwl/cwlTest.py::TestCWLv10Conformance::test_mesos_cwl_conformance src/toil/test/cwl/cwlTest.py::TestCWLv10Conformance::test_mesos_cwl_conformance_with_caching src/toil/test/src/promisedRequirementTest.py::TestMesosPromisedRequirements src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"
 
 batchsystem:
   rules:
@@ -780,15 +791,4 @@ cactus_integration:
     - chmod 400 /root/.ssh/id_rsa
     - make test threads="${TEST_THREADS}" tests=src/toil/test/cactus/test_cactus_integration.py
 
-# Plugin tests
-plugin:
-  rules:
-    - if: $CI_PIPELINE_SOURCE != "schedule"
-  stage: integration
-  cache:
-    key: cache-$MAIN_PYTHON_PKG
-    paths:
-      - .cache/pip
-  script:
-    - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
-    - make test threads="${TEST_THREADS}" tests=src/toil/test/batchSystems/batch_system_plugin_test.py
+

{toil-8.2.0 → toil-9.1.0}/Makefile

@@ -65,7 +65,7 @@ the appliance images to, for example:
 You might also want to build just for one architecture and load into your
 Docker daemon. We have a 'load_docker' target for this.
 
-    make load_docker arch=amd64
+    make load_docker arch=linux/amd64
 
 If Docker is not installed, Docker-related targets tasks and tests will be skipped. The
 same can be achieved by setting TOIL_DOCKER_REGISTRY to an empty string.
@@ -85,7 +85,8 @@ help:
 
 # This Makefile uses bash features like printf and <()
 SHELL=bash
-tests=src/toil/test
+# We need the default tests to include the doctests, but not htcondor which isn't on mac.
+tests=src/toil --ignore src/toil/batchSystems/htcondor.py
 arch=linux/amd64,linux/arm64
 cov=--cov=toil
 logging=--log-format="%(asctime)s %(levelname)s %(message)s" --log-level DEBUG -o log_cli=true --log-cli-level INFO
@@ -109,7 +110,7 @@ marker=""
 threads:="auto"
 
 dist:="worksteal"
-pytest_args:=""
+pytest_args:="--randomly-dont-reorganize"
 
 # Only pass the threading options if running parallel tests. Otherwise we lose
 # live logging. See <https://stackoverflow.com/q/62533239>
@@ -129,7 +130,7 @@ develop: check_venv
 
 clean_develop: check_venv
 	- rm -rf src/*.egg-info
-	- rm src/toil/version.py
+	- rm -f src/toil/version.py
 
 uninstall:
 	- pip uninstall -y toil
@@ -149,7 +150,7 @@ dist/$(sdist_name):
 
 clean_sdist:
 	- rm -rf dist
-	- rm src/toil/version.py
+	- rm -f src/toil/version.py
 
 download_cwl_spec:
 	git clone https://github.com/common-workflow-language/cwl-v1.2.git src/toil/test/cwl/spec_v12 || true && cd src/toil/test/cwl/spec_v12 && git checkout 0d538a0dbc5518f3c6083ce4571926f65cb84f76
@@ -169,10 +170,20 @@ test: check_venv check_build_reqs
 	TOIL_HISTORY=0 \
 	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(cov) $(tests) $(pytest_args)
 
+# When running doctests, we need to not capture output, because on CI we can
+# get failures where doctest saw no output and we report captured output, as in
+# <https://ucsc-ci.com/databiosphere/toil/-/jobs/96131>. doctest and pytest's
+# captures might not work properly together.
+doctest: check_venv check_build_reqs
+	TOIL_OWNER_TAG="shared" \
+	TOIL_HISTORY=0 \
+	    python -m pytest --capture=no $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(cov) $(tests) --ignore src/toil/test $(pytest_args)
+
 test_debug: check_venv check_build_reqs
+	# Don't include threadopts to make sure we can see our live logging.
 	TOIL_OWNER_TAG="$(whoami)" \
 	TOIL_HISTORY=0 \
-	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(tests) $(pytest_args) --maxfail=1
+	    python -m pytest $(verbose) $(durations) -m "$(marker)" $(logging) $(tests) $(pytest_args) --maxfail=1
 
 
 # This target will skip building docker and all docker based tests
@@ -184,11 +195,12 @@ test_offline: check_venv check_build_reqs
 	TOIL_HISTORY=0 \
 	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(cov) $(tests) $(pytest_args)
 
-# This target will run about 1 minute of tests, and stop at the first failure
+# This target will run about 1 minute of tests, and stop at the first failure.
+# There can be more than one failure because of parallel execution.
 test_1min: check_venv check_build_reqs
-	TOIL_SKIP_DOCKER=True \
+	TOIL_SKIP_DOCKER=False \
 	TOIL_HISTORY=0 \
-	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=10  --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
+	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=30 --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
 
 ifdef TOIL_DOCKER_REGISTRY
 
@@ -206,7 +218,7 @@ endef
 
 docker: toil_docker prometheus_docker grafana_docker mtail_docker
 
-toil_docker: docker/Dockerfile
+toil_docker: docker/Dockerfile src/toil/version.py
 	mkdir -p .docker_cache
 	@set -ex \
 	; cd docker \
@@ -377,7 +389,7 @@ preflight: mypy touched_pylint
 		develop clean_develop \
 		sdist clean_sdist \
 		download_cwl_spec \
-		test test_offline test_1min \
+		test doctest test_offline test_1min \
 		docs clean_docs \
 		clean \
 		sort_imports remove_unused_imports remove_trailing_whitespace \

{toil-8.2.0 → toil-9.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toil
-Version: 8.2.0
+Version: 9.1.0
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community
@@ -33,7 +33,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: dill<0.4,>=0.3.2
-Requires-Dist: requests<=2.31.0
+Requires-Dist: requests<=2.32.5
 Requires-Dist: docker<8,>=6.1.0
 Requires-Dist: urllib3<3,>=1.26.0
 Requires-Dist: python-dateutil
@@ -53,12 +53,12 @@ Requires-Dist: boto3-stubs[autoscaling,boto3,ec2,iam,s3,sdb,sts]<2,>=1.28.3.post
 Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: moto<6,>=5.0.3; extra == "aws"
-Requires-Dist: ec2_metadata<3; extra == "aws"
+Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "aws"
 Provides-Extra: cwl
-Requires-Dist: cwltool==3.1.20250110105449; extra == "cwl"
+Requires-Dist: cwltool==3.1.20250715140722; extra == "cwl"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
-Requires-Dist: galaxy-tool-util<25; extra == "cwl"
-Requires-Dist: galaxy-util<25; extra == "cwl"
+Requires-Dist: galaxy-tool-util<26; extra == "cwl"
+Requires-Dist: galaxy-util<26; extra == "cwl"
 Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
 Requires-Dist: networkx!=2.8.1,<4; extra == "cwl"
@@ -67,28 +67,28 @@ Requires-Dist: cwl-utils>=0.36; extra == "cwl"
 Provides-Extra: encryption
 Requires-Dist: pynacl<2,>=1.4.0; extra == "encryption"
 Provides-Extra: google
-Requires-Dist: apache-libcloud<3,>=2.2.1; extra == "google"
+Requires-Dist: apache-libcloud<4,>=3.6.1; extra == "google"
 Requires-Dist: google-cloud-storage<=2.8.0,>=2; extra == "google"
 Requires-Dist: google-auth<3,>=2.18.1; extra == "google"
 Provides-Extra: kubernetes
-Requires-Dist: kubernetes<33,>=12.0.1; extra == "kubernetes"
+Requires-Dist: kubernetes<34,>=12.0.1; extra == "kubernetes"
 Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "kubernetes"
 Requires-Dist: types-urllib3; extra == "kubernetes"
 Requires-Dist: types-PyYAML; extra == "kubernetes"
 Requires-Dist: idna>=2; extra == "kubernetes"
 Provides-Extra: wdl
-Requires-Dist: miniwdl==1.12.1; extra == "wdl"
+Requires-Dist: miniwdl==1.13.0; extra == "wdl"
 Requires-Dist: wdlparse==0.1.0; extra == "wdl"
 Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
 Provides-Extra: server
-Requires-Dist: connexion[swagger-ui]<4,>=2.10.0; extra == "server"
-Requires-Dist: flask<2.3,>=2.0; extra == "server"
-Requires-Dist: werkzeug<4,>=2.0; extra == "server"
-Requires-Dist: flask-cors==5.0.1; extra == "server"
-Requires-Dist: gunicorn==23.0.0; extra == "server"
-Requires-Dist: celery<6,>=5.1.0; extra == "server"
-Requires-Dist: wes-service<5,>=4.0.0; extra == "server"
-Requires-Dist: ruamel.yaml<0.19,>=0.15; extra == "server"
+Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "server"
+Requires-Dist: flask<4,>=3; extra == "server"
+Requires-Dist: werkzeug<4,>=3; extra == "server"
+Requires-Dist: flask-cors<7,>=5.0.1; extra == "server"
+Requires-Dist: gunicorn<24,>=23.0.0; extra == "server"
+Requires-Dist: celery<6,>=5.4.0; extra == "server"
+Requires-Dist: wes-service<6,>=5.0.0; extra == "server"
+Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "server"
 Provides-Extra: htcondor
 Requires-Dist: htcondor<25,>=23.6.0; sys_platform != "darwin" and extra == "htcondor"
 Provides-Extra: mesos
@@ -98,36 +98,36 @@ Requires-Dist: boto3-stubs[autoscaling,boto3,ec2,iam,s3,sdb,sts]<2,>=1.28.3.post
 Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: moto<6,>=5.0.3; extra == "all"
-Requires-Dist: ec2_metadata<3; extra == "all"
-Requires-Dist: cwltool==3.1.20250110105449; extra == "all"
+Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "all"
+Requires-Dist: cwltool==3.1.20250715140722; extra == "all"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
-Requires-Dist: galaxy-tool-util<25; extra == "all"
-Requires-Dist: galaxy-util<25; extra == "all"
+Requires-Dist: galaxy-tool-util<26; extra == "all"
+Requires-Dist: galaxy-util<26; extra == "all"
 Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
 Requires-Dist: networkx!=2.8.1,<4; extra == "all"
 Requires-Dist: CacheControl[filecache]; extra == "all"
 Requires-Dist: cwl-utils>=0.36; extra == "all"
 Requires-Dist: pynacl<2,>=1.4.0; extra == "all"
-Requires-Dist: apache-libcloud<3,>=2.2.1; extra == "all"
+Requires-Dist: apache-libcloud<4,>=3.6.1; extra == "all"
 Requires-Dist: google-cloud-storage<=2.8.0,>=2; extra == "all"
 Requires-Dist: google-auth<3,>=2.18.1; extra == "all"
-Requires-Dist: kubernetes<33,>=12.0.1; extra == "all"
+Requires-Dist: kubernetes<34,>=12.0.1; extra == "all"
 Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
 Requires-Dist: types-urllib3; extra == "all"
 Requires-Dist: types-PyYAML; extra == "all"
 Requires-Dist: idna>=2; extra == "all"
-Requires-Dist: miniwdl==1.12.1; extra == "all"
+Requires-Dist: miniwdl==1.13.0; extra == "all"
 Requires-Dist: wdlparse==0.1.0; extra == "all"
 Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
-Requires-Dist: connexion[swagger-ui]<4,>=2.10.0; extra == "all"
-Requires-Dist: flask<2.3,>=2.0; extra == "all"
-Requires-Dist: werkzeug<4,>=2.0; extra == "all"
-Requires-Dist: flask-cors==5.0.1; extra == "all"
-Requires-Dist: gunicorn==23.0.0; extra == "all"
-Requires-Dist: celery<6,>=5.1.0; extra == "all"
-Requires-Dist: wes-service<5,>=4.0.0; extra == "all"
-Requires-Dist: ruamel.yaml<0.19,>=0.15; extra == "all"
+Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "all"
+Requires-Dist: flask<4,>=3; extra == "all"
+Requires-Dist: werkzeug<4,>=3; extra == "all"
+Requires-Dist: flask-cors<7,>=5.0.1; extra == "all"
+Requires-Dist: gunicorn<24,>=23.0.0; extra == "all"
+Requires-Dist: celery<6,>=5.4.0; extra == "all"
+Requires-Dist: wes-service<6,>=5.0.0; extra == "all"
+Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "all"
 Requires-Dist: pymesos<0.4,>=0.3.15; python_version < "3.11" and extra == "all"
 Dynamic: author
 Dynamic: author-email

{toil-8.2.0 → toil-9.1.0}/contrib/admin/cleanup_aws_resources.py

@@ -70,8 +70,11 @@ def contains_toil_test_patterns(string):
 
 
 def matches(resource_name):
-    if (resource_name.endswith('--files') or resource_name.endswith('--jobs') or resource_name.endswith('_toil')
-            or resource_name.endswith('--internal') or resource_name.startswith('toil-s3test-')):
+    if (
+        resource_name.endswith('--files') or resource_name.endswith('--jobs')
+        or resource_name.endswith('--toil') or resource_name.endswith('_toil')
+        or resource_name.endswith('--internal') or resource_name.startswith('toil-s3test-')
+    ):
         if contains_toil_test_patterns(resource_name):
             return resource_name
 

{toil-8.2.0 → toil-9.1.0}/contrib/admin/mypy-with-ignore.py

@@ -22,7 +22,8 @@ def main():
 
     # TODO: Remove these paths as typing is added and mypy conflicts are addressed.
     # These are handled as path prefixes.
-    ignore_paths = [os.path.abspath(f) for f in [
+
+    ignore_paths = [os.path.join(pkg_root, f) for f in [
         'docs/_build',
         'docker/Dockerfile.py',
         'docs/conf.py',
@@ -61,12 +62,8 @@ def main():
         'src/toil/jobStores/fileJobStore.py',
         'src/toil/jobStores/__init__.py',
         'src/toil/jobStores/googleJobStore.py',
-        'src/toil/jobStores/aws/utils.py',
-        'src/toil/jobStores/aws/jobStore.py',
-        'src/toil/jobStores/aws/__init__.py',
         'src/toil/utils/__init__.py',
         'src/toil/lib/throttle.py',
-        'src/toil/lib/iterables.py',
         'src/toil/lib/bioio.py',
         'src/toil/lib/ec2.py',
         'src/toil/lib/expando.py',
@@ -87,7 +84,6 @@ def main():
         'src/toil/test/cactus',
         'src/toil/test/docs',
         'src/toil/test/jobStores',
-        'src/toil/test/lib/aws',
         'src/toil/test/lib/test_conversions.py',
         'src/toil/test/lib/test_ec2.py',
         'src/toil/test/lib/test_misc.py',

{toil-8.2.0 → toil-9.1.0}/docker/Dockerfile.py

@@ -72,6 +72,21 @@ dependencies = ' '.join(python_packages[python] +
                          # Dependencies for singularity on kubernetes
                          'tzdata'])
 
+# pymesos's http-parser dependency can't build on Python later than 3.10, as
+# released in 0.9.0. The upstream pymesos can, but we write it out of Toil's
+# dependencies on later Python versions, since a working http-parser is not
+# available in PyPI. So we need to manually inject a working http-parser, and
+# pymesos, into the Docker images.
+extra_mesos_python_modules = {
+    'python3.9': [],
+    'python3.10': [],
+    'python3.11': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
+    'python3.12': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
+    'python3.13': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15']
+}
+
+extra_python_modules = " ".join(extra_mesos_python_modules[python])
+
 
 def heredoc(s):
     s = textwrap.dedent(s).format(**globals())
@@ -80,11 +95,10 @@ def heredoc(s):
 
 motd = heredoc('''
 
-    This is the Toil appliance. You can run your Toil script directly on the appliance.
-    Run toil <workflow>.py --help to see all options for running your workflow.
+    This is the Toil appliance. You can run your Toil workflow directly on the appliance.
     For more information see http://toil.readthedocs.io/en/latest/
 
-    Copyright (C) 2015-2022 Regents of the University of California
+    Copyright (C) 2015-2025 Regents of the University of California
 
     Version: {applianceSelf}
 
@@ -182,18 +196,22 @@ print(heredoc('''
     # The stock pip is too old and can't install from sdist with extras
     RUN curl -sS https://bootstrap.pypa.io/get-pip.py | {python}
 
-    # Include virtualenv, as it is still the recommended way to deploy pipelines
-    RUN {pip} install --upgrade virtualenv==20.25.1
+    # Include virtualenv, as it is still the recommended way to deploy
+    # pipelines.
+    #
+    # We need to --ignore-installed here to allow shadowing system packages
+    # from apt in /usr/lib/python3/dist-packages when the installed package
+    # needs newer versions. We just hope that doesn't break the Ubuntu system
+    # too badly when we're actually on the system Python, or if Toil needs to
+    # upgrade a distutils or setuptools dependency. On the deadsnakes Pythons,
+    # installations into the version-specific package directory won't be seen
+    # by the system Python which is a different version.
+    #
+    # TODO: Change to nested virtual environments and .pth files and teach Toil
+    # to just ship the user-level one for hot deploy.
+    RUN {pip} install --ignore-installed --upgrade 'virtualenv>=20.25.1,<21'
 
-    # Install s3am (--never-download prevents silent upgrades to pip, wheel and setuptools)
-    # Install setuptools within the virtual environment to properly access distutils due to PEP 632 and gh-95299 in Python 3.12 release notes
-    # https://docs.python.org/3/whatsnew/3.12.html#summary-release-highlights
-    RUN virtualenv --python {python} --never-download /home/s3am \
-        && /home/s3am/bin/pip install setuptools \
-        && /home/s3am/bin/pip install s3am==2.0 \
-        && ln -s /home/s3am/bin/s3am /usr/local/bin/
-    
-    RUN {pip} install --upgrade setuptools==69.2.0
+    RUN {pip} install --ignore-installed --upgrade 'setuptools>=80,<81'
 
     # Fix for https://issues.apache.org/jira/browse/MESOS-3793
     ENV MESOS_LAUNCHER=posix
@@ -222,7 +240,7 @@ print(heredoc('''
 
     # This component changes most frequently and keeping it last maximizes Docker cache hits.
     COPY {sdistName} .
-    RUN {pip} install {sdistName}[all]
+    RUN {pip} install --ignore-installed --upgrade {sdistName}[all] {extra_python_modules}
     RUN rm {sdistName}
 
     # We intentionally inherit the default ENTRYPOINT and CMD from the base image, to the effect

{toil-8.2.0 → toil-9.1.0}/docs/appendices/aws_min_permissions.rst

@@ -18,7 +18,6 @@ with the ``--awsEc2ProfileArn`` parameter.
                 "Action": [
                     "ec2:*",
                     "s3:*",
-                    "sdb:*",
                     "iam:PassRole"
                 ],
                 "Resource": "*"

{toil-8.2.0 → toil-9.1.0}/docs/cwl/conformance.rst

@@ -64,6 +64,8 @@ CWL 1.2 Optional Features
 
 .. |docker| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/docker.json%3Fjob=cwl_badge
 
+.. |env_var| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/env_var.json%3Fjob=cwl_badge
+
 .. |expression_tool| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/expression_tool.json%3Fjob=cwl_badge
 
 .. |format_checking| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/format_checking.json%3Fjob=cwl_badge
@@ -82,16 +84,26 @@ CWL 1.2 Optional Features
 
 .. |multiple_input| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/multiple_input.json%3Fjob=cwl_badge
 
+.. |multiple| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/multiple.json%3Fjob=cwl_badge
+
 .. |networkaccess| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/networkaccess.json%3Fjob=cwl_badge
 
+.. |require| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/require.json%3Fjob=cwl_badge
+
 .. |resource| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/resource.json%3Fjob=cwl_badge
 
 .. |scatter| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/scatter.json%3Fjob=cwl_badge
 
 .. |schema_def| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/schema_def.json%3Fjob=cwl_badge
 
+.. |secondary_files| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/secondary_files.json%3Fjob=cwl_badge
+
+.. |shell_command| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/shell_command.json%3Fjob=cwl_badge
+
 .. |step_input| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/step_input.json%3Fjob=cwl_badge
 
+.. |subworkflow| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/subworkflow.json%3Fjob=cwl_badge
+
 .. |timelimit| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/timelimit.json%3Fjob=cwl_badge
 
 .. |workflow| image:: https://flat.badgen.net/https/ucsc-ci.com/api/v4/projects/3/jobs/artifacts/master/raw/badges1.2/workflow.json%3Fjob=cwl_badge

{toil-8.2.0 → toil-9.1.0}/docs/cwl/running.rst

@@ -100,8 +100,8 @@ Besides the normal Toil options and the options supported by cwltool, toil-cwl-r
   --reference-inputs    Do not copy remote inputs into Toil's file store and assume they are accessible in place from all nodes.
   --disable-streaming   Do not allow streaming of job input files. By default, files marked with ``streamable`` True are streamed from remote job stores.
   --cwl-default-ram     Apply CWL specification default ramMin.
-  --no-cwl-default-ram  Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies.
-
+  --no-cwl-default-ram  Do not apply CWL specification default ramMin, so that Toil --defaultMemory applies. This can help jobs get to Slurm with no memory limit assigned.
+  --cwl-min-ram BYTES   Specify a minimum memory allocation for all tasks. If ``--no-cwl-default-ram`` is passed, this does not apply to tools that do not specify a memory requirement; ``--defaultMemory`` is used for those tools in that case.
 
 Running CWL in the Cloud
 ------------------------