toil 9.0.0__tar.gz → 9.1.1__tar.gz

{toil-9.0.0 → toil-9.1.1}/.github/workflows/codeql-analysis.yml

@@ -39,7 +39,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v5
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

{toil-9.0.0 → toil-9.1.1}/.gitlab-ci.yml

@@ -164,7 +164,8 @@ quick_test_offline:
     make develop extras=[aws,google,wdl]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the offline tests"
-    TOIL_TEST_QUICK=True make test_offline threads="${TEST_THREADS}"
+    # Make sure not to run the doctests here because we didn't install all possible extras
+    TOIL_TEST_QUICK=True make test_offline tests=src/toil/test threads="${TEST_THREADS}"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
 py39_appliance_build:
@@ -208,6 +209,7 @@ py39_main:
     python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
+    make doctest threads="${TEST_THREADS}"
     make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
@@ -253,6 +255,7 @@ py310_main:
     python3.10 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 && pip install -U pip wheel && make prepare && pip install pycparser && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
+    make doctest threads="${TEST_THREADS}"
     make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
@@ -298,6 +301,7 @@ py311_main:
     python3.11 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
+    make doctest threads="${TEST_THREADS}"
     make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
@@ -343,6 +347,7 @@ py312_main:
     python3.12 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
+    make doctest threads="${TEST_THREADS}"
     make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
@@ -379,11 +384,11 @@ py313_main:
     python3.13 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 && pip install -U pip wheel && make prepare && make develop extras=[all]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
+    make doctest threads="${TEST_THREADS}"
     make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
-
 slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -458,7 +463,7 @@ cwl_badge:
     rm -rf badges1.2
     echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
-    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
+    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   allow_failure: true
   artifacts:
@@ -491,7 +496,7 @@ cwl_badge_release:
     rm -rf badges1.2
     echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
-    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
+    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   allow_failure: true
   artifacts:
@@ -685,7 +690,7 @@ provisioner_integration:
     - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
     # This reads GITLAB_SECRET_FILE_SSH_KEYS
     - python setup_gitlab_ssh.py
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::TestUtils::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest src/toil/test/wdl/wdltoil_test_kubernetes.py::WDLKubernetesClusterTest"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::TestUtils::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::TestAWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/wdl/wdltoil_test_kubernetes.py::WDLKubernetesClusterTest"
 #    - make test tests=src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run
 
 google_jobstore:

{toil-9.0.0 → toil-9.1.1}/Makefile

@@ -85,7 +85,8 @@ help:
 
 # This Makefile uses bash features like printf and <()
 SHELL=bash
-tests=src/toil/test
+# We need the default tests to include the doctests, but not htcondor which isn't on mac.
+tests=src/toil --ignore src/toil/batchSystems/htcondor.py
 arch=linux/amd64,linux/arm64
 cov=--cov=toil
 logging=--log-format="%(asctime)s %(levelname)s %(message)s" --log-level DEBUG -o log_cli=true --log-cli-level INFO
@@ -109,7 +110,7 @@ marker=""
 threads:="auto"
 
 dist:="worksteal"
-pytest_args:=""
+pytest_args:="--randomly-dont-reorganize"
 
 # Only pass the threading options if running parallel tests. Otherwise we lose
 # live logging. See <https://stackoverflow.com/q/62533239>
@@ -169,10 +170,20 @@ test: check_venv check_build_reqs
 	TOIL_HISTORY=0 \
 	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(cov) $(tests) $(pytest_args)
 
+# When running doctests, we need to not capture output, because on CI we can
+# get failures where doctest saw no output and we report captured output, as in
+# <https://ucsc-ci.com/databiosphere/toil/-/jobs/96131>. doctest and pytest's
+# captures might not work properly together.
+doctest: check_venv check_build_reqs
+	TOIL_OWNER_TAG="shared" \
+	TOIL_HISTORY=0 \
+	    python -m pytest --capture=no $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(cov) $(tests) --ignore src/toil/test $(pytest_args)
+
 test_debug: check_venv check_build_reqs
+	# Don't include threadopts to make sure we can see our live logging.
 	TOIL_OWNER_TAG="$(whoami)" \
 	TOIL_HISTORY=0 \
-	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) $(tests) $(pytest_args) --maxfail=1
+	    python -m pytest $(verbose) $(durations) -m "$(marker)" $(logging) $(tests) $(pytest_args) --maxfail=1
 
 
 # This target will skip building docker and all docker based tests
@@ -189,7 +200,7 @@ test_offline: check_venv check_build_reqs
 test_1min: check_venv check_build_reqs
 	TOIL_SKIP_DOCKER=False \
 	TOIL_HISTORY=0 \
-	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=30  --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
+	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=30 --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
 
 ifdef TOIL_DOCKER_REGISTRY
 
@@ -378,7 +389,7 @@ preflight: mypy touched_pylint
 		develop clean_develop \
 		sdist clean_sdist \
 		download_cwl_spec \
-		test test_offline test_1min \
+		test doctest test_offline test_1min \
 		docs clean_docs \
 		clean \
 		sort_imports remove_unused_imports remove_trailing_whitespace \

{toil-9.0.0 → toil-9.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toil
-Version: 9.0.0
+Version: 9.1.1
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community
@@ -33,7 +33,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: dill<0.4,>=0.3.2
-Requires-Dist: requests<=2.31.0
+Requires-Dist: requests<=2.32.5
 Requires-Dist: docker<8,>=6.1.0
 Requires-Dist: urllib3<3,>=1.26.0
 Requires-Dist: python-dateutil
@@ -55,10 +55,10 @@ Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: moto<6,>=5.0.3; extra == "aws"
 Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "aws"
 Provides-Extra: cwl
-Requires-Dist: cwltool==3.1.20250110105449; extra == "cwl"
+Requires-Dist: cwltool==3.1.20250715140722; extra == "cwl"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
 Requires-Dist: galaxy-tool-util<26; extra == "cwl"
-Requires-Dist: galaxy-util<25; extra == "cwl"
+Requires-Dist: galaxy-util<26; extra == "cwl"
 Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
 Requires-Dist: networkx!=2.8.1,<4; extra == "cwl"
@@ -67,11 +67,11 @@ Requires-Dist: cwl-utils>=0.36; extra == "cwl"
 Provides-Extra: encryption
 Requires-Dist: pynacl<2,>=1.4.0; extra == "encryption"
 Provides-Extra: google
-Requires-Dist: apache-libcloud<3,>=2.2.1; extra == "google"
+Requires-Dist: apache-libcloud<4,>=3.6.1; extra == "google"
 Requires-Dist: google-cloud-storage<=2.8.0,>=2; extra == "google"
 Requires-Dist: google-auth<3,>=2.18.1; extra == "google"
 Provides-Extra: kubernetes
-Requires-Dist: kubernetes<33,>=12.0.1; extra == "kubernetes"
+Requires-Dist: kubernetes<34,>=12.0.1; extra == "kubernetes"
 Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "kubernetes"
 Requires-Dist: types-urllib3; extra == "kubernetes"
 Requires-Dist: types-PyYAML; extra == "kubernetes"
@@ -84,7 +84,7 @@ Provides-Extra: server
 Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "server"
 Requires-Dist: flask<4,>=3; extra == "server"
 Requires-Dist: werkzeug<4,>=3; extra == "server"
-Requires-Dist: flask-cors<6,>=5.0.1; extra == "server"
+Requires-Dist: flask-cors<7,>=5.0.1; extra == "server"
 Requires-Dist: gunicorn<24,>=23.0.0; extra == "server"
 Requires-Dist: celery<6,>=5.4.0; extra == "server"
 Requires-Dist: wes-service<6,>=5.0.0; extra == "server"
@@ -99,20 +99,20 @@ Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: moto<6,>=5.0.3; extra == "all"
 Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "all"
-Requires-Dist: cwltool==3.1.20250110105449; extra == "all"
+Requires-Dist: cwltool==3.1.20250715140722; extra == "all"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
 Requires-Dist: galaxy-tool-util<26; extra == "all"
-Requires-Dist: galaxy-util<25; extra == "all"
+Requires-Dist: galaxy-util<26; extra == "all"
 Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
 Requires-Dist: networkx!=2.8.1,<4; extra == "all"
 Requires-Dist: CacheControl[filecache]; extra == "all"
 Requires-Dist: cwl-utils>=0.36; extra == "all"
 Requires-Dist: pynacl<2,>=1.4.0; extra == "all"
-Requires-Dist: apache-libcloud<3,>=2.2.1; extra == "all"
+Requires-Dist: apache-libcloud<4,>=3.6.1; extra == "all"
 Requires-Dist: google-cloud-storage<=2.8.0,>=2; extra == "all"
 Requires-Dist: google-auth<3,>=2.18.1; extra == "all"
-Requires-Dist: kubernetes<33,>=12.0.1; extra == "all"
+Requires-Dist: kubernetes<34,>=12.0.1; extra == "all"
 Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
 Requires-Dist: types-urllib3; extra == "all"
 Requires-Dist: types-PyYAML; extra == "all"
@@ -123,7 +123,7 @@ Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
 Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "all"
 Requires-Dist: flask<4,>=3; extra == "all"
 Requires-Dist: werkzeug<4,>=3; extra == "all"
-Requires-Dist: flask-cors<6,>=5.0.1; extra == "all"
+Requires-Dist: flask-cors<7,>=5.0.1; extra == "all"
 Requires-Dist: gunicorn<24,>=23.0.0; extra == "all"
 Requires-Dist: celery<6,>=5.4.0; extra == "all"
 Requires-Dist: wes-service<6,>=5.0.0; extra == "all"

{toil-9.0.0 → toil-9.1.1}/contrib/admin/cleanup_aws_resources.py

@@ -70,8 +70,11 @@ def contains_toil_test_patterns(string):
 
 
 def matches(resource_name):
-    if (resource_name.endswith('--files') or resource_name.endswith('--jobs') or resource_name.endswith('_toil')
-            or resource_name.endswith('--internal') or resource_name.startswith('toil-s3test-')):
+    if (
+        resource_name.endswith('--files') or resource_name.endswith('--jobs')
+        or resource_name.endswith('--toil') or resource_name.endswith('_toil')
+        or resource_name.endswith('--internal') or resource_name.startswith('toil-s3test-')
+    ):
         if contains_toil_test_patterns(resource_name):
             return resource_name
 

{toil-9.0.0 → toil-9.1.1}/contrib/admin/mypy-with-ignore.py

@@ -22,7 +22,8 @@ def main():
 
     # TODO: Remove these paths as typing is added and mypy conflicts are addressed.
     # These are handled as path prefixes.
-    ignore_paths = [os.path.abspath(f) for f in [
+
+    ignore_paths = [os.path.join(pkg_root, f) for f in [
         'docs/_build',
         'docker/Dockerfile.py',
         'docs/conf.py',
@@ -61,12 +62,8 @@ def main():
         'src/toil/jobStores/fileJobStore.py',
         'src/toil/jobStores/__init__.py',
         'src/toil/jobStores/googleJobStore.py',
-        'src/toil/jobStores/aws/utils.py',
-        'src/toil/jobStores/aws/jobStore.py',
-        'src/toil/jobStores/aws/__init__.py',
         'src/toil/utils/__init__.py',
         'src/toil/lib/throttle.py',
-        'src/toil/lib/iterables.py',
         'src/toil/lib/bioio.py',
         'src/toil/lib/ec2.py',
         'src/toil/lib/expando.py',
@@ -87,7 +84,6 @@ def main():
         'src/toil/test/cactus',
         'src/toil/test/docs',
         'src/toil/test/jobStores',
-        'src/toil/test/lib/aws',
         'src/toil/test/lib/test_conversions.py',
         'src/toil/test/lib/test_ec2.py',
         'src/toil/test/lib/test_misc.py',

{toil-9.0.0 → toil-9.1.1}/docker/Dockerfile.py

@@ -95,11 +95,10 @@ def heredoc(s):
 
 motd = heredoc('''
 
-    This is the Toil appliance. You can run your Toil script directly on the appliance.
-    Run toil <workflow>.py --help to see all options for running your workflow.
+    This is the Toil appliance. You can run your Toil workflow directly on the appliance.
     For more information see http://toil.readthedocs.io/en/latest/
 
-    Copyright (C) 2015-2022 Regents of the University of California
+    Copyright (C) 2015-2025 Regents of the University of California
 
     Version: {applianceSelf}
 

{toil-9.0.0 → toil-9.1.1}/docs/appendices/aws_min_permissions.rst

@@ -18,7 +18,6 @@ with the ``--awsEc2ProfileArn`` parameter.
                 "Action": [
                     "ec2:*",
                     "s3:*",
-                    "sdb:*",
                     "iam:PassRole"
                 ],
                 "Resource": "*"

{toil-9.0.0 → toil-9.1.1}/requirements-cwl.txt

@@ -1,7 +1,7 @@
-cwltool==3.1.20250110105449
+cwltool==3.1.20250715140722
 schema-salad>=8.4.20230128170514,<9
 galaxy-tool-util<26
-galaxy-util<25
+galaxy-util<26
 ruamel.yaml>=0.15,<=0.19
 ruamel.yaml.clib>=0.2.6
 networkx!=2.8.1,<4

{toil-9.0.0 → toil-9.1.1}/requirements-dev.txt

@@ -1,8 +1,9 @@
 pytest>=6.2.1,<9
-pytest-cov>=2.12.1,<7
+pytest-cov>=2.12.1,<8
 pytest-timeout>=1.4.2,<3
 pytest_httpserver>=1.1.3,<2
-pytest-subtests
+pytest-subtests>=0.14.2,<0.15
+pytest-randomly>=3.16.0,<5
 stubserver>=1.1,<2
 setuptools>=65.5.1,<81
 sphinx>=7,<9
@@ -11,7 +12,7 @@ astroid>=3,<4
 sphinx-autodoc-typehints>=1.24.0,<4
 sphinxcontrib-autoprogram==0.1.9
 cwltest>=2.6.20250314152537
-mypy==1.16.1
+mypy==1.18.1
 types-aws-xray-sdk
 types-boto<2.49.18.20241020
 types-Flask-Cors
@@ -29,7 +30,7 @@ autoflake
 isort
 diff_cover
 lxml
-docutils>=0.16,<0.22
+docutils>=0.16,<0.23
 pyupgrade
 pytest-xdist
 build

{toil-9.0.0 → toil-9.1.1}/requirements-google.txt

@@ -1,3 +1,3 @@
-apache-libcloud>=2.2.1,<3
+apache-libcloud>=3.6.1,<4
 google-cloud-storage>=2,<=2.8.0
 google-auth>=2.18.1,<3

{toil-9.0.0 → toil-9.1.1}/requirements-kubernetes.txt

@@ -1,4 +1,4 @@
-kubernetes>=12.0.1, <33
+kubernetes>=12.0.1, <34
 # Nobody ever published stubs that support Python back to before 3.9, until
 # version 22.6.0 of the Kubernetes module. So we have to use the mismatched
 # stubs.

{toil-9.0.0 → toil-9.1.1}/requirements-server.txt

@@ -1,7 +1,7 @@
 connexion[swagger-ui]>=3,<4
 flask>=3,<4
 werkzeug>=3,<4
-flask-cors>=5.0.1,<6
+flask-cors>=5.0.1,<7
 gunicorn>=23.0.0,<24
 celery>=5.4.0,<6
 wes-service>=5.0.0,<6

{toil-9.0.0 → toil-9.1.1}/requirements.txt

@@ -1,5 +1,5 @@
 dill>=0.3.2, <0.4
-requests<=2.31.0
+requests<=2.32.5
 docker>=6.1.0, <8
 urllib3>=1.26.0,<3
 python-dateutil

{toil-9.0.0 → toil-9.1.1}/src/toil/batchSystems/abstractBatchSystem.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from dataclasses import dataclass
 import enum
 import logging
 import os
@@ -72,10 +73,13 @@ class BatchJobExitReason(enum.IntEnum):
         except ValueError:
             return str(value)
 
-
-class UpdatedBatchJobInfo(NamedTuple):
+@dataclass
+class UpdatedBatchJobInfo:
     jobID: int
-    exitStatus: int
+    """
+    The Toil batch system ID of the job.
+    """
+    exitStatus: int = EXIT_STATUS_UNAVAILABLE_VALUE
     """
     The exit status (integer value) of the job. 0 implies successful.
 
@@ -83,8 +87,12 @@ class UpdatedBatchJobInfo(NamedTuple):
     (e.g. job is lost, or otherwise died but actual exit code was not reported).
     """
 
-    exitReason: Optional[BatchJobExitReason]
-    wallTime: Union[float, int, None]
+    exitReason: Optional[BatchJobExitReason] = None
+    wallTime: Union[float, int, None] = None
+    backing_id: Optional[str] = None
+    """
+    The identifier for the job in the backing scheduler, if available.
+    """
 
 
 # Information required for worker cleanup on shutdown of the batch system.

{toil-9.0.0 → toil-9.1.1}/src/toil/batchSystems/abstractGridEngineBatchSystem.py

@@ -159,14 +159,21 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                 logger.debug("Running %r", subLine)
                 batchJobID = self.boss.with_retries(self.submitJob, subLine)
                 if self.boss._outbox is not None:
-                    # JobID corresponds to the toil version of the jobID, dif from jobstore idea of the id, batchjobid is what we get from slurm
+                    # JobID corresponds to the toil version of the jobID,
+                    # different from the jobstore's idea of the id. batchjobid
+                    # is what we get from e.g. slurm
                     self.boss._outbox.publish(
                         ExternalBatchIdMessage(
                             jobID, batchJobID, self.boss.__class__.__name__
                         )
                     )
 
-                logger.debug("Submitted job %s", str(batchJobID))
+                logger.info(
+                    "Job %s with batch system ID %s queued as job %s",
+                    jobName,
+                    jobID,
+                    str(batchJobID)
+                )
 
                 # Store dict for mapping Toil job ID to batch job ID
                 # TODO: Note that this currently stores a tuple of (batch system
@@ -251,8 +258,8 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                     self.coalesce_job_exit_codes, batch_job_id_list
                 )
                 # We got the statuses as a batch
-                for running_job_id, status in zip(running_job_list, statuses):
-                    activity = self._handle_job_status(running_job_id, status, activity)
+                for running_job_id, status, backing_id in zip(running_job_list, statuses, batch_job_id_list):
+                    activity = self._handle_job_status(running_job_id, status, activity, backing_id)
 
             self._checkOnJobsCache = activity
             self._checkOnJobsTimestamp = datetime.now()
@@ -263,6 +270,7 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
             job_id: int,
             status: Union[int, tuple[int, Optional[BatchJobExitReason]], None],
             activity: bool,
+            backing_id: str,
         ) -> bool:
             """
             Helper method for checkOnJobs to handle job statuses
@@ -275,7 +283,11 @@ class AbstractGridEngineBatchSystem(BatchSystemCleanupSupport):
                     code, reason = status
                 self.updatedJobsQueue.put(
                     UpdatedBatchJobInfo(
-                        jobID=job_id, exitStatus=code, exitReason=reason, wallTime=None
+                        jobID=job_id,
+                        exitStatus=code,
+                        exitReason=reason,
+                        wallTime=None,
+                        backing_id=backing_id,
                     )
                 )
                 self.forgetJob(job_id)

{toil-9.0.0 → toil-9.1.1}/src/toil/batchSystems/kubernetes.py

@@ -37,6 +37,7 @@ from threading import Condition, Event, RLock, Thread
 from typing import Any, Callable, Literal, Optional, TypeVar, Union, cast, overload
 
 from toil.lib.conversions import opt_strtobool
+from toil.lib.throttle import LocalThrottle
 
 if sys.version_info < (3, 10):
     from typing_extensions import ParamSpec
@@ -281,6 +282,10 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
         # in the queue or any resource becomes available.
         self._work_available: Condition = Condition(lock=self._mutex)
 
+        # To make sure we don't spam the log when the metrics server is down,
+        # we use a throttle
+        self._metrics_throttle: LocalThrottle = LocalThrottle(600)
+
         self.schedulingThread: Thread = Thread(target=self._scheduler, daemon=True)
         self.schedulingThread.start()
 
@@ -1363,7 +1368,8 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                 # This is the sort of error we would expect from an overloaded
                 # Kubernetes or a dead metrics service.
                 # We can't tell that the pod is stuck, so say that it isn't.
-                logger.warning("Could not query metrics service: %s", e)
+                if self._metrics_throttle.throttle(False):
+                    logger.warning("Kubernetes metrics service is not available: %s", e)
                 return False
             else:
                 raise
@@ -1602,6 +1608,7 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
                         exitStatus=exitCode,
                         wallTime=runtime,
                         exitReason=exitReason,
+                        backing_id=jobObject.metadata.name,
                     )
 
                     if (exitReason == BatchJobExitReason.FAILED) or (
@@ -1855,7 +1862,11 @@ class KubernetesBatchSystem(BatchSystemCleanupSupport):
 
         # Return the one finished job we found
         return UpdatedBatchJobInfo(
-            jobID=jobID, exitStatus=exitCode, wallTime=runtime, exitReason=None
+            jobID=jobID,
+            exitStatus=exitCode,
+            wallTime=runtime,
+            exitReason=None,
+            backing_id=jobObject.metadata.name,
         )
 
     def _waitForJobDeath(self, jobName: str) -> None:

{toil-9.0.0 → toil-9.1.1}/src/toil/batchSystems/mesos/batchSystem.py

@@ -103,6 +103,9 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
         if config.mesos_framework_id is not None:
             self.mesos_framework_id = config.mesos_framework_id
 
+        # How long in seconds to wait to register before declaring Mesos unreachable.
+        self.mesos_timeout = 60
+
         # Written to when Mesos kills tasks, as directed by Toil.
         # Jobs must not enter this set until they are removed from runningJobMap.
         self.killedJobIds = set()
@@ -345,17 +348,38 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
             framework.roles = config.mesos_role
             framework.capabilities = [dict(type="MULTI_ROLE")]
 
+        endpoint = self._resolveAddress(self.mesos_endpoint)
+        log.info("Connecting to Mesos at %s...", self.mesos_endpoint)
+
         # Make the driver which implements most of the scheduler logic and calls back to us for the user-defined parts.
         # Make sure it will call us with nice namespace-y addicts
         self.driver = MesosSchedulerDriver(
             self,
             framework,
-            self._resolveAddress(self.mesos_endpoint),
+            endpoint,
             use_addict=True,
             implicit_acknowledgements=True,
         )
         self.driver.start()
 
+        wait_count = 0
+        while self.frameworkId is None:
+            # Wait to register with Mesos, and eventually fail if it just isn't
+            # responding.
+
+            # TODO: Use a condition instead of a spin wait.
+
+            if wait_count >= self.mesos_timeout:
+                error_message = f"Could not connect to Mesos endpoint at {self.mesos_endpoint}"
+                log.error(error_message)
+                self.shutdown()
+                raise RuntimeError(error_message)
+            elif wait_count > 1 and wait_count % 10 == 0:
+                log.warning("Waiting for Mesos registration (try %s/%s)", wait_count, self.mesos_timeout)
+            time.sleep(1)
+            wait_count += 1
+
+
     @staticmethod
     def _resolveAddress(address):
         """
@@ -394,10 +418,17 @@ class MesosBatchSystem(BatchSystemLocalSupport, AbstractScalableBatchSystem, Sch
         """
         Invoked when the scheduler successfully registers with a Mesos master
         """
-        log.debug("Registered with framework ID %s", frameworkId.value)
+        log.info("Registered with Mesos as framework ID %s", frameworkId.value)
         # Save the framework ID
         self.frameworkId = frameworkId.value
 
+    def error(self, driver, message):
+        """
+        Invoked when Mesos reports an unrecoverable error.
+        """
+        log.error("Mesos error: %s", message)
+        super().error(driver, message)
+
     def _declineAllOffers(self, driver, offers):
         for offer in offers:
             driver.declineOffer(offer.id)