toil 9.1.1__tar.gz → 9.2.0__tar.gz

{toil-9.1.1 → toil-9.2.0}/.git-blame-ignore-revs

@@ -6,3 +6,5 @@ a307a57651296e576d55f9159869dc9130da2b7c
 1c6217e11a2815476ede1cc3ee584d5da3d5242a
 # make pyupgrade (3.9+)
 00f54f8417b3a0bc5d1ea46650aa2e6ce4792159
+# make pyupgrade (3.10+) & reformat
+e08b52cf3a8e0eb0ca8eee7b1f0223ba08c4a2ac

{toil-9.1.1 → toil-9.2.0}/.github/workflows/codeql-analysis.yml

@@ -39,11 +39,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v4

{toil-9.1.1 → toil-9.2.0}/.gitlab-ci.yml

@@ -88,21 +88,21 @@ lint:
     # make diff_pydocstyle_report
 
 # We make sure to also lint with our oldest supported Python version on every PR.
-py39_lint:
+py310_lint:
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: linting_and_dependencies
   cache:
-    key: cache-python3.9
+    key: cache-python3.10
     paths:
       - .cache/pip
       - .mypy_cache
   script: |
     pwd
     echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all dependencies (except htcondor)"
-    python3.9 -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
-    python3.9 -m pip freeze
-    python3.9 --version
+    python3.10 -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[all]
+    python3.10 -m pip freeze
+    python3.10 --version
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:mypy\r\e[0KCheck the Python types with mypy"
     make mypy
@@ -118,7 +118,7 @@ cwl_dependency_is_stand_alone:
     - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: linting_and_dependencies
   cache:
-    key: cache-$MAIN_PYTHON_PKG
+    key: cache-$MAIN_PYTHON_PKG-cwl
     paths:
       - .cache/pip
   script: |
@@ -135,7 +135,7 @@ wdl_dependency_is_stand_alone:
     - if: $CI_PIPELINE_SOURCE != "schedule"
   stage: linting_and_dependencies
   cache:
-    key: cache-$MAIN_PYTHON_PKG
+    key: cache-$MAIN_PYTHON_PKG-wdl
     paths:
       - .cache/pip
   script: |
@@ -168,52 +168,6 @@ quick_test_offline:
     TOIL_TEST_QUICK=True make test_offline tests=src/toil/test threads="${TEST_THREADS}"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
-py39_appliance_build:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_TAG
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
-    - if: $CI_COMMIT_BRANCH =~ /.*3\.9.*/
-  stage: basic_tests
-  cache:
-    key: cache-python3.9
-    paths:
-      - .cache/pip
-  script: |
-    pwd
-    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
-    python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all,htcondor]
-    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
-    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
-    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
-    python setup_gitlab_docker.py
-    make push_docker
-    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"
-
-py39_main:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_TAG
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
-    - if: $CI_COMMIT_BRANCH =~ /.*3\.9.*/
-  stage: basic_tests
-  cache:
-    key: cache-python3.9
-    paths:
-      - .cache/pip
-  script: |
-    pwd
-    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
-    python3.9 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all,htcondor]
-    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
-    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
-    make doctest threads="${TEST_THREADS}"
-    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
-    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
-    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
-
 py310_appliance_build:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -389,6 +343,43 @@ py313_main:
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
+py314_appliance_build:
+  stage: basic_tests
+  cache:
+    key: cache-python3.14
+    paths:
+      - .cache/pip
+  script: |
+    pwd
+    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, including htcondor"
+    python3.14 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.14 &&  pip install -U pip wheel && pip install -U build && make prepare && pip install pycparser && make develop extras=[all]
+    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
+    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
+    echo -e "\e[0Ksection_start:`date +%s`:docker\r\e[0KBuild a source distribution and then build the docker containers"
+    python setup_gitlab_docker.py
+    make push_docker
+    echo -e "\e[0Ksection_end:`date +%s`:docker\r\e[0K"
+
+py314_main:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+  stage: basic_tests
+  cache:
+    key: cache-python3.14
+    paths:
+      - .cache/pip
+  script: |
+    pwd
+    # todo: htcondor is not out for python 3.14 yet, we don't actively test htcondor batchsystems but should still test an htcondor install
+    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install all the dependencies, except htcondor"
+    python3.14 -m virtualenv venv && . venv/bin/activate && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.14 && pip install -U pip wheel && make prepare && make develop extras=[all]
+    echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
+    echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the tests"
+    make doctest threads="${TEST_THREADS}"
+    make test threads="${TEST_THREADS}" tests="src/toil/test/src src/toil/test/utils src/toil/test/server"
+    TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
+    echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
+
 slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -584,7 +575,7 @@ wdl:
       - .cache/pip
   script: |
     pwd
-    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install default-jew, and all the Python dependencies except htcondor"
+    echo -e "\e[0Ksection_start:`date +%s`:prepare\r\e[0KDownload and install default-jre, and all the Python dependencies except htcondor"
     apt update && apt install -y default-jre
     ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[all]
     echo -e "\e[0Ksection_end:`date +%s`:prepare\r\e[0K"
@@ -782,7 +773,7 @@ cactus_integration:
     paths:
       - .cache/pip
   script:
-    - export CACTUS_COMMIT_SHA=03295e9af99e2e9168ccd02e78a9f4c0c8dcd490
+    - export CACTUS_COMMIT_SHA=b51ff106fa8510d33b09551d338ace48b5c5fa5a
     - set -e
     - ${MAIN_PYTHON_PKG} -m virtualenv venv && . venv/bin/activate && make prepare && make develop extras=[aws]
     - python setup_gitlab_docker.py  # login to increase the docker.io rate limit

{toil-9.1.1 → toil-9.2.0}/.readthedocs.yaml

@@ -7,7 +7,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.14"
   apt_packages:
     - graphviz
 

{toil-9.1.1 → toil-9.2.0}/Makefile

@@ -376,7 +376,7 @@ diff_mypy:
 	diff-cover --fail-under=100 --compare-branch origin/master cobertura.xml
 
 pyupgrade: $(PYSOURCES)
-	pyupgrade --exit-zero-even-if-changed --py39-plus $^
+	pyupgrade --exit-zero-even-if-changed --py310-plus $^
 
 flake8: $(PYSOURCES)
 	flake8 --ignore=E501,W293,W291,E265,E302,E722,E126,E303,E261,E201,E202,W503,W504,W391,E128,E301,E127,E502,E129,E262,E111,E117,E306,E203,E231,E226,E741,E122,E251,E305,E701,E222,E225,E241,E305,E123,E121,E703,E704,E125,E402 $^

{toil-9.1.1 → toil-9.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toil
-Version: 9.1.1
+Version: 9.2.0
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community
@@ -16,11 +16,11 @@ Classifier: Natural Language :: English
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: POSIX
 Classifier: Operating System :: POSIX :: Linux
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: Topic :: Scientific/Engineering :: Astronomy
@@ -29,7 +29,7 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
 Classifier: Topic :: System :: Distributed Computing
 Classifier: Topic :: Utilities
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: dill<0.4,>=0.3.2
@@ -40,7 +40,6 @@ Requires-Dist: python-dateutil
 Requires-Dist: psutil<8,>=6.1.0
 Requires-Dist: PyPubSub<5,>=4.0.3
 Requires-Dist: addict<2.5,>=2.2.1
-Requires-Dist: backports.zoneinfo[tzdata]; python_version < "3.9"
 Requires-Dist: enlighten<2,>=1.5.2
 Requires-Dist: configargparse<2,>=1.7
 Requires-Dist: ruamel.yaml>=0.15
@@ -55,11 +54,11 @@ Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "aws"
 Requires-Dist: moto<6,>=5.0.3; extra == "aws"
 Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "aws"
 Provides-Extra: cwl
-Requires-Dist: cwltool==3.1.20250715140722; extra == "cwl"
+Requires-Dist: cwltool==3.1.20260108082145; extra == "cwl"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "cwl"
 Requires-Dist: galaxy-tool-util<26; extra == "cwl"
 Requires-Dist: galaxy-util<26; extra == "cwl"
-Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "cwl"
+Requires-Dist: ruamel.yaml<=0.19.1,>=0.15; extra == "cwl"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "cwl"
 Requires-Dist: networkx!=2.8.1,<4; extra == "cwl"
 Requires-Dist: CacheControl[filecache]; extra == "cwl"
@@ -77,15 +76,14 @@ Requires-Dist: types-urllib3; extra == "kubernetes"
 Requires-Dist: types-PyYAML; extra == "kubernetes"
 Requires-Dist: idna>=2; extra == "kubernetes"
 Provides-Extra: wdl
-Requires-Dist: miniwdl==1.13.0; extra == "wdl"
+Requires-Dist: miniwdl==1.13.1; extra == "wdl"
 Requires-Dist: wdlparse==0.1.0; extra == "wdl"
-Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "wdl"
 Provides-Extra: server
 Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "server"
 Requires-Dist: flask<4,>=3; extra == "server"
 Requires-Dist: werkzeug<4,>=3; extra == "server"
 Requires-Dist: flask-cors<7,>=5.0.1; extra == "server"
-Requires-Dist: gunicorn<24,>=23.0.0; extra == "server"
+Requires-Dist: gunicorn<26,>=23.0.0; extra == "server"
 Requires-Dist: celery<6,>=5.4.0; extra == "server"
 Requires-Dist: wes-service<6,>=5.0.0; extra == "server"
 Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "server"
@@ -99,11 +97,11 @@ Requires-Dist: mypy-boto3-iam<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: mypy-boto3-s3<2,>=1.28.3.post2; extra == "all"
 Requires-Dist: moto<6,>=5.0.3; extra == "all"
 Requires-Dist: ec2_metadata<3,>=2.14.0; extra == "all"
-Requires-Dist: cwltool==3.1.20250715140722; extra == "all"
+Requires-Dist: cwltool==3.1.20260108082145; extra == "all"
 Requires-Dist: schema-salad<9,>=8.4.20230128170514; extra == "all"
 Requires-Dist: galaxy-tool-util<26; extra == "all"
 Requires-Dist: galaxy-util<26; extra == "all"
-Requires-Dist: ruamel.yaml<=0.19,>=0.15; extra == "all"
+Requires-Dist: ruamel.yaml<=0.19.1,>=0.15; extra == "all"
 Requires-Dist: ruamel.yaml.clib>=0.2.6; extra == "all"
 Requires-Dist: networkx!=2.8.1,<4; extra == "all"
 Requires-Dist: CacheControl[filecache]; extra == "all"
@@ -117,14 +115,13 @@ Requires-Dist: kubernetes-stubs==v22.6.0post1; extra == "all"
 Requires-Dist: types-urllib3; extra == "all"
 Requires-Dist: types-PyYAML; extra == "all"
 Requires-Dist: idna>=2; extra == "all"
-Requires-Dist: miniwdl==1.13.0; extra == "all"
+Requires-Dist: miniwdl==1.13.1; extra == "all"
 Requires-Dist: wdlparse==0.1.0; extra == "all"
-Requires-Dist: graphlib-backport==1.0; python_version < "3.9" and extra == "all"
 Requires-Dist: connexion[swagger-ui]<4,>=3; extra == "all"
 Requires-Dist: flask<4,>=3; extra == "all"
 Requires-Dist: werkzeug<4,>=3; extra == "all"
 Requires-Dist: flask-cors<7,>=5.0.1; extra == "all"
-Requires-Dist: gunicorn<24,>=23.0.0; extra == "all"
+Requires-Dist: gunicorn<26,>=23.0.0; extra == "all"
 Requires-Dist: celery<6,>=5.4.0; extra == "all"
 Requires-Dist: wes-service<6,>=5.0.0; extra == "all"
 Requires-Dist: ruamel.yaml<1,>=0.18.7; extra == "all"

{toil-9.1.1 → toil-9.2.0}/contrib/mypy-stubs/pubsub/core/topictreetraverser.pyi

@@ -1,7 +1,8 @@
-from .topicobj import Topic as Topic
 from enum import IntEnum
 from typing import Optional
 
+from .topicobj import Topic as Topic
+
 class ITopicTreeVisitor: ...
 
 class TreeTraversal(IntEnum):
@@ -12,4 +13,6 @@ class TreeTraversal(IntEnum):
 class TopicTreeTraverser:
     def __init__(self, visitor: Optional[ITopicTreeVisitor] = None) -> None: ...
     def setVisitor(self, visitor: ITopicTreeVisitor) -> None: ...
-    def traverse(self, topicObj: Topic, how: TreeTraversal = ..., onlyFiltered: bool = True) -> None: ...
+    def traverse(
+        self, topicObj: Topic, how: TreeTraversal = ..., onlyFiltered: bool = True
+    ) -> None: ...

toil-9.2.0/contrib/slurm-test/docker-compose.yml

@@ -0,0 +1,100 @@
+# This is a v3 compose file
+services:
+  slurmmaster:
+        # This should be https://github.com/jlundos/slurm-cluster/tree/devel
+        # Commit ad53b77bc74de768b627185a012e0bcb1b44ba1c
+        image: quay.io/adamnovak/slurm-master:latest
+        hostname: slurmmaster
+        user: admin
+        volumes:
+                - shared-vol:/home/admin
+                - ${PWD}/slurm.conf:/etc/slurm/slurm.conf
+
+        environment:
+                - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/admin/.local/bin
+        ports:
+                - 6817:6817
+                - 6818:6818
+                - 6819:6819
+        deploy:
+          resources:
+            limits:
+              memory: 2G
+        healthcheck:
+          # When we start up, we write a key to a volume that the workers need to authenticate us.
+          # So don't be healthy until that key exists with the right content.
+          # The leading space maybe keeps this out of shell history? It's just
+          # stolen from
+          # <https://www.baeldung.com/linux/docker-compose-container-interdependence>
+          test: ["CMD-SHELL", " sudo diff /etc/munge/munge.key /home/admin/shared/mungesetup/munge.key"]
+          interval: 2s
+          timeout: 10s
+          retries: 3
+          start_period: 2s
+
+  slurmnode1:
+        image: quay.io/adamnovak/slurm-node:latest
+        # TODO: The worker container must be privileged or running a job that uses --export, like:
+        #   sbatch --export=OMP_NUM_THREADS=1 --wrap="ls"
+        # will cause the executing slurmd to crash with a message in the log like:
+        #   fatal: clone: Operation not permitted
+        privileged: true
+        hostname: slurmnode1
+        user: admin
+        volumes:
+                - shared-vol:/home/admin
+                - ${PWD}/slurm.conf:/etc/slurm/slurm.conf
+        environment:
+                - SLURM_NODENAME=slurmnode1
+        links:
+                - slurmmaster
+        deploy:
+          resources:
+            limits:
+              memory: 2G
+        depends_on:
+          slurmmaster:
+            condition: service_healthy
+
+  slurmnode2:
+        image: quay.io/adamnovak/slurm-node:latest
+        privileged: true
+        hostname: slurmnode2
+        user: admin
+        volumes:
+                - shared-vol:/home/admin
+                - ${PWD}/slurm.conf:/etc/slurm/slurm.conf
+        environment:
+                - SLURM_NODENAME=slurmnode2
+        links:
+                - slurmmaster
+        deploy:
+          resources:
+            limits:
+              memory: 2G
+        depends_on:
+          slurmmaster:
+            condition: service_healthy
+
+  slurmnode3:
+        image: quay.io/adamnovak/slurm-node:latest
+        privileged: true
+        hostname: slurmnode3
+        user: admin
+        volumes:
+                - shared-vol:/home/admin
+                - ${PWD}/slurm.conf:/etc/slurm/slurm.conf
+        environment:
+                - SLURM_NODENAME=slurmnode3
+        links:
+                - slurmmaster
+        deploy:
+          resources:
+            limits:
+              memory: 2G
+        depends_on:
+          slurmmaster:
+            condition: service_healthy
+
+volumes:
+        shared-vol:

{toil-9.1.1 → toil-9.2.0}/contrib/slurm-test/slurm.conf

@@ -48,11 +48,13 @@ SlurmUser=root
 #SlurmdUser=root
 #SrunEpilog=
 #SrunProlog=
-StateSaveLocation=/var/spool
+StateSaveLocation=/var/spool/slurmctld
 SwitchType=switch/none
 #TaskEpilog=
-TaskPlugin=task/affinity
-TaskPluginParam=Sched
+#TaskPlugin=task/cgroup,task/affinity
+TaskPlugin=task/none
+#TaskPlugin=task/affinity
+#TaskPluginParam=Sched # This doesn't work with current slurm!
 #TaskProlog=
 #TopologyPlugin=topology/tree
 #TmpFS=/tmp
@@ -87,7 +89,7 @@ Waittime=0
 #MaxMemPerCPU=0
 #SchedulerTimeSlice=30
 SchedulerType=sched/backfill
-SelectType=select/cons_res
+SelectType=select/cons_tres
 SelectTypeParameters=CR_Core
 #
 #
@@ -114,7 +116,7 @@ SelectTypeParameters=CR_Core
 #AccountingStoragePort=
 AccountingStorageType=accounting_storage/none
 #AccountingStorageUser=
-AccountingStoreJobComment=YES
+AccountingStoreFlags=job_comment
 ClusterName=cluster
 #DebugFlags=
 #JobCompHost=
@@ -127,9 +129,9 @@ JobCompType=jobcomp/none
 JobAcctGatherFrequency=30
 JobAcctGatherType=jobacct_gather/none
 SlurmctldDebug=error
-SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log
+SlurmctldLogFile=/var/log/slurm/slurmctld.log
 SlurmdDebug=error
-SlurmdLogFile=/var/log/slurm-llnl/slurmd.log
+SlurmdLogFile=/var/log/slurm/slurmd.log
 #SlurmSchedLogFile=
 #SlurmSchedLogLevel=
 #
@@ -147,5 +149,6 @@ SlurmdLogFile=/var/log/slurm-llnl/slurmd.log
 #
 #
 # COMPUTE NODES
+#
 NodeName=slurmnode[1-10] CPUs=1 RealMemory=2000 State=UNKNOWN
-PartitionName=slurmpar Nodes=slurmnode[1-10] Default=YES MaxTime=00:02:00 State=UP
+PartitionName=slurmpar Nodes=slurmnode[1-10] Default=YES MaxTime=00:05:00 State=UP

toil-9.2.0/contrib/slurm-test/slurm_test.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -ex
+# With the docker compose plugin, containers are named like slurm-test-slurmmaster-1
+# If your containers are named like ${LEADER} you have the old docker-compose Python version instead.
+# Try running with NAME_SEP=_
+NAME_SEP=${CONTAINER_NAME_SEP:--}
+LEADER="slurm-test${NAME_SEP}slurmmaster${NAME_SEP}1"
+docker compose up -d
+docker compose ps
+docker cp toil_workflow.py ${LEADER}:/home/admin
+docker cp -L sort.py ${LEADER}:/home/admin
+docker cp fileToSort.txt ${LEADER}:/home/admin
+docker cp toil_workflow.py ${LEADER}:/home/admin
+GIT_COMMIT=$(git rev-parse HEAD)
+# The base cluster image doesn't ship a working venv, or git
+docker exec -e DEBIAN_FRONTEND=noninteractive ${LEADER} sudo apt-get update
+docker exec -e DEBIAN_FRONTEND=noninteractive ${LEADER} sudo apt-get -qq -y install python3-venv git >/dev/null
+docker exec ${LEADER} python3.12 -m venv /home/admin/venv
+docker exec ${LEADER} /home/admin/venv/bin/python -m pip install "git+https://github.com/DataBiosphere/toil.git@${GIT_COMMIT}"
+# This can sometimes report:
+#   slurm_load_partitions: Unexpected message received
+# In that case we need to wait and try again.
+DELAY=1
+docker exec ${LEADER} sinfo -N -l && STATUS=0 || STATUS="${?}"
+while [[ "${STATUS}" != "0" && "${LOOP_COUNT}" != "10" ]] ; do
+    echo "Waiting for Slurm to be up"
+    sleep "${DELAY}"
+    docker exec ${LEADER} sinfo -N -l && STATUS=0 || STATUS="${?}"
+    ((LOOP_COUNT+=1))
+    ((DELAY+=DELAY))
+done
+if [[ "${STATUS}" != "0" ]] ; then
+    echo >&2 "Could not get Slurm info; did Slurm start successfully?"
+    exit 1
+fi
+# Test 1: A really basic workflow to check Slurm is working correctly
+docker exec -e TOIL_CHECK_ENV=True ${LEADER} /home/admin/venv/bin/python /home/admin/toil_workflow.py file:my-job-store --batchSystem slurm --slurmTime 2:00 --disableCaching --retryCount 0 --batchLogsDir ./nonexistent/paths
+docker cp ${LEADER}:/home/admin/output.txt output_Docker.txt
+# Test 2: Make sure that "sort" workflow runs under slurm
+docker exec -e TOIL_CHECK_ENV=True ${LEADER} /home/admin/venv/bin/python /home/admin/sort.py file:my-job-store --batchSystem slurm --slurmTime 2:00 --disableCaching --retryCount 0
+docker cp ${LEADER}:/home/admin/sortedFile.txt sortedFile.txt
+docker compose down -v
+./check_out.sh
+echo "Sucessfully ran workflow on slurm cluster"

{toil-9.1.1/src/toil/test/sort → toil-9.2.0/contrib/slurm-test}/sort.py

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort.
-"""
+"""A demonstration of toil. Sorts the lines of a file into ascending order by doing a parallel merge sort."""
 import codecs
 import os
 import random
@@ -64,9 +63,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
     length = os.path.getsize(inputFile)
     if length > N:
         # We will subdivide the file
-        RealtimeLogger.critical(
-            "Splitting file: %s of size: %s" % (inputFileStoreID, length)
-        )
+        RealtimeLogger.critical(f"Splitting file: {inputFileStoreID} of size: {length}")
         # Split the file into two copies
         midPoint = getMidPoint(inputFile, 0, length)
         t1 = job.fileStore.getLocalTempFile()
@@ -108,9 +105,7 @@ def down(job, inputFileStoreID, N, path, downCheckpoints, options, memory=sortMe
         ).rv()
     else:
         # We can sort this bit of the file
-        RealtimeLogger.critical(
-            "Sorting file: %s of size: %s" % (inputFileStoreID, length)
-        )
+        RealtimeLogger.critical(f"Sorting file: {inputFileStoreID} of size: {length}")
         # Sort the copy and write back to the fileStore
         shutil.copyfile(inputFile, inputFile + ".sort")
         sort(inputFile + ".sort")

{toil-9.1.1 → toil-9.2.0}/contrib/toil-ci-prebake/Dockerfile

@@ -26,23 +26,22 @@ RUN apt-get -q -y update && \
     apt-get -q -y install \
         make git software-properties-common build-essential virtualenv libssl-dev libffi-dev \
         libncurses5-dev curl zip unzip cmake wget docker.io python3-pip python3-virtualenv \
-        python-dev npm nodejs tzdata jq awscli graphviz && \
+        python3-dev npm nodejs tzdata jq awscli graphviz && \
     apt-get clean
 
 RUN sudo add-apt-repository -y ppa:deadsnakes/ppa && \
     sudo apt-get update && \
     sudo apt-get install -y \
-        python3.8 python3.8-dev python3.8-venv python3.8-distutils \
         python3.9 python3.9-dev python3.9-venv python3.9-distutils \
         python3.10 python3.10-dev python3.10-venv python3.10-distutils \
         python3.11 python3.11-dev python3.11-venv python3.11-distutils \
-        python3.12 python3.12-dev python3.12-venv python3.12-distutils \
-        python3.13 python3.13-dev python3.13-venv
+        python3.12 python3.12-dev python3.12-venv \
+        python3.13 python3.13-dev python3.13-venv \
+        python3.14 python3.14-dev python3.14-venv
 
 RUN rm -f /usr/bin/python
 RUN ln -s /usr/bin/python3.11 /usr/bin/python
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.8 get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py && python3.13 get-pip.py # for python 3.12, get-pip seems to remove other pip versions, python 3.13 needs get-pip.py or ensure-pip as older versions of pip relies on distutils which is removed in 3.13
-RUN python3.13 -m pip install virtualenv -U && python3.12 -m pip install virtualenv -U && python3.8 -m pip install virtualenv && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv # python3.12 needs latest version of virtualenv
-
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3.9 get-pip.py && python3.10 get-pip.py && python3.11 get-pip.py && python3.12 get-pip.py && python3.13 get-pip.py && python3.14 get-pip.py # for python 3.12, get-pip seems to remove other pip versions, python 3.13 needs get-pip.py or ensure-pip as older versions of pip relies on distutils which is removed in 3.13
+RUN python3.14 -m pip install virtualenv && python3.14 -m pip install virtualenv -U && python3.13 -m pip install virtualenv -U && python3.12 -m pip install virtualenv -U && python3.9 -m pip install virtualenv && python3.10 -m pip install virtualenv && python3.11 -m pip install virtualenv # python3.12 needs latest version of virtualenv
 
 LABEL maintainers="anovak@soe.ucsc.edu lblauvel@ucsc.edu"

{toil-9.1.1 → toil-9.2.0}/docker/Dockerfile.py

@@ -25,11 +25,12 @@ python = f'python{sys.version_info[0]}.{sys.version_info[1]}'
 pip = f'{python} -m pip'
 
 # Debian and Ubuntu don't package ensurepip by default so the python-venv package must be installed
-python_packages = {'python3.9': ['python3.9-distutils', 'python3.9-venv'],
-                   'python3.10': ['python3.10-distutils', 'python3.10-venv'],
+python_packages = {'python3.10': ['python3.10-distutils', 'python3.10-venv'],
                    'python3.11': ['python3.11-distutils', 'python3.11-venv'],
                    'python3.12': ['python3.12-venv'],
-                   'python3.13': ['python3.13-venv']}  # python3.13 removed distutils
+                   'python3.13': ['python3.13-venv'],
+                   'python3.14': ['python3.14-venv'],
+                   }  # python3.13 removed distutils
 
 dependencies = ' '.join(python_packages[python] +
                         ['libffi-dev',  # For client side encryption for extras with PyNACL
@@ -78,11 +79,11 @@ dependencies = ' '.join(python_packages[python] +
 # available in PyPI. So we need to manually inject a working http-parser, and
 # pymesos, into the Docker images.
 extra_mesos_python_modules = {
-    'python3.9': [],
     'python3.10': [],
     'python3.11': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
     'python3.12': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
-    'python3.13': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15']
+    'python3.13': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15'],
+    'python3.14': ['http-parser@git+https://github.com/adamnovak/http-parser.git@5a63516597bb4c93a7ba178b1e4bab939da5afb3', 'pymesos==0.3.15']
 }
 
 extra_python_modules = " ".join(extra_mesos_python_modules[python])

{toil-9.1.1 → toil-9.2.0}/docs/appendices/deploy.rst

@@ -41,7 +41,7 @@ From here, you can install a project and its dependencies::
 
    3 directories, 5 files
    $ pip install matplotlib
-   $ cp -R workflow util venv/lib/python3.9/site-packages
+   $ cp -R workflow util venv/lib/python3.14/site-packages
 
 Ideally, your project would have a ``setup.py`` file (see `setuptools`_) which streamlines the installation process::
 

{toil-9.1.1 → toil-9.2.0}/docs/running/hpcEnvironments.rst

@@ -28,7 +28,7 @@ For example, to run the sort example :ref:`sort example <sortExample>` on Slurm,
 
 By default, this does not include any time limit or particular Slurm partition. If your Slurm cluster requires time limits, add the ``--slurmTime`` option to set the time limit to use for jobs.
 
-If you do specify a time limit, a partition will be automatically selected that can acommodate jobs of that duration, and a partition will be automatically selected for jobs that need GPUs. To use a particular partition, use the ``--slurmPartition`` argument. If you are running GPU jobs and they need to go to a different partition, use the ``--slurmGPUPartition`` argument. For example, to :ref:`run a WDL workflow from Dockstore <runWdl>` using GPUs on Slurm, with a time limit of 4 hours per job and partitions manually specified, you would run::
+If you do specify a time limit, a partition will be automatically selected that can accommodate jobs of that duration, and a partition will be automatically selected for jobs that need GPUs. To use a particular partition, use the ``--slurmPartition`` argument. If you are running GPU jobs and they need to go to a different partition, use the ``--slurmGPUPartition`` argument. For example, to :ref:`run a WDL workflow from Dockstore <runWdl>` using GPUs on Slurm, with a time limit of 4 hours per job and partitions manually specified, you would run::
 
     $ toil-wdl-runner '#workflow/github.com/vgteam/vg_wdl/GiraffeDeepVariantFromGAF:gbz' \
       https://raw.githubusercontent.com/vgteam/vg_wdl/refs/heads/gbz/params/giraffe_and_deepvariant_gaf.json \