toil 6.0.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/worker.py

@@ -50,7 +50,7 @@ logger = logging.getLogger(__name__)
 class StatsDict(MagicExpando):
     """Subclass of MagicExpando for type-checking purposes."""
 
-    jobs: List[str]
+    jobs: List[MagicExpando]
 
 
 def nextChainable(predecessor: JobDescription, jobStore: AbstractJobStore, config: Config) -> Optional[JobDescription]:
@@ -287,13 +287,14 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
 
     jobAttemptFailed = False
     failure_exit_code = 1
+    first_job_cores = None
     statsDict = StatsDict()  # type: ignore[no-untyped-call]
     statsDict.jobs = []
-    statsDict.workers.logsToMaster = []
+    statsDict.workers.logs_to_leader = []
+    statsDict.workers.logging_user_streams = []
 
     def blockFn() -> bool:
         return True
-    listOfJobs = [jobName]
     job = None
     try:
 
@@ -313,7 +314,6 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
         ##########################################
 
         jobDesc = jobStore.load_job(jobStoreID)
-        listOfJobs[0] = str(jobDesc)
         logger.debug("Parsed job description")
 
         ##########################################
@@ -363,6 +363,8 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
         ##########################################
 
         if config.stats:
+            # Remember the cores from the first job, which is how many we have reserved for us.
+            statsDict.workers.requested_cores = jobDesc.cores
             startClock = get_total_cpu_time()
 
         startTime = time.time()
@@ -411,7 +413,8 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
                             # job body cut.
 
                 # Accumulate messages from this job & any subsequent chained jobs
-                statsDict.workers.logsToMaster += fileStore.loggingMessages
+                statsDict.workers.logs_to_leader += fileStore.logging_messages
+                statsDict.workers.logging_user_streams += fileStore.logging_user_streams
 
                 logger.info("Completed body for %s", jobDesc)
 
@@ -457,9 +460,6 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
             # body) up after we finish executing it.
             successorID = successor.jobStoreID
 
-            # add the successor to the list of jobs run
-            listOfJobs.append(str(successor))
-
             # Now we need to become that successor, under the original ID.
             successor.replace(jobDesc)
             jobDesc = successor
@@ -489,6 +489,16 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
             statsDict.workers.time = str(time.time() - startTime)
             statsDict.workers.clock = str(totalCPUTime - startClock)
             statsDict.workers.memory = str(totalMemoryUsage)
+            # Say the worker used the max disk we saw from any job
+            max_bytes = 0
+            for job_stats in statsDict.jobs:
+                if "disk" in job_stats:
+                    max_bytes = max(max_bytes, int(job_stats.disk))
+            statsDict.workers.disk = str(max_bytes)
+            # Count the jobs executed.
+            # TODO: toil stats could compute this but its parser is too general to hook into simply.
+            statsDict.workers.jobs_run  = len(statsDict.jobs)
+
 
         # log the worker log path here so that if the file is truncated the path can still be found
         if redirectOutputToLogFile:
@@ -499,13 +509,16 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
     ##########################################
     #Trapping where worker goes wrong
     ##########################################
-    except Exception as e: #Case that something goes wrong in worker
+    except BaseException as e: #Case that something goes wrong in worker, or we are asked to stop
         traceback.print_exc()
         logger.error("Exiting the worker because of a failed job on host %s", socket.gethostname())
         if isinstance(e, CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION):
             # We need to inform the leader that this is a CWL workflow problem
             # and it needs to inform its caller.
             failure_exit_code = CWL_UNSUPPORTED_REQUIREMENT_EXIT_CODE
+        elif isinstance(e, SystemExit) and isinstance(e.code, int) and e.code != 0:
+            # We're meant to be exiting with a particular code.
+            failure_exit_code = e.code
         AbstractFileStore._terminateEvent.set()
     finally:
         # Get rid of our deferred function manager now so we can't mistake it
@@ -581,7 +594,6 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
         jobDesc.logJobStoreFileID = logJobStoreFileID = jobStore.getEmptyFileStoreID(
             jobDesc.jobStoreID, cleanup=True
         )
-        jobDesc.chainedJobs = listOfJobs
         with jobStore.update_file_stream(logJobStoreFileID) as w:
             with open(tempWorkerLogPath, 'rb') as f:
                 if os.path.getsize(tempWorkerLogPath) > logFileByteReportLimit !=0:
@@ -605,10 +617,13 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
             # Make sure lines are Unicode so they can be JSON serialized as part of the dict.
             # We may have damaged the Unicode text by cutting it at an arbitrary byte so we drop bad characters.
             logMessages = [line.decode('utf-8', 'skip') for line in logFile.read().splitlines()]
-        statsDict.logs.names = listOfJobs
+        statsDict.logs.names = [names.stats_name for names in jobDesc.get_chain()]
         statsDict.logs.messages = logMessages
 
-    if (debugging or config.stats or statsDict.workers.logsToMaster) and not jobAttemptFailed:  # We have stats/logging to report back
+    if debugging or config.stats or statsDict.workers.logs_to_leader or statsDict.workers.logging_user_streams:
+        # We have stats/logging to report back.
+        # We report even if the job attempt failed.
+        # TODO: Will that upset analysis of the stats?
         jobStore.write_logs(json.dumps(statsDict, ensure_ascii=True))
 
     # Remove the temp dir
@@ -631,10 +646,10 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
 
     # This must happen after the log file is done with, else there is no place to put the log
     if (not jobAttemptFailed) and jobDesc.is_subtree_done():
-        # We can now safely get rid of the JobDescription, and all jobs it chained up
-        for otherID in jobDesc.merged_jobs:
-            jobStore.delete_job(otherID)
-        jobStore.delete_job(str(jobDesc.jobStoreID))
+        for merged_in in jobDesc.get_chain():
+            # We can now safely get rid of the JobDescription, and all jobs it chained up
+            jobStore.delete_job(merged_in.job_store_id)
+        
 
     if jobAttemptFailed:
         return failure_exit_code

{toil-6.0.0.dist-info → toil-6.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: toil
-Version: 6.0.0
+Version: 6.1.0
 Summary: Pipeline management software for clusters.
 Home-page: https://github.com/DataBiosphere/toil
 Author: Benedict Paten and the Toil community
@@ -32,7 +32,7 @@ Requires-Python: >=3.8
 License-File: LICENSE
 Requires-Dist: dill <0.4,>=0.3.2
 Requires-Dist: requests <3,>=2
-Requires-Dist: docker <8,>=3.7.2
+Requires-Dist: docker <8,>=6.1.0
 Requires-Dist: urllib3 <3,>=1.26.0
 Requires-Dist: python-dateutil
 Requires-Dist: psutil <6,>=3.0.1
@@ -42,7 +42,6 @@ Requires-Dist: pytz >=2012
 Requires-Dist: enlighten <2,>=1.5.2
 Requires-Dist: configargparse <2,>=1.7
 Requires-Dist: ruamel.yaml >=0.15
-Requires-Dist: ruamel.yaml.string >=0.1.1
 Requires-Dist: pyyaml <7,>=6
 Requires-Dist: typing-extensions <5,>=4.6.2
 Provides-Extra: all
@@ -54,7 +53,7 @@ Requires-Dist: cwltool ==3.1.20240112164112 ; extra == 'all'
 Requires-Dist: schema-salad <9,>=8.4.20230128170514 ; extra == 'all'
 Requires-Dist: galaxy-tool-util <23 ; extra == 'all'
 Requires-Dist: galaxy-util <23 ; extra == 'all'
-Requires-Dist: ruamel.yaml <=0.18.3,>=0.15 ; extra == 'all'
+Requires-Dist: ruamel.yaml <=0.19,>=0.15 ; extra == 'all'
 Requires-Dist: ruamel.yaml.clib >=0.2.6 ; extra == 'all'
 Requires-Dist: networkx !=2.8.1,<4 ; extra == 'all'
 Requires-Dist: CacheControl[filecache] ; extra == 'all'
@@ -76,7 +75,7 @@ Requires-Dist: flask-cors ==4.0.0 ; extra == 'all'
 Requires-Dist: gunicorn ==21.2.0 ; extra == 'all'
 Requires-Dist: celery <6,>=5.1.0 ; extra == 'all'
 Requires-Dist: wes-service <5,>=4.0.0 ; extra == 'all'
-Requires-Dist: ruamel.yaml <0.18.4,>=0.15 ; extra == 'all'
+Requires-Dist: ruamel.yaml <0.19,>=0.15 ; extra == 'all'
 Requires-Dist: pymesos <0.4,>=0.3.15 ; (python_version < "3.11") and extra == 'all'
 Requires-Dist: graphlib-backport ==1.0 ; (python_version < "3.9") and extra == 'all'
 Provides-Extra: aws
@@ -89,7 +88,7 @@ Requires-Dist: cwltool ==3.1.20240112164112 ; extra == 'cwl'
 Requires-Dist: schema-salad <9,>=8.4.20230128170514 ; extra == 'cwl'
 Requires-Dist: galaxy-tool-util <23 ; extra == 'cwl'
 Requires-Dist: galaxy-util <23 ; extra == 'cwl'
-Requires-Dist: ruamel.yaml <=0.18.3,>=0.15 ; extra == 'cwl'
+Requires-Dist: ruamel.yaml <=0.19,>=0.15 ; extra == 'cwl'
 Requires-Dist: ruamel.yaml.clib >=0.2.6 ; extra == 'cwl'
 Requires-Dist: networkx !=2.8.1,<4 ; extra == 'cwl'
 Requires-Dist: CacheControl[filecache] ; extra == 'cwl'
@@ -117,7 +116,7 @@ Requires-Dist: flask-cors ==4.0.0 ; extra == 'server'
 Requires-Dist: gunicorn ==21.2.0 ; extra == 'server'
 Requires-Dist: celery <6,>=5.1.0 ; extra == 'server'
 Requires-Dist: wes-service <5,>=4.0.0 ; extra == 'server'
-Requires-Dist: ruamel.yaml <0.18.4,>=0.15 ; extra == 'server'
+Requires-Dist: ruamel.yaml <0.19,>=0.15 ; extra == 'server'
 Provides-Extra: wdl
 Requires-Dist: miniwdl ==1.11.1 ; extra == 'wdl'
 Requires-Dist: wdlparse ==0.1.0 ; extra == 'wdl'

{toil-6.0.0.dist-info → toil-6.1.0.dist-info}/RECORD

@@ -1,33 +1,33 @@
 toil/__init__.py,sha256=7Rj4nfBg1bZoiUGi5qbCQKPBydtj-ulj735-AWx8EHY,31427
-toil/bus.py,sha256=zwC-TbdeGLoUNPBCOk_jxGVrBeV3Ws3mz2nrFlXssKs,28140
-toil/common.py,sha256=cK2kiFt2QP0_vqz0MfLPJ-ncPX_YANf7_iI4GGHWAkM,70888
+toil/bus.py,sha256=I3H7nYzbBjCm7L3QkNmI-S_JAXUyy60DXXFURgVfMIg,29159
+toil/common.py,sha256=rMc9_ECeecbma18uwN1PS_HEKAAdi_fdtgjf4Ax_23M,70917
 toil/deferred.py,sha256=vGr0IrCvpTiCDR05d5wwmqs6WrifgqDRjDjS24ozJPA,13953
-toil/exceptions.py,sha256=Ym1y-oBMB0tYgbKqDfhU8hnuiPgbQisU0bC0SVJ_iLU,1815
-toil/job.py,sha256=nkGuvqHS0s28FjF7npNT9x8i6kl1uVA6bGIGl5IgOfo,148226
-toil/leader.py,sha256=u9Cq0vISKQ4Gwe1GHsXwr6gitqZV68OUGa043CBLX3M,76819
+toil/exceptions.py,sha256=3c87Jo1NKBz8CcQA3GEm-r57DnJAfWiGFMKQFYAlJOo,1835
+toil/job.py,sha256=GDMObf7tCyND99cLb4XR07lXGJ_REWlzKKOVK8s0QgE,149335
+toil/leader.py,sha256=Xkwy9g3BiU0Cx9vJeBSrvWqM4XFpZTU8vRbZpYDNaCM,77216
 toil/realtimeLogger.py,sha256=eB_6E9lJYMrsjAswsdI6EiFBJeYO1iMk6S3OHBYVPsk,9733
 toil/resource.py,sha256=l_Kdf63fHkWl8eUFFTQc7Br4k56LUd4bgB1LnvhbveM,24869
 toil/serviceManager.py,sha256=86GYynCH7wh7nDswvQc4F1OdGl2OQmC6wiW_si3uatA,19084
-toil/statsAndLogging.py,sha256=bfcgJt1qhCkGI_rU48YAVXmJugIcvYxE_p0FMKSWwlY,13164
+toil/statsAndLogging.py,sha256=1HNcE35CFeBX-1Ikpj7oqHsSDd7UlBJDKeSAhrdlfTc,14455
 toil/toilState.py,sha256=EIfKb8DSfm33DNBXZIXAa_TZJdkMcygGI3CmdWp0sEY,14907
-toil/version.py,sha256=MXXtrcm9hfmM_jIcBQINBwiurpKcH0Un9cctsKEH6qc,486
-toil/worker.py,sha256=4jx6iyJspzBVU7Vawhzgw5i2SmqscADPmi13hcR4KPc,31579
+toil/version.py,sha256=Yzmq-I9oCvrZhIDnlptJ_iJ4ucMUrnMip0ug8ngzVkw,491
+toil/worker.py,sha256=qzSfEgvpU7_G3-r7BNrN7hRQDuTtvZV59QLB8wOnlp0,32574
 toil/batchSystems/__init__.py,sha256=UqxHmoYxHGYSnlEpmZCw1GnnpGFASzZlVG365E8_muY,1035
-toil/batchSystems/abstractBatchSystem.py,sha256=w7SyMkSn1G0AQS1YhUqs6oYlVNlUF1ZUuvF1uXAUhQQ,32347
-toil/batchSystems/abstractGridEngineBatchSystem.py,sha256=jLdFWGHW-NAbCsl_sPATQDoO8edENcCU_xqlYwEGpmI,20870
+toil/batchSystems/abstractBatchSystem.py,sha256=zk13Pw8f86wTtbRX_tWHv_B-uqOlwZVs20PaWfNEn4Y,32794
+toil/batchSystems/abstractGridEngineBatchSystem.py,sha256=TPhtWOu0t2W7ntFSA3XUfPBrJD12AsAL_-kxPafVC8g,20963
 toil/batchSystems/awsBatch.py,sha256=aPxHzBsf_9PCQuEkkRz5wrNOwh4Yf2Wr0jaP1M6SYmQ,25296
-toil/batchSystems/cleanup_support.py,sha256=cPC83xxEuE0ckV7a1Z-cSBkaT68trIDMg2AZr1K5dJ4,4002
+toil/batchSystems/cleanup_support.py,sha256=SKx9804ZvcdgnuY1a7kU47G8Yj-1G13MVVx8dxTnj8o,4069
 toil/batchSystems/contained_executor.py,sha256=582lofKCez9RWfQ3H5k1vpNVYgZ2YavooiWnH-nrBBk,5193
 toil/batchSystems/gridengine.py,sha256=Njkh4FJDvz3nqnLTaPgExSBGbOadhM7ua-ofQyFhhyc,7752
 toil/batchSystems/htcondor.py,sha256=a1KyCJmtGGPPaVxcCdDFvTcaX19CTjTlgKxDtgTCEqo,17293
 toil/batchSystems/kubernetes.py,sha256=AnNB5LxkB-9FmofWlXpF8eySaW1qVq2a1BXKrOMzkHc,84697
 toil/batchSystems/local_support.py,sha256=EI3R6pkGRQXQodAqDHOVAqKa9Zytlh1hHjjnciydGu0,3903
-toil/batchSystems/lsf.py,sha256=-OQDDwD4myH-6MsJMzauS4dZStu42x_9V1dUJ8gb8bk,17044
+toil/batchSystems/lsf.py,sha256=HNJdmUG6_uOe6UsnD0ecUiilnKRM2-2jydHnx_gGX7c,17376
 toil/batchSystems/lsfHelper.py,sha256=JJE3OHoye-V55TGMN03s-t56Lf2ijrhAjZqInLccF7Q,7205
 toil/batchSystems/options.py,sha256=bsqrv8qBvPdaGcyDCESzArLmqB-b1GdE98n058niPMY,7654
 toil/batchSystems/registry.py,sha256=7Mp_KJWIgU0YEXVELITifPX0yZugkP0sV36W8fa1RZk,6659
 toil/batchSystems/singleMachine.py,sha256=A9zsbYj6TqASVMboQKDaYeTijjE1rqVwxQNSO5Lc9rs,38113
-toil/batchSystems/slurm.py,sha256=UmXeWS3WmQMlHFms7253ovQNzFPTmW01IWfe8wyjSrA,19103
+toil/batchSystems/slurm.py,sha256=FyTrugG9maY-wfsW10q8MIqMgKAaxSAnBpXKr4-shug,22635
 toil/batchSystems/torque.py,sha256=rEvrvKnw2FUAMPpwO-9h2qpifLd1pFv2zzDxiBKD0yQ,11625
 toil/batchSystems/mesos/__init__.py,sha256=_klpvdKQWxDTeL1fbPdItd-Lx3KJnNMAzHJtUqsbykU,3354
 toil/batchSystems/mesos/batchSystem.py,sha256=Iosg58Uy3BnFk8VoM7vMM-MStKC-Kgoe0CAb7YKXOpY,38900
@@ -36,41 +36,41 @@ toil/batchSystems/mesos/executor.py,sha256=KfreVYrjMaRRzpQGhxbi2zbBQj6fUQ0EniiS6
 toil/batchSystems/mesos/test/__init__.py,sha256=IZoexyiZcAPdv_L39v7mMO7xVHcMuEn7f_M4gyeh-Fk,4670
 toil/cwl/__init__.py,sha256=bIVIPfacZlHZVPrzjXDD5cV4a1VCGpwErJpO5_ta9Bk,2131
 toil/cwl/conftest.py,sha256=R6Jw5-PeTCK0rapfHmV_VJwVcqlVbO5w3cFUbaIWUnk,848
-toil/cwl/cwltoil.py,sha256=UY2XxyMLfQXRO76jiduUtunZgQ_tvUwuWWsTnikJjDY,157397
+toil/cwl/cwltoil.py,sha256=PhORC-4EzKEIwqoDu7nIIVqlvfOAJ_Hr6iObBuLdqGA,159478
 toil/cwl/utils.py,sha256=l-B9zy9cTiOEi0kybE3CzUzkX1VtnIPTkBvGwXtxUB4,8014
 toil/fileStores/__init__.py,sha256=kW1QFBZLJoagBdmxoaABN6peG46f5owJurISVXkbEIg,2390
-toil/fileStores/abstractFileStore.py,sha256=reuGRuYM_iSur6M2Mfw0zxqpNvR4HGvRkfg9KGsVNHA,27904
-toil/fileStores/cachingFileStore.py,sha256=oJSMZ4JwZ16jTVAC1BTd4_Uy9prGovlOaIIvJFkvA-U,92926
-toil/fileStores/nonCachingFileStore.py,sha256=ZxLE0sTu-tzg0dKxHSAaBcaooBAMX_CBII6yQf3tr9Q,16037
+toil/fileStores/abstractFileStore.py,sha256=EgksAy4Y2kUFYV-63I-6sD5FGwsFe3el8dcCvyyweJs,30374
+toil/fileStores/cachingFileStore.py,sha256=N4hXE1j89w8lXy1hR4O1cvlma4pXRnniiw6e7Zy3iXA,91851
+toil/fileStores/nonCachingFileStore.py,sha256=WyIJO0r90aukWBX1Gho8OQ8vuinrrilxN2fA3aQrGhs,15355
 toil/jobStores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-toil/jobStores/abstractJobStore.py,sha256=WPJwNXQII0LlCsSqXMqHKZzfg8bqPLrlkLdZZ9WEce8,77298
+toil/jobStores/abstractJobStore.py,sha256=VSYCgIovwj6McZehRAAvApUoQfhxH_el2NReDCmKwpw,77125
 toil/jobStores/conftest.py,sha256=n4dffd8QPoA_kenbHE_XzA2bf1ZTNlXux8ffukB1qCk,834
-toil/jobStores/fileJobStore.py,sha256=xp3kzoGvLuwvf-vhVlKpHLQdoarPeQUuMGEJDMvITNA,48325
+toil/jobStores/fileJobStore.py,sha256=F4FAJun6oV6GgV7Uj6JcYMdbXT2OcvcFikXt0Hdhm9I,48316
 toil/jobStores/googleJobStore.py,sha256=psqJPvhX_H5WVu9UQr7KlqNdh4hjdcaxHFYaGkSCHFs,24101
 toil/jobStores/utils.py,sha256=tg_MG-3GFVKZxILr7FykBq-NfVugD3j7apkKJLXZtOs,15235
 toil/jobStores/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/jobStores/aws/jobStore.py,sha256=KNaESQm_vs2fMRT9NQLEoc1dxCRnzV8Txj_MTwj_Mnk,78343
-toil/jobStores/aws/utils.py,sha256=lsCEFhEDlU-sxFYBRRwLdWkXFdX9dJNYLyyzFe5AFdI,19299
+toil/jobStores/aws/utils.py,sha256=XNrXsyZhnoZUf-jE3hdP3cejO5wnW2A1in2TmtO7xy8,19340
 toil/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/lib/accelerators.py,sha256=8zkcZREND57xKWsONfV81o0_zyiTbMYL7VI7RElH94M,5551
 toil/lib/bioio.py,sha256=c6B22vMF9w4VjOdZDug07ksJJ0SrqmV8NVJGDdNbLnc,2328
 toil/lib/compatibility.py,sha256=RKk9hpI_KfBZR_SFqLcIf_3vC-oE4cgyi0T8vP-S0xg,1515
-toil/lib/conversions.py,sha256=t5JjGTE-9lN-ufmeQk85GM0Ep6-jHwzrI9YWWY8N5tQ,4186
+toil/lib/conversions.py,sha256=ykZ_md_CtJojKwkPJM1UEDkNC5-xwwdcBNio0Rs_c5Q,4832
 toil/lib/docker.py,sha256=RnEI8sZZmZ_zMYHOMDqEYu_BItJ-s-6319SR9Xcr7v8,21104
 toil/lib/ec2.py,sha256=3057UA-ECEgp1gNUbvq2jrf61iB5FQ7BCSYQNGySIcM,22527
 toil/lib/ec2nodes.py,sha256=RZsIARyrRrWQIOHseG3691ExMEDatNmMtv3mov6u7aw,14167
 toil/lib/exceptions.py,sha256=vxILY_EJUU_Tvg0uDKvtaFjOYE8cJiCtgrGLEOm3qRg,2271
 toil/lib/expando.py,sha256=a_q6CeB7HNZRSbg0teMZN1Qz_vrwGo1lBN7onOyoLvQ,2925
-toil/lib/generatedEC2Lists.py,sha256=KUqsPIDKIS8KtaFh6b-6RiHKhChD70w05i6djmAlPog,242608
+toil/lib/generatedEC2Lists.py,sha256=gcid9noAlzctnH-OKYYuVTgtWgswI7C2MxqM1JALaWU,244245
 toil/lib/humanize.py,sha256=gozIsspDv0LP_lZ8FN5EA9t1_SYoudeciQ0O0FRGzxM,1239
-toil/lib/io.py,sha256=os5akOPSrzsiGTx3nCewlaTT7xFFqs3k4FRZklhd3A4,8997
+toil/lib/io.py,sha256=QLKoZWBV4Ddf61_A5fD5LuQGfgP4tcTW0_hgCZ_CVvk,9645
 toil/lib/iterables.py,sha256=IEb68dyazYOSRVjnK0AFNCymA8M17Ux37sPQAbkdrcw,3429
 toil/lib/memoize.py,sha256=gS58oA-XzrpGeucRA6s8mtwN9BK63w6JTIDnNovMJQI,3223
 toil/lib/misc.py,sha256=Qm8SUEYkLKZeH5ZuGzsUxlEvWDC-H_4yBI96LS_SPxo,5729
 toil/lib/objects.py,sha256=yrprtKobB5n1_d5oMu6SBzTAp4PX7uzw9lTkBkR1ldw,5384
-toil/lib/resources.py,sha256=YYx-OOKT6rAVvbNB5t7VHVDop2hFhaEXlwx_ryG6qcY,2185
+toil/lib/resources.py,sha256=SK1ghdZkT3FsvKzFqh5QxfRRqBnhCfA2FMvLPg1An10,2501
 toil/lib/retry.py,sha256=iIASaiOh26ipVL1QtapNAdoGaA7iB6bVLqvNotaRIWo,22366
-toil/lib/threading.py,sha256=_2Zbgoejme07yQdRzWDE16rz5Qq0kZbaon_QA8SZIdE,23527
+toil/lib/threading.py,sha256=NQtvJZofxThZxAC-z6OX3Ai6QvO-E38iSzC7GfkHap8,24222
 toil/lib/throttle.py,sha256=2tofeq1rFkXyBZiew3ECUhcicOF01DupIF9HtXuFqD0,4977
 toil/lib/aws/__init__.py,sha256=Sfp0FIMI_EeUwYcUK9DwivajWY6fYUZsGiAn9eNiY_w,7157
 toil/lib/aws/ami.py,sha256=hhfH8mkE8RCPo9V_hWCFg4G8_V--Lu_iKG1fbwGDB6U,9259
@@ -82,11 +82,11 @@ toil/lib/encryption/_dummy.py,sha256=TBmPDmt1FLUtwUtb8e1EKZ-8tYRC3lyrxAp4Rkaw1A8
 toil/lib/encryption/_nacl.py,sha256=FMiBDQ5WlmbbBGWWMQMl2kKd5m7aKvUVODHUhV09dEA,3831
 toil/lib/encryption/conftest.py,sha256=PF4PbhiUftyUcNx2uU0RtX8-LcR0YAFFI5z4fBlzF5M,226
 toil/options/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-toil/options/common.py,sha256=YDj7zV99MqFXLASEn4iXz9IQc_0XE8h5pkjJH-CO3Js,47766
+toil/options/common.py,sha256=5Z7R6kgQ5czsnei2NUNdojFUEWLRHOdnmh8FRzJZdtM,47719
 toil/options/cwl.py,sha256=kbevYHXwmv0iQJnmKwusl235PY_ZlHFWG1nTbdlA8Xo,12939
-toil/options/wdl.py,sha256=12E5G0VFUP3NF4ZDgf4inDuGR35KoK5H8yI0tPYUQRM,2191
+toil/options/wdl.py,sha256=IcngBeScBRoGiqqkpApIUPDwvqp76tY6TLIoTOGJb00,2663
 toil/provisioners/__init__.py,sha256=5_mxQo62vAcjCAk-ulYH0d9EcLAjmb3IP_MTYRL_D3k,9067
-toil/provisioners/abstractProvisioner.py,sha256=JCbUGP47LfG8eWhczb8bItBbUfANoTHRR92_l_nXRJ4,55367
+toil/provisioners/abstractProvisioner.py,sha256=475PQvSKr1hchKPsnzKWHv0W2rKAdVANV6ZLD_D4oxA,55930
 toil/provisioners/clusterScaler.py,sha256=jiSTAXaR6IhQahYsQuADKxk5kzSLRGaK6KIrGR9mkNE,62095
 toil/provisioners/gceProvisioner.py,sha256=sHvdCK390OYaOTGAdd2a1bCaZgzP612rHf6qfrhpQsQ,23554
 toil/provisioners/node.py,sha256=De_-7GOGrZiFVmHfGH-vWDOK9JT4W9IySQESy3R0D5I,13777
@@ -110,7 +110,9 @@ toil/test/__init__.py,sha256=Fhqh_Diu_x5bPLmKUiEbFTfKEZqqClnZgJ4WTFEaToI,44366
 toil/test/batchSystems/__init__.py,sha256=9h1rcqYjjnn97DpDEJf8tlNOIDx0XiXyQwh0F_81y0k,612
 toil/test/batchSystems/batchSystemTest.py,sha256=IwpxaYCYO-VJ0RkEnYXUNFAzwwiosob2EzEOmWOKaA0,53561
 toil/test/batchSystems/test_lsf_helper.py,sha256=vgWORofYN63TIUOUeMv0Zbjm8EbJiRM_-aHppERLwr4,4936
-toil/test/batchSystems/test_slurm.py,sha256=XDOFHq5DVT8YLNRPeZPwWRTbpXTNylc7KsMTLUIEHq8,18582
+toil/test/batchSystems/test_slurm.py,sha256=7C9xj0SnA-Pq7ltZYRXJ7Lvgw4ffsASa40bWolfqtRo,19162
+toil/test/cactus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+toil/test/cactus/test_cactus_integration.py,sha256=jzntInJFDcags78cKrrUnO6noFq0ZfJ9MgBdLcE3iGE,2417
 toil/test/cwl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/test/cwl/alwaysfails.cwl,sha256=EWDfzxMG4WqH2YQErhIHYPVc7-1eyucEUboj8r7yVbA,210
 toil/test/cwl/colon_test_output.cwl,sha256=8evFOnHvpJEPn1lBI2zuEKueMUbh2p8oSNOKl-M4Ojw,401
@@ -118,7 +120,7 @@ toil/test/cwl/colon_test_output_job.yaml,sha256=BbXFRuWsRJV6JpJAutnHywJl_AbwEcwN
 toil/test/cwl/conditional_wf.cwl,sha256=SAtUSmD5ta-DVG0SRHpT8EtOp8ztm-6v9Vskcl09c3s,357
 toil/test/cwl/conditional_wf.yaml,sha256=WsS9dlpbRUll9M56KspQdOJ8h6SASTW7eRELefa274E,24
 toil/test/cwl/conftest.py,sha256=okwzv8y-qDSIz--eauonzQXxj6DxyT-u8sXIUMmjxlE,698
-toil/test/cwl/cwlTest.py,sha256=TPkCRgjLoFAx-A_3Al_864Be40edchoecVf8aFHwdU0,55846
+toil/test/cwl/cwlTest.py,sha256=Afjxd1vSv8VEXq9w26uoICXZ-bjwf4Sq7Vn0LffMMUg,59717
 toil/test/cwl/directory_from_directory.cwl,sha256=-hjZuUs5fMpKMnbj-FZAgcns4iWsCFJV6Qh7-fsRn1I,556
 toil/test/cwl/download.cwl,sha256=XQvz8or_-k6pwyhgeUeJOew0_b3BeHcF08EaalbySz4,296
 toil/test/cwl/download_directory.cwl,sha256=RdPmElFeHJqXGWVqEqR5VFMmXDiJubhTcM9hIrAhKY4,535
@@ -147,13 +149,13 @@ toil/test/cwl/test_filename_conflict_detection.cwl,sha256=wVrc9EXmO74tgaEe4oE-EM
 toil/test/cwl/test_filename_conflict_detection_at_root.cwl,sha256=G3clUTFeeGZt5xPi2SNs05W7I-RT0OEN2C8xKHzQ300,695
 toil/test/cwl/test_filename_conflict_resolution.cwl,sha256=sIoXi61RzCkzD1MTCoglRYAXo8xaE1JiC6irvSANK0I,717
 toil/test/docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-toil/test/docs/scriptsTest.py,sha256=A3wSrEIRpU7f_T3yxvSnNt-BYTw0E8PC5djFH2XNZYc,6116
+toil/test/docs/scriptsTest.py,sha256=LsFnNH_nrDF_xiDhhXmgi__8eZnspQSW3edgVWnI_h8,6100
 toil/test/jobStores/__init__.py,sha256=9h1rcqYjjnn97DpDEJf8tlNOIDx0XiXyQwh0F_81y0k,612
-toil/test/jobStores/jobStoreTest.py,sha256=Nw90EjAbliCr8IkEm1OjFAKiLH33WV-DXOtabg7kYVk,67474
+toil/test/jobStores/jobStoreTest.py,sha256=iqYW-PBaOMim7dCibvzEK_OxLaVsiu4wHLiZmTd0vu8,67693
 toil/test/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/test/lib/dockerTest.py,sha256=qUqNhaHj_rG7NA0EmKtcm58IH3mEuqlxitvsoiMsuDk,17108
 toil/test/lib/test_conversions.py,sha256=uXd9_sqqUAeXRcqfTEnQKyKofIMBJAPjWf43VK4vyVw,7317
-toil/test/lib/test_ec2.py,sha256=49SkFtjRk_7Mw8Wu7KdIPd7hZP6ejvI_-DsS3MAOxBk,4244
+toil/test/lib/test_ec2.py,sha256=I9jmLY7_5oJtFR7vObBS0pnzf-GCCm9CfTLrHp9q2Xs,4252
 toil/test/lib/test_misc.py,sha256=OD4VBFgNOHt6krjyjxvvS9TqPmKorMz6MsGDlG4nAY8,2759
 toil/test/lib/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/test/lib/aws/test_iam.py,sha256=6YQDMxj73HZ86q_gu0ues76dYHbuxHhBgWpJ8Gt3yvM,5026
@@ -163,9 +165,11 @@ toil/test/mesos/MesosDataStructuresTest.py,sha256=EJ0kFy5rRHvNOJ_1bn-4nqldXNtlZC
 toil/test/mesos/__init__.py,sha256=9h1rcqYjjnn97DpDEJf8tlNOIDx0XiXyQwh0F_81y0k,612
 toil/test/mesos/helloWorld.py,sha256=4_KgsdftwdAYHi79KbyWWIUm2JK0AaT5qZ1vdON8LDY,2203
 toil/test/mesos/stress.py,sha256=x6hAyfO2cSJ6BkGu1g97k4rOxOZigAw4g_DXTgpB7Ic,2129
+toil/test/options/__init__.py,sha256=9h1rcqYjjnn97DpDEJf8tlNOIDx0XiXyQwh0F_81y0k,612
+toil/test/options/options.py,sha256=oJJTBkAUs3lgClhujHiG0jGTiZ38ZgKdVoZ7D_Yh534,1426
 toil/test/provisioners/__init__.py,sha256=9h1rcqYjjnn97DpDEJf8tlNOIDx0XiXyQwh0F_81y0k,612
 toil/test/provisioners/clusterScalerTest.py,sha256=ObsCEM9lkvN59iR4Cgztqpw58swv_6ChdNOq4qUm_y8,44279
-toil/test/provisioners/clusterTest.py,sha256=Fgc0sxoLuoxopDjn77m8Pu6dOC0dgnROYyGfTHZu1xY,6301
+toil/test/provisioners/clusterTest.py,sha256=_yXE8zNcj7iH0NgAZyhOmixJ25XkdZcJ0j6qP98Idic,6434
 toil/test/provisioners/gceProvisionerTest.py,sha256=yNieaSx5g5H5NtAw5b-lw6IL-8OeC10LMgDMIcwhTkw,12996
 toil/test/provisioners/provisionerTest.py,sha256=M6JXn57yYsMuU33QkZ92Q9wMS-OZB4KygyMum9zvyqk,2040
 toil/test/provisioners/restartScript.py,sha256=IHOXV0dez52utDJlIDO-O6yzGocK8wHyRpeFWYtjJUE,415
@@ -206,15 +210,15 @@ toil/test/src/toilContextManagerTest.py,sha256=JjyPzIb6JOXmbcJnGrJKkKfKL3U_OFr13
 toil/test/src/userDefinedJobArgTypeTest.py,sha256=SXb3WmFWNtsIKKP-78eHL6tfabu4T8C0Spo9OK1v6xM,2848
 toil/test/src/workerTest.py,sha256=LDHYZ-Vsw1iW-HDC57h0FDdZy2KBD9ooBazxsGSITJc,4609
 toil/test/utils/__init__.py,sha256=9h1rcqYjjnn97DpDEJf8tlNOIDx0XiXyQwh0F_81y0k,612
-toil/test/utils/toilDebugTest.py,sha256=-tvsXaJ8oXswIoE8fcyuHWh68GNcmzla36LLlmAzfQs,6440
+toil/test/utils/toilDebugTest.py,sha256=s9xGdt1qFPWR-48U17JpNFIBTaLSkwyIp3xFMSarhxw,6435
 toil/test/utils/toilKillTest.py,sha256=Wu0hANeRspLpqzdZUnWt-yt8wBhbxjEijOkjCWfNrkg,3541
-toil/test/utils/utilsTest.py,sha256=KqD2DsUb1RAjbDrny-6d74pJCl4giVVu7J8eWhk5d_A,16514
+toil/test/utils/utilsTest.py,sha256=TH_Z9-0EeqDCpLoETCyVbEW__Y5ZaF_u6VAUjabuJAM,16518
 toil/test/wdl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-toil/test/wdl/wdltoil_test.py,sha256=ukHCGnQRZia6_o9hw3Higl2hICwv4xgHzJ3yOB8PnU8,14363
+toil/test/wdl/wdltoil_test.py,sha256=s6bxlQIGDBOzlE0cfQSs5ls8lp-J2_oeVdfqXmgJSt4,16886
 toil/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/utils/toilClean.py,sha256=X6Fws6LVqa-ScUITMTomV5iohXa3yz_rAIFnUfXlqZU,1850
 toil/utils/toilConfig.py,sha256=aOCGkke9ayqGgN26lzjs5rthruHS68v3R9gytkP0mvo,1487
-toil/utils/toilDebugFile.py,sha256=j3vPg6RDBvmD30HqjTPUtzDymSNkkg65ypO5ceaB-h8,6406
+toil/utils/toilDebugFile.py,sha256=4EHKyMbYvlhjj-8Qwlhrf3SEsj-eytZMX393yXk8BLw,6412
 toil/utils/toilDebugJob.py,sha256=NvVsOd4NzItx3oSYCKnkqNNmiMfztqd_cjmkFKy_uDo,2991
 toil/utils/toilDestroyCluster.py,sha256=Z9PQxqycsYeU0rjTTzZKWhKYuqP19wD988aGmfC3Vxc,1430
 toil/utils/toilKill.py,sha256=0XK8K1Tk4Lla7eBPHI8PFmcOBZu_g0GWLjK1IkDYoXw,2714
@@ -223,15 +227,15 @@ toil/utils/toilMain.py,sha256=an4LFEIzCXY8IjoAcjgF5OgO0KdTt1Nay9LEs8UOWu4,2456
 toil/utils/toilRsyncCluster.py,sha256=MkRGcM2wG4BaZnAMDgI1lLoEUA4Adx0jxoOeGWkQWAY,2059
 toil/utils/toilServer.py,sha256=4XbyOHQHgMWWrCWj9Uz1M-jLQheV2Ju-DQ_fNdnBg1o,1178
 toil/utils/toilSshCluster.py,sha256=T9lNGBTiC_1TMo-QZNMbw5teoYZLdL6mN7C_e2FwSDU,2678
-toil/utils/toilStats.py,sha256=MrT9iIa0XnS3scMuWzREIxMjDXcIhITQDVoMGwrcclc,23436
-toil/utils/toilStatus.py,sha256=kFebVEvaU3U0I_HyLtL0m7Gvo6fERe-zTaXcr-j4pa8,16381
+toil/utils/toilStats.py,sha256=kDC-b1xGSX1C_hPGGL3nh7uYxiyKRhZOzX1QwA9LCW4,25163
+toil/utils/toilStatus.py,sha256=6-bTaYFstxbMhIlCkzjhkGtwTfs20XZ2c-NWGyzSa2I,16384
 toil/utils/toilUpdateEC2Instances.py,sha256=e1EE60o8lu0Gqd8n1H-rTC94fraDsZnqPy4HdStnK6o,1273
 toil/wdl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 toil/wdl/utils.py,sha256=z617Jhhbx4pCnBTrX7a5v-3q1Fxqff8LpwjM8Ohiibw,1307
-toil/wdl/wdltoil.py,sha256=35G115rdDduPJCZbnIt1e4Jfe1OYjsXioID7GLCvSFY,132580
-toil-6.0.0.dist-info/LICENSE,sha256=FPaTNB9xyeCT60XXXq2Bn4FGN9392OxN5jixnV_XXyo,11516
-toil-6.0.0.dist-info/METADATA,sha256=Gm56tbutYr_qgEochdv0BCvmn_yFiIylsEqjpljE2KY,6120
-toil-6.0.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-toil-6.0.0.dist-info/entry_points.txt,sha256=EF2yFhV2UYuJGr-OjMkUOd3RyOCgRWQbS6XJtBJ25eM,436
-toil-6.0.0.dist-info/top_level.txt,sha256=1ydj7IXvHS9tMT5OVTSSpub6ZOaQeIn3KGCgJqaikF0,5
-toil-6.0.0.dist-info/RECORD,,
+toil/wdl/wdltoil.py,sha256=mNKYsykqRSyPFiAoIYXmciyloIPsQ5YARGM49SZ6coc,140229
+toil-6.1.0.dist-info/LICENSE,sha256=FPaTNB9xyeCT60XXXq2Bn4FGN9392OxN5jixnV_XXyo,11516
+toil-6.1.0.dist-info/METADATA,sha256=2dZlmgBCNmzUtKDUm4UI53In42U0tad_wjwmUAznKAk,6070
+toil-6.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+toil-6.1.0.dist-info/entry_points.txt,sha256=EF2yFhV2UYuJGr-OjMkUOd3RyOCgRWQbS6XJtBJ25eM,436
+toil-6.1.0.dist-info/top_level.txt,sha256=1ydj7IXvHS9tMT5OVTSSpub6ZOaQeIn3KGCgJqaikF0,5
+toil-6.1.0.dist-info/RECORD,,