toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

toil/test/cwl/glob_dir.cwl

@@ -0,0 +1,15 @@
+cwlVersion: v1.2
+class: CommandLineTool
+inputs: []
+outputs:
+  shouldmake:
+    type: Directory
+    outputBinding:
+      glob: "shouldmake"
+baseCommand: [ bash, -c ]
+arguments:
+ - |
+   mkdir -p shouldmake;
+   touch shouldmake/test.txt;
+
+

toil/test/cwl/preemptible.cwl

@@ -0,0 +1,21 @@
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+
+class: CommandLineTool
+doc: "Be preemptible"
+cwlVersion: v1.0
+
+inputs: []
+
+outputs:
+  output:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+hints:
+  arv:UsePreemptible:
+    usePreemptible: true
+
+baseCommand: ["echo", "hello"]
+stdout: output.txt

toil/test/cwl/preemptible_expression.cwl

@@ -0,0 +1,28 @@
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+
+class: CommandLineTool
+doc: "Be preemptible"
+cwlVersion: v1.0
+
+inputs:
+  - id: preemptible
+    type: boolean
+
+outputs:
+  output:
+    type: File
+    outputBinding:
+      glob: output.txt
+
+requirements:
+  InlineJavascriptRequirement: {}
+
+hints:
+  arv:UsePreemptible:
+    # This isn't allowed; this always has to be a real boolean and can't be a cwl:Expression. See
+    # https://github.com/arvados/arvados/blob/48a0d575e6de34bcda91c489e4aa98df291a8cca/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml#L345
+    usePreemptible: $(inputs.preemptible)
+
+baseCommand: ["echo", "hello"]
+stdout: output.txt

toil/test/cwl/revsort.cwl

@@ -10,7 +10,7 @@ cwlVersion: v1.0
 # in which the command line tools will execute.
 hints:
   - class: DockerRequirement
-    dockerPull: debian:8
+    dockerPull: debian:12
 
 
 # The inputs array defines the structure of the input object that describes

toil/test/cwl/revsort2.cwl

@@ -10,7 +10,7 @@ cwlVersion: v1.0
 # in which the command line tools will execute.
 hints:
   - class: DockerRequirement
-    dockerPull: debian:8
+    dockerPull: debian:12
 
 
 # The inputs array defines the structure of the input object that describes

toil/test/docs/scriptsTest.py

@@ -2,7 +2,6 @@ import os
 import re
 import shutil
 import subprocess
-import tempfile
 import sys
 import unittest
 
@@ -70,8 +69,8 @@ class ToilDocumentationTest(ToilTest):
     def testCwlexample(self):
         self.checkExitCode("tutorial_cwlexample.py")
 
-    def testDiscoverfiles(self):
-        self.checkExitCode("tutorial_discoverfiles.py")
+    def testStats(self):
+        self.checkExitCode("tutorial_stats.py")
 
     def testDynamic(self):
         self.checkExitCode("tutorial_dynamic.py")

toil/test/jobStores/jobStoreTest.py

@@ -18,7 +18,6 @@ import logging
 import os
 import shutil
 import socketserver
-import tempfile
 import threading
 import time
 import urllib.parse as urlparse
@@ -27,6 +26,7 @@ from abc import ABCMeta, abstractmethod
 from io import BytesIO
 from itertools import chain, islice
 from queue import Queue
+from tempfile import mkstemp
 from threading import Thread
 from typing import Any, Tuple
 from urllib.request import Request, urlopen
@@ -40,7 +40,7 @@ from toil.job import Job, JobDescription, TemporaryID
 from toil.jobStores.abstractJobStore import (NoSuchFileException,
                                              NoSuchJobException)
 from toil.jobStores.fileJobStore import FileJobStore
-from toil.lib.aws.utils import create_s3_bucket, get_object_for_url
+from toil.lib.io import mkdtemp
 from toil.lib.memoize import memoize
 from toil.lib.retry import retry
 from toil.statsAndLogging import StatsAndLogging
@@ -101,7 +101,8 @@ class AbstractJobStoreTest:
             return super().__new__(cls)
 
         def _createConfig(self):
-            return Config()
+            config = Config()
+            return config
 
         @abstractmethod
         def _createJobStore(self):
@@ -224,11 +225,14 @@ class AbstractJobStoreTest:
 
         def testPersistantFilesToDelete(self):
             """
-            Make sure that updating a job carries over filesToDelete.
-
-            The following demonstrates the job update pattern, where files to be deleted are referenced in
-            "filesToDelete" array, which is persisted to disk first. If things go wrong during the update, this list of
-            files to delete is used to remove the unneeded files.
+            Make sure that updating a job persists filesToDelete.
+
+            The following demonstrates the job update pattern, where files to
+            be deleted atomically with a job update are referenced in
+            "filesToDelete" array, which is persisted to disk first. If things
+            go wrong during the update, this list of files to delete is used to
+            ensure that the updated job and the files are never both visible at
+            the same time.
             """
 
             # Create a job.
@@ -439,7 +443,7 @@ class AbstractJobStoreTest:
                 self.assertEqual(f.read(), one)
 
             # ... and copy it to a temporary physical file on the jobstore1.
-            fh, path = tempfile.mkstemp()
+            fh, path = mkstemp()
             try:
                 os.close(fh)
                 tmpPath = path + '.read-only'
@@ -543,14 +547,16 @@ class AbstractJobStoreTest:
             jobNames = ['testStatsAndLogging_writeLogFiles']
             jobLogList = ['string', b'bytes', '', b'newline\n']
             config = self._createConfig()
-            setattr(config, 'writeLogs', '.')
+            setattr(config, 'writeLogs', self._createTempDir())
             setattr(config, 'writeLogsGzip', None)
             StatsAndLogging.writeLogFiles(jobNames, jobLogList, config)
-            jobLogFile = os.path.join(config.writeLogs, jobNames[0] + '000.log')
+            jobLogFile = os.path.join(config.writeLogs, jobNames[0] + '_000.log')
+            # The log directory should get exactly one file, names after this
+            # easy job name with no replacements needed.
+            self.assertEqual(os.listdir(config.writeLogs), [os.path.basename(jobLogFile)])
             self.assertTrue(os.path.isfile(jobLogFile))
             with open(jobLogFile) as f:
                 self.assertEqual(f.read(), 'string\nbytes\n\nnewline\n')
-            os.remove(jobLogFile)
 
         def testBatchCreate(self):
             """Test creation of many jobs."""
@@ -854,7 +860,7 @@ class AbstractJobStoreTest:
 
                 # Multi-part upload from file
                 checksum = hashlib.md5()
-                fh, path = tempfile.mkstemp()
+                fh, path = mkstemp()
                 try:
                     with os.fdopen(fh, 'wb+') as writable:
                         with open('/dev/urandom', 'rb') as readable:
@@ -1042,7 +1048,7 @@ class AbstractJobStoreTest:
         def testEmptyFileStoreIDIsReadable(self):
             """Simply creates an empty fileStoreID and attempts to read from it."""
             id = self.jobstore_initialized.get_empty_file_store_id()
-            fh, path = tempfile.mkstemp()
+            fh, path = mkstemp()
             try:
                 self.jobstore_initialized.read_file(id, path)
                 self.assertTrue(os.path.isfile(path))
@@ -1071,7 +1077,7 @@ class AbstractEncryptedJobStoreTest:
 
         def setUp(self):
             # noinspection PyAttributeOutsideInit
-            self.sseKeyDir = tempfile.mkdtemp()
+            self.sseKeyDir = mkdtemp()
             super().setUp()
 
         def tearDown(self):
@@ -1139,14 +1145,14 @@ class FileJobStoreTest(AbstractJobStoreTest.Test):
             return hashlib.md5(f.read()).hexdigest()
 
     def _createExternalStore(self):
-        return tempfile.mkdtemp()
+        return mkdtemp()
 
     def _cleanUpExternalStore(self, dirPath):
-        shutil.rmtree(dirPath)
+        shutil.rmtree(dirPath, ignore_errors=True)
 
     def testPreserveFileName(self):
         """Check that the fileID ends with the given file name."""
-        fh, path = tempfile.mkstemp()
+        fh, path = mkstemp()
         try:
             os.close(fh)
             job = self.arbitraryJob()
@@ -1458,6 +1464,8 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
 
     def _hashTestFile(self, url: str) -> str:
         from toil.jobStores.aws.jobStore import AWSJobStore
+        from toil.lib.aws.utils import get_object_for_url
+        str(AWSJobStore)  # to prevent removal of that import
         key = get_object_for_url(urlparse.urlparse(url), existing=True)
         contents = key.get().get('Body').read()
         return hashlib.md5(contents).hexdigest()
@@ -1465,7 +1473,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
     def _createExternalStore(self):
         """A S3.Bucket instance is returned"""
         from toil.jobStores.aws.jobStore import establish_boto3_session
-        from toil.lib.aws.utils import retry_s3
+        from toil.lib.aws.utils import retry_s3, create_s3_bucket
 
         resource = establish_boto3_session().resource(
             "s3", region_name=self.awsRegion()
@@ -1480,8 +1488,13 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
                 return bucket
 
     def _cleanUpExternalStore(self, bucket):
-        bucket.objects.all().delete()
-        bucket.delete()
+        from toil.jobStores.aws.jobStore import establish_boto3_session
+        from toil.lib.aws.utils import delete_s3_bucket
+
+        resource = establish_boto3_session().resource(
+            "s3", region_name=self.awsRegion()
+        )
+        delete_s3_bucket(resource, bucket.name)
 
     def _largeLogEntrySize(self):
         from toil.jobStores.aws.jobStore import AWSJobStore

toil/test/lib/aws/test_iam.py

@@ -11,25 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import json
 import logging
-import os
-import uuid
-from typing import Optional
-
-import pytest
-
-from toil.jobStores.aws.jobStore import AWSJobStore
-from toil.lib.aws import iam
-from toil.lib.aws.utils import create_s3_bucket
-from toil.lib.ec2 import establish_boto3_session
-from toil.test import ToilTest, needs_aws_s3
 
-import moto
 import boto3
-import json
-
 from moto import mock_iam
 
+from toil.lib.aws import iam
+from toil.test import ToilTest
+
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 

toil/test/lib/aws/test_utils.py

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import os
-import uuid
-from typing import Optional
 
 import pytest
 

toil/test/lib/dockerTest.py

@@ -20,8 +20,8 @@ from threading import Thread
 
 from docker.errors import ContainerError
 from toil.common import Toil
-from toil.job import Job
 from toil.exceptions import FailedJobsException
+from toil.job import Job
 from toil.lib.docker import (FORGO,
                              RM,
                              STOP,
@@ -248,12 +248,12 @@ class DockerTest(ToilTest):
                              deferParam=None)
 
     def testDockerPipeChain(self, caching=False):
-        """
+        r"""
         Test for piping API for dockerCall().  Using this API (activated when
         list of argument lists is given as parameters), commands a piped
         together into a chain.
-        ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
-        printf 'x\n y\n' | wc -l
+        ex:  ``parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ]`` should execute:
+        ``printf 'x\n y\n' | wc -l``
         """
         options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
         options.logLevel = self.dockerTestLogLevel

toil/test/lib/test_ec2.py

@@ -13,26 +13,29 @@
 # limitations under the License.
 import logging
 import os
+
 import pytest
 
 from toil.lib.aws.ami import (aws_marketplace_flatcar_ami_search,
-                              get_flatcar_ami,
                               feed_flatcar_ami_release,
-                              flatcar_release_feed_amis)
-from toil.lib.aws.session import establish_boto3_session
-from toil.test import ToilTest, needs_aws_ec2
+                              flatcar_release_feed_amis,
+                              get_flatcar_ami)
+from toil.test import ToilTest, needs_aws_ec2, needs_online
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
-
+@needs_online
 class FlatcarFeedTest(ToilTest):
-    """Test accessing the FLatcar AMI release feed, independent of the AWS API"""
-    
+    """Test accessing the Flatcar AMI release feed, independent of the AWS API"""
+
+    # Note that we need to support getting an empty list back, because
+    # sometimes the Flatcar feeds are just down, and we can't fail CI at those
+    # times.
+
     def test_parse_archive_feed(self):
         """Make sure we can get a Flatcar release from the Internet Archive."""
         amis = list(flatcar_release_feed_amis('us-west-2', 'amd64', 'archive'))
-        self.assertTrue(len(amis) > 0)
         for ami in amis:
             self.assertEqual(len(ami), len('ami-02b46c73fed689d1c'))
             self.assertTrue(ami.startswith('ami-'))
@@ -40,30 +43,22 @@ class FlatcarFeedTest(ToilTest):
     def test_parse_beta_feed(self):
         """Make sure we can get a Flatcar release from the beta channel."""
         amis = list(flatcar_release_feed_amis('us-west-2', 'amd64', 'beta'))
-        self.assertTrue(len(amis) > 0)
         for ami in amis:
             self.assertEqual(len(ami), len('ami-02b46c73fed689d1c'))
             self.assertTrue(ami.startswith('ami-'))
     
-    # TODO: This will fail until https://github.com/flatcar/Flatcar/issues/962 is fixed
-    @pytest.mark.xfail
     def test_parse_stable_feed(self):
         """Make sure we can get a Flatcar release from the stable channel."""
         amis = list(flatcar_release_feed_amis('us-west-2', 'amd64', 'stable'))
-        self.assertTrue(len(amis) > 0)
         for ami in amis:
             self.assertEqual(len(ami), len('ami-02b46c73fed689d1c'))
             self.assertTrue(ami.startswith('ami-'))
             
-    def test_bypass_stable_feed(self):
-        """Make sure we can either get or safely not get a Flatcar release from the stable channel."""
-        list(flatcar_release_feed_amis('us-west-2', 'amd64', 'stable'))
-        # Ifd we get here we safely managed to iterate everything.
-
 @needs_aws_ec2
 class AMITest(ToilTest):
     @classmethod
     def setUpClass(cls):
+        from toil.lib.aws.session import establish_boto3_session
         session = establish_boto3_session(region_name='us-west-2')
         cls.ec2_client = session.client('ec2')
 

toil/test/mesos/helloWorld.py

@@ -15,7 +15,7 @@
 A simple user script for Toil
 """
 
-import argparse
+from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job
@@ -25,14 +25,13 @@ parentMessage = "The parent job is now running!"
 
 def hello_world(job):
 
-    job.fileStore.logToMaster(parentMessage)
+    job.fileStore.log_to_leader(parentMessage)
     with open('foo_bam.txt', 'w') as handle:
         handle.write('\nThis is a triumph...\n')
 
     # Assign FileStoreID to a given file
     foo_bam = job.fileStore.writeGlobalFile('foo_bam.txt')
 
-
     # Spawn child
     job.addChildJobFn(hello_world_child, foo_bam, memory=100, cores=0.5, disk="3G")
 
@@ -40,7 +39,7 @@ def hello_world(job):
 def hello_world_child(job, hw):
 
     path = job.fileStore.readGlobalFile(hw)
-    job.fileStore.logToMaster(childMessage)
+    job.fileStore.log_to_leader(childMessage)
     # NOTE: path and the udpated file are stored to /tmp
     # If we want to SAVE our changes to this tmp file, we must write it out.
     with open(path) as r:
@@ -56,7 +55,7 @@ def hello_world_child(job, hw):
 def main():
     # Boilerplate -- startToil requires options
 
-    parser = argparse.ArgumentParser()
+    parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
     options = parser.parse_args()
 

toil/test/mesos/stress.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from argparse import ArgumentParser
+from configargparse import ArgumentParser
 
 from toil.job import Job
 

toil/test/{wdl/conftest.py → options/__init__.py}

@@ -11,13 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-# https://pytest.org/latest/example/pythoncollection.html
-
-collect_ignore = []
-
-try:
-    import wdlparse
-    print(wdlparse.__file__)  # keep this import from being removed
-except ImportError:
-    collect_ignore.append("toilwdl.py")

toil/test/options/options.py

@@ -0,0 +1,37 @@
+from configargparse import ArgParser
+
+from toil.common import addOptions, Toil
+from toil.test import ToilTest
+
+
+class OptionsTest(ToilTest):
+    """
+    Class to test functionality of all Toil options
+    """
+    def test_default_caching_slurm(self):
+        """
+        Test to ensure that caching will be set to false when running on Slurm
+        :return:
+        """
+        parser = ArgParser()
+        addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
+        test_args = ["--jobstore=example-jobstore", "--batchSystem=slurm"]
+        options = parser.parse_args(test_args)
+        with Toil(options) as toil:
+            caching_value = toil.config.caching
+        self.assertEqual(caching_value, False)
+
+    def test_caching_option_priority(self):
+        """
+        Test to ensure that the --caching option takes priority over the default_caching() return value
+        :return:
+        """
+        parser = ArgParser()
+        addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
+        # the kubernetes batchsystem (and I think all batchsystems including singlemachine) return False
+        # for default_caching
+        test_args = ["--jobstore=example-jobstore", "--batchSystem=kubernetes", "--caching=True"]
+        options = parser.parse_args(test_args)
+        with Toil(options) as toil:
+            caching_value = toil.config.caching
+        self.assertEqual(caching_value, True)

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -29,6 +29,7 @@ from toil.test import (ToilTest,
                        integrative,
                        needs_aws_ec2,
                        needs_fetchable_appliance,
+                       needs_mesos,
                        slow,
                        timeLimit)
 from toil.test.provisioners.clusterTest import AbstractClusterTest
@@ -171,9 +172,6 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
         venv_command = ['virtualenv', '--system-site-packages', '--python', exactPython, '--never-download', self.venvDir]
         self.sshUtil(venv_command)
 
-        upgrade_command = [self.pip(), 'install', 'setuptools==28.7.1', 'pyyaml==3.12']
-        self.sshUtil(upgrade_command)
-
         log.info('Set up script...')
         self._getScript()
 
@@ -214,6 +212,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1800)
 class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
     def __init__(self, name):
@@ -282,6 +281,7 @@ class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(2400)
 class AWSStaticAutoscaleTest(AWSAutoscaleTest):
     """Runs the tests on a statically provisioned cluster with autoscaling enabled."""
@@ -357,6 +357,7 @@ class AWSManagedAutoscaleTest(AWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
     def __init__(self, name):
@@ -396,6 +397,7 @@ class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class AWSRestartTest(AbstractAWSAutoscaleTest):
     """This test insures autoscaling works on a restarted Toil run."""
@@ -412,9 +414,10 @@ class AWSRestartTest(AbstractAWSAutoscaleTest):
 
     def _getScript(self):
         def restartScript():
-            import argparse
             import os
 
+            from configargparse import ArgumentParser
+
             from toil.job import Job
 
             def f0(job):
@@ -422,7 +425,7 @@ class AWSRestartTest(AbstractAWSAutoscaleTest):
                     raise RuntimeError('failed on purpose')
 
             if __name__ == '__main__':
-                parser = argparse.ArgumentParser()
+                parser = ArgumentParser()
                 Job.Runner.addToilOptions(parser)
                 options = parser.parse_args()
                 rootJob = Job.wrapJobFn(f0, cores=0.5, memory='50 M', disk='50 M')
@@ -458,6 +461,7 @@ class AWSRestartTest(AbstractAWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class PreemptibleDeficitCompensationTest(AbstractAWSAutoscaleTest):
     def __init__(self, name):

toil/test/provisioners/clusterScalerTest.py

@@ -19,16 +19,16 @@ import time
 import uuid
 from argparse import Namespace
 from collections import defaultdict
-from contextlib import contextmanager
 from queue import Empty, Queue
 from threading import Event, Thread
-from typing import Dict, List, Optional, Set, Tuple
+from typing import List, Optional, Set, Tuple
 from unittest.mock import MagicMock
 
 from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
                                                    AbstractScalableBatchSystem,
                                                    NodeInfo)
-from toil.common import Config, defaultTargetTime
+from toil.common import Config
+from toil.options.common import defaultTargetTime
 from toil.job import JobDescription
 from toil.lib.conversions import human2bytes as h2b
 from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
@@ -873,7 +873,9 @@ class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisi
             return self.jobQueue.qsize()
 
     def getJobs(self):
-        return self.jobBatchSystemIDToIssuedJob.values()
+        # jobBatchSystemIDToIssuedJob may be modified while we are working.
+        # So copy it.
+        return dict(self.jobBatchSystemIDToIssuedJob).values()
 
     # AbstractScalableBatchSystem functionality
     def getNodes(self, preemptible: Optional[bool] = False, timeout: int = 600):