toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

toil/test/docs/scriptsTest.py

@@ -2,7 +2,6 @@ import os
 import re
 import shutil
 import subprocess
-import tempfile
 import sys
 import unittest
 

toil/test/jobStores/jobStoreTest.py

@@ -18,7 +18,6 @@ import logging
 import os
 import shutil
 import socketserver
-import tempfile
 import threading
 import time
 import urllib.parse as urlparse
@@ -27,6 +26,7 @@ from abc import ABCMeta, abstractmethod
 from io import BytesIO
 from itertools import chain, islice
 from queue import Queue
+from tempfile import mkstemp
 from threading import Thread
 from typing import Any, Tuple
 from urllib.request import Request, urlopen
@@ -41,6 +41,7 @@ from toil.jobStores.abstractJobStore import (NoSuchFileException,
                                              NoSuchJobException)
 from toil.jobStores.fileJobStore import FileJobStore
 from toil.lib.aws.utils import create_s3_bucket, get_object_for_url
+from toil.lib.io import mkdtemp
 from toil.lib.memoize import memoize
 from toil.lib.retry import retry
 from toil.statsAndLogging import StatsAndLogging
@@ -101,7 +102,8 @@ class AbstractJobStoreTest:
             return super().__new__(cls)
 
         def _createConfig(self):
-            return Config()
+            config = Config()
+            return config
 
         @abstractmethod
         def _createJobStore(self):
@@ -224,11 +226,14 @@ class AbstractJobStoreTest:
 
         def testPersistantFilesToDelete(self):
             """
-            Make sure that updating a job carries over filesToDelete.
-
-            The following demonstrates the job update pattern, where files to be deleted are referenced in
-            "filesToDelete" array, which is persisted to disk first. If things go wrong during the update, this list of
-            files to delete is used to remove the unneeded files.
+            Make sure that updating a job persists filesToDelete.
+
+            The following demonstrates the job update pattern, where files to
+            be deleted atomically with a job update are referenced in
+            "filesToDelete" array, which is persisted to disk first. If things
+            go wrong during the update, this list of files to delete is used to
+            ensure that the updated job and the files are never both visible at
+            the same time.
             """
 
             # Create a job.
@@ -439,7 +444,7 @@ class AbstractJobStoreTest:
                 self.assertEqual(f.read(), one)
 
             # ... and copy it to a temporary physical file on the jobstore1.
-            fh, path = tempfile.mkstemp()
+            fh, path = mkstemp()
             try:
                 os.close(fh)
                 tmpPath = path + '.read-only'
@@ -854,7 +859,7 @@ class AbstractJobStoreTest:
 
                 # Multi-part upload from file
                 checksum = hashlib.md5()
-                fh, path = tempfile.mkstemp()
+                fh, path = mkstemp()
                 try:
                     with os.fdopen(fh, 'wb+') as writable:
                         with open('/dev/urandom', 'rb') as readable:
@@ -1042,7 +1047,7 @@ class AbstractJobStoreTest:
         def testEmptyFileStoreIDIsReadable(self):
             """Simply creates an empty fileStoreID and attempts to read from it."""
             id = self.jobstore_initialized.get_empty_file_store_id()
-            fh, path = tempfile.mkstemp()
+            fh, path = mkstemp()
             try:
                 self.jobstore_initialized.read_file(id, path)
                 self.assertTrue(os.path.isfile(path))
@@ -1071,7 +1076,7 @@ class AbstractEncryptedJobStoreTest:
 
         def setUp(self):
             # noinspection PyAttributeOutsideInit
-            self.sseKeyDir = tempfile.mkdtemp()
+            self.sseKeyDir = mkdtemp()
             super().setUp()
 
         def tearDown(self):
@@ -1139,14 +1144,14 @@ class FileJobStoreTest(AbstractJobStoreTest.Test):
             return hashlib.md5(f.read()).hexdigest()
 
     def _createExternalStore(self):
-        return tempfile.mkdtemp()
+        return mkdtemp()
 
     def _cleanUpExternalStore(self, dirPath):
-        shutil.rmtree(dirPath)
+        shutil.rmtree(dirPath, ignore_errors=True)
 
     def testPreserveFileName(self):
         """Check that the fileID ends with the given file name."""
-        fh, path = tempfile.mkstemp()
+        fh, path = mkstemp()
         try:
             os.close(fh)
             job = self.arbitraryJob()
@@ -1458,6 +1463,7 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
 
     def _hashTestFile(self, url: str) -> str:
         from toil.jobStores.aws.jobStore import AWSJobStore
+        str(AWSJobStore)  # to prevent removal of that import
         key = get_object_for_url(urlparse.urlparse(url), existing=True)
         contents = key.get().get('Body').read()
         return hashlib.md5(contents).hexdigest()
@@ -1480,8 +1486,13 @@ class AWSJobStoreTest(AbstractJobStoreTest.Test):
                 return bucket
 
     def _cleanUpExternalStore(self, bucket):
-        bucket.objects.all().delete()
-        bucket.delete()
+        from toil.jobStores.aws.jobStore import establish_boto3_session
+        from toil.lib.aws.utils import delete_s3_bucket
+
+        resource = establish_boto3_session().resource(
+            "s3", region_name=self.awsRegion()
+        )
+        delete_s3_bucket(resource, bucket.name)
 
     def _largeLogEntrySize(self):
         from toil.jobStores.aws.jobStore import AWSJobStore

toil/test/lib/aws/test_iam.py

@@ -11,25 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import json
 import logging
-import os
-import uuid
-from typing import Optional
-
-import pytest
-
-from toil.jobStores.aws.jobStore import AWSJobStore
-from toil.lib.aws import iam
-from toil.lib.aws.utils import create_s3_bucket
-from toil.lib.ec2 import establish_boto3_session
-from toil.test import ToilTest, needs_aws_s3
 
-import moto
 import boto3
-import json
-
 from moto import mock_iam
 
+from toil.lib.aws import iam
+from toil.test import ToilTest
+
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 

toil/test/lib/aws/test_utils.py

@@ -12,9 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-import os
-import uuid
-from typing import Optional
 
 import pytest
 

toil/test/lib/dockerTest.py

@@ -20,8 +20,8 @@ from threading import Thread
 
 from docker.errors import ContainerError
 from toil.common import Toil
-from toil.job import Job
 from toil.exceptions import FailedJobsException
+from toil.job import Job
 from toil.lib.docker import (FORGO,
                              RM,
                              STOP,
@@ -248,12 +248,12 @@ class DockerTest(ToilTest):
                              deferParam=None)
 
     def testDockerPipeChain(self, caching=False):
-        """
+        r"""
         Test for piping API for dockerCall().  Using this API (activated when
         list of argument lists is given as parameters), commands a piped
         together into a chain.
-        ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
-        printf 'x\n y\n' | wc -l
+        ex:  ``parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ]`` should execute:
+        ``printf 'x\n y\n' | wc -l``
         """
         options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
         options.logLevel = self.dockerTestLogLevel

toil/test/lib/test_ec2.py

@@ -13,26 +13,30 @@
 # limitations under the License.
 import logging
 import os
+
 import pytest
 
 from toil.lib.aws.ami import (aws_marketplace_flatcar_ami_search,
-                              get_flatcar_ami,
                               feed_flatcar_ami_release,
-                              flatcar_release_feed_amis)
+                              flatcar_release_feed_amis,
+                              get_flatcar_ami)
 from toil.lib.aws.session import establish_boto3_session
-from toil.test import ToilTest, needs_aws_ec2
+from toil.test import ToilTest, needs_aws_ec2, needs_online
 
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.DEBUG)
 
-
+@needs_online
 class FlatcarFeedTest(ToilTest):
-    """Test accessing the FLatcar AMI release feed, independent of the AWS API"""
-    
+    """Test accessing the Flatcar AMI release feed, independent of the AWS API"""
+
+    # Note that we need to support getting an empty list back, because
+    # sometimes the Flatcar feeds are just down, and we can't fail CI at those
+    # times.
+
     def test_parse_archive_feed(self):
         """Make sure we can get a Flatcar release from the Internet Archive."""
         amis = list(flatcar_release_feed_amis('us-west-2', 'amd64', 'archive'))
-        self.assertTrue(len(amis) > 0)
         for ami in amis:
             self.assertEqual(len(ami), len('ami-02b46c73fed689d1c'))
             self.assertTrue(ami.startswith('ami-'))
@@ -40,26 +44,17 @@ class FlatcarFeedTest(ToilTest):
     def test_parse_beta_feed(self):
         """Make sure we can get a Flatcar release from the beta channel."""
         amis = list(flatcar_release_feed_amis('us-west-2', 'amd64', 'beta'))
-        self.assertTrue(len(amis) > 0)
         for ami in amis:
             self.assertEqual(len(ami), len('ami-02b46c73fed689d1c'))
             self.assertTrue(ami.startswith('ami-'))
     
-    # TODO: This will fail until https://github.com/flatcar/Flatcar/issues/962 is fixed
-    @pytest.mark.xfail
     def test_parse_stable_feed(self):
         """Make sure we can get a Flatcar release from the stable channel."""
         amis = list(flatcar_release_feed_amis('us-west-2', 'amd64', 'stable'))
-        self.assertTrue(len(amis) > 0)
         for ami in amis:
             self.assertEqual(len(ami), len('ami-02b46c73fed689d1c'))
             self.assertTrue(ami.startswith('ami-'))
             
-    def test_bypass_stable_feed(self):
-        """Make sure we can either get or safely not get a Flatcar release from the stable channel."""
-        list(flatcar_release_feed_amis('us-west-2', 'amd64', 'stable'))
-        # Ifd we get here we safely managed to iterate everything.
-
 @needs_aws_ec2
 class AMITest(ToilTest):
     @classmethod

toil/test/mesos/helloWorld.py

@@ -15,7 +15,7 @@
 A simple user script for Toil
 """
 
-import argparse
+from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job
@@ -25,14 +25,13 @@ parentMessage = "The parent job is now running!"
 
 def hello_world(job):
 
-    job.fileStore.logToMaster(parentMessage)
+    job.fileStore.log_to_leader(parentMessage)
     with open('foo_bam.txt', 'w') as handle:
         handle.write('\nThis is a triumph...\n')
 
     # Assign FileStoreID to a given file
     foo_bam = job.fileStore.writeGlobalFile('foo_bam.txt')
 
-
     # Spawn child
     job.addChildJobFn(hello_world_child, foo_bam, memory=100, cores=0.5, disk="3G")
 
@@ -40,7 +39,7 @@ def hello_world(job):
 def hello_world_child(job, hw):
 
     path = job.fileStore.readGlobalFile(hw)
-    job.fileStore.logToMaster(childMessage)
+    job.fileStore.log_to_leader(childMessage)
     # NOTE: path and the udpated file are stored to /tmp
     # If we want to SAVE our changes to this tmp file, we must write it out.
     with open(path) as r:
@@ -56,7 +55,7 @@ def hello_world_child(job, hw):
 def main():
     # Boilerplate -- startToil requires options
 
-    parser = argparse.ArgumentParser()
+    parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
     options = parser.parse_args()
 

toil/test/mesos/stress.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from argparse import ArgumentParser
+from configargparse import ArgumentParser
 
 from toil.job import Job
 

toil/test/provisioners/aws/awsProvisionerTest.py

@@ -29,6 +29,7 @@ from toil.test import (ToilTest,
                        integrative,
                        needs_aws_ec2,
                        needs_fetchable_appliance,
+                       needs_mesos,
                        slow,
                        timeLimit)
 from toil.test.provisioners.clusterTest import AbstractClusterTest
@@ -171,9 +172,6 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
         venv_command = ['virtualenv', '--system-site-packages', '--python', exactPython, '--never-download', self.venvDir]
         self.sshUtil(venv_command)
 
-        upgrade_command = [self.pip(), 'install', 'setuptools==28.7.1', 'pyyaml==3.12']
-        self.sshUtil(upgrade_command)
-
         log.info('Set up script...')
         self._getScript()
 
@@ -214,6 +212,7 @@ class AbstractAWSAutoscaleTest(AbstractClusterTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1800)
 class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
     def __init__(self, name):
@@ -282,6 +281,7 @@ class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(2400)
 class AWSStaticAutoscaleTest(AWSAutoscaleTest):
     """Runs the tests on a statically provisioned cluster with autoscaling enabled."""
@@ -357,6 +357,7 @@ class AWSManagedAutoscaleTest(AWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
     def __init__(self, name):
@@ -396,6 +397,7 @@ class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class AWSRestartTest(AbstractAWSAutoscaleTest):
     """This test insures autoscaling works on a restarted Toil run."""
@@ -412,9 +414,10 @@ class AWSRestartTest(AbstractAWSAutoscaleTest):
 
     def _getScript(self):
         def restartScript():
-            import argparse
             import os
 
+            from configargparse import ArgumentParser
+
             from toil.job import Job
 
             def f0(job):
@@ -422,7 +425,7 @@ class AWSRestartTest(AbstractAWSAutoscaleTest):
                     raise RuntimeError('failed on purpose')
 
             if __name__ == '__main__':
-                parser = argparse.ArgumentParser()
+                parser = ArgumentParser()
                 Job.Runner.addToilOptions(parser)
                 options = parser.parse_args()
                 rootJob = Job.wrapJobFn(f0, cores=0.5, memory='50 M', disk='50 M')
@@ -458,6 +461,7 @@ class AWSRestartTest(AbstractAWSAutoscaleTest):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class PreemptibleDeficitCompensationTest(AbstractAWSAutoscaleTest):
     def __init__(self, name):

toil/test/provisioners/clusterScalerTest.py

@@ -19,16 +19,16 @@ import time
 import uuid
 from argparse import Namespace
 from collections import defaultdict
-from contextlib import contextmanager
 from queue import Empty, Queue
 from threading import Event, Thread
-from typing import Dict, List, Optional, Set, Tuple
+from typing import List, Optional, Set, Tuple
 from unittest.mock import MagicMock
 
 from toil.batchSystems.abstractBatchSystem import (AbstractBatchSystem,
                                                    AbstractScalableBatchSystem,
                                                    NodeInfo)
-from toil.common import Config, defaultTargetTime
+from toil.common import Config
+from toil.options.common import defaultTargetTime
 from toil.job import JobDescription
 from toil.lib.conversions import human2bytes as h2b
 from toil.provisioners.abstractProvisioner import AbstractProvisioner, Shape
@@ -873,7 +873,9 @@ class MockBatchSystemAndProvisioner(AbstractScalableBatchSystem, AbstractProvisi
             return self.jobQueue.qsize()
 
     def getJobs(self):
-        return self.jobBatchSystemIDToIssuedJob.values()
+        # jobBatchSystemIDToIssuedJob may be modified while we are working.
+        # So copy it.
+        return dict(self.jobBatchSystemIDToIssuedJob).values()
 
     # AbstractScalableBatchSystem functionality
     def getNodes(self, preemptible: Optional[bool] = False, timeout: int = 600):

toil/test/provisioners/clusterTest.py

@@ -18,8 +18,6 @@ import subprocess
 import time
 from uuid import uuid4
 
-import boto.ec2
-
 from toil.lib.aws import zone_to_region
 from toil.lib.retry import retry
 from toil.provisioners.aws import get_best_aws_zone
@@ -38,7 +36,9 @@ class AbstractClusterTest(ToilTest):
         self.clusterType = 'mesos'
         self.zone = get_best_aws_zone()
         assert self.zone is not None, "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
-        # We need a boto2 connection to EC2 to check on the cluster
+        # We need a boto2 connection to EC2 to check on the cluster.
+        # Since we are protected by needs_aws_ec2 we can import from boto.
+        import boto.ec2
         self.boto2_ec2 = boto.ec2.connect_to_region(zone_to_region(self.zone))
         # Where should we put our virtualenv?
         self.venvDir = '/tmp/venv'
@@ -143,6 +143,17 @@ class AbstractClusterTest(ToilTest):
             log.error("Failed to run %s.", str(cmd))
             raise subprocess.CalledProcessError(p.returncode, ' '.join(cmd))
 
+    @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
+    def rsync_util(self, from_file: str, to_file: str) -> None:
+        """
+        Transfer a file to/from the cluster.
+
+        The cluster-side path should have a ':' in front of it.
+        """
+        cmd = ['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName, from_file, to_file]
+        log.info("Running %s.", str(cmd))
+        subprocess.check_call(cmd)
+
     @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
     def createClusterUtil(self, args=None):
         args = [] if args is None else args

toil/test/provisioners/gceProvisionerTest.py

@@ -136,12 +136,6 @@ class AbstractGCEAutoscaleTest(ToilTest):
                         '--python', exactPython, '/home/venv']
         self.sshUtil(venv_command)
 
-        upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
-        self.sshUtil(upgrade_command)
-
-        yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
-        self.sshUtil(yaml_command)
-
         self._getScript()
 
         toilOptions = [self.jobStore,

toil/test/provisioners/restartScript.py

@@ -1,6 +1,7 @@
-import argparse
 import os
 
+from configargparse import ArgumentParser
+
 from toil.job import Job
 
 
@@ -9,7 +10,7 @@ def f0(job):
         raise RuntimeError('failed on purpose')
 
 if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
+    parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
     options = parser.parse_args()
     rootJob = Job.wrapJobFn(f0, cores=0.5, memory='50 M', disk='50 M')

toil/test/server/serverTest.py

@@ -21,7 +21,7 @@ import uuid
 import zipfile
 from abc import abstractmethod
 from io import BytesIO
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from typing import Optional
 from urllib.parse import urlparse
 
 try:

toil/test/sort/restart_sort.py

@@ -20,7 +20,8 @@ import codecs
 import os
 import random
 import shutil
-from argparse import ArgumentParser
+
+from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job

toil/test/sort/sort.py

@@ -18,7 +18,8 @@ import codecs
 import os
 import random
 import shutil
-from argparse import ArgumentParser
+
+from configargparse import ArgumentParser
 
 from toil.common import Toil
 from toil.job import Job

toil/test/sort/sortTest.py

@@ -23,10 +23,10 @@ from uuid import uuid4
 from toil import resolveEntryPoint
 from toil.batchSystems.mesos.test import MesosTestSupport
 from toil.common import Toil
+from toil.exceptions import FailedJobsException
 from toil.job import Job
 from toil.jobStores.abstractJobStore import (JobStoreExistsException,
                                              NoSuchJobStoreException)
-from toil.exceptions import FailedJobsException
 from toil.lib.bioio import root_logger
 from toil.test import (ToilTest,
                        needs_aws_ec2,
@@ -34,10 +34,8 @@ from toil.test import (ToilTest,
                        needs_google_storage,
                        needs_gridengine,
                        needs_mesos,
-                       needs_parasol,
                        needs_torque,
                        slow)
-from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport
 from toil.test.sort.sort import (copySubRangeOfFile,
                                  getMidPoint,
                                  main,
@@ -64,7 +62,7 @@ def runMain(options):
 
 
 @slow
-class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
+class SortTest(ToilTest, MesosTestSupport):
     """
     Tests Toil by sorting a file in parallel on various combinations of job stores and batch
     systems.
@@ -238,15 +236,6 @@ class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
     def testFileTorqueEngine(self):
         self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='torque')
 
-    @needs_parasol
-    @unittest.skip("skipping until parasol support is less flaky (see github issue #449")
-    def testFileParasol(self):
-        self._startParasol()
-        try:
-            self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='parasol')
-        finally:
-            self._stopParasol()
-
     testNo = 5
 
     def testSort(self):