toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/provisioners/gceProvisionerTest.py

@@ -19,13 +19,15 @@ from uuid import uuid4
 
 import pytest
 
-from toil.test import (ToilTest,
-                       integrative,
-                       needs_fetchable_appliance,
-                       needs_google_project,
-                       needs_google_storage,
-                       slow,
-                       timeLimit)
+from toil.test import (
+    ToilTest,
+    integrative,
+    needs_fetchable_appliance,
+    needs_google_project,
+    needs_google_storage,
+    slow,
+    timeLimit,
+)
 from toil.version import exactPython
 
 log = logging.getLogger(__name__)
@@ -37,50 +39,62 @@ log = logging.getLogger(__name__)
 @needs_fetchable_appliance
 @slow
 class AbstractGCEAutoscaleTest(ToilTest):
-    projectID = os.getenv('TOIL_GOOGLE_PROJECTID')
+    projectID = os.getenv("TOIL_GOOGLE_PROJECTID")
 
     def sshUtil(self, command):
-        baseCommand = ['toil', 'ssh-cluster', '--insecure', '-p=gce', self.clusterName]
+        baseCommand = ["toil", "ssh-cluster", "--insecure", "-p=gce", self.clusterName]
         callCommand = baseCommand + command
         subprocess.check_call(callCommand)
 
     def rsyncUtil(self, src, dest):
-        baseCommand = ['toil', 'rsync-cluster', '--insecure', '-p=gce', self.clusterName]
+        baseCommand = [
+            "toil",
+            "rsync-cluster",
+            "--insecure",
+            "-p=gce",
+            self.clusterName,
+        ]
         callCommand = baseCommand + [src, dest]
         subprocess.check_call(callCommand)
 
     def destroyClusterUtil(self):
-        callCommand = ['toil', 'destroy-cluster', '-p=gce', self.clusterName]
+        callCommand = ["toil", "destroy-cluster", "-p=gce", self.clusterName]
         subprocess.check_call(callCommand)
 
     def createClusterUtil(self, args=None):
         if args is None:
             args = []
-        callCommand = ['toil', 'launch-cluster', self.clusterName, '-p=gce', '--keyPairName=%s' % self.keyName,
-                       '--leaderNodeType=%s' % self.leaderInstanceType, '--zone=%s' % self.googleZone]
+        callCommand = [
+            "toil",
+            "launch-cluster",
+            self.clusterName,
+            "-p=gce",
+            "--keyPairName=%s" % self.keyName,
+            "--leaderNodeType=%s" % self.leaderInstanceType,
+            "--zone=%s" % self.googleZone,
+        ]
         if self.botoDir is not None:
-            callCommand += ['--boto=%s' % self.botoDir]
+            callCommand += ["--boto=%s" % self.botoDir]
         callCommand = callCommand + args if args else callCommand
-        log.info("createClusterUtil: %s" % ''.join(callCommand))
+        log.info("createClusterUtil: %s" % "".join(callCommand))
         subprocess.check_call(callCommand)
 
     def cleanJobStoreUtil(self):
-        callCommand = ['toil', 'clean', self.jobStore]
+        callCommand = ["toil", "clean", self.jobStore]
         subprocess.check_call(callCommand)
 
     def __init__(self, methodName):
         super().__init__(methodName=methodName)
         # TODO: add TOIL_GOOGLE_KEYNAME to needs_google_project or ssh with SA account
-        self.keyName = os.getenv('TOIL_GOOGLE_KEYNAME')
+        self.keyName = os.getenv("TOIL_GOOGLE_KEYNAME")
         # TODO: remove this when switching to google jobstore
-        self.botoDir = os.getenv('TOIL_BOTO_DIR')
+        self.botoDir = os.getenv("TOIL_BOTO_DIR")
         # TODO: get this from SA account or add an environment variable
-        self.googleZone = 'us-west1-a'
+        self.googleZone = "us-west1-a"
 
-
-        self.leaderInstanceType = 'n1-standard-1'
+        self.leaderInstanceType = "n1-standard-1"
         self.instanceTypes = ["n1-standard-2"]
-        self.numWorkers = ['2']
+        self.numWorkers = ["2"]
         self.numSamples = 2
         self.spotBid = 0.15
 
@@ -92,7 +106,7 @@ class AbstractGCEAutoscaleTest(ToilTest):
         self.destroyClusterUtil()
         self.cleanJobStoreUtil()
 
-    #def getMatchingRoles(self, clusterName):
+    # def getMatchingRoles(self, clusterName):
     #    ctx = AWSProvisioner._buildContext(clusterName)
     #    roles = list(ctx.local_roles())
     #    return roles
@@ -107,7 +121,6 @@ class AbstractGCEAutoscaleTest(ToilTest):
         """
         raise NotImplementedError()
 
-
     @abstractmethod
     def _runScript(self, toilOptions):
         """
@@ -127,82 +140,103 @@ class AbstractGCEAutoscaleTest(ToilTest):
         self.launchCluster()
 
         # TODO: What is the point of this test?
-        #assert len(self.getMatchingRoles(self.clusterName)) == 1
+        # assert len(self.getMatchingRoles(self.clusterName)) == 1
 
         # TODO: Add a check of leader and node storage size if set.
 
         # --never-download prevents silent upgrades to pip, wheel and setuptools
-        venv_command = ['virtualenv', '--system-site-packages', '--never-download',
-                        '--python', exactPython, '/home/venv']
+        venv_command = [
+            "virtualenv",
+            "--system-site-packages",
+            "--never-download",
+            "--python",
+            exactPython,
+            "/home/venv",
+        ]
         self.sshUtil(venv_command)
 
         self._getScript()
 
-        toilOptions = [self.jobStore,
-                       '--batchSystem=mesos',
-                       '--workDir=/var/lib/toil',
-                       '--clean=always',
-                       '--retryCount=2',
-                       '--clusterStats=/home/',
-                       '--logDebug',
-                       '--logFile=/home/sort.log',
-                       '--provisioner=gce']
-
-        toilOptions.extend(['--nodeTypes=' + ",".join(self.instanceTypes),
-                            '--maxNodes=%s' % ",".join(self.numWorkers)])
+        toilOptions = [
+            self.jobStore,
+            "--batchSystem=mesos",
+            "--workDir=/var/lib/toil",
+            "--clean=always",
+            "--retryCount=2",
+            "--clusterStats=/home/",
+            "--logDebug",
+            "--logFile=/home/sort.log",
+            "--provisioner=gce",
+        ]
+
+        toilOptions.extend(
+            [
+                "--nodeTypes=" + ",".join(self.instanceTypes),
+                "--maxNodes=%s" % ",".join(self.numWorkers),
+            ]
+        )
         if preemptibleJobs:
-            toilOptions.extend(['--defaultPreemptible'])
+            toilOptions.extend(["--defaultPreemptible"])
 
         self._runScript(toilOptions)
 
-        #TODO: Does this just check if it is still running?
-        #assert len(self.getMatchingRoles(self.clusterName)) == 1
+        # TODO: Does this just check if it is still running?
+        # assert len(self.getMatchingRoles(self.clusterName)) == 1
 
-        checkStatsCommand = ['/home/venv/bin/python', '-c',
-                             'import json; import os; '
-                             'json.load(open("/home/" + [f for f in os.listdir("/home/") '
-                                                   'if f.endswith(".json")].pop()))'
-                             ]
+        checkStatsCommand = [
+            "/home/venv/bin/python",
+            "-c",
+            "import json; import os; "
+            'json.load(open("/home/" + [f for f in os.listdir("/home/") '
+            'if f.endswith(".json")].pop()))',
+        ]
 
         self.sshUtil(checkStatsCommand)
 
-
         # TODO: Add a check to make sure everything is cleaned up.
 
 
-
 @pytest.mark.timeout(1600)
 class GCEAutoscaleTest(AbstractGCEAutoscaleTest):
 
     def __init__(self, name):
         super().__init__(name)
-        self.clusterName = 'provisioner-test-' + str(uuid4())
+        self.clusterName = "provisioner-test-" + str(uuid4())
         self.requestedLeaderStorage = 80
 
     def setUp(self):
         super().setUp()
-        self.jobStore = f'google:{self.projectID}:autoscale-{uuid4()}'
+        self.jobStore = f"google:{self.projectID}:autoscale-{uuid4()}"
 
     def _getScript(self):
         # TODO: Isn't this the key file?
         fileToSort = os.path.join(os.getcwd(), str(uuid4()))
-        with open(fileToSort, 'w') as f:
+        with open(fileToSort, "w") as f:
             # Fixme: making this file larger causes the test to hang
-            f.write('01234567890123456789012345678901')
-        self.rsyncUtil(os.path.join(self._projectRootPath(), 'src/toil/test/sort/sort.py'), ':/home/sort.py')
-        self.rsyncUtil(fileToSort, ':/home/sortFile')
+            f.write("01234567890123456789012345678901")
+        self.rsyncUtil(
+            os.path.join(self._projectRootPath(), "src/toil/test/sort/sort.py"),
+            ":/home/sort.py",
+        )
+        self.rsyncUtil(fileToSort, ":/home/sortFile")
         os.unlink(fileToSort)
 
     def _runScript(self, toilOptions):
-        runCommand = ['/home/venv/bin/python', '/home/sort.py', '--fileToSort=/home/sortFile']
+        runCommand = [
+            "/home/venv/bin/python",
+            "/home/sort.py",
+            "--fileToSort=/home/sortFile",
+        ]
         #'--sseKey=/home/sortFile']
         runCommand.extend(toilOptions)
-        log.info("_runScript: %s" % ''.join(runCommand))
+        log.info("_runScript: %s" % "".join(runCommand))
         self.sshUtil(runCommand)
 
     def launchCluster(self):
         # add arguments to test that we can specify leader storage
-        self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage)])
+        self.createClusterUtil(
+            args=["--leaderStorage", str(self.requestedLeaderStorage)]
+        )
 
     # TODO: aren't these checks inherited?
     @integrative
@@ -210,7 +244,7 @@ class GCEAutoscaleTest(AbstractGCEAutoscaleTest):
     @needs_google_storage
     def testAutoScale(self):
         self.instanceTypes = ["n1-standard-2"]
-        self.numWorkers = ['2']
+        self.numWorkers = ["2"]
         self._test()
 
     @integrative
@@ -220,7 +254,7 @@ class GCEAutoscaleTest(AbstractGCEAutoscaleTest):
         self.instanceTypes = ["n1-standard-2:%f" % self.spotBid]
         # Some spot workers have a stopped state after being started, strangely.
         # This could be the natural preemption process, but it seems too rapid.
-        self.numWorkers = ['3'] # Try 3 to account for a stopped node.
+        self.numWorkers = ["3"]  # Try 3 to account for a stopped node.
         self._test(preemptibleJobs=True)
 
 
@@ -229,35 +263,49 @@ class GCEStaticAutoscaleTest(GCEAutoscaleTest):
     """
     Runs the tests on a statically provisioned cluster with autoscaling enabled.
     """
+
     def __init__(self, name):
         super().__init__(name)
         self.requestedNodeStorage = 20
 
     def launchCluster(self):
-        self.createClusterUtil(args=['--leaderStorage', str(self.requestedLeaderStorage),
-                                     '--nodeTypes', ",".join(self.instanceTypes), '-w', ",".join(self.numWorkers),
-                                     '--nodeStorage', str(self.requestedLeaderStorage)])
+        self.createClusterUtil(
+            args=[
+                "--leaderStorage",
+                str(self.requestedLeaderStorage),
+                "--nodeTypes",
+                ",".join(self.instanceTypes),
+                "-w",
+                ",".join(self.numWorkers),
+                "--nodeStorage",
+                str(self.requestedLeaderStorage),
+            ]
+        )
 
         # TODO: check the number of workers and their storage
-        #nodes = AWSProvisioner._getNodesInCluster(ctx, self.clusterName, both=True)
-        #nodes.sort(key=lambda x: x.launch_time)
+        # nodes = AWSProvisioner._getNodesInCluster(ctx, self.clusterName, both=True)
+        # nodes.sort(key=lambda x: x.launch_time)
         # assuming that leader is first
-        #workers = nodes[1:]
+        # workers = nodes[1:]
         # test that two worker nodes were created
-        #self.assertEqual(2, len(workers))
+        # self.assertEqual(2, len(workers))
         # test that workers have expected storage size
         # just use the first worker
-        #worker = workers[0]
-        #worker = next(wait_instances_running(ctx.ec2, [worker]))
-        #rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
-        #self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
-        #rootVolume = ctx.ec2.get_all_volumes(volume_ids=[rootBlockDevice.volume_id])[0]
-        #self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
+        # worker = workers[0]
+        # worker = next(wait_instances_running(ctx.ec2, [worker]))
+        # rootBlockDevice = worker.block_device_mapping["/dev/xvda"]
+        # self.assertTrue(isinstance(rootBlockDevice, BlockDeviceType))
+        # rootVolume = ctx.ec2.get_all_volumes(volume_ids=[rootBlockDevice.volume_id])[0]
+        # self.assertGreaterEqual(rootVolume.size, self.requestedNodeStorage)
 
     def _runScript(self, toilOptions):
-        runCommand = ['/home/venv/bin/python', '/home/sort.py', '--fileToSort=/home/sortFile']
+        runCommand = [
+            "/home/venv/bin/python",
+            "/home/sort.py",
+            "--fileToSort=/home/sortFile",
+        ]
         runCommand.extend(toilOptions)
-        log.info("_runScript: %s" % ''.join(runCommand))
+        log.info("_runScript: %s" % "".join(runCommand))
         self.sshUtil(runCommand)
 
 
@@ -266,28 +314,37 @@ class GCEAutoscaleTestMultipleNodeTypes(AbstractGCEAutoscaleTest):
 
     def __init__(self, name):
         super().__init__(name)
-        self.clusterName = 'provisioner-test-' + str(uuid4())
+        self.clusterName = "provisioner-test-" + str(uuid4())
 
     def setUp(self):
         super().setUp()
-        self.jobStore = f'google:{self.projectID}:multinode-{uuid4()}'
+        self.jobStore = f"google:{self.projectID}:multinode-{uuid4()}"
 
     def _getScript(self):
-        sseKeyFile = os.path.join(os.getcwd(), 'keyFile')
-        with open(sseKeyFile, 'w') as f:
-            f.write('01234567890123456789012345678901')
-        self.rsyncUtil(os.path.join(self._projectRootPath(), 'src/toil/test/sort/sort.py'), ':/home/sort.py')
-        self.rsyncUtil(sseKeyFile, ':/home/keyFile')
+        sseKeyFile = os.path.join(os.getcwd(), "keyFile")
+        with open(sseKeyFile, "w") as f:
+            f.write("01234567890123456789012345678901")
+        self.rsyncUtil(
+            os.path.join(self._projectRootPath(), "src/toil/test/sort/sort.py"),
+            ":/home/sort.py",
+        )
+        self.rsyncUtil(sseKeyFile, ":/home/keyFile")
         os.unlink(sseKeyFile)
 
     def _runScript(self, toilOptions):
-        #Set memory requirements so that sort jobs can be run
+        # Set memory requirements so that sort jobs can be run
         # on small instances, but merge jobs must be run on large
         # instances
-        runCommand = ['/home/venv/bin/python', '/home/sort.py', '--fileToSort=/home/s3am/bin/asadmin', '--sortMemory=0.6G', '--mergeMemory=3.0G']
+        runCommand = [
+            "/home/venv/bin/python",
+            "/home/sort.py",
+            "--fileToSort=/home/s3am/bin/asadmin",
+            "--sortMemory=0.6G",
+            "--mergeMemory=3.0G",
+        ]
         runCommand.extend(toilOptions)
-        #runCommand.append('--sseKey=/home/keyFile')
-        log.info("_runScript: %s" % ''.join(runCommand))
+        # runCommand.append('--sseKey=/home/keyFile')
+        log.info("_runScript: %s" % "".join(runCommand))
         self.sshUtil(runCommand)
 
     @integrative
@@ -295,9 +352,10 @@ class GCEAutoscaleTestMultipleNodeTypes(AbstractGCEAutoscaleTest):
     @needs_google_storage
     def testAutoScale(self):
         self.instanceTypes = ["n1-standard-2", "n1-standard-4"]
-        self.numWorkers = ['2','1']
+        self.numWorkers = ["2", "1"]
         self._test()
 
+
 @pytest.mark.timeout(1800)
 class GCERestartTest(AbstractGCEAutoscaleTest):
     """
@@ -306,37 +364,53 @@ class GCERestartTest(AbstractGCEAutoscaleTest):
 
     def __init__(self, name):
         super().__init__(name)
-        self.clusterName = 'restart-test-' + str(uuid4())
+        self.clusterName = "restart-test-" + str(uuid4())
 
     def setUp(self):
         super().setUp()
-        self.instanceTypes = ['n1-standard-1']
-        self.numWorkers = ['1']
+        self.instanceTypes = ["n1-standard-1"]
+        self.numWorkers = ["1"]
         self.scriptName = "/home/restartScript.py"
         # TODO: replace this with a google job store
-        zone = 'us-west-2'
-        self.jobStore = f'google:{self.projectID}:restart-{uuid4()}'
+        zone = "us-west-2"
+        self.jobStore = f"google:{self.projectID}:restart-{uuid4()}"
 
     def _getScript(self):
-        self.rsyncUtil(os.path.join(self._projectRootPath(), 'src/toil/test/provisioners/restartScript.py'),
-                        ':'+self.scriptName)
-
+        self.rsyncUtil(
+            os.path.join(
+                self._projectRootPath(), "src/toil/test/provisioners/restartScript.py"
+            ),
+            ":" + self.scriptName,
+        )
 
     def _runScript(self, toilOptions):
         # clean = onSuccess
-        disallowedOptions = ['--clean=always', '--retryCount=2']
-        newOptions = [option for option in toilOptions if option not in disallowedOptions]
+        disallowedOptions = ["--clean=always", "--retryCount=2"]
+        newOptions = [
+            option for option in toilOptions if option not in disallowedOptions
+        ]
         try:
             # include a default memory - on restart the minimum memory requirement is the default, usually 2 GB
-            command = ['/home/venv/bin/python', self.scriptName, '-e', 'FAIL=true', '--defaultMemory=50000000']
+            command = [
+                "/home/venv/bin/python",
+                self.scriptName,
+                "-e",
+                "FAIL=true",
+                "--defaultMemory=50000000",
+            ]
             command.extend(newOptions)
             self.sshUtil(command)
         except subprocess.CalledProcessError:
             pass
         else:
-            self.fail('Command succeeded when we expected failure')
+            self.fail("Command succeeded when we expected failure")
         with timeLimit(1200):
-            command = ['/home/venv/bin/python', self.scriptName, '--restart', '--defaultMemory=50000000']
+            command = [
+                "/home/venv/bin/python",
+                self.scriptName,
+                "--restart",
+                "--defaultMemory=50000000",
+            ]
             command.extend(toilOptions)
             self.sshUtil(command)
 

toil/test/provisioners/provisionerTest.py

@@ -25,21 +25,33 @@ class ProvisionerTest(ToilTest):
 
     def test_node_type_parsing(self) -> None:
         assert parse_node_types(None) == []
-        assert parse_node_types('') == []
-        assert parse_node_types('red beans') == [({'red beans'}, None)]
-        assert parse_node_types('red beans,rice') == [({'red beans'}, None), ({'rice'}, None)]
-        assert parse_node_types('red beans/black beans,rice') == [({'red beans', 'black beans'}, None), ({'rice'}, None)]
-        assert parse_node_types('frankfurters:0.05') == [({'frankfurters'}, 0.05)]
-        assert parse_node_types('red beans/black beans:999,rice,red beans/black beans') == [({'red beans', 'black beans'}, 999), ({'rice'}, None), ({'red beans', 'black beans'}, None)]
+        assert parse_node_types("") == []
+        assert parse_node_types("red beans") == [({"red beans"}, None)]
+        assert parse_node_types("red beans,rice") == [
+            ({"red beans"}, None),
+            ({"rice"}, None),
+        ]
+        assert parse_node_types("red beans/black beans,rice") == [
+            ({"red beans", "black beans"}, None),
+            ({"rice"}, None),
+        ]
+        assert parse_node_types("frankfurters:0.05") == [({"frankfurters"}, 0.05)]
+        assert parse_node_types(
+            "red beans/black beans:999,rice,red beans/black beans"
+        ) == [
+            ({"red beans", "black beans"}, 999),
+            ({"rice"}, None),
+            ({"red beans", "black beans"}, None),
+        ]
         with pytest.raises(ValueError):
-            parse_node_types('your thoughts:penny')
+            parse_node_types("your thoughts:penny")
         with pytest.raises(ValueError) as err:
-            parse_node_types(',,,')
-        assert 'empty' in str(err.value)
+            parse_node_types(",,,")
+        assert "empty" in str(err.value)
         with pytest.raises(ValueError):
-            parse_node_types('now hear this:')
+            parse_node_types("now hear this:")
         with pytest.raises(ValueError) as err:
-            parse_node_types('miles I will walk:500:500')
-        assert 'multiple' in str(err.value)
+            parse_node_types("miles I will walk:500:500")
+        assert "multiple" in str(err.value)
         with pytest.raises(ValueError):
-            parse_node_types('red beans:500/black beans:500,rice')
+            parse_node_types("red beans:500/black beans:500,rice")

toil/test/provisioners/restartScript.py

@@ -6,12 +6,13 @@ from toil.job import Job
 
 
 def f0(job):
-    if 'FAIL' in os.environ:
-        raise RuntimeError('failed on purpose')
+    if "FAIL" in os.environ:
+        raise RuntimeError("failed on purpose")
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
     options = parser.parse_args()
-    rootJob = Job.wrapJobFn(f0, cores=0.5, memory='50 M', disk='50 M')
+    rootJob = Job.wrapJobFn(f0, cores=0.5, memory="50 M", disk="50 M")
     Job.Runner.startToil(rootJob, options)