toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

toil/test/provisioners/clusterTest.py

@@ -15,16 +15,21 @@ import logging
 import os
 import subprocess
 import time
-
+from typing import Optional
 from uuid import uuid4
-from typing import Optional, List
 
-from toil.lib.retry import retry
-from toil.test import ToilTest, needs_aws_ec2, needs_fetchable_appliance, slow, needs_env_var
 from toil.lib.aws import zone_to_region
 from toil.lib.aws.session import AWSConnectionManager
+from toil.lib.retry import retry
 from toil.provisioners import cluster_factory
 from toil.provisioners.aws import get_best_aws_zone
+from toil.test import (
+    ToilTest,
+    needs_aws_ec2,
+    needs_env_var,
+    needs_fetchable_appliance,
+    slow,
+)
 
 log = logging.getLogger(__name__)
 
@@ -34,31 +39,33 @@ log = logging.getLogger(__name__)
 class AbstractClusterTest(ToilTest):
     def __init__(self, methodName: str) -> None:
         super().__init__(methodName=methodName)
-        self.keyName = os.getenv('TOIL_AWS_KEYNAME').strip() or 'id_rsa'
-        self.clusterName = f'aws-provisioner-test-{uuid4()}'
-        self.leaderNodeType = 't2.medium'
-        self.clusterType = 'mesos'
+        self.keyName = os.getenv("TOIL_AWS_KEYNAME").strip() or "id_rsa"
+        self.clusterName = f"aws-provisioner-test-{uuid4()}"
+        self.leaderNodeType = "t2.medium"
+        self.clusterType = "mesos"
         self.zone = get_best_aws_zone()
-        assert self.zone is not None, "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
+        assert (
+            self.zone is not None
+        ), "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
         self.region = zone_to_region(self.zone)
 
         # Get connection to AWS
         self.aws = AWSConnectionManager()
 
         # Where should we put our virtualenv?
-        self.venvDir = '/tmp/venv'
+        self.venvDir = "/tmp/venv"
 
     def python(self) -> str:
         """
         Return the full path to the venv Python on the leader.
         """
-        return os.path.join(self.venvDir, 'bin/python')
+        return os.path.join(self.venvDir, "bin/python")
 
     def pip(self) -> str:
         """
         Return the full path to the venv pip on the leader.
         """
-        return os.path.join(self.venvDir, 'bin/pip')
+        return os.path.join(self.venvDir, "bin/pip")
 
     def destroyCluster(self) -> None:
         """
@@ -66,7 +73,9 @@ class AbstractClusterTest(ToilTest):
 
         Succeeds if the cluster does not currently exist.
         """
-        subprocess.check_call(['toil', 'destroy-cluster', '-p=aws', '-z', self.zone, self.clusterName])
+        subprocess.check_call(
+            ["toil", "destroy-cluster", "-p=aws", "-z", self.zone, self.clusterName]
+        )
 
     def setUp(self) -> None:
         """
@@ -82,71 +91,83 @@ class AbstractClusterTest(ToilTest):
         # Note that teardown will run even if the test crashes.
         super().tearDown()
         self.destroyCluster()
-        subprocess.check_call(['toil', 'clean', self.jobStore])
+        subprocess.check_call(["toil", "clean", self.jobStore])
 
-    def sshUtil(self, command: List[str]) -> None:
+    def sshUtil(self, command: list[str]) -> None:
         """
         Run the given command on the cluster.
         Raise subprocess.CalledProcessError if it fails.
         """
 
-        cmd = ['toil', 'ssh-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName] + command
+        cmd = [
+            "toil",
+            "ssh-cluster",
+            "--insecure",
+            "-p=aws",
+            "-z",
+            self.zone,
+            self.clusterName,
+        ] + command
         log.info("Running %s.", str(cmd))
         p = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
         # Put in non-blocking mode. See https://stackoverflow.com/a/59291466
         os.set_blocking(p.stdout.fileno(), False)
         os.set_blocking(p.stderr.fileno(), False)
 
-        out_buffer = b''
-        err_buffer = b''
+        out_buffer = b""
+        err_buffer = b""
 
         loops_since_line = 0
 
         running = True
         while running:
             # While the process is running, see if it stopped
-            running = (p.poll() is None)
+            running = p.poll() is None
 
             # Also collect its output
             out_data = p.stdout.read()
             if out_data:
                 out_buffer += out_data
 
-            while out_buffer.find(b'\n') != -1:
+            while out_buffer.find(b"\n") != -1:
                 # And log every full line
-                cut = out_buffer.find(b'\n')
-                log.info('STDOUT: %s', out_buffer[0:cut].decode('utf-8', errors='ignore'))
+                cut = out_buffer.find(b"\n")
+                log.info(
+                    "STDOUT: %s", out_buffer[0:cut].decode("utf-8", errors="ignore")
+                )
                 loops_since_line = 0
-                out_buffer = out_buffer[cut+1:]
+                out_buffer = out_buffer[cut + 1 :]
 
             # Same for the error
             err_data = p.stderr.read()
             if err_data:
                 err_buffer += err_data
 
-            while err_buffer.find(b'\n') != -1:
-                cut = err_buffer.find(b'\n')
-                log.info('STDERR: %s', err_buffer[0:cut].decode('utf-8', errors='ignore'))
+            while err_buffer.find(b"\n") != -1:
+                cut = err_buffer.find(b"\n")
+                log.info(
+                    "STDERR: %s", err_buffer[0:cut].decode("utf-8", errors="ignore")
+                )
                 loops_since_line = 0
-                err_buffer = err_buffer[cut+1:]
+                err_buffer = err_buffer[cut + 1 :]
 
             loops_since_line += 1
             if loops_since_line > 60:
-                log.debug('...waiting...')
+                log.debug("...waiting...")
                 loops_since_line = 0
 
             time.sleep(1)
 
         # At the end, log the last lines
         if out_buffer:
-            log.info('STDOUT: %s', out_buffer.decode('utf-8', errors='ignore'))
+            log.info("STDOUT: %s", out_buffer.decode("utf-8", errors="ignore"))
         if err_buffer:
-            log.info('STDERR: %s', err_buffer.decode('utf-8', errors='ignore'))
+            log.info("STDERR: %s", err_buffer.decode("utf-8", errors="ignore"))
 
         if p.returncode != 0:
             # It failed
             log.error("Failed to run %s.", str(cmd))
-            raise subprocess.CalledProcessError(p.returncode, ' '.join(cmd))
+            raise subprocess.CalledProcessError(p.returncode, " ".join(cmd))
 
     @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
     def rsync_util(self, from_file: str, to_file: str) -> None:
@@ -155,18 +176,38 @@ class AbstractClusterTest(ToilTest):
 
         The cluster-side path should have a ':' in front of it.
         """
-        cmd = ['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName, from_file, to_file]
+        cmd = [
+            "toil",
+            "rsync-cluster",
+            "--insecure",
+            "-p=aws",
+            "-z",
+            self.zone,
+            self.clusterName,
+            from_file,
+            to_file,
+        ]
         log.info("Running %s.", str(cmd))
         subprocess.check_call(cmd)
 
     @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
-    def createClusterUtil(self, args: Optional[List[str]]=None) -> None:
+    def createClusterUtil(self, args: Optional[list[str]] = None) -> None:
         args = [] if args is None else args
 
-        command = ['toil', 'launch-cluster', '-p=aws', '-z', self.zone, f'--keyPairName={self.keyName}',
-                   f'--leaderNodeType={self.leaderNodeType}', f'--clusterType={self.clusterType}', '--logDebug', self.clusterName] + args
-
-        log.debug('Launching cluster: %s', command)
+        command = [
+            "toil",
+            "launch-cluster",
+            "-p=aws",
+            "-z",
+            self.zone,
+            f"--keyPairName={self.keyName}",
+            f"--leaderNodeType={self.leaderNodeType}",
+            f"--clusterType={self.clusterType}",
+            "--logDebug",
+            self.clusterName,
+        ] + args
+
+        log.debug("Launching cluster: %s", command)
 
         # Try creating the cluster
         subprocess.check_call(command)
@@ -181,9 +222,10 @@ class AbstractClusterTest(ToilTest):
 @slow
 class CWLOnARMTest(AbstractClusterTest):
     """Run the CWL 1.2 conformance tests on ARM specifically."""
+
     def __init__(self, methodName: str) -> None:
         super().__init__(methodName=methodName)
-        self.clusterName = f'cwl-test-{uuid4()}'
+        self.clusterName = f"cwl-test-{uuid4()}"
         self.leaderNodeType = "t4g.2xlarge"
         self.clusterType = "kubernetes"
         # We need to be running in a directory which Flatcar and the Toil Appliance both have
@@ -247,8 +289,5 @@ class CWLOnARMTest(AbstractClusterTest):
         # Bring it back to be an artifact.
         self.rsync_util(
             f":{self.cwl_test_dir}/toil/conformance-1.2.junit.xml",
-            os.path.join(
-                self._projectRootPath(),
-                "arm-conformance-1.2.junit.xml"
-            )
+            os.path.join(self._projectRootPath(), "arm-conformance-1.2.junit.xml"),
         )