toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/src/promisedRequirementTest.py

@@ -32,46 +32,77 @@ class hidden:
     http://stackoverflow.com/questions/1323455/python-unit-test-with-base-and-sub-class#answer-25695512
     """
 
-    class AbstractPromisedRequirementsTest(batchSystemTest.hidden.AbstractBatchSystemJobTest):
+    class AbstractPromisedRequirementsTest(
+        batchSystemTest.hidden.AbstractBatchSystemJobTest
+    ):
         """An abstract base class for testing Toil workflows with promised requirements."""
+
         @slow
         def testConcurrencyDynamic(self):
             """
             Asserts that promised core resources are allocated properly using a dynamic Toil workflow
             """
             for coresPerJob in self.allocatedCores:
-                log.debug('Testing %d cores per job with CPU count %d', coresPerJob, self.cpuCount)
-                tempDir = self._createTempDir('testFiles')
+                log.debug(
+                    "Testing %d cores per job with CPU count %d",
+                    coresPerJob,
+                    self.cpuCount,
+                )
+                tempDir = self._createTempDir("testFiles")
                 counterPath = self.getCounterPath(tempDir)
 
-                root = Job.wrapJobFn(maxConcurrency, self.cpuCount, counterPath, coresPerJob,
-                                     cores=1, memory='1M', disk='1M')
+                root = Job.wrapJobFn(
+                    maxConcurrency,
+                    self.cpuCount,
+                    counterPath,
+                    coresPerJob,
+                    cores=1,
+                    memory="1M",
+                    disk="1M",
+                )
                 values = Job.Runner.startToil(root, self.getOptions(tempDir))
                 maxValue = max(values)
                 self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
 
         @slow
-        @retry_flaky_test(prepare=[batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
-                                   batchSystemTest.hidden.AbstractBatchSystemJobTest.setUp])
+        @retry_flaky_test(
+            prepare=[
+                batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
+                batchSystemTest.hidden.AbstractBatchSystemJobTest.setUp,
+            ]
+        )
         def testConcurrencyStatic(self):
             """
             Asserts that promised core resources are allocated properly using a static DAG
             """
             for coresPerJob in self.allocatedCores:
-                log.debug('Testing %d cores per job with CPU count %d', coresPerJob, self.cpuCount)
-                tempDir = self._createTempDir('testFiles')
+                log.debug(
+                    "Testing %d cores per job with CPU count %d",
+                    coresPerJob,
+                    self.cpuCount,
+                )
+                tempDir = self._createTempDir("testFiles")
                 counterPath = self.getCounterPath(tempDir)
 
                 root = Job()
-                one = Job.wrapFn(getOne, cores=0.1, memory='32M', disk='1M')
-                thirtyTwoMb = Job.wrapFn(getThirtyTwoMb, cores=0.1, memory='32M', disk='1M')
+                one = Job.wrapFn(getOne, cores=0.1, memory="32M", disk="1M")
+                thirtyTwoMb = Job.wrapFn(
+                    getThirtyTwoMb, cores=0.1, memory="32M", disk="1M"
+                )
                 root.addChild(one)
                 root.addChild(thirtyTwoMb)
                 for _ in range(self.cpuCount):
-                    root.addFollowOn(Job.wrapFn(batchSystemTest.measureConcurrency, counterPath,
-                                                cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()),
-                                                memory=PromisedRequirement(thirtyTwoMb.rv()),
-                                                disk='1M'))
+                    root.addFollowOn(
+                        Job.wrapFn(
+                            batchSystemTest.measureConcurrency,
+                            counterPath,
+                            cores=PromisedRequirement(
+                                lambda x: x * coresPerJob, one.rv()
+                            ),
+                            memory=PromisedRequirement(thirtyTwoMb.rv()),
+                            disk="1M",
+                        )
+                    )
                 Job.Runner.startToil(root, self.getOptions(tempDir))
                 _, maxValue = batchSystemTest.getCounters(counterPath)
                 self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
@@ -90,7 +121,7 @@ class hidden:
             :param str tempDir: path to test directory
             :return: path to counter file
             """
-            counterPath = os.path.join(tempDir, 'counter')
+            counterPath = os.path.join(tempDir, "counter")
             batchSystemTest.resetCounters(counterPath)
             minValue, maxValue = batchSystemTest.getCounters(counterPath)
             assert (minValue, maxValue) == (0, 0)
@@ -108,13 +139,19 @@ class hidden:
             file2 = 512
             F1 = Job.wrapJobFn(_writer, file1)
             F2 = Job.wrapJobFn(_writer, file2)
-            G = Job.wrapJobFn(_follower, file1 + file2,
-                              disk=PromisedRequirement(lambda x, y: x.size + y.size,
-                                                       F1.rv(), F2.rv()))
+            G = Job.wrapJobFn(
+                _follower,
+                file1 + file2,
+                disk=PromisedRequirement(
+                    lambda x, y: x.size + y.size, F1.rv(), F2.rv()
+                ),
+            )
             F1.addChild(F2)
             F2.addChild(G)
 
-            Job.Runner.startToil(F1, self.getOptions(self._createTempDir('testFiles'), caching=caching))
+            Job.Runner.startToil(
+                F1, self.getOptions(self._createTempDir("testFiles"), caching=caching)
+            )
 
         def testPromisesWithNonCachingFileStore(self):
             self.testPromisesWithJobStoreFileObjects(caching=False)
@@ -124,14 +161,18 @@ class hidden:
             """
             Checks for a race condition when using promised requirements and child job functions.
             """
-            A = Job.wrapJobFn(logDiskUsage, 'A', sleep=5, disk=PromisedRequirement(1024))
-            B = Job.wrapJobFn(logDiskUsage, 'B', disk=PromisedRequirement(lambda x: x + 1024, A.rv()))
+            A = Job.wrapJobFn(
+                logDiskUsage, "A", sleep=5, disk=PromisedRequirement(1024)
+            )
+            B = Job.wrapJobFn(
+                logDiskUsage, "B", disk=PromisedRequirement(lambda x: x + 1024, A.rv())
+            )
             A.addChild(B)
-            Job.Runner.startToil(A, self.getOptions(self._createTempDir('testFiles')))
+            Job.Runner.startToil(A, self.getOptions(self._createTempDir("testFiles")))
 
 
 def _writer(job, fileSize):
-    '''
+    """
     Write a local file and return the FileID obtained from running writeGlobalFile on
     it.
 
@@ -139,8 +180,8 @@ def _writer(job, fileSize):
     :param int fileSize: Size of the file in bytes
     :returns: the result of writeGlobalFile on a locally created file
     :rtype: job.FileID
-    '''
-    with open(job.fileStore.getLocalTempFileName(), 'wb') as fH:
+    """
+    with open(job.fileStore.getLocalTempFileName(), "wb") as fH:
         fH.write(os.urandom(fileSize))
     return job.fileStore.writeGlobalFile(fH.name)
 
@@ -166,15 +207,18 @@ def maxConcurrency(job, cpuCount, filename, coresPerJob):
     :param int coresPerJob: number of cores assigned to each job
     :return int max concurrency value:
     """
-    one = job.addChildFn(getOne, cores=0.1, memory='32M', disk='1M')
-    thirtyTwoMb = job.addChildFn(getThirtyTwoMb, cores=0.1, memory='32M', disk='1M')
+    one = job.addChildFn(getOne, cores=0.1, memory="32M", disk="1M")
+    thirtyTwoMb = job.addChildFn(getThirtyTwoMb, cores=0.1, memory="32M", disk="1M")
 
     values = []
     for _ in range(cpuCount):
-        value = job.addFollowOnFn(batchSystemTest.measureConcurrency, filename,
-                                  cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()),
-                                  memory=PromisedRequirement(thirtyTwoMb.rv()),
-                                  disk='1M').rv()
+        value = job.addFollowOnFn(
+            batchSystemTest.measureConcurrency,
+            filename,
+            cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()),
+            memory=PromisedRequirement(thirtyTwoMb.rv()),
+            disk="1M",
+        ).rv()
         values.append(value)
     return values
 
@@ -184,7 +228,7 @@ def getOne():
 
 
 def getThirtyTwoMb():
-    return '32M'
+    return "32M"
 
 
 def logDiskUsage(job, funcName, sleep=0):
@@ -194,7 +238,7 @@ def logDiskUsage(job, funcName, sleep=0):
     :return: job function's disk usage
     """
     diskUsage = job.disk
-    job.fileStore.log_to_leader(f'{funcName}: {diskUsage}')
+    job.fileStore.log_to_leader(f"{funcName}: {diskUsage}")
     time.sleep(sleep)
     return diskUsage
 
@@ -212,14 +256,16 @@ class SingleMachinePromisedRequirementsTest(hidden.AbstractPromisedRequirementsT
 
 
 @needs_mesos
-class MesosPromisedRequirementsTest(hidden.AbstractPromisedRequirementsTest, MesosTestSupport):
+class MesosPromisedRequirementsTest(
+    hidden.AbstractPromisedRequirementsTest, MesosTestSupport
+):
     """
     Tests against the Mesos batch system
     """
 
     def getOptions(self, tempDir, caching=True):
         options = super().getOptions(tempDir, caching=caching)
-        options.mesos_endpoint = 'localhost:5050'
+        options.mesos_endpoint = "localhost:5050"
         return options
 
     def getBatchSystemName(self):

toil/test/src/promisesTest.py

@@ -23,7 +23,7 @@ class CachedUnpicklingJobStoreTest(ToilTest):
         """
         for _ in range(2):
             options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-            options.logLevel = 'INFO'
+            options.logLevel = "INFO"
             root = Job.wrapJobFn(parent)
             Job.Runner.startToil(root, options)
 
@@ -40,7 +40,7 @@ class ChainedIndexedPromisesTest(ToilTest):
     # https://github.com/BD2KGenomics/toil/issues/1021
     def test(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.logLevel = 'INFO'
+        options.logLevel = "INFO"
         root = Job.wrapJobFn(a)
         self.assertEqual(Job.Runner.startToil(root, options), 42)
 
@@ -62,17 +62,18 @@ class PathIndexingPromiseTest(ToilTest):
     Test support for indexing promises of arbitrarily nested data structures of lists, dicts and
     tuples, or any other object supporting the __getitem__() protocol.
     """
+
     def test(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
-        options.logLevel = 'INFO'
+        options.logLevel = "INFO"
         root = Job.wrapJobFn(d)
-        self.assertEqual(Job.Runner.startToil(root, options), ('b', 43, 3))
+        self.assertEqual(Job.Runner.startToil(root, options), ("b", 43, 3))
 
 
 def d(job):
     child = job.addChild(job.wrapFn(e))
-    return child.rv('a'), child.rv(42), child.rv('c', 2)
+    return child.rv("a"), child.rv(42), child.rv("c", 2)
 
 
 def e():
-    return {'a': 'b', 42: 43, 'c': [1, 2, 3]}
+    return {"a": "b", 42: 43, "c": [1, 2, 3]}

toil/test/src/realtimeLoggerTest.py

@@ -23,13 +23,16 @@ class RealtimeLoggerTest(ToilTest):
     def testRealtimeLogger(self):
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.realTimeLogging = True
-        options.logLevel = 'INFO'
+        options.logLevel = "INFO"
 
         detector = MessageDetector()
 
         # Set up a log message detector to the root logger
         logging.getLogger().addHandler(detector)
 
+        # I believe coloredlogs replaces handlers with its own when doing handler formatting, preserving only filters
+        # https://github.com/xolox/python-coloredlogs/blob/65bdfe976ac0bf81e8c0bd9a98242b9d666b2859/coloredlogs/__init__.py#L453-L459
+        options.colored_logs = False
         Job.Runner.startToil(LogTest(), options)
 
         # We need the message we're supposed to see
@@ -46,19 +49,20 @@ class MessageDetector(logging.StreamHandler):
     def __init__(self):
         self.detected = False  # Have we seen the message we want?
         self.overLogged = False  # Have we seen the message we don't want?
+
         super().__init__()
 
     def emit(self, record):
-        if record.msg == 'This should be logged at info level':
+        if record.msg == "This should be logged at info level":
             self.detected = True
-        if record.msg == 'This should be logged at debug level':
+        if record.msg == "This should be logged at debug level":
             self.overLogged = True
 
 
 class LogTest(Job):
     def __init__(self):
-        Job.__init__(self, memory=100000, cores=1, disk='3G')
+        Job.__init__(self, memory=100000, cores=1, disk="3G")
 
     def run(self, fileStore):
-        RealtimeLogger.info('This should be logged at info level')
-        RealtimeLogger.debug('This should be logged at debug level')
+        RealtimeLogger.info("This should be logged at info level")
+        RealtimeLogger.debug("This should be logged at debug level")

toil/test/src/regularLogTest.py

@@ -28,10 +28,14 @@ class RegularLogTest(ToilTest):
 
     def setUp(self) -> None:
         super().setUp()
-        self.tempDir = self._createTempDir(purpose='tempDir')
+        self.tempDir = self._createTempDir(purpose="tempDir")
 
     def _getFiles(self, dir):
-        return [os.path.join(dir, f) for f in os.listdir(dir) if os.path.isfile(os.path.join(dir, f))]
+        return [
+            os.path.join(dir, f)
+            for f in os.listdir(dir)
+            if os.path.isfile(os.path.join(dir, f))
+        ]
 
     def _assertFileTypeExists(self, dir, extension, encoding=None):
         # an encoding of None implies no compression
@@ -45,56 +49,86 @@ class RegularLogTest(ToilTest):
         if encoding is not None:
             for log in onlyLogs:
                 with open(log, "rb") as f:
-                    logger.info("Checking for encoding %s on file %s", str(encoding), log)
+                    logger.info(
+                        "Checking for encoding %s on file %s", str(encoding), log
+                    )
                     if encoding == "gzip":
                         # Check for gzip magic header '\x1f\x8b'
-                        assert f.read().startswith(b'\x1f\x8b')
+                        assert f.read().startswith(b"\x1f\x8b")
                     else:
                         mime = mimetypes.guess_type(log)
                         self.assertEqual(mime[1], encoding)
 
     @slow
     def testLogToMaster(self):
-        toilOutput = subprocess.check_output([sys.executable,
-                                              '-m', helloWorld.__name__,
-                                              './toilTest',
-                                              '--clean=always',
-                                              '--logLevel=info'], stderr=subprocess.STDOUT)
-        assert helloWorld.childMessage in toilOutput.decode('utf-8')
+        toilOutput = subprocess.check_output(
+            [
+                sys.executable,
+                "-m",
+                helloWorld.__name__,
+                "./toilTest",
+                "--clean=always",
+                "--logLevel=info",
+            ],
+            stderr=subprocess.STDOUT,
+        )
+        assert helloWorld.childMessage in toilOutput.decode("utf-8")
 
     def testWriteLogs(self):
-        subprocess.check_call([sys.executable,
-                               '-m', helloWorld.__name__,
-                               './toilTest',
-                               '--clean=always',
-                               '--logLevel=debug',
-                               '--writeLogs=%s' % self.tempDir])
-        self._assertFileTypeExists(self.tempDir, '.log')
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                helloWorld.__name__,
+                "./toilTest",
+                "--clean=always",
+                "--logLevel=debug",
+                "--writeLogs=%s" % self.tempDir,
+            ]
+        )
+        self._assertFileTypeExists(self.tempDir, ".log")
 
     @slow
     def testWriteGzipLogs(self):
-        subprocess.check_call([sys.executable,
-                               '-m', helloWorld.__name__,
-                               './toilTest',
-                               '--clean=always',
-                               '--logLevel=debug',
-                               '--writeLogsGzip=%s' % self.tempDir])
-        self._assertFileTypeExists(self.tempDir, '.log.gz', 'gzip')
+        subprocess.check_call(
+            [
+                sys.executable,
+                "-m",
+                helloWorld.__name__,
+                "./toilTest",
+                "--clean=always",
+                "--logLevel=debug",
+                "--writeLogsGzip=%s" % self.tempDir,
+            ]
+        )
+        self._assertFileTypeExists(self.tempDir, ".log.gz", "gzip")
 
     @slow
     def testMultipleLogToMaster(self):
-        toilOutput = subprocess.check_output([sys.executable,
-                                              '-m', helloWorld.__name__,
-                                              './toilTest',
-                                              '--clean=always',
-                                              '--logLevel=info'], stderr=subprocess.STDOUT)
-        assert helloWorld.parentMessage in toilOutput.decode('utf-8')
+        toilOutput = subprocess.check_output(
+            [
+                sys.executable,
+                "-m",
+                helloWorld.__name__,
+                "./toilTest",
+                "--clean=always",
+                "--logLevel=info",
+            ],
+            stderr=subprocess.STDOUT,
+        )
+        assert helloWorld.parentMessage in toilOutput.decode("utf-8")
 
     def testRegularLog(self):
-        toilOutput = subprocess.check_output([sys.executable,
-                                              '-m', helloWorld.__name__,
-                                              './toilTest',
-                                              '--clean=always',
-                                              '--batchSystem=single_machine',
-                                              '--logLevel=debug'], stderr=subprocess.STDOUT)
-        assert "single machine batch system" in toilOutput.decode('utf-8')
+        toilOutput = subprocess.check_output(
+            [
+                sys.executable,
+                "-m",
+                helloWorld.__name__,
+                "./toilTest",
+                "--clean=always",
+                "--batchSystem=single_machine",
+                "--logLevel=debug",
+            ],
+            stderr=subprocess.STDOUT,
+        )
+        assert "single machine batch system" in toilOutput.decode("utf-8")