toil 6.1.0a1__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/test/src/jobServiceTest.py

@@ -46,10 +46,10 @@ class JobServiceTest(ToilTest):
         """
         job = Job()
         service = ToySerializableService("woot")
-        startValue = job.addService(service) # Add a first service to job
-        subService = ToySerializableService(startValue) # Now create a child of
+        startValue = job.addService(service)  # Add a first service to job
+        subService = ToySerializableService(startValue)  # Now create a child of
         # that service that takes the start value promise from the parent service
-        job.addService(subService, parentService=service) # This should work if
+        job.addService(subService, parentService=service)  # This should work if
         # serialization on services is working correctly.
 
         self.runToil(job)
@@ -60,11 +60,13 @@ class JobServiceTest(ToilTest):
         Tests the creation of a Job.Service with random failures of the worker.
         """
         for test in range(2):
-            outFile = get_temp_file(rootDir=self._createTempDir()) # Temporary file
+            outFile = get_temp_file(rootDir=self._createTempDir())  # Temporary file
             messageInt = random.randint(1, sys.maxsize)
             try:
                 # Wire up the services/jobs
-                t = Job.wrapJobFn(serviceTest, outFile, messageInt, checkpoint=checkpoint)
+                t = Job.wrapJobFn(
+                    serviceTest, outFile, messageInt, checkpoint=checkpoint
+                )
 
                 # Run the workflow repeatedly until success
                 self.runToil(t)
@@ -75,24 +77,30 @@ class JobServiceTest(ToilTest):
                 os.remove(outFile)
 
     @slow
-    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
+    @skipIf(
+        SingleMachineBatchSystem.numCores < 4,
+        "Need at least four cores to run this test",
+    )
     def testServiceDeadlock(self):
         """
         Creates a job with more services than maxServices, checks that deadlock is detected.
         """
         outFile = get_temp_file(rootDir=self._createTempDir())
         try:
+
             def makeWorkflow():
                 job = Job()
                 r1 = job.addService(ToySerializableService("woot1"))
                 r2 = job.addService(ToySerializableService("woot2"))
                 r3 = job.addService(ToySerializableService("woot3"))
-                job.addChildFn(fnTest, [ r1, r2, r3 ], outFile)
+                job.addChildFn(fnTest, [r1, r2, r3], outFile)
                 return job
 
             # This should fail as too few services available
             try:
-                self.runToil(makeWorkflow(), badWorker=0.0, maxServiceJobs=2, deadlockWait=5)
+                self.runToil(
+                    makeWorkflow(), badWorker=0.0, maxServiceJobs=2, deadlockWait=5
+                )
             except DeadlockException:
                 print("Got expected deadlock exception")
             else:
@@ -113,7 +121,10 @@ class JobServiceTest(ToilTest):
         self.testService(checkpoint=True)
 
     @slow
-    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
+    @skipIf(
+        SingleMachineBatchSystem.numCores < 4,
+        "Need at least four cores to run this test",
+    )
     def testServiceRecursive(self, checkpoint=True):
         """
         Tests the creation of a Job.Service, creating a chain of services and accessing jobs.
@@ -122,10 +133,12 @@ class JobServiceTest(ToilTest):
         for test in range(1):
             # Temporary file
             outFile = get_temp_file(rootDir=self._createTempDir())
-            messages = [ random.randint(1, sys.maxsize) for i in range(3) ]
+            messages = [random.randint(1, sys.maxsize) for i in range(3)]
             try:
                 # Wire up the services/jobs
-                t = Job.wrapJobFn(serviceTestRecursive, outFile, messages, checkpoint=checkpoint)
+                t = Job.wrapJobFn(
+                    serviceTestRecursive, outFile, messages, checkpoint=checkpoint
+                )
 
                 # Run the workflow repeatedly until success
                 self.runToil(t)
@@ -136,32 +149,76 @@ class JobServiceTest(ToilTest):
                 os.remove(outFile)
 
     @slow
-    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
-    @retry_flaky_test(prepare=[ToilTest.tearDown, ToilTest.setUp])
+    @skipIf(
+        SingleMachineBatchSystem.numCores < 4,
+        "Need at least four cores to run this test",
+    )
     @pytest.mark.timeout(1200)
     def testServiceParallelRecursive(self, checkpoint=True):
         """
         Tests the creation of a Job.Service, creating parallel chains of services and accessing jobs.
         Randomly fails the worker.
         """
+
+        # This test needs to have something like 10 jobs succeed
+
+        BUNDLE_SIZE = 3
+        BUNDLE_COUNT = 2
+        RETRY_COUNT = 4
+        FAIL_FRACTION = 0.5
+        MAX_ATTEMPTS = 10
+
+        total_jobs = BUNDLE_SIZE * BUNDLE_COUNT * 2 + 1
+        p_complete_job_failure = FAIL_FRACTION ** (RETRY_COUNT + 1)
+        p_workflow_success = (1 - p_complete_job_failure) ** total_jobs
+        logger.info("Going to run %s total jobs, each of which completely fails %s of the time, so the workflow will succeed with probability %s", total_jobs, p_complete_job_failure, p_workflow_success)
+        p_test_failure = (1 - p_workflow_success) ** MAX_ATTEMPTS
+        logger.info("This test will fail spuriously with probability %s", p_test_failure)
+
+        # We want to run the workflow through several times to test restarting, so we need it to often fail but reliably sometimes succeed, and almost always succeed when repeated.
+        
+        self.assertGreater(0.8, p_workflow_success)
+        self.assertGreater(p_workflow_success, 0.2)
+        self.assertGreater(0.001, p_test_failure)
+        
+
         for test in range(1):
             # Temporary file
-            outFiles = [get_temp_file(rootDir=self._createTempDir()) for j in range(2)]
-            messageBundles = [ [ random.randint(1, sys.maxsize) for i in range(3) ] for j in range(2) ]
+            outFiles = [get_temp_file(rootDir=self._createTempDir()) for j in range(BUNDLE_COUNT)]
+            # We send 3 messages each in 2 sets, each of which needs a service and a client
+            messageBundles = [
+                [random.randint(1, sys.maxsize) for i in range(BUNDLE_SIZE)] for j in range(BUNDLE_COUNT)
+            ]
             try:
                 # Wire up the services/jobs
-                t = Job.wrapJobFn(serviceTestParallelRecursive, outFiles, messageBundles, checkpoint=True)
+                t = Job.wrapJobFn(
+                    serviceTestParallelRecursive,
+                    outFiles,
+                    messageBundles,
+                    checkpoint=True,
+                )
 
                 # Run the workflow repeatedly until success
-                self.runToil(t, retryCount=2)
+                self.runToil(t, retryCount=RETRY_COUNT, badWorker=FAIL_FRACTION, max_attempts=MAX_ATTEMPTS)
 
                 # Check output
-                for (messages, outFile) in zip(messageBundles, outFiles):
-                    self.assertEqual(list(map(int, open(outFile).readlines())), messages)
+                for messages, outFile in zip(messageBundles, outFiles):
+                    self.assertEqual(
+                        list(map(int, open(outFile).readlines())), messages
+                    )
             finally:
                 list(map(os.remove, outFiles))
 
-    def runToil(self, rootJob, retryCount=1, badWorker=0.5, badWorkedFailInterval=0.1, maxServiceJobs=sys.maxsize, deadlockWait=60):
+    def runToil(
+        self,
+        rootJob,
+        retryCount=1,
+        badWorker=0.5,
+        badWorkedFailInterval=0.1,
+        maxServiceJobs=sys.maxsize,
+        deadlockWait=60,
+        max_attempts=50
+    ):
         # Create the runner for the workflow.
         options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
         options.logLevel = "DEBUG"
@@ -171,27 +228,38 @@ class JobServiceTest(ToilTest):
         options.badWorkerFailInterval = badWorkedFailInterval
         options.servicePollingInterval = 1
         options.maxServiceJobs = maxServiceJobs
-        options.deadlockWait=deadlockWait
+        options.deadlockWait = deadlockWait
 
         # Run the workflow
-        totalTrys = 0
+        total_tries = 1
         while True:
             try:
                 Job.Runner.startToil(rootJob, options)
                 break
             except FailedJobsException as e:
                 i = e.numberOfFailedJobs
-                if totalTrys > 50: #p(fail after this many restarts) = 0.5**32
-                    self.fail() #Exceeded a reasonable number of restarts
-                totalTrys += 1
+                logger.info("Workflow attempt %s/%s failed with %s failed jobs", total_tries, max_attempts, i)
+                if total_tries == max_attempts:
+                    self.fail()  # Exceeded a reasonable number of restarts
+                total_tries += 1
                 options.restart = True
+        logger.info("Succeeded after %s/%s attempts", total_tries, max_attempts)
+
 
 class PerfectServiceTest(JobServiceTest):
-    def runToil(self, rootJob, retryCount=1, badWorker=0, badWorkedFailInterval=1000, maxServiceJobs=sys.maxsize, deadlockWait=60):
+    def runToil(
+        self,
+        *args,
+        **kwargs
+    ):
         """
         Let us run all the tests in the other service test class, but without worker failures.
         """
-        super().runToil(rootJob, retryCount, badWorker, badWorkedFailInterval, maxServiceJobs, deadlockWait)
+        kwargs["badWorker"] = 0
+        super().runToil(
+            *args,
+            **kwargs
+        )
 
 
 def serviceTest(job, outFile, messageInt):
@@ -199,51 +267,76 @@ def serviceTest(job, outFile, messageInt):
     Creates one service and one accessing job, which communicate with two files to establish
     that both run concurrently.
     """
-    #Clean out out-file
-    open(outFile, 'w').close()
-    randInt = random.randint(1, sys.maxsize) # We create a random number that is added to messageInt and subtracted by the serviceAccessor, to prove that
-    # when service test is checkpointed and restarted there is never a connection made between an earlier service and later serviceAccessor, or vice versa.
-    job.addChildJobFn(serviceAccessor, job.addService(ToyService(messageInt + randInt)), outFile, randInt)
+    # Clean out out-file
+    open(outFile, "w").close()
+    # We create a random number that is added to messageInt and subtracted by
+    # the serviceAccessor, to prove that when service test is checkpointed and
+    # restarted there is never a connection made between an earlier service and
+    # later serviceAccessor, or vice versa.
+    to_subtract = random.randint(
+        1, sys.maxsize
+    )
+    job.addChildJobFn(
+        serviceAccessor,
+        job.addService(ToyService(messageInt + to_subtract)),
+        outFile,
+        to_subtract,
+    )
+
 
 def serviceTestRecursive(job, outFile, messages):
     """
     Creates a chain of services and accessing jobs, each paired together.
     """
     if len(messages) > 0:
-        #Clean out out-file
-        open(outFile, 'w').close()
-        randInt = random.randint(1, sys.maxsize)
-        service = ToyService(messages[0] + randInt)
-        child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)
+        # Clean out out-file
+        open(outFile, "w").close()
+        to_add = random.randint(1, sys.maxsize)
+        service = ToyService(messages[0] + to_add)
+        child = job.addChildJobFn(
+            serviceAccessor, job.addService(service), outFile, to_add
+        )
 
         for i in range(1, len(messages)):
-            randInt = random.randint(1, sys.maxsize)
-            service2 = ToyService(messages[i] + randInt, cores=0.1)
-            child = child.addChildJobFn(serviceAccessor,
-                                        job.addService(service2, parentService=service),
-                                        outFile, randInt, cores=0.1)
+            to_add = random.randint(1, sys.maxsize)
+            service2 = ToyService(messages[i] + to_add, cores=0.1)
+            child = child.addChildJobFn(
+                serviceAccessor,
+                job.addService(service2, parentService=service),
+                outFile,
+                to_add,
+                cores=0.1,
+            )
             service = service2
 
+
 def serviceTestParallelRecursive(job, outFiles, messageBundles):
     """
     Creates multiple chains of services and accessing jobs.
     """
     for messages, outFile in zip(messageBundles, outFiles):
-        #Clean out out-file
-        open(outFile, 'w').close()
+        # Clean out out-file
+        open(outFile, "w").close()
         if len(messages) > 0:
-            randInt = random.randint(1, sys.maxsize)
-            service = ToyService(messages[0] + randInt)
-            child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)
+            to_add = random.randint(1, sys.maxsize)
+            service = ToyService(messages[0] + to_add)
+            child = job.addChildJobFn(
+                serviceAccessor, job.addService(service), outFile, to_add
+            )
 
             for i in range(1, len(messages)):
-                randInt = random.randint(1, sys.maxsize)
-                service2 = ToyService(messages[i] + randInt, cores=0.1)
-                child = child.addChildJobFn(serviceAccessor,
-                                            job.addService(service2, parentService=service),
-                                            outFile, randInt, cores=0.1)
+                to_add = random.randint(1, sys.maxsize)
+                service2 = ToyService(messages[i] + to_add, cores=0.1)
+                child = child.addChildJobFn(
+                    serviceAccessor,
+                    job.addService(service2, parentService=service),
+                    outFile,
+                    to_add,
+                    cores=0.1,
+                )
                 service = service2
 
+
 class ToyService(Job.Service):
     def __init__(self, messageInt, *args, **kwargs):
         """
@@ -264,10 +357,17 @@ class ToyService(Job.Service):
         # So we don't associate these files with this job.
         inJobStoreID = job.fileStore.jobStore.get_empty_file_store_id()
         outJobStoreID = job.fileStore.jobStore.get_empty_file_store_id()
-        self.serviceThread = Thread(target=self.serviceWorker,
-                                    args=(job.fileStore.jobStore, self.terminate, self.error,
-                                          inJobStoreID, outJobStoreID,
-                                          self.messageInt))
+        self.serviceThread = Thread(
+            target=self.serviceWorker,
+            args=(
+                job.fileStore.jobStore,
+                self.terminate,
+                self.error,
+                inJobStoreID,
+                outJobStoreID,
+                self.messageInt,
+            ),
+        )
         self.serviceThread.start()
         return (inJobStoreID, outJobStoreID)
 
@@ -281,22 +381,27 @@ class ToyService(Job.Service):
         return True
 
     @staticmethod
-    def serviceWorker(jobStore, terminate, error, inJobStoreID, outJobStoreID, messageInt):
+    def serviceWorker(
+        jobStore, terminate, error, inJobStoreID, outJobStoreID, messageInt
+    ):
         try:
             while True:
-                if terminate.isSet(): # Quit if we've got the terminate signal
+                if terminate.isSet():  # Quit if we've got the terminate signal
                     logger.debug("Service worker being told to quit")
                     return
 
-                time.sleep(0.2) # Sleep to avoid thrashing
+                time.sleep(0.2)  # Sleep to avoid thrashing
 
                 # Try reading a line from the input file
                 try:
                     with jobStore.read_file_stream(inJobStoreID) as f:
-                        f = codecs.getreader('utf-8')(f)
+                        f = codecs.getreader("utf-8")(f)
                         line = f.readline()
                 except:
-                    logger.debug("Something went wrong reading a line: %s", traceback.format_exc())
+                    logger.debug(
+                        "Something went wrong reading a line: %s",
+                        traceback.format_exc(),
+                    )
                     raise
 
                 if len(line.strip()) == 0:
@@ -307,7 +412,11 @@ class ToyService(Job.Service):
                 try:
                     inputInt = int(line)
                 except ValueError:
-                    logger.debug("Tried casting input line '%s' to integer but got error: %s", line, traceback.format_exc())
+                    logger.debug(
+                        "Tried casting input line '%s' to integer but got error: %s",
+                        line,
+                        traceback.format_exc(),
+                    )
                     continue
 
                 # Write out the resulting read integer and the message
@@ -318,28 +427,29 @@ class ToyService(Job.Service):
             error.set()
             raise
 
-def serviceAccessor(job, communicationFiles, outFile, randInt):
+
+def serviceAccessor(job, communicationFiles, outFile, to_subtract):
     """
     Writes a random integer iinto the inJobStoreFileID file, then tries 10 times reading
     from outJobStoreFileID to get a pair of integers, the first equal to i the second written into the outputFile.
     """
     inJobStoreFileID, outJobStoreFileID = communicationFiles
 
-    # Get a random integer
+    # Get a random integer to advertise ourselves
     key = random.randint(1, sys.maxsize)
 
     # Write the integer into the file
     logger.debug("Writing key to inJobStoreFileID")
     with job.fileStore.jobStore.update_file_stream(inJobStoreFileID) as fH:
-        fH.write(("%s\n" % key).encode('utf-8'))
+        fH.write(("%s\n" % key).encode("utf-8"))
 
     logger.debug("Trying to read key and message from outJobStoreFileID")
-    for i in range(10): # Try 10 times over
-        time.sleep(0.2) #Avoid thrashing
+    for i in range(10):  # Try 10 times over
+        time.sleep(0.2)  # Avoid thrashing
 
         # Try reading an integer from the input file and writing out the message
         with job.fileStore.jobStore.read_file_stream(outJobStoreFileID) as fH:
-            fH = codecs.getreader('utf-8')(fH)
+            fH = codecs.getreader("utf-8")(fH)
             line = fH.readline()
 
         tokens = line.split()
@@ -349,12 +459,15 @@ def serviceAccessor(job, communicationFiles, outFile, randInt):
         key2, message = tokens
 
         if int(key2) == key:
-            logger.debug(f"Matched key's: {key}, writing message: {int(message) - randInt} with randInt: {randInt}")
-            with open(outFile, 'a') as fH:
-                fH.write("%s\n" % (int(message) - randInt))
+            logger.debug(
+                f"Matched key's: {key}, writing message: {int(message) - to_subtract} with to_subtract: {to_subtract}"
+            )
+            with open(outFile, "a") as fH:
+                fH.write("%s\n" % (int(message) - to_subtract))
             return
 
-    assert 0 # Job failed to get info from the service
+    assert 0  # Job failed to get info from the service
+
 
 class ToySerializableService(Job.Service):
     def __init__(self, messageInt, *args, **kwargs):
@@ -373,9 +486,10 @@ class ToySerializableService(Job.Service):
     def check(self):
         return True
 
+
 def fnTest(strings, outputFile):
     """
     Function concatenates the strings together and writes them to the output file
     """
-    with open(outputFile, 'w') as fH:
+    with open(outputFile, "w") as fH:
         fH.write(" ".join(strings))