toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +21 -10
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/contained_executor.py +3 -3
  6. toil/batchSystems/htcondor.py +0 -1
  7. toil/batchSystems/kubernetes.py +34 -31
  8. toil/batchSystems/local_support.py +3 -1
  9. toil/batchSystems/mesos/batchSystem.py +7 -7
  10. toil/batchSystems/options.py +32 -83
  11. toil/batchSystems/registry.py +104 -23
  12. toil/batchSystems/singleMachine.py +16 -13
  13. toil/batchSystems/slurm.py +3 -3
  14. toil/batchSystems/torque.py +0 -1
  15. toil/bus.py +6 -8
  16. toil/common.py +532 -743
  17. toil/cwl/__init__.py +28 -32
  18. toil/cwl/cwltoil.py +523 -520
  19. toil/cwl/utils.py +55 -10
  20. toil/fileStores/__init__.py +2 -2
  21. toil/fileStores/abstractFileStore.py +36 -11
  22. toil/fileStores/cachingFileStore.py +607 -530
  23. toil/fileStores/nonCachingFileStore.py +43 -10
  24. toil/job.py +140 -75
  25. toil/jobStores/abstractJobStore.py +147 -79
  26. toil/jobStores/aws/jobStore.py +23 -9
  27. toil/jobStores/aws/utils.py +1 -2
  28. toil/jobStores/fileJobStore.py +117 -19
  29. toil/jobStores/googleJobStore.py +16 -7
  30. toil/jobStores/utils.py +5 -6
  31. toil/leader.py +71 -43
  32. toil/lib/accelerators.py +10 -5
  33. toil/lib/aws/__init__.py +3 -14
  34. toil/lib/aws/ami.py +22 -9
  35. toil/lib/aws/iam.py +21 -13
  36. toil/lib/aws/session.py +2 -16
  37. toil/lib/aws/utils.py +4 -5
  38. toil/lib/compatibility.py +1 -1
  39. toil/lib/conversions.py +7 -3
  40. toil/lib/docker.py +22 -23
  41. toil/lib/ec2.py +10 -6
  42. toil/lib/ec2nodes.py +106 -100
  43. toil/lib/encryption/_nacl.py +2 -1
  44. toil/lib/generatedEC2Lists.py +325 -18
  45. toil/lib/io.py +21 -0
  46. toil/lib/misc.py +1 -1
  47. toil/lib/resources.py +1 -1
  48. toil/lib/threading.py +74 -26
  49. toil/options/common.py +738 -0
  50. toil/options/cwl.py +336 -0
  51. toil/options/wdl.py +32 -0
  52. toil/provisioners/abstractProvisioner.py +1 -4
  53. toil/provisioners/aws/__init__.py +3 -6
  54. toil/provisioners/aws/awsProvisioner.py +6 -0
  55. toil/provisioners/clusterScaler.py +3 -2
  56. toil/provisioners/gceProvisioner.py +2 -2
  57. toil/realtimeLogger.py +2 -1
  58. toil/resource.py +24 -18
  59. toil/server/app.py +2 -3
  60. toil/server/cli/wes_cwl_runner.py +4 -4
  61. toil/server/utils.py +1 -1
  62. toil/server/wes/abstract_backend.py +3 -2
  63. toil/server/wes/amazon_wes_utils.py +5 -4
  64. toil/server/wes/tasks.py +2 -3
  65. toil/server/wes/toil_backend.py +2 -10
  66. toil/server/wsgi_app.py +2 -0
  67. toil/serviceManager.py +12 -10
  68. toil/statsAndLogging.py +5 -1
  69. toil/test/__init__.py +29 -54
  70. toil/test/batchSystems/batchSystemTest.py +11 -111
  71. toil/test/batchSystems/test_slurm.py +3 -2
  72. toil/test/cwl/cwlTest.py +213 -90
  73. toil/test/cwl/glob_dir.cwl +15 -0
  74. toil/test/cwl/preemptible.cwl +21 -0
  75. toil/test/cwl/preemptible_expression.cwl +28 -0
  76. toil/test/cwl/revsort.cwl +1 -1
  77. toil/test/cwl/revsort2.cwl +1 -1
  78. toil/test/docs/scriptsTest.py +0 -1
  79. toil/test/jobStores/jobStoreTest.py +27 -16
  80. toil/test/lib/aws/test_iam.py +4 -14
  81. toil/test/lib/aws/test_utils.py +0 -3
  82. toil/test/lib/dockerTest.py +4 -4
  83. toil/test/lib/test_ec2.py +11 -16
  84. toil/test/mesos/helloWorld.py +4 -5
  85. toil/test/mesos/stress.py +1 -1
  86. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  87. toil/test/provisioners/clusterScalerTest.py +6 -4
  88. toil/test/provisioners/clusterTest.py +14 -3
  89. toil/test/provisioners/gceProvisionerTest.py +0 -6
  90. toil/test/provisioners/restartScript.py +3 -2
  91. toil/test/server/serverTest.py +1 -1
  92. toil/test/sort/restart_sort.py +2 -1
  93. toil/test/sort/sort.py +2 -1
  94. toil/test/sort/sortTest.py +2 -13
  95. toil/test/src/autoDeploymentTest.py +45 -45
  96. toil/test/src/busTest.py +5 -5
  97. toil/test/src/checkpointTest.py +2 -2
  98. toil/test/src/deferredFunctionTest.py +1 -1
  99. toil/test/src/fileStoreTest.py +32 -16
  100. toil/test/src/helloWorldTest.py +1 -1
  101. toil/test/src/importExportFileTest.py +1 -1
  102. toil/test/src/jobDescriptionTest.py +2 -1
  103. toil/test/src/jobServiceTest.py +1 -1
  104. toil/test/src/jobTest.py +18 -18
  105. toil/test/src/miscTests.py +5 -3
  106. toil/test/src/promisedRequirementTest.py +3 -3
  107. toil/test/src/realtimeLoggerTest.py +1 -1
  108. toil/test/src/resourceTest.py +2 -2
  109. toil/test/src/restartDAGTest.py +1 -1
  110. toil/test/src/resumabilityTest.py +36 -2
  111. toil/test/src/retainTempDirTest.py +1 -1
  112. toil/test/src/systemTest.py +2 -2
  113. toil/test/src/toilContextManagerTest.py +2 -2
  114. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  115. toil/test/utils/toilDebugTest.py +98 -32
  116. toil/test/utils/toilKillTest.py +2 -2
  117. toil/test/utils/utilsTest.py +20 -0
  118. toil/test/wdl/wdltoil_test.py +148 -45
  119. toil/toilState.py +7 -6
  120. toil/utils/toilClean.py +1 -1
  121. toil/utils/toilConfig.py +36 -0
  122. toil/utils/toilDebugFile.py +60 -33
  123. toil/utils/toilDebugJob.py +39 -12
  124. toil/utils/toilDestroyCluster.py +1 -1
  125. toil/utils/toilKill.py +1 -1
  126. toil/utils/toilLaunchCluster.py +13 -2
  127. toil/utils/toilMain.py +3 -2
  128. toil/utils/toilRsyncCluster.py +1 -1
  129. toil/utils/toilSshCluster.py +1 -1
  130. toil/utils/toilStats.py +240 -143
  131. toil/utils/toilStatus.py +1 -4
  132. toil/version.py +11 -11
  133. toil/wdl/utils.py +2 -122
  134. toil/wdl/wdltoil.py +999 -386
  135. toil/worker.py +25 -31
  136. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
  137. toil-6.1.0a1.dist-info/RECORD +237 -0
  138. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
  139. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
  140. toil/batchSystems/parasol.py +0 -379
  141. toil/batchSystems/tes.py +0 -459
  142. toil/test/batchSystems/parasolTestSupport.py +0 -117
  143. toil/test/wdl/builtinTest.py +0 -506
  144. toil/test/wdl/conftest.py +0 -23
  145. toil/test/wdl/toilwdlTest.py +0 -522
  146. toil/wdl/toilwdl.py +0 -141
  147. toil/wdl/versions/dev.py +0 -107
  148. toil/wdl/versions/draft2.py +0 -980
  149. toil/wdl/versions/v1.py +0 -794
  150. toil/wdl/wdl_analysis.py +0 -116
  151. toil/wdl/wdl_functions.py +0 -997
  152. toil/wdl/wdl_synthesis.py +0 -1011
  153. toil/wdl/wdl_types.py +0 -243
  154. toil-5.12.0.dist-info/RECORD +0 -244
  155. /toil/{wdl/versions → options}/__init__.py +0 -0
  156. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
  157. {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
@@ -3,13 +3,13 @@ import subprocess
3
3
  import time
4
4
  from contextlib import contextmanager
5
5
 
6
+ from toil.exceptions import FailedJobsException
6
7
  from toil.lib.iterables import concat
7
8
  from toil.test import (ApplianceTestSupport,
8
9
  needs_local_appliance,
9
10
  needs_mesos,
10
11
  slow)
11
12
  from toil.version import exactPython
12
- from toil.exceptions import FailedJobsException
13
13
 
14
14
  logger = logging.getLogger(__name__)
15
15
 
@@ -207,28 +207,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
207
207
 
208
208
  def testDeferralWithConcurrentEncapsulation(self):
209
209
  """
210
- Ensure that the following DAG succeeds:
211
-
212
- ┌───────────┐
213
- │ Root (W1) │
214
- └───────────┘
215
-
216
- ┌──────────┴─────────┐
217
- ▼ ▼
218
- ┌────────────────┐ ┌────────────────────┐
219
- │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════╗
220
- └────────────────┘ └────────────────────┘ ║
221
- │ ║
222
- ▼ ▼
223
- ┌───────────────────┐ ┌────────────────┐
224
- │ Encapsulated (W3) │ │ Follow-on (W6) │
225
- └───────────────────┘ └────────────────┘
226
- │ │
227
- ┌───────┴────────┐ │
228
- ▼ ▼ ▼
229
- ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
230
- │ Dummy 1 (W4) │ │ Dummy 2 (W5) │ │ Last (W6) │
231
- └──────────────┘ └──────────────┘ └──────────────┘
210
+ Ensure that the following DAG succeeds::
211
+
212
+ ┌───────────┐
213
+ │ Root (W1) │
214
+ └───────────┘
215
+
216
+ ┌──────────┴─────────┐
217
+ ▼ ▼
218
+ ┌────────────────┐ ┌────────────────────┐
219
+ │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════╗
220
+ └────────────────┘ └────────────────────┘ ║
221
+ │ ║
222
+ ▼ ▼
223
+ ┌───────────────────┐ ┌────────────────┐
224
+ │ Encapsulated (W3) │ │ Follow-on (W6) │
225
+ └───────────────────┘ └────────────────┘
226
+ │ │
227
+ ┌───────┴────────┐ │
228
+ ▼ ▼ ▼
229
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
230
+ │ Dummy 1 (W4) │ │ Dummy 2 (W5) │ │ Last (W6) │
231
+ └──────────────┘ └──────────────┘ └──────────────┘
232
232
 
233
233
  The Wn numbers denote the worker processes that a particular job is run in. `Deferring`
234
234
  adds a deferred function and then runs for a long time. The deferred function will be
@@ -313,28 +313,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
313
313
 
314
314
  def testDeferralWithFailureAndEncapsulation(self):
315
315
  """
316
- Ensure that the following DAG succeeds:
317
-
318
- ┌───────────┐
319
- │ Root (W1) │
320
- └───────────┘
321
-
322
- ┌──────────┴─────────┐
323
- ▼ ▼
324
- ┌────────────────┐ ┌────────────────────┐
325
- │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════════════╗
326
- └────────────────┘ └────────────────────┘ ║
327
- │ ║
328
- ▼ ▼
329
- ┌───────────────────┐ ┌────────────────┐
330
- │ Encapsulated (W3) │════════════╗ │ Follow-on (W7) │
331
- └───────────────────┘ ║ └────────────────┘
332
- │ ║
333
- ┌──────┴──────┐ ║
334
- ▼ ▼ ▼
335
- ┌────────────┐┌────────────┐ ┌──────────────┐
336
- │ Dummy (W4) ││ Dummy (W5) │ │ Trigger (W6) │
337
- └────────────┘└────────────┘ └──────────────┘
316
+ Ensure that the following DAG succeeds::
317
+
318
+ ┌───────────┐
319
+ │ Root (W1) │
320
+ └───────────┘
321
+
322
+ ┌──────────┴─────────┐
323
+ ▼ ▼
324
+ ┌────────────────┐ ┌────────────────────┐
325
+ │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════════════╗
326
+ └────────────────┘ └────────────────────┘ ║
327
+ │ ║
328
+ ▼ ▼
329
+ ┌───────────────────┐ ┌────────────────┐
330
+ │ Encapsulated (W3) │════════════╗ │ Follow-on (W7) │
331
+ └───────────────────┘ ║ └────────────────┘
332
+ │ ║
333
+ ┌──────┴──────┐ ║
334
+ ▼ ▼ ▼
335
+ ┌────────────┐┌────────────┐ ┌──────────────┐
336
+ │ Dummy (W4) ││ Dummy (W5) │ │ Trigger (W6) │
337
+ └────────────┘└────────────┘ └──────────────┘
338
338
 
339
339
  `Trigger` causes `Deferring` to crash. `Follow-on` runs next, detects `Deferring`'s
340
340
  left-overs and runs the deferred function. `Follow-on` is an instance of `Job` and the
toil/test/src/busTest.py CHANGED
@@ -15,17 +15,17 @@
15
15
  import logging
16
16
  import os
17
17
  from threading import Thread, current_thread
18
- from typing import Optional
19
18
 
20
19
  from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
21
- from toil.bus import JobCompletedMessage, JobIssuedMessage, MessageBus, replay_message_bus
20
+ from toil.bus import (JobCompletedMessage,
21
+ JobIssuedMessage,
22
+ MessageBus,
23
+ replay_message_bus)
22
24
  from toil.common import Toil
23
- from toil.job import Job
24
25
  from toil.exceptions import FailedJobsException
26
+ from toil.job import Job
25
27
  from toil.test import ToilTest, get_temp_file
26
28
 
27
-
28
-
29
29
  logger = logging.getLogger(__name__)
30
30
 
31
31
  class MessageBusTest(ToilTest):
@@ -12,9 +12,9 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
+ from toil.exceptions import FailedJobsException
15
16
  from toil.job import Job
16
17
  from toil.jobStores.abstractJobStore import NoSuchFileException
17
- from toil.exceptions import FailedJobsException
18
18
  from toil.test import ToilTest, slow
19
19
 
20
20
 
@@ -82,7 +82,7 @@ class CheckRetryCount(Job):
82
82
 
83
83
  def run(self, fileStore):
84
84
  retryCount = self.getNumRetries(fileStore)
85
- fileStore.logToMaster(str(retryCount))
85
+ fileStore.log_to_leader(str(retryCount))
86
86
  if retryCount < self.numFailuresBeforeSuccess:
87
87
  self.addChild(AlwaysFail())
88
88
 
@@ -20,8 +20,8 @@ from uuid import uuid4
20
20
 
21
21
  import psutil
22
22
 
23
- from toil.job import Job
24
23
  from toil.exceptions import FailedJobsException
24
+ from toil.job import Job
25
25
  from toil.lib.threading import cpu_count
26
26
  from toil.test import ToilTest, slow
27
27
 
@@ -29,14 +29,18 @@ from uuid import uuid4
29
29
  import pytest
30
30
 
31
31
  from toil.common import Toil
32
+ from toil.exceptions import FailedJobsException
32
33
  from toil.fileStores import FileID
33
34
  from toil.fileStores.cachingFileStore import (CacheUnbalancedError,
34
35
  IllegalDeletionCacheError)
35
36
  from toil.job import Job
36
37
  from toil.jobStores.abstractJobStore import NoSuchFileException
37
- from toil.exceptions import FailedJobsException
38
38
  from toil.realtimeLogger import RealtimeLogger
39
- from toil.test import ToilTest, needs_aws_ec2, needs_google_project, needs_google_storage, slow
39
+ from toil.test import (ToilTest,
40
+ needs_aws_ec2,
41
+ needs_google_project,
42
+ needs_google_storage,
43
+ slow)
40
44
 
41
45
  # Some tests take too long on the AWS jobstore and are unquitable for CI. They can be
42
46
  # be run during manual tests by setting this to False.
@@ -397,6 +401,7 @@ class hidden:
397
401
  self.options.caching = True
398
402
 
399
403
  @slow
404
+ @pytest.mark.xfail(reason="Cannot succeed in time on small CI runners")
400
405
  def testExtremeCacheSetup(self):
401
406
  """
402
407
  Try to create the cache with bad worker active and then have 10 child jobs try to run in
@@ -472,7 +477,15 @@ class hidden:
472
477
  # the cache hence this test is redundant (caching will be free).
473
478
  if not self.options.jobStore.startswith(('aws', 'google')):
474
479
  workDirDev = os.stat(self.options.workDir).st_dev
475
- jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
480
+ if self.options.jobStore.startswith("file:"):
481
+ # Before #4538, options.jobStore would have the raw path while the Config object would prepend the
482
+ # filesystem to the path (/path/to/file vs file:/path/to/file)
483
+ # The options namespace and the Config object now have the exact same behavior
484
+ # which means parse_jobstore will be called with argparse rather than with the config object
485
+ # so remove the prepended file: scheme
486
+ jobStoreDev = os.stat(os.path.dirname(self.options.jobStore[5:])).st_dev
487
+ else:
488
+ jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
476
489
  if workDirDev == jobStoreDev:
477
490
  self.skipTest('Job store and working directory are on the same filesystem.')
478
491
 
@@ -633,6 +646,8 @@ class hidden:
633
646
 
634
647
  Attempting to get the file from the jobstore should not fail.
635
648
  """
649
+ print("Testing")
650
+ logger.debug("Testing testing 123")
636
651
  self.options.retryCount = 0
637
652
  self.options.logLevel = 'DEBUG'
638
653
  A = Job.wrapJobFn(self._adjustCacheLimit, newTotalMB=1024, disk='1G')
@@ -655,20 +670,20 @@ class hidden:
655
670
  :param fileMB: File Size
656
671
  :return: Job store file ID for second written file
657
672
  """
658
- job.fileStore.logToMaster('Double writing a file into job store')
673
+ job.fileStore.log_to_leader('Double writing a file into job store')
659
674
  work_dir = job.fileStore.getLocalTempDir()
660
675
  with open(os.path.join(work_dir, str(uuid4())), 'wb') as testFile:
661
676
  testFile.write(os.urandom(fileMB * 1024 * 1024))
662
677
 
663
- job.fileStore.logToMaster('Writing copy 1 and discarding ID')
678
+ job.fileStore.log_to_leader('Writing copy 1 and discarding ID')
664
679
  job.fileStore.writeGlobalFile(testFile.name)
665
- job.fileStore.logToMaster('Writing copy 2 and saving ID')
680
+ job.fileStore.log_to_leader('Writing copy 2 and saving ID')
666
681
  fsID = job.fileStore.writeGlobalFile(testFile.name)
667
- job.fileStore.logToMaster(f'Copy 2 ID: {fsID}')
682
+ job.fileStore.log_to_leader(f'Copy 2 ID: {fsID}')
668
683
 
669
684
  hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAssertions(job, fsID)
670
685
 
671
- job.fileStore.logToMaster('Writing copy 3 and returning ID')
686
+ job.fileStore.log_to_leader('Writing copy 3 and returning ID')
672
687
  return job.fileStore.writeGlobalFile(testFile.name)
673
688
 
674
689
  @staticmethod
@@ -680,7 +695,7 @@ class hidden:
680
695
  :param fsID: Job store file ID for the read file
681
696
  :return: None
682
697
  """
683
- job.fileStore.logToMaster('Reading the written file')
698
+ job.fileStore.log_to_leader('Reading the written file')
684
699
  job.fileStore.readGlobalFile(fsID)
685
700
 
686
701
  # writeGlobalFile tests
@@ -794,8 +809,7 @@ class hidden:
794
809
  """
795
810
  Write a local file to the job store (hence adding a copy to cache), then have 10 jobs
796
811
  read it. Assert cached file size never goes up, assert unused job
797
- required disk space is always:
798
- (a multiple of job reqs) - (number of current file readers * filesize).
812
+ required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
799
813
  At the end, assert the cache shows unused job-required space = 0.
800
814
  """
801
815
  self._testMultipleJobsReadGlobalFileFunction(cacheHit=True)
@@ -805,8 +819,7 @@ class hidden:
805
819
  """
806
820
  Write a non-local file to the job store(hence no cached copy), then have 10 jobs read
807
821
  it. Assert cached file size never goes up, assert unused job
808
- required disk space is always:
809
- (a multiple of job reqs) - (number of current file readers * filesize).
822
+ required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
810
823
  At the end, assert the cache shows unused job-required space = 0.
811
824
  """
812
825
  self._testMultipleJobsReadGlobalFileFunction(cacheHit=False)
@@ -834,7 +847,10 @@ class hidden:
834
847
  jobs[i].addChild(B)
835
848
  Job.Runner.startToil(A, self.options)
836
849
  with open(x.name) as y:
837
- assert int(y.read()) > 2
850
+ # At least one job at a time should have been observed.
851
+ # We can't actually guarantee that any of our jobs will
852
+ # see each other currently running.
853
+ assert int(y.read()) > 1
838
854
 
839
855
  @staticmethod
840
856
  def _multipleFileReader(job, diskMB, fsID, maxWriteFile):
@@ -1121,7 +1137,7 @@ class hidden:
1121
1137
  """
1122
1138
 
1123
1139
  # Make sure we actually have the disk size we are supposed to
1124
- job.fileStore.logToMaster('Job is running with %d bytes of disk, %d requested' % (job.disk, jobDisk))
1140
+ job.fileStore.log_to_leader('Job is running with %d bytes of disk, %d requested' % (job.disk, jobDisk))
1125
1141
  assert job.disk == jobDisk, 'Job was scheduled with %d bytes but requested %d' % (job.disk, jobDisk)
1126
1142
 
1127
1143
  cls = hidden.AbstractCachingFileStoreTest
@@ -1187,7 +1203,7 @@ class hidden:
1187
1203
  try:
1188
1204
  job.fileStore.deleteLocalFile(fileToDelete)
1189
1205
  except IllegalDeletionCacheError:
1190
- job.fileStore.logToMaster('Detected a deleted file %s.' % fileToDelete)
1206
+ job.fileStore.log_to_leader('Detected a deleted file %s.' % fileToDelete)
1191
1207
  os.rename(tempfile, outfile)
1192
1208
  else:
1193
1209
  # If we are processing the write test, or if we are testing the immutably read
@@ -24,7 +24,7 @@ class HelloWorldTest(ToilTest):
24
24
 
25
25
  class HelloWorld(Job):
26
26
  def __init__(self):
27
- Job.__init__(self, memory=100000, cores=2, disk="3G")
27
+ Job.__init__(self, memory=100000, cores=1, disk="3G")
28
28
 
29
29
  def run(self, fileStore):
30
30
  fileID = self.addChildJobFn(childFn, cores=1, memory="1M", disk="3G").rv()
@@ -18,9 +18,9 @@ import stat
18
18
  import uuid
19
19
 
20
20
  from toil.common import Toil
21
+ from toil.exceptions import FailedJobsException
21
22
  from toil.fileStores import FileID
22
23
  from toil.job import Job
23
- from toil.exceptions import FailedJobsException
24
24
  from toil.test import ToilTest, slow
25
25
 
26
26
 
@@ -13,7 +13,8 @@
13
13
  # limitations under the License.
14
14
 
15
15
  import os
16
- from argparse import ArgumentParser
16
+
17
+ from configargparse import ArgumentParser
17
18
 
18
19
  from toil.common import Toil
19
20
  from toil.job import Job, JobDescription, TemporaryID
@@ -24,9 +24,9 @@ from unittest import skipIf
24
24
  import pytest
25
25
 
26
26
  from toil.batchSystems.singleMachine import SingleMachineBatchSystem
27
+ from toil.exceptions import FailedJobsException
27
28
  from toil.job import Job
28
29
  from toil.leader import DeadlockException
29
- from toil.exceptions import FailedJobsException
30
30
  from toil.lib.retry import retry_flaky_test
31
31
  from toil.test import ToilTest, get_temp_file, slow
32
32
 
toil/test/src/jobTest.py CHANGED
@@ -20,8 +20,8 @@ import unittest
20
20
  import pytest
21
21
 
22
22
  from toil.common import Toil
23
- from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
24
23
  from toil.exceptions import FailedJobsException
24
+ from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
25
25
  from toil.test import ToilTest, get_temp_file, slow
26
26
 
27
27
  logger = logging.getLogger(__name__)
@@ -37,15 +37,15 @@ class JobTest(ToilTest):
37
37
  @slow
38
38
  def testStatic(self):
39
39
  r"""
40
- Create a DAG of jobs non-dynamically and run it. DAG is:
40
+ Create a DAG of jobs non-dynamically and run it. DAG is::
41
41
 
42
- A -> F
43
- \-------
44
- B -> D \
45
- \ \
46
- ------- C -> E
42
+ A -> F
43
+ \-------
44
+ B -> D \
45
+ \ \
46
+ ------- C -> E
47
47
 
48
- Follow on is marked by ->
48
+ Follow on is marked by ``->``
49
49
  """
50
50
  outFile = get_temp_file(rootDir=self._createTempDir())
51
51
  try:
@@ -81,15 +81,15 @@ class JobTest(ToilTest):
81
81
 
82
82
  def testStatic2(self):
83
83
  r"""
84
- Create a DAG of jobs non-dynamically and run it. DAG is:
84
+ Create a DAG of jobs non-dynamically and run it. DAG is::
85
85
 
86
- A -> F
87
- \-------
88
- B -> D \
89
- \ \
90
- ------- C -> E
86
+ A -> F
87
+ \-------
88
+ B -> D \
89
+ \ \
90
+ ------- C -> E
91
91
 
92
- Follow on is marked by ->
92
+ Follow on is marked by ``->``
93
93
  """
94
94
  outFile = get_temp_file(rootDir=self._createTempDir())
95
95
  try:
@@ -269,7 +269,7 @@ class JobTest(ToilTest):
269
269
  identifies leaf vertices incorrectly
270
270
 
271
271
  Test verification of new checkpoint jobs being leaf verticies,
272
- starting with the following baseline workflow:
272
+ starting with the following baseline workflow::
273
273
 
274
274
  Parent
275
275
  |
@@ -291,7 +291,7 @@ class JobTest(ToilTest):
291
291
  omits the workflow root job
292
292
 
293
293
  Test verification of a new checkpoint job being leaf vertex,
294
- starting with a baseline workflow of a single, root job:
294
+ starting with a baseline workflow of a single, root job::
295
295
 
296
296
  Root # Checkpoint=True
297
297
 
@@ -623,7 +623,7 @@ class JobTest(ToilTest):
623
623
  return True
624
624
 
625
625
  def simpleJobFn(job, value):
626
- job.fileStore.logToMaster(value)
626
+ job.fileStore.log_to_leader(value)
627
627
 
628
628
  def fn1Test(string, outputFile):
629
629
  """
@@ -16,12 +16,14 @@ import logging
16
16
  import os
17
17
  import random
18
18
  import sys
19
- import tempfile
20
19
  from uuid import uuid4
21
20
 
22
21
  from toil.common import getNodeID
23
22
  from toil.lib.exceptions import panic, raise_
24
- from toil.lib.io import AtomicFileCreate, atomic_install, atomic_tmp_file
23
+ from toil.lib.io import (AtomicFileCreate,
24
+ atomic_install,
25
+ atomic_tmp_file,
26
+ mkdtemp)
25
27
  from toil.lib.misc import CalledProcessErrorStderr, call_command
26
28
  from toil.test import ToilTest, slow
27
29
 
@@ -63,7 +65,7 @@ class MiscTests(ToilTest):
63
65
  files = {}
64
66
  # Create a random directory structure
65
67
  for i in range(0,10):
66
- directories.append(tempfile.mkdtemp(dir=random.choice(directories), prefix='test'))
68
+ directories.append(mkdtemp(dir=random.choice(directories), prefix='test'))
67
69
  # Create 50 random file entries in different locations in the directories. 75% of the time
68
70
  # these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
69
71
  # files.
@@ -48,7 +48,7 @@ class hidden:
48
48
  cores=1, memory='1M', disk='1M')
49
49
  values = Job.Runner.startToil(root, self.getOptions(tempDir))
50
50
  maxValue = max(values)
51
- self.assertEqual(maxValue, self.cpuCount // coresPerJob)
51
+ self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
52
52
 
53
53
  @slow
54
54
  @retry_flaky_test(prepare=[batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
@@ -74,7 +74,7 @@ class hidden:
74
74
  disk='1M'))
75
75
  Job.Runner.startToil(root, self.getOptions(tempDir))
76
76
  _, maxValue = batchSystemTest.getCounters(counterPath)
77
- self.assertEqual(maxValue, self.cpuCount // coresPerJob)
77
+ self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
78
78
 
79
79
  def getOptions(self, tempDir, caching=True):
80
80
  options = super().getOptions(tempDir)
@@ -194,7 +194,7 @@ def logDiskUsage(job, funcName, sleep=0):
194
194
  :return: job function's disk usage
195
195
  """
196
196
  diskUsage = job.disk
197
- job.fileStore.logToMaster(f'{funcName}: {diskUsage}')
197
+ job.fileStore.log_to_leader(f'{funcName}: {diskUsage}')
198
198
  time.sleep(sleep)
199
199
  return diskUsage
200
200
 
@@ -57,7 +57,7 @@ class MessageDetector(logging.StreamHandler):
57
57
 
58
58
  class LogTest(Job):
59
59
  def __init__(self):
60
- Job.__init__(self, memory=100000, cores=2, disk='3G')
60
+ Job.__init__(self, memory=100000, cores=1, disk='3G')
61
61
 
62
62
  def run(self, fileStore):
63
63
  RealtimeLogger.info('This should be logged at info level')
@@ -209,7 +209,7 @@ class ResourceTest(ToilTest):
209
209
  """
210
210
 
211
211
  def script():
212
- import argparse
212
+ from configargparse import ArgumentParser
213
213
 
214
214
  from toil.common import Toil
215
215
  from toil.job import Job
@@ -218,7 +218,7 @@ class ResourceTest(ToilTest):
218
218
  pass
219
219
 
220
220
  if __name__ == '__main__':
221
- parser = argparse.ArgumentParser()
221
+ parser = ArgumentParser()
222
222
  Job.Runner.addToilOptions(parser)
223
223
  options = parser.parse_args()
224
224
  job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
@@ -19,8 +19,8 @@ import shutil
19
19
  import signal
20
20
 
21
21
  from toil.common import Toil
22
- from toil.job import Job
23
22
  from toil.exceptions import FailedJobsException
23
+ from toil.job import Job
24
24
  from toil.test import ToilTest, slow
25
25
 
26
26
  logger = logging.getLogger(__name__)
@@ -14,17 +14,17 @@
14
14
 
15
15
  import os
16
16
 
17
+ from toil.exceptions import FailedJobsException
17
18
  from toil.job import Job
18
19
  from toil.jobStores.abstractJobStore import NoSuchFileException
19
- from toil.exceptions import FailedJobsException
20
20
  from toil.test import ToilTest, slow
21
21
 
22
22
 
23
- @slow
24
23
  class ResumabilityTest(ToilTest):
25
24
  """
26
25
  https://github.com/BD2KGenomics/toil/issues/808
27
26
  """
27
+ @slow
28
28
  def test(self):
29
29
  """
30
30
  Tests that a toil workflow that fails once can be resumed without a NoSuchJobException.
@@ -53,6 +53,34 @@ class ResumabilityTest(ToilTest):
53
53
  # store ID: n/t/jobwbijqL failed with exit value 1"
54
54
  self.assertTrue("failed with exit value" not in logString)
55
55
 
56
+ def test_chaining(self):
57
+ """
58
+ Tests that a job which is chained to and fails can resume and succeed.
59
+ """
60
+
61
+ options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
62
+ options.logLevel = "DEBUG"
63
+ options.retryCount = 0
64
+ tempDir = self._createTempDir()
65
+ options.logFile = os.path.join(tempDir, "log.txt")
66
+
67
+ root = Job.wrapJobFn(chaining_parent)
68
+
69
+ with self.assertRaises(FailedJobsException):
70
+ # This one is intended to fail.
71
+ Job.Runner.startToil(root, options)
72
+
73
+ with open(options.logFile, 'r') as f:
74
+ log_content = f.read()
75
+ # Make sure we actually did do chaining
76
+ assert "Chaining from" in log_content
77
+
78
+ # Because of the chaining, the problem we are looking for is the job
79
+ # with the root ID not being able to load the body of a job with a
80
+ # different ID. That doesn't look like a job deleted despite failure.
81
+ options.restart = True
82
+ Job.Runner.startToil(root, options)
83
+
56
84
  def parent(job):
57
85
  """
58
86
  Set up a bunch of dummy child jobs, and a bad job that needs to be
@@ -62,6 +90,12 @@ def parent(job):
62
90
  job.addChildJobFn(goodChild)
63
91
  job.addFollowOnJobFn(badChild)
64
92
 
93
+ def chaining_parent(job):
94
+ """
95
+ Set up a failing job to chain to.
96
+ """
97
+ job.addFollowOnJobFn(badChild)
98
+
65
99
  def goodChild(job):
66
100
  """
67
101
  Does nothing.
@@ -14,8 +14,8 @@
14
14
  import os
15
15
  import shutil
16
16
 
17
- from toil.job import Job
18
17
  from toil.exceptions import FailedJobsException
18
+ from toil.job import Job
19
19
  from toil.test import ToilTest
20
20
 
21
21
 
@@ -1,9 +1,9 @@
1
1
  import errno
2
2
  import multiprocessing
3
3
  import os
4
- import tempfile
5
4
  from functools import partial
6
5
 
6
+ from toil.lib.io import mkdtemp
7
7
  from toil.lib.threading import cpu_count
8
8
  from toil.test import ToilTest
9
9
 
@@ -37,7 +37,7 @@ class SystemTest(ToilTest):
37
37
 
38
38
 
39
39
  def _testAtomicityOfNonEmptyDirectoryRenamesTask(parent, child, _):
40
- tmpChildDir = tempfile.mkdtemp(dir=parent, prefix='child', suffix='.tmp')
40
+ tmpChildDir = mkdtemp(dir=parent, prefix='child', suffix='.tmp')
41
41
  grandChild = os.path.join(tmpChildDir, 'grandChild')
42
42
  open(grandChild, 'w').close()
43
43
  grandChildId = os.stat(grandChild).st_ino
@@ -61,10 +61,10 @@ class ToilContextManagerTest(ToilTest):
61
61
 
62
62
  class HelloWorld(Job):
63
63
  def __init__(self):
64
- Job.__init__(self, memory=100000, cores=2, disk='1M')
64
+ Job.__init__(self, memory=100000, disk='1M')
65
65
 
66
66
  def run(self, fileStore):
67
- fileID = self.addChildJobFn(childFn, cores=1, memory='1M', disk='1M').rv()
67
+ fileID = self.addChildJobFn(childFn, memory='1M', disk='1M').rv()
68
68
  return self.addFollowOn(FollowOn(fileID)).rv()
69
69
 
70
70
 
@@ -59,7 +59,7 @@ class UserDefinedJobArgTypeTest(ToilTest):
59
59
 
60
60
  class JobClass(Job):
61
61
  def __init__(self, level, foo):
62
- Job.__init__(self, memory=100000, cores=2, disk="300M")
62
+ Job.__init__(self, memory=100000, cores=1, disk="300M")
63
63
  self.level = level
64
64
  self.foo = foo
65
65