toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. toil/__init__.py +121 -83
  2. toil/batchSystems/__init__.py +1 -0
  3. toil/batchSystems/abstractBatchSystem.py +137 -77
  4. toil/batchSystems/abstractGridEngineBatchSystem.py +211 -101
  5. toil/batchSystems/awsBatch.py +237 -128
  6. toil/batchSystems/cleanup_support.py +22 -16
  7. toil/batchSystems/contained_executor.py +30 -26
  8. toil/batchSystems/gridengine.py +85 -49
  9. toil/batchSystems/htcondor.py +164 -87
  10. toil/batchSystems/kubernetes.py +622 -386
  11. toil/batchSystems/local_support.py +17 -12
  12. toil/batchSystems/lsf.py +132 -79
  13. toil/batchSystems/lsfHelper.py +13 -11
  14. toil/batchSystems/mesos/__init__.py +41 -29
  15. toil/batchSystems/mesos/batchSystem.py +288 -149
  16. toil/batchSystems/mesos/executor.py +77 -49
  17. toil/batchSystems/mesos/test/__init__.py +31 -23
  18. toil/batchSystems/options.py +38 -29
  19. toil/batchSystems/registry.py +53 -19
  20. toil/batchSystems/singleMachine.py +293 -123
  21. toil/batchSystems/slurm.py +489 -137
  22. toil/batchSystems/torque.py +46 -32
  23. toil/bus.py +141 -73
  24. toil/common.py +630 -359
  25. toil/cwl/__init__.py +1 -1
  26. toil/cwl/cwltoil.py +1114 -532
  27. toil/cwl/utils.py +17 -22
  28. toil/deferred.py +62 -41
  29. toil/exceptions.py +5 -3
  30. toil/fileStores/__init__.py +5 -5
  31. toil/fileStores/abstractFileStore.py +88 -57
  32. toil/fileStores/cachingFileStore.py +711 -247
  33. toil/fileStores/nonCachingFileStore.py +113 -75
  34. toil/job.py +988 -315
  35. toil/jobStores/abstractJobStore.py +387 -243
  36. toil/jobStores/aws/jobStore.py +727 -403
  37. toil/jobStores/aws/utils.py +161 -109
  38. toil/jobStores/conftest.py +1 -0
  39. toil/jobStores/fileJobStore.py +289 -151
  40. toil/jobStores/googleJobStore.py +137 -70
  41. toil/jobStores/utils.py +36 -15
  42. toil/leader.py +614 -269
  43. toil/lib/accelerators.py +115 -18
  44. toil/lib/aws/__init__.py +55 -28
  45. toil/lib/aws/ami.py +122 -87
  46. toil/lib/aws/iam.py +284 -108
  47. toil/lib/aws/s3.py +31 -0
  48. toil/lib/aws/session.py +193 -58
  49. toil/lib/aws/utils.py +238 -218
  50. toil/lib/bioio.py +13 -5
  51. toil/lib/compatibility.py +11 -6
  52. toil/lib/conversions.py +83 -49
  53. toil/lib/docker.py +131 -103
  54. toil/lib/ec2.py +322 -209
  55. toil/lib/ec2nodes.py +174 -106
  56. toil/lib/encryption/_dummy.py +5 -3
  57. toil/lib/encryption/_nacl.py +10 -6
  58. toil/lib/encryption/conftest.py +1 -0
  59. toil/lib/exceptions.py +26 -7
  60. toil/lib/expando.py +4 -2
  61. toil/lib/ftp_utils.py +217 -0
  62. toil/lib/generatedEC2Lists.py +127 -19
  63. toil/lib/humanize.py +6 -2
  64. toil/lib/integration.py +341 -0
  65. toil/lib/io.py +99 -11
  66. toil/lib/iterables.py +4 -2
  67. toil/lib/memoize.py +12 -8
  68. toil/lib/misc.py +65 -18
  69. toil/lib/objects.py +2 -2
  70. toil/lib/resources.py +19 -7
  71. toil/lib/retry.py +115 -77
  72. toil/lib/threading.py +282 -80
  73. toil/lib/throttle.py +15 -14
  74. toil/options/common.py +834 -401
  75. toil/options/cwl.py +175 -90
  76. toil/options/runner.py +50 -0
  77. toil/options/wdl.py +70 -19
  78. toil/provisioners/__init__.py +111 -46
  79. toil/provisioners/abstractProvisioner.py +322 -157
  80. toil/provisioners/aws/__init__.py +62 -30
  81. toil/provisioners/aws/awsProvisioner.py +980 -627
  82. toil/provisioners/clusterScaler.py +541 -279
  83. toil/provisioners/gceProvisioner.py +282 -179
  84. toil/provisioners/node.py +147 -79
  85. toil/realtimeLogger.py +34 -22
  86. toil/resource.py +137 -75
  87. toil/server/app.py +127 -61
  88. toil/server/celery_app.py +3 -1
  89. toil/server/cli/wes_cwl_runner.py +82 -53
  90. toil/server/utils.py +54 -28
  91. toil/server/wes/abstract_backend.py +64 -26
  92. toil/server/wes/amazon_wes_utils.py +21 -15
  93. toil/server/wes/tasks.py +121 -63
  94. toil/server/wes/toil_backend.py +142 -107
  95. toil/server/wsgi_app.py +4 -3
  96. toil/serviceManager.py +58 -22
  97. toil/statsAndLogging.py +148 -64
  98. toil/test/__init__.py +263 -179
  99. toil/test/batchSystems/batchSystemTest.py +438 -195
  100. toil/test/batchSystems/batch_system_plugin_test.py +18 -7
  101. toil/test/batchSystems/test_gridengine.py +173 -0
  102. toil/test/batchSystems/test_lsf_helper.py +67 -58
  103. toil/test/batchSystems/test_slurm.py +93 -47
  104. toil/test/cactus/test_cactus_integration.py +20 -22
  105. toil/test/cwl/cwlTest.py +271 -71
  106. toil/test/cwl/measure_default_memory.cwl +12 -0
  107. toil/test/cwl/not_run_required_input.cwl +29 -0
  108. toil/test/cwl/scatter_duplicate_outputs.cwl +40 -0
  109. toil/test/docs/scriptsTest.py +60 -34
  110. toil/test/jobStores/jobStoreTest.py +412 -235
  111. toil/test/lib/aws/test_iam.py +116 -48
  112. toil/test/lib/aws/test_s3.py +16 -9
  113. toil/test/lib/aws/test_utils.py +5 -6
  114. toil/test/lib/dockerTest.py +118 -141
  115. toil/test/lib/test_conversions.py +113 -115
  116. toil/test/lib/test_ec2.py +57 -49
  117. toil/test/lib/test_integration.py +104 -0
  118. toil/test/lib/test_misc.py +12 -5
  119. toil/test/mesos/MesosDataStructuresTest.py +23 -10
  120. toil/test/mesos/helloWorld.py +7 -6
  121. toil/test/mesos/stress.py +25 -20
  122. toil/test/options/options.py +7 -2
  123. toil/test/provisioners/aws/awsProvisionerTest.py +293 -140
  124. toil/test/provisioners/clusterScalerTest.py +440 -250
  125. toil/test/provisioners/clusterTest.py +81 -42
  126. toil/test/provisioners/gceProvisionerTest.py +174 -100
  127. toil/test/provisioners/provisionerTest.py +25 -13
  128. toil/test/provisioners/restartScript.py +5 -4
  129. toil/test/server/serverTest.py +188 -141
  130. toil/test/sort/restart_sort.py +137 -68
  131. toil/test/sort/sort.py +134 -66
  132. toil/test/sort/sortTest.py +91 -49
  133. toil/test/src/autoDeploymentTest.py +140 -100
  134. toil/test/src/busTest.py +20 -18
  135. toil/test/src/checkpointTest.py +8 -2
  136. toil/test/src/deferredFunctionTest.py +49 -35
  137. toil/test/src/dockerCheckTest.py +33 -26
  138. toil/test/src/environmentTest.py +20 -10
  139. toil/test/src/fileStoreTest.py +538 -271
  140. toil/test/src/helloWorldTest.py +7 -4
  141. toil/test/src/importExportFileTest.py +61 -31
  142. toil/test/src/jobDescriptionTest.py +32 -17
  143. toil/test/src/jobEncapsulationTest.py +2 -0
  144. toil/test/src/jobFileStoreTest.py +74 -50
  145. toil/test/src/jobServiceTest.py +187 -73
  146. toil/test/src/jobTest.py +120 -70
  147. toil/test/src/miscTests.py +19 -18
  148. toil/test/src/promisedRequirementTest.py +82 -36
  149. toil/test/src/promisesTest.py +7 -6
  150. toil/test/src/realtimeLoggerTest.py +6 -6
  151. toil/test/src/regularLogTest.py +71 -37
  152. toil/test/src/resourceTest.py +80 -49
  153. toil/test/src/restartDAGTest.py +36 -22
  154. toil/test/src/resumabilityTest.py +9 -2
  155. toil/test/src/retainTempDirTest.py +45 -14
  156. toil/test/src/systemTest.py +12 -8
  157. toil/test/src/threadingTest.py +44 -25
  158. toil/test/src/toilContextManagerTest.py +10 -7
  159. toil/test/src/userDefinedJobArgTypeTest.py +8 -5
  160. toil/test/src/workerTest.py +33 -16
  161. toil/test/utils/toilDebugTest.py +70 -58
  162. toil/test/utils/toilKillTest.py +4 -5
  163. toil/test/utils/utilsTest.py +239 -102
  164. toil/test/wdl/wdltoil_test.py +789 -148
  165. toil/test/wdl/wdltoil_test_kubernetes.py +37 -23
  166. toil/toilState.py +52 -26
  167. toil/utils/toilConfig.py +13 -4
  168. toil/utils/toilDebugFile.py +44 -27
  169. toil/utils/toilDebugJob.py +85 -25
  170. toil/utils/toilDestroyCluster.py +11 -6
  171. toil/utils/toilKill.py +8 -3
  172. toil/utils/toilLaunchCluster.py +251 -145
  173. toil/utils/toilMain.py +37 -16
  174. toil/utils/toilRsyncCluster.py +27 -14
  175. toil/utils/toilSshCluster.py +45 -22
  176. toil/utils/toilStats.py +75 -36
  177. toil/utils/toilStatus.py +226 -119
  178. toil/utils/toilUpdateEC2Instances.py +3 -1
  179. toil/version.py +11 -11
  180. toil/wdl/utils.py +5 -5
  181. toil/wdl/wdltoil.py +3513 -1052
  182. toil/worker.py +269 -128
  183. toil-8.0.0.dist-info/METADATA +173 -0
  184. toil-8.0.0.dist-info/RECORD +253 -0
  185. {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/WHEEL +1 -1
  186. toil-7.0.0.dist-info/METADATA +0 -158
  187. toil-7.0.0.dist-info/RECORD +0 -244
  188. {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/LICENSE +0 -0
  189. {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/entry_points.txt +0 -0
  190. {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/top_level.txt +0 -0
@@ -23,14 +23,15 @@ from toil.job import Job
23
23
  childMessage = "The child job is now running!"
24
24
  parentMessage = "The parent job is now running!"
25
25
 
26
+
26
27
  def hello_world(job):
27
28
 
28
29
  job.fileStore.log_to_leader(parentMessage)
29
- with open('foo_bam.txt', 'w') as handle:
30
- handle.write('\nThis is a triumph...\n')
30
+ with open("foo_bam.txt", "w") as handle:
31
+ handle.write("\nThis is a triumph...\n")
31
32
 
32
33
  # Assign FileStoreID to a given file
33
- foo_bam = job.fileStore.writeGlobalFile('foo_bam.txt')
34
+ foo_bam = job.fileStore.writeGlobalFile("foo_bam.txt")
34
35
 
35
36
  # Spawn child
36
37
  job.addChildJobFn(hello_world_child, foo_bam, memory=100, cores=0.5, disk="3G")
@@ -43,13 +44,13 @@ def hello_world_child(job, hw):
43
44
  # NOTE: path and the udpated file are stored to /tmp
44
45
  # If we want to SAVE our changes to this tmp file, we must write it out.
45
46
  with open(path) as r:
46
- with open('bar_bam.txt', 'w') as handle:
47
+ with open("bar_bam.txt", "w") as handle:
47
48
  for line in r.readlines():
48
49
  handle.write(line)
49
50
 
50
51
  # Assign FileStoreID to a given file
51
52
  # can also use: job.updateGlobalFile() given the FileStoreID instantiation.
52
- job.fileStore.writeGlobalFile('bar_bam.txt')
53
+ job.fileStore.writeGlobalFile("bar_bam.txt")
53
54
 
54
55
 
55
56
  def main():
@@ -67,5 +68,5 @@ def main():
67
68
  toil.start(i)
68
69
 
69
70
 
70
- if __name__ == '__main__':
71
+ if __name__ == "__main__":
71
72
  main()
toil/test/mesos/stress.py CHANGED
@@ -17,58 +17,63 @@ from configargparse import ArgumentParser
17
17
  from toil.job import Job
18
18
 
19
19
 
20
- def touchFile( fileStore ):
20
+ def touchFile(fileStore):
21
21
  with fileStore.writeGlobalFileStream() as (f, id):
22
- f.write( "This is a triumph" )
22
+ f.write("This is a triumph")
23
+
23
24
 
24
25
  class LongTestJob(Job):
25
26
  def __init__(self, numJobs):
26
- Job.__init__(self, memory=100000, cores=0.01)
27
+ Job.__init__(self, memory=100000, cores=0.01)
27
28
  self.numJobs = numJobs
28
29
 
29
30
  def run(self, fileStore):
30
- for i in range(0,self.numJobs):
31
+ for i in range(0, self.numJobs):
31
32
  self.addChild(HelloWorldJob(i))
32
33
  self.addFollowOn(LongTestFollowOn())
33
34
 
35
+
34
36
  class LongTestFollowOn(Job):
35
37
 
36
38
  def __init__(self):
37
- Job.__init__(self, memory=1000000, cores=0.01)
39
+ Job.__init__(self, memory=1000000, cores=0.01)
38
40
 
39
41
  def run(self, fileStore):
40
- touchFile( fileStore )
42
+ touchFile(fileStore)
41
43
 
42
- class HelloWorldJob(Job):
43
44
 
44
- def __init__(self,i):
45
- Job.__init__(self, memory=100000, cores=0.01)
46
- self.i=i
45
+ class HelloWorldJob(Job):
47
46
 
47
+ def __init__(self, i):
48
+ Job.__init__(self, memory=100000, cores=0.01)
49
+ self.i = i
48
50
 
49
51
  def run(self, fileStore):
50
- touchFile( fileStore )
52
+ touchFile(fileStore)
51
53
  self.addFollowOn(HelloWorldFollowOn(self.i))
52
54
 
55
+
53
56
  class HelloWorldFollowOn(Job):
54
57
 
55
- def __init__(self,i):
56
- Job.__init__(self, memory=200000, cores=0.01)
58
+ def __init__(self, i):
59
+ Job.__init__(self, memory=200000, cores=0.01)
57
60
  self.i = i
58
61
 
59
62
  def run(self, fileStore):
60
- touchFile( fileStore)
63
+ touchFile(fileStore)
64
+
61
65
 
62
66
  def main(numJobs):
63
67
  # Boilerplate -- startToil requires options
64
68
  parser = ArgumentParser()
65
69
  Job.Runner.addToilOptions(parser)
66
- options = parser.parse_args( args=['./toilTest'] )
67
- options.batchSystem="mesos"
68
- options.mesos_endpoint="localhost:5050"
70
+ options = parser.parse_args(args=["./toilTest"])
71
+ options.batchSystem = "mesos"
72
+ options.mesos_endpoint = "localhost:5050"
69
73
  # Launch first toil Job
70
- i = LongTestJob( numJobs )
71
- Job.Runner.startToil(i, options )
74
+ i = LongTestJob(numJobs)
75
+ Job.Runner.startToil(i, options)
76
+
72
77
 
73
- if __name__=="__main__":
78
+ if __name__ == "__main__":
74
79
  main(numJobs=5)
@@ -1,6 +1,6 @@
1
1
  from configargparse import ArgParser
2
2
 
3
- from toil.common import addOptions, Toil
3
+ from toil.common import Toil, addOptions
4
4
  from toil.test import ToilTest
5
5
 
6
6
 
@@ -8,6 +8,7 @@ class OptionsTest(ToilTest):
8
8
  """
9
9
  Class to test functionality of all Toil options
10
10
  """
11
+
11
12
  def test_default_caching_slurm(self):
12
13
  """
13
14
  Test to ensure that caching will be set to false when running on Slurm
@@ -30,7 +31,11 @@ class OptionsTest(ToilTest):
30
31
  addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
31
32
  # the kubernetes batchsystem (and I think all batchsystems including singlemachine) return False
32
33
  # for default_caching
33
- test_args = ["--jobstore=example-jobstore", "--batchSystem=kubernetes", "--caching=True"]
34
+ test_args = [
35
+ "--jobstore=example-jobstore",
36
+ "--batchSystem=kubernetes",
37
+ "--caching=True",
38
+ ]
34
39
  options = parser.parse_args(test_args)
35
40
  with Toil(options) as toil:
36
41
  caching_value = toil.config.caching