toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. toil/__init__.py +18 -13
  2. toil/batchSystems/abstractBatchSystem.py +39 -13
  3. toil/batchSystems/abstractGridEngineBatchSystem.py +24 -24
  4. toil/batchSystems/awsBatch.py +14 -14
  5. toil/batchSystems/cleanup_support.py +7 -3
  6. toil/batchSystems/contained_executor.py +3 -3
  7. toil/batchSystems/htcondor.py +0 -1
  8. toil/batchSystems/kubernetes.py +34 -31
  9. toil/batchSystems/local_support.py +3 -1
  10. toil/batchSystems/lsf.py +7 -7
  11. toil/batchSystems/mesos/batchSystem.py +7 -7
  12. toil/batchSystems/options.py +32 -83
  13. toil/batchSystems/registry.py +104 -23
  14. toil/batchSystems/singleMachine.py +16 -13
  15. toil/batchSystems/slurm.py +87 -16
  16. toil/batchSystems/torque.py +0 -1
  17. toil/bus.py +44 -8
  18. toil/common.py +544 -753
  19. toil/cwl/__init__.py +28 -32
  20. toil/cwl/cwltoil.py +595 -574
  21. toil/cwl/utils.py +55 -10
  22. toil/exceptions.py +1 -1
  23. toil/fileStores/__init__.py +2 -2
  24. toil/fileStores/abstractFileStore.py +88 -14
  25. toil/fileStores/cachingFileStore.py +610 -549
  26. toil/fileStores/nonCachingFileStore.py +46 -22
  27. toil/job.py +182 -101
  28. toil/jobStores/abstractJobStore.py +161 -95
  29. toil/jobStores/aws/jobStore.py +23 -9
  30. toil/jobStores/aws/utils.py +6 -6
  31. toil/jobStores/fileJobStore.py +116 -18
  32. toil/jobStores/googleJobStore.py +16 -7
  33. toil/jobStores/utils.py +5 -6
  34. toil/leader.py +87 -56
  35. toil/lib/accelerators.py +10 -5
  36. toil/lib/aws/__init__.py +3 -14
  37. toil/lib/aws/ami.py +22 -9
  38. toil/lib/aws/iam.py +21 -13
  39. toil/lib/aws/session.py +2 -16
  40. toil/lib/aws/utils.py +4 -5
  41. toil/lib/compatibility.py +1 -1
  42. toil/lib/conversions.py +26 -3
  43. toil/lib/docker.py +22 -23
  44. toil/lib/ec2.py +10 -6
  45. toil/lib/ec2nodes.py +106 -100
  46. toil/lib/encryption/_nacl.py +2 -1
  47. toil/lib/generatedEC2Lists.py +325 -18
  48. toil/lib/io.py +49 -2
  49. toil/lib/misc.py +1 -1
  50. toil/lib/resources.py +9 -2
  51. toil/lib/threading.py +101 -38
  52. toil/options/common.py +736 -0
  53. toil/options/cwl.py +336 -0
  54. toil/options/wdl.py +37 -0
  55. toil/provisioners/abstractProvisioner.py +9 -4
  56. toil/provisioners/aws/__init__.py +3 -6
  57. toil/provisioners/aws/awsProvisioner.py +6 -0
  58. toil/provisioners/clusterScaler.py +3 -2
  59. toil/provisioners/gceProvisioner.py +2 -2
  60. toil/realtimeLogger.py +2 -1
  61. toil/resource.py +24 -18
  62. toil/server/app.py +2 -3
  63. toil/server/cli/wes_cwl_runner.py +4 -4
  64. toil/server/utils.py +1 -1
  65. toil/server/wes/abstract_backend.py +3 -2
  66. toil/server/wes/amazon_wes_utils.py +5 -4
  67. toil/server/wes/tasks.py +2 -3
  68. toil/server/wes/toil_backend.py +2 -10
  69. toil/server/wsgi_app.py +2 -0
  70. toil/serviceManager.py +12 -10
  71. toil/statsAndLogging.py +41 -9
  72. toil/test/__init__.py +29 -54
  73. toil/test/batchSystems/batchSystemTest.py +11 -111
  74. toil/test/batchSystems/test_slurm.py +24 -8
  75. toil/test/cactus/__init__.py +0 -0
  76. toil/test/cactus/test_cactus_integration.py +58 -0
  77. toil/test/cwl/cwlTest.py +438 -223
  78. toil/test/cwl/glob_dir.cwl +15 -0
  79. toil/test/cwl/preemptible.cwl +21 -0
  80. toil/test/cwl/preemptible_expression.cwl +28 -0
  81. toil/test/cwl/revsort.cwl +1 -1
  82. toil/test/cwl/revsort2.cwl +1 -1
  83. toil/test/docs/scriptsTest.py +2 -3
  84. toil/test/jobStores/jobStoreTest.py +34 -21
  85. toil/test/lib/aws/test_iam.py +4 -14
  86. toil/test/lib/aws/test_utils.py +0 -3
  87. toil/test/lib/dockerTest.py +4 -4
  88. toil/test/lib/test_ec2.py +12 -17
  89. toil/test/mesos/helloWorld.py +4 -5
  90. toil/test/mesos/stress.py +1 -1
  91. toil/test/{wdl/conftest.py → options/__init__.py} +0 -10
  92. toil/test/options/options.py +37 -0
  93. toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
  94. toil/test/provisioners/clusterScalerTest.py +6 -4
  95. toil/test/provisioners/clusterTest.py +23 -11
  96. toil/test/provisioners/gceProvisionerTest.py +0 -6
  97. toil/test/provisioners/restartScript.py +3 -2
  98. toil/test/server/serverTest.py +1 -1
  99. toil/test/sort/restart_sort.py +2 -1
  100. toil/test/sort/sort.py +2 -1
  101. toil/test/sort/sortTest.py +2 -13
  102. toil/test/src/autoDeploymentTest.py +45 -45
  103. toil/test/src/busTest.py +5 -5
  104. toil/test/src/checkpointTest.py +2 -2
  105. toil/test/src/deferredFunctionTest.py +1 -1
  106. toil/test/src/fileStoreTest.py +32 -16
  107. toil/test/src/helloWorldTest.py +1 -1
  108. toil/test/src/importExportFileTest.py +1 -1
  109. toil/test/src/jobDescriptionTest.py +2 -1
  110. toil/test/src/jobServiceTest.py +1 -1
  111. toil/test/src/jobTest.py +18 -18
  112. toil/test/src/miscTests.py +5 -3
  113. toil/test/src/promisedRequirementTest.py +3 -3
  114. toil/test/src/realtimeLoggerTest.py +1 -1
  115. toil/test/src/resourceTest.py +2 -2
  116. toil/test/src/restartDAGTest.py +1 -1
  117. toil/test/src/resumabilityTest.py +36 -2
  118. toil/test/src/retainTempDirTest.py +1 -1
  119. toil/test/src/systemTest.py +2 -2
  120. toil/test/src/toilContextManagerTest.py +2 -2
  121. toil/test/src/userDefinedJobArgTypeTest.py +1 -1
  122. toil/test/utils/toilDebugTest.py +98 -32
  123. toil/test/utils/toilKillTest.py +2 -2
  124. toil/test/utils/utilsTest.py +23 -3
  125. toil/test/wdl/wdltoil_test.py +223 -45
  126. toil/toilState.py +7 -6
  127. toil/utils/toilClean.py +1 -1
  128. toil/utils/toilConfig.py +36 -0
  129. toil/utils/toilDebugFile.py +60 -33
  130. toil/utils/toilDebugJob.py +39 -12
  131. toil/utils/toilDestroyCluster.py +1 -1
  132. toil/utils/toilKill.py +1 -1
  133. toil/utils/toilLaunchCluster.py +13 -2
  134. toil/utils/toilMain.py +3 -2
  135. toil/utils/toilRsyncCluster.py +1 -1
  136. toil/utils/toilSshCluster.py +1 -1
  137. toil/utils/toilStats.py +445 -305
  138. toil/utils/toilStatus.py +2 -5
  139. toil/version.py +10 -10
  140. toil/wdl/utils.py +2 -122
  141. toil/wdl/wdltoil.py +1257 -492
  142. toil/worker.py +55 -46
  143. toil-6.1.0.dist-info/METADATA +124 -0
  144. toil-6.1.0.dist-info/RECORD +241 -0
  145. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/WHEEL +1 -1
  146. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/entry_points.txt +0 -1
  147. toil/batchSystems/parasol.py +0 -379
  148. toil/batchSystems/tes.py +0 -459
  149. toil/test/batchSystems/parasolTestSupport.py +0 -117
  150. toil/test/wdl/builtinTest.py +0 -506
  151. toil/test/wdl/toilwdlTest.py +0 -522
  152. toil/wdl/toilwdl.py +0 -141
  153. toil/wdl/versions/dev.py +0 -107
  154. toil/wdl/versions/draft2.py +0 -980
  155. toil/wdl/versions/v1.py +0 -794
  156. toil/wdl/wdl_analysis.py +0 -116
  157. toil/wdl/wdl_functions.py +0 -997
  158. toil/wdl/wdl_synthesis.py +0 -1011
  159. toil/wdl/wdl_types.py +0 -243
  160. toil-5.12.0.dist-info/METADATA +0 -118
  161. toil-5.12.0.dist-info/RECORD +0 -244
  162. /toil/{wdl/versions → options}/__init__.py +0 -0
  163. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/LICENSE +0 -0
  164. {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/top_level.txt +0 -0
@@ -17,8 +17,7 @@ import os
17
17
  import subprocess
18
18
  import time
19
19
  from uuid import uuid4
20
-
21
- import boto.ec2
20
+ from typing import Optional, List
22
21
 
23
22
  from toil.lib.aws import zone_to_region
24
23
  from toil.lib.retry import retry
@@ -30,7 +29,7 @@ log = logging.getLogger(__name__)
30
29
  @needs_aws_ec2
31
30
  @needs_fetchable_appliance
32
31
  class AbstractClusterTest(ToilTest):
33
- def __init__(self, methodName):
32
+ def __init__(self, methodName: str) -> None:
34
33
  super().__init__(methodName=methodName)
35
34
  self.keyName = os.getenv('TOIL_AWS_KEYNAME').strip() or 'id_rsa'
36
35
  self.clusterName = 'aws-provisioner-test-' + str(uuid4())
@@ -38,18 +37,20 @@ class AbstractClusterTest(ToilTest):
38
37
  self.clusterType = 'mesos'
39
38
  self.zone = get_best_aws_zone()
40
39
  assert self.zone is not None, "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
41
- # We need a boto2 connection to EC2 to check on the cluster
40
+ # We need a boto2 connection to EC2 to check on the cluster.
41
+ # Since we are protected by needs_aws_ec2 we can import from boto.
42
+ import boto.ec2
42
43
  self.boto2_ec2 = boto.ec2.connect_to_region(zone_to_region(self.zone))
43
44
  # Where should we put our virtualenv?
44
45
  self.venvDir = '/tmp/venv'
45
46
 
46
- def python(self):
47
+ def python(self) -> str:
47
48
  """
48
49
  Return the full path to the venv Python on the leader.
49
50
  """
50
51
  return os.path.join(self.venvDir, 'bin/python')
51
52
 
52
- def pip(self):
53
+ def pip(self) -> str:
53
54
  """
54
55
  Return the full path to the venv pip on the leader.
55
56
  """
@@ -63,7 +64,7 @@ class AbstractClusterTest(ToilTest):
63
64
  """
64
65
  subprocess.check_call(['toil', 'destroy-cluster', '-p=aws', '-z', self.zone, self.clusterName])
65
66
 
66
- def setUp(self):
67
+ def setUp(self) -> None:
67
68
  """
68
69
  Set up for the test.
69
70
  Must be overridden to call this method and set self.jobStore.
@@ -73,13 +74,13 @@ class AbstractClusterTest(ToilTest):
73
74
  # If this fails, no tests will run.
74
75
  self.destroyCluster()
75
76
 
76
- def tearDown(self):
77
+ def tearDown(self) -> None:
77
78
  # Note that teardown will run even if the test crashes.
78
79
  super().tearDown()
79
80
  self.destroyCluster()
80
81
  subprocess.check_call(['toil', 'clean', self.jobStore])
81
82
 
82
- def sshUtil(self, command):
83
+ def sshUtil(self, command: List[str]) -> None:
83
84
  """
84
85
  Run the given command on the cluster.
85
86
  Raise subprocess.CalledProcessError if it fails.
@@ -144,7 +145,18 @@ class AbstractClusterTest(ToilTest):
144
145
  raise subprocess.CalledProcessError(p.returncode, ' '.join(cmd))
145
146
 
146
147
  @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
147
- def createClusterUtil(self, args=None):
148
+ def rsync_util(self, from_file: str, to_file: str) -> None:
149
+ """
150
+ Transfer a file to/from the cluster.
151
+
152
+ The cluster-side path should have a ':' in front of it.
153
+ """
154
+ cmd = ['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName, from_file, to_file]
155
+ log.info("Running %s.", str(cmd))
156
+ subprocess.check_call(cmd)
157
+
158
+ @retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
159
+ def createClusterUtil(self, args: Optional[List[str]]=None) -> None:
148
160
  args = [] if args is None else args
149
161
 
150
162
  command = ['toil', 'launch-cluster', '-p=aws', '-z', self.zone, f'--keyPairName={self.keyName}',
@@ -156,5 +168,5 @@ class AbstractClusterTest(ToilTest):
156
168
  subprocess.check_call(command)
157
169
  # If we fail, tearDown will destroy the cluster.
158
170
 
159
- def launchCluster(self):
171
+ def launchCluster(self) -> None:
160
172
  self.createClusterUtil()
@@ -136,12 +136,6 @@ class AbstractGCEAutoscaleTest(ToilTest):
136
136
  '--python', exactPython, '/home/venv']
137
137
  self.sshUtil(venv_command)
138
138
 
139
- upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
140
- self.sshUtil(upgrade_command)
141
-
142
- yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
143
- self.sshUtil(yaml_command)
144
-
145
139
  self._getScript()
146
140
 
147
141
  toilOptions = [self.jobStore,
@@ -1,6 +1,7 @@
1
- import argparse
2
1
  import os
3
2
 
3
+ from configargparse import ArgumentParser
4
+
4
5
  from toil.job import Job
5
6
 
6
7
 
@@ -9,7 +10,7 @@ def f0(job):
9
10
  raise RuntimeError('failed on purpose')
10
11
 
11
12
  if __name__ == '__main__':
12
- parser = argparse.ArgumentParser()
13
+ parser = ArgumentParser()
13
14
  Job.Runner.addToilOptions(parser)
14
15
  options = parser.parse_args()
15
16
  rootJob = Job.wrapJobFn(f0, cores=0.5, memory='50 M', disk='50 M')
@@ -21,7 +21,7 @@ import uuid
21
21
  import zipfile
22
22
  from abc import abstractmethod
23
23
  from io import BytesIO
24
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
24
+ from typing import Optional
25
25
  from urllib.parse import urlparse
26
26
 
27
27
  try:
@@ -20,7 +20,8 @@ import codecs
20
20
  import os
21
21
  import random
22
22
  import shutil
23
- from argparse import ArgumentParser
23
+
24
+ from configargparse import ArgumentParser
24
25
 
25
26
  from toil.common import Toil
26
27
  from toil.job import Job
toil/test/sort/sort.py CHANGED
@@ -18,7 +18,8 @@ import codecs
18
18
  import os
19
19
  import random
20
20
  import shutil
21
- from argparse import ArgumentParser
21
+
22
+ from configargparse import ArgumentParser
22
23
 
23
24
  from toil.common import Toil
24
25
  from toil.job import Job
@@ -23,10 +23,10 @@ from uuid import uuid4
23
23
  from toil import resolveEntryPoint
24
24
  from toil.batchSystems.mesos.test import MesosTestSupport
25
25
  from toil.common import Toil
26
+ from toil.exceptions import FailedJobsException
26
27
  from toil.job import Job
27
28
  from toil.jobStores.abstractJobStore import (JobStoreExistsException,
28
29
  NoSuchJobStoreException)
29
- from toil.exceptions import FailedJobsException
30
30
  from toil.lib.bioio import root_logger
31
31
  from toil.test import (ToilTest,
32
32
  needs_aws_ec2,
@@ -34,10 +34,8 @@ from toil.test import (ToilTest,
34
34
  needs_google_storage,
35
35
  needs_gridengine,
36
36
  needs_mesos,
37
- needs_parasol,
38
37
  needs_torque,
39
38
  slow)
40
- from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport
41
39
  from toil.test.sort.sort import (copySubRangeOfFile,
42
40
  getMidPoint,
43
41
  main,
@@ -64,7 +62,7 @@ def runMain(options):
64
62
 
65
63
 
66
64
  @slow
67
- class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
65
+ class SortTest(ToilTest, MesosTestSupport):
68
66
  """
69
67
  Tests Toil by sorting a file in parallel on various combinations of job stores and batch
70
68
  systems.
@@ -238,15 +236,6 @@ class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
238
236
  def testFileTorqueEngine(self):
239
237
  self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='torque')
240
238
 
241
- @needs_parasol
242
- @unittest.skip("skipping until parasol support is less flaky (see github issue #449")
243
- def testFileParasol(self):
244
- self._startParasol()
245
- try:
246
- self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='parasol')
247
- finally:
248
- self._stopParasol()
249
-
250
239
  testNo = 5
251
240
 
252
241
  def testSort(self):
@@ -3,13 +3,13 @@ import subprocess
3
3
  import time
4
4
  from contextlib import contextmanager
5
5
 
6
+ from toil.exceptions import FailedJobsException
6
7
  from toil.lib.iterables import concat
7
8
  from toil.test import (ApplianceTestSupport,
8
9
  needs_local_appliance,
9
10
  needs_mesos,
10
11
  slow)
11
12
  from toil.version import exactPython
12
- from toil.exceptions import FailedJobsException
13
13
 
14
14
  logger = logging.getLogger(__name__)
15
15
 
@@ -207,28 +207,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
207
207
 
208
208
  def testDeferralWithConcurrentEncapsulation(self):
209
209
  """
210
- Ensure that the following DAG succeeds:
211
-
212
- ┌───────────┐
213
- │ Root (W1) │
214
- └───────────┘
215
-
216
- ┌──────────┴─────────┐
217
- ▼ ▼
218
- ┌────────────────┐ ┌────────────────────┐
219
- │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════╗
220
- └────────────────┘ └────────────────────┘ ║
221
- │ ║
222
- ▼ ▼
223
- ┌───────────────────┐ ┌────────────────┐
224
- │ Encapsulated (W3) │ │ Follow-on (W6) │
225
- └───────────────────┘ └────────────────┘
226
- │ │
227
- ┌───────┴────────┐ │
228
- ▼ ▼ ▼
229
- ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
230
- │ Dummy 1 (W4) │ │ Dummy 2 (W5) │ │ Last (W6) │
231
- └──────────────┘ └──────────────┘ └──────────────┘
210
+ Ensure that the following DAG succeeds::
211
+
212
+ ┌───────────┐
213
+ │ Root (W1) │
214
+ └───────────┘
215
+
216
+ ┌──────────┴─────────┐
217
+ ▼ ▼
218
+ ┌────────────────┐ ┌────────────────────┐
219
+ │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════╗
220
+ └────────────────┘ └────────────────────┘ ║
221
+ │ ║
222
+ ▼ ▼
223
+ ┌───────────────────┐ ┌────────────────┐
224
+ │ Encapsulated (W3) │ │ Follow-on (W6) │
225
+ └───────────────────┘ └────────────────┘
226
+ │ │
227
+ ┌───────┴────────┐ │
228
+ ▼ ▼ ▼
229
+ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐
230
+ │ Dummy 1 (W4) │ │ Dummy 2 (W5) │ │ Last (W6) │
231
+ └──────────────┘ └──────────────┘ └──────────────┘
232
232
 
233
233
  The Wn numbers denote the worker processes that a particular job is run in. `Deferring`
234
234
  adds a deferred function and then runs for a long time. The deferred function will be
@@ -313,28 +313,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
313
313
 
314
314
  def testDeferralWithFailureAndEncapsulation(self):
315
315
  """
316
- Ensure that the following DAG succeeds:
317
-
318
- ┌───────────┐
319
- │ Root (W1) │
320
- └───────────┘
321
-
322
- ┌──────────┴─────────┐
323
- ▼ ▼
324
- ┌────────────────┐ ┌────────────────────┐
325
- │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════════════╗
326
- └────────────────┘ └────────────────────┘ ║
327
- │ ║
328
- ▼ ▼
329
- ┌───────────────────┐ ┌────────────────┐
330
- │ Encapsulated (W3) │════════════╗ │ Follow-on (W7) │
331
- └───────────────────┘ ║ └────────────────┘
332
- │ ║
333
- ┌──────┴──────┐ ║
334
- ▼ ▼ ▼
335
- ┌────────────┐┌────────────┐ ┌──────────────┐
336
- │ Dummy (W4) ││ Dummy (W5) │ │ Trigger (W6) │
337
- └────────────┘└────────────┘ └──────────────┘
316
+ Ensure that the following DAG succeeds::
317
+
318
+ ┌───────────┐
319
+ │ Root (W1) │
320
+ └───────────┘
321
+
322
+ ┌──────────┴─────────┐
323
+ ▼ ▼
324
+ ┌────────────────┐ ┌────────────────────┐
325
+ │ Deferring (W2) │ │ Encapsulating (W3) │═══════════════════════╗
326
+ └────────────────┘ └────────────────────┘ ║
327
+ │ ║
328
+ ▼ ▼
329
+ ┌───────────────────┐ ┌────────────────┐
330
+ │ Encapsulated (W3) │════════════╗ │ Follow-on (W7) │
331
+ └───────────────────┘ ║ └────────────────┘
332
+ │ ║
333
+ ┌──────┴──────┐ ║
334
+ ▼ ▼ ▼
335
+ ┌────────────┐┌────────────┐ ┌──────────────┐
336
+ │ Dummy (W4) ││ Dummy (W5) │ │ Trigger (W6) │
337
+ └────────────┘└────────────┘ └──────────────┘
338
338
 
339
339
  `Trigger` causes `Deferring` to crash. `Follow-on` runs next, detects `Deferring`'s
340
340
  left-overs and runs the deferred function. `Follow-on` is an instance of `Job` and the
toil/test/src/busTest.py CHANGED
@@ -15,17 +15,17 @@
15
15
  import logging
16
16
  import os
17
17
  from threading import Thread, current_thread
18
- from typing import Optional
19
18
 
20
19
  from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
21
- from toil.bus import JobCompletedMessage, JobIssuedMessage, MessageBus, replay_message_bus
20
+ from toil.bus import (JobCompletedMessage,
21
+ JobIssuedMessage,
22
+ MessageBus,
23
+ replay_message_bus)
22
24
  from toil.common import Toil
23
- from toil.job import Job
24
25
  from toil.exceptions import FailedJobsException
26
+ from toil.job import Job
25
27
  from toil.test import ToilTest, get_temp_file
26
28
 
27
-
28
-
29
29
  logger = logging.getLogger(__name__)
30
30
 
31
31
  class MessageBusTest(ToilTest):
@@ -12,9 +12,9 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
+ from toil.exceptions import FailedJobsException
15
16
  from toil.job import Job
16
17
  from toil.jobStores.abstractJobStore import NoSuchFileException
17
- from toil.exceptions import FailedJobsException
18
18
  from toil.test import ToilTest, slow
19
19
 
20
20
 
@@ -82,7 +82,7 @@ class CheckRetryCount(Job):
82
82
 
83
83
  def run(self, fileStore):
84
84
  retryCount = self.getNumRetries(fileStore)
85
- fileStore.logToMaster(str(retryCount))
85
+ fileStore.log_to_leader(str(retryCount))
86
86
  if retryCount < self.numFailuresBeforeSuccess:
87
87
  self.addChild(AlwaysFail())
88
88
 
@@ -20,8 +20,8 @@ from uuid import uuid4
20
20
 
21
21
  import psutil
22
22
 
23
- from toil.job import Job
24
23
  from toil.exceptions import FailedJobsException
24
+ from toil.job import Job
25
25
  from toil.lib.threading import cpu_count
26
26
  from toil.test import ToilTest, slow
27
27
 
@@ -29,14 +29,18 @@ from uuid import uuid4
29
29
  import pytest
30
30
 
31
31
  from toil.common import Toil
32
+ from toil.exceptions import FailedJobsException
32
33
  from toil.fileStores import FileID
33
34
  from toil.fileStores.cachingFileStore import (CacheUnbalancedError,
34
35
  IllegalDeletionCacheError)
35
36
  from toil.job import Job
36
37
  from toil.jobStores.abstractJobStore import NoSuchFileException
37
- from toil.exceptions import FailedJobsException
38
38
  from toil.realtimeLogger import RealtimeLogger
39
- from toil.test import ToilTest, needs_aws_ec2, needs_google_project, needs_google_storage, slow
39
+ from toil.test import (ToilTest,
40
+ needs_aws_ec2,
41
+ needs_google_project,
42
+ needs_google_storage,
43
+ slow)
40
44
 
41
45
  # Some tests take too long on the AWS jobstore and are unquitable for CI. They can be
42
46
  # be run during manual tests by setting this to False.
@@ -397,6 +401,7 @@ class hidden:
397
401
  self.options.caching = True
398
402
 
399
403
  @slow
404
+ @pytest.mark.xfail(reason="Cannot succeed in time on small CI runners")
400
405
  def testExtremeCacheSetup(self):
401
406
  """
402
407
  Try to create the cache with bad worker active and then have 10 child jobs try to run in
@@ -472,7 +477,15 @@ class hidden:
472
477
  # the cache hence this test is redundant (caching will be free).
473
478
  if not self.options.jobStore.startswith(('aws', 'google')):
474
479
  workDirDev = os.stat(self.options.workDir).st_dev
475
- jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
480
+ if self.options.jobStore.startswith("file:"):
481
+ # Before #4538, options.jobStore would have the raw path while the Config object would prepend the
482
+ # filesystem to the path (/path/to/file vs file:/path/to/file)
483
+ # The options namespace and the Config object now have the exact same behavior
484
+ # which means parse_jobstore will be called with argparse rather than with the config object
485
+ # so remove the prepended file: scheme
486
+ jobStoreDev = os.stat(os.path.dirname(self.options.jobStore[5:])).st_dev
487
+ else:
488
+ jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
476
489
  if workDirDev == jobStoreDev:
477
490
  self.skipTest('Job store and working directory are on the same filesystem.')
478
491
 
@@ -633,6 +646,8 @@ class hidden:
633
646
 
634
647
  Attempting to get the file from the jobstore should not fail.
635
648
  """
649
+ print("Testing")
650
+ logger.debug("Testing testing 123")
636
651
  self.options.retryCount = 0
637
652
  self.options.logLevel = 'DEBUG'
638
653
  A = Job.wrapJobFn(self._adjustCacheLimit, newTotalMB=1024, disk='1G')
@@ -655,20 +670,20 @@ class hidden:
655
670
  :param fileMB: File Size
656
671
  :return: Job store file ID for second written file
657
672
  """
658
- job.fileStore.logToMaster('Double writing a file into job store')
673
+ job.fileStore.log_to_leader('Double writing a file into job store')
659
674
  work_dir = job.fileStore.getLocalTempDir()
660
675
  with open(os.path.join(work_dir, str(uuid4())), 'wb') as testFile:
661
676
  testFile.write(os.urandom(fileMB * 1024 * 1024))
662
677
 
663
- job.fileStore.logToMaster('Writing copy 1 and discarding ID')
678
+ job.fileStore.log_to_leader('Writing copy 1 and discarding ID')
664
679
  job.fileStore.writeGlobalFile(testFile.name)
665
- job.fileStore.logToMaster('Writing copy 2 and saving ID')
680
+ job.fileStore.log_to_leader('Writing copy 2 and saving ID')
666
681
  fsID = job.fileStore.writeGlobalFile(testFile.name)
667
- job.fileStore.logToMaster(f'Copy 2 ID: {fsID}')
682
+ job.fileStore.log_to_leader(f'Copy 2 ID: {fsID}')
668
683
 
669
684
  hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAssertions(job, fsID)
670
685
 
671
- job.fileStore.logToMaster('Writing copy 3 and returning ID')
686
+ job.fileStore.log_to_leader('Writing copy 3 and returning ID')
672
687
  return job.fileStore.writeGlobalFile(testFile.name)
673
688
 
674
689
  @staticmethod
@@ -680,7 +695,7 @@ class hidden:
680
695
  :param fsID: Job store file ID for the read file
681
696
  :return: None
682
697
  """
683
- job.fileStore.logToMaster('Reading the written file')
698
+ job.fileStore.log_to_leader('Reading the written file')
684
699
  job.fileStore.readGlobalFile(fsID)
685
700
 
686
701
  # writeGlobalFile tests
@@ -794,8 +809,7 @@ class hidden:
794
809
  """
795
810
  Write a local file to the job store (hence adding a copy to cache), then have 10 jobs
796
811
  read it. Assert cached file size never goes up, assert unused job
797
- required disk space is always:
798
- (a multiple of job reqs) - (number of current file readers * filesize).
812
+ required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
799
813
  At the end, assert the cache shows unused job-required space = 0.
800
814
  """
801
815
  self._testMultipleJobsReadGlobalFileFunction(cacheHit=True)
@@ -805,8 +819,7 @@ class hidden:
805
819
  """
806
820
  Write a non-local file to the job store(hence no cached copy), then have 10 jobs read
807
821
  it. Assert cached file size never goes up, assert unused job
808
- required disk space is always:
809
- (a multiple of job reqs) - (number of current file readers * filesize).
822
+ required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
810
823
  At the end, assert the cache shows unused job-required space = 0.
811
824
  """
812
825
  self._testMultipleJobsReadGlobalFileFunction(cacheHit=False)
@@ -834,7 +847,10 @@ class hidden:
834
847
  jobs[i].addChild(B)
835
848
  Job.Runner.startToil(A, self.options)
836
849
  with open(x.name) as y:
837
- assert int(y.read()) > 2
850
+ # At least one job at a time should have been observed.
851
+ # We can't actually guarantee that any of our jobs will
852
+ # see each other currently running.
853
+ assert int(y.read()) > 1
838
854
 
839
855
  @staticmethod
840
856
  def _multipleFileReader(job, diskMB, fsID, maxWriteFile):
@@ -1121,7 +1137,7 @@ class hidden:
1121
1137
  """
1122
1138
 
1123
1139
  # Make sure we actually have the disk size we are supposed to
1124
- job.fileStore.logToMaster('Job is running with %d bytes of disk, %d requested' % (job.disk, jobDisk))
1140
+ job.fileStore.log_to_leader('Job is running with %d bytes of disk, %d requested' % (job.disk, jobDisk))
1125
1141
  assert job.disk == jobDisk, 'Job was scheduled with %d bytes but requested %d' % (job.disk, jobDisk)
1126
1142
 
1127
1143
  cls = hidden.AbstractCachingFileStoreTest
@@ -1187,7 +1203,7 @@ class hidden:
1187
1203
  try:
1188
1204
  job.fileStore.deleteLocalFile(fileToDelete)
1189
1205
  except IllegalDeletionCacheError:
1190
- job.fileStore.logToMaster('Detected a deleted file %s.' % fileToDelete)
1206
+ job.fileStore.log_to_leader('Detected a deleted file %s.' % fileToDelete)
1191
1207
  os.rename(tempfile, outfile)
1192
1208
  else:
1193
1209
  # If we are processing the write test, or if we are testing the immutably read
@@ -24,7 +24,7 @@ class HelloWorldTest(ToilTest):
24
24
 
25
25
  class HelloWorld(Job):
26
26
  def __init__(self):
27
- Job.__init__(self, memory=100000, cores=2, disk="3G")
27
+ Job.__init__(self, memory=100000, cores=1, disk="3G")
28
28
 
29
29
  def run(self, fileStore):
30
30
  fileID = self.addChildJobFn(childFn, cores=1, memory="1M", disk="3G").rv()
@@ -18,9 +18,9 @@ import stat
18
18
  import uuid
19
19
 
20
20
  from toil.common import Toil
21
+ from toil.exceptions import FailedJobsException
21
22
  from toil.fileStores import FileID
22
23
  from toil.job import Job
23
- from toil.exceptions import FailedJobsException
24
24
  from toil.test import ToilTest, slow
25
25
 
26
26
 
@@ -13,7 +13,8 @@
13
13
  # limitations under the License.
14
14
 
15
15
  import os
16
- from argparse import ArgumentParser
16
+
17
+ from configargparse import ArgumentParser
17
18
 
18
19
  from toil.common import Toil
19
20
  from toil.job import Job, JobDescription, TemporaryID
@@ -24,9 +24,9 @@ from unittest import skipIf
24
24
  import pytest
25
25
 
26
26
  from toil.batchSystems.singleMachine import SingleMachineBatchSystem
27
+ from toil.exceptions import FailedJobsException
27
28
  from toil.job import Job
28
29
  from toil.leader import DeadlockException
29
- from toil.exceptions import FailedJobsException
30
30
  from toil.lib.retry import retry_flaky_test
31
31
  from toil.test import ToilTest, get_temp_file, slow
32
32
 
toil/test/src/jobTest.py CHANGED
@@ -20,8 +20,8 @@ import unittest
20
20
  import pytest
21
21
 
22
22
  from toil.common import Toil
23
- from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
24
23
  from toil.exceptions import FailedJobsException
24
+ from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
25
25
  from toil.test import ToilTest, get_temp_file, slow
26
26
 
27
27
  logger = logging.getLogger(__name__)
@@ -37,15 +37,15 @@ class JobTest(ToilTest):
37
37
  @slow
38
38
  def testStatic(self):
39
39
  r"""
40
- Create a DAG of jobs non-dynamically and run it. DAG is:
40
+ Create a DAG of jobs non-dynamically and run it. DAG is::
41
41
 
42
- A -> F
43
- \-------
44
- B -> D \
45
- \ \
46
- ------- C -> E
42
+ A -> F
43
+ \-------
44
+ B -> D \
45
+ \ \
46
+ ------- C -> E
47
47
 
48
- Follow on is marked by ->
48
+ Follow on is marked by ``->``
49
49
  """
50
50
  outFile = get_temp_file(rootDir=self._createTempDir())
51
51
  try:
@@ -81,15 +81,15 @@ class JobTest(ToilTest):
81
81
 
82
82
  def testStatic2(self):
83
83
  r"""
84
- Create a DAG of jobs non-dynamically and run it. DAG is:
84
+ Create a DAG of jobs non-dynamically and run it. DAG is::
85
85
 
86
- A -> F
87
- \-------
88
- B -> D \
89
- \ \
90
- ------- C -> E
86
+ A -> F
87
+ \-------
88
+ B -> D \
89
+ \ \
90
+ ------- C -> E
91
91
 
92
- Follow on is marked by ->
92
+ Follow on is marked by ``->``
93
93
  """
94
94
  outFile = get_temp_file(rootDir=self._createTempDir())
95
95
  try:
@@ -269,7 +269,7 @@ class JobTest(ToilTest):
269
269
  identifies leaf vertices incorrectly
270
270
 
271
271
  Test verification of new checkpoint jobs being leaf verticies,
272
- starting with the following baseline workflow:
272
+ starting with the following baseline workflow::
273
273
 
274
274
  Parent
275
275
  |
@@ -291,7 +291,7 @@ class JobTest(ToilTest):
291
291
  omits the workflow root job
292
292
 
293
293
  Test verification of a new checkpoint job being leaf vertex,
294
- starting with a baseline workflow of a single, root job:
294
+ starting with a baseline workflow of a single, root job::
295
295
 
296
296
  Root # Checkpoint=True
297
297
 
@@ -623,7 +623,7 @@ class JobTest(ToilTest):
623
623
  return True
624
624
 
625
625
  def simpleJobFn(job, value):
626
- job.fileStore.logToMaster(value)
626
+ job.fileStore.log_to_leader(value)
627
627
 
628
628
  def fn1Test(string, outputFile):
629
629
  """