toil 5.12.0__py3-none-any.whl → 6.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +39 -13
- toil/batchSystems/abstractGridEngineBatchSystem.py +24 -24
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/cleanup_support.py +7 -3
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/lsf.py +7 -7
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +87 -16
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +44 -8
- toil/common.py +544 -753
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +595 -574
- toil/cwl/utils.py +55 -10
- toil/exceptions.py +1 -1
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +88 -14
- toil/fileStores/cachingFileStore.py +610 -549
- toil/fileStores/nonCachingFileStore.py +46 -22
- toil/job.py +182 -101
- toil/jobStores/abstractJobStore.py +161 -95
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +6 -6
- toil/jobStores/fileJobStore.py +116 -18
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +87 -56
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +26 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +49 -2
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +9 -2
- toil/lib/threading.py +101 -38
- toil/options/common.py +736 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +37 -0
- toil/provisioners/abstractProvisioner.py +9 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +41 -9
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +24 -8
- toil/test/cactus/__init__.py +0 -0
- toil/test/cactus/test_cactus_integration.py +58 -0
- toil/test/cwl/cwlTest.py +438 -223
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +2 -3
- toil/test/jobStores/jobStoreTest.py +34 -21
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +12 -17
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/{wdl/conftest.py → options/__init__.py} +0 -10
- toil/test/options/options.py +37 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +23 -11
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +23 -3
- toil/test/wdl/wdltoil_test.py +223 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +445 -305
- toil/utils/toilStatus.py +2 -5
- toil/version.py +10 -10
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +1257 -492
- toil/worker.py +55 -46
- toil-6.1.0.dist-info/METADATA +124 -0
- toil-6.1.0.dist-info/RECORD +241 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/METADATA +0 -118
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0.dist-info}/top_level.txt +0 -0
|
@@ -17,8 +17,7 @@ import os
|
|
|
17
17
|
import subprocess
|
|
18
18
|
import time
|
|
19
19
|
from uuid import uuid4
|
|
20
|
-
|
|
21
|
-
import boto.ec2
|
|
20
|
+
from typing import Optional, List
|
|
22
21
|
|
|
23
22
|
from toil.lib.aws import zone_to_region
|
|
24
23
|
from toil.lib.retry import retry
|
|
@@ -30,7 +29,7 @@ log = logging.getLogger(__name__)
|
|
|
30
29
|
@needs_aws_ec2
|
|
31
30
|
@needs_fetchable_appliance
|
|
32
31
|
class AbstractClusterTest(ToilTest):
|
|
33
|
-
def __init__(self, methodName):
|
|
32
|
+
def __init__(self, methodName: str) -> None:
|
|
34
33
|
super().__init__(methodName=methodName)
|
|
35
34
|
self.keyName = os.getenv('TOIL_AWS_KEYNAME').strip() or 'id_rsa'
|
|
36
35
|
self.clusterName = 'aws-provisioner-test-' + str(uuid4())
|
|
@@ -38,18 +37,20 @@ class AbstractClusterTest(ToilTest):
|
|
|
38
37
|
self.clusterType = 'mesos'
|
|
39
38
|
self.zone = get_best_aws_zone()
|
|
40
39
|
assert self.zone is not None, "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
|
|
41
|
-
# We need a boto2 connection to EC2 to check on the cluster
|
|
40
|
+
# We need a boto2 connection to EC2 to check on the cluster.
|
|
41
|
+
# Since we are protected by needs_aws_ec2 we can import from boto.
|
|
42
|
+
import boto.ec2
|
|
42
43
|
self.boto2_ec2 = boto.ec2.connect_to_region(zone_to_region(self.zone))
|
|
43
44
|
# Where should we put our virtualenv?
|
|
44
45
|
self.venvDir = '/tmp/venv'
|
|
45
46
|
|
|
46
|
-
def python(self):
|
|
47
|
+
def python(self) -> str:
|
|
47
48
|
"""
|
|
48
49
|
Return the full path to the venv Python on the leader.
|
|
49
50
|
"""
|
|
50
51
|
return os.path.join(self.venvDir, 'bin/python')
|
|
51
52
|
|
|
52
|
-
def pip(self):
|
|
53
|
+
def pip(self) -> str:
|
|
53
54
|
"""
|
|
54
55
|
Return the full path to the venv pip on the leader.
|
|
55
56
|
"""
|
|
@@ -63,7 +64,7 @@ class AbstractClusterTest(ToilTest):
|
|
|
63
64
|
"""
|
|
64
65
|
subprocess.check_call(['toil', 'destroy-cluster', '-p=aws', '-z', self.zone, self.clusterName])
|
|
65
66
|
|
|
66
|
-
def setUp(self):
|
|
67
|
+
def setUp(self) -> None:
|
|
67
68
|
"""
|
|
68
69
|
Set up for the test.
|
|
69
70
|
Must be overridden to call this method and set self.jobStore.
|
|
@@ -73,13 +74,13 @@ class AbstractClusterTest(ToilTest):
|
|
|
73
74
|
# If this fails, no tests will run.
|
|
74
75
|
self.destroyCluster()
|
|
75
76
|
|
|
76
|
-
def tearDown(self):
|
|
77
|
+
def tearDown(self) -> None:
|
|
77
78
|
# Note that teardown will run even if the test crashes.
|
|
78
79
|
super().tearDown()
|
|
79
80
|
self.destroyCluster()
|
|
80
81
|
subprocess.check_call(['toil', 'clean', self.jobStore])
|
|
81
82
|
|
|
82
|
-
def sshUtil(self, command):
|
|
83
|
+
def sshUtil(self, command: List[str]) -> None:
|
|
83
84
|
"""
|
|
84
85
|
Run the given command on the cluster.
|
|
85
86
|
Raise subprocess.CalledProcessError if it fails.
|
|
@@ -144,7 +145,18 @@ class AbstractClusterTest(ToilTest):
|
|
|
144
145
|
raise subprocess.CalledProcessError(p.returncode, ' '.join(cmd))
|
|
145
146
|
|
|
146
147
|
@retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
|
|
147
|
-
def
|
|
148
|
+
def rsync_util(self, from_file: str, to_file: str) -> None:
|
|
149
|
+
"""
|
|
150
|
+
Transfer a file to/from the cluster.
|
|
151
|
+
|
|
152
|
+
The cluster-side path should have a ':' in front of it.
|
|
153
|
+
"""
|
|
154
|
+
cmd = ['toil', 'rsync-cluster', '--insecure', '-p=aws', '-z', self.zone, self.clusterName, from_file, to_file]
|
|
155
|
+
log.info("Running %s.", str(cmd))
|
|
156
|
+
subprocess.check_call(cmd)
|
|
157
|
+
|
|
158
|
+
@retry(errors=[subprocess.CalledProcessError], intervals=[1, 1])
|
|
159
|
+
def createClusterUtil(self, args: Optional[List[str]]=None) -> None:
|
|
148
160
|
args = [] if args is None else args
|
|
149
161
|
|
|
150
162
|
command = ['toil', 'launch-cluster', '-p=aws', '-z', self.zone, f'--keyPairName={self.keyName}',
|
|
@@ -156,5 +168,5 @@ class AbstractClusterTest(ToilTest):
|
|
|
156
168
|
subprocess.check_call(command)
|
|
157
169
|
# If we fail, tearDown will destroy the cluster.
|
|
158
170
|
|
|
159
|
-
def launchCluster(self):
|
|
171
|
+
def launchCluster(self) -> None:
|
|
160
172
|
self.createClusterUtil()
|
|
@@ -136,12 +136,6 @@ class AbstractGCEAutoscaleTest(ToilTest):
|
|
|
136
136
|
'--python', exactPython, '/home/venv']
|
|
137
137
|
self.sshUtil(venv_command)
|
|
138
138
|
|
|
139
|
-
upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
|
|
140
|
-
self.sshUtil(upgrade_command)
|
|
141
|
-
|
|
142
|
-
yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
|
|
143
|
-
self.sshUtil(yaml_command)
|
|
144
|
-
|
|
145
139
|
self._getScript()
|
|
146
140
|
|
|
147
141
|
toilOptions = [self.jobStore,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import argparse
|
|
2
1
|
import os
|
|
3
2
|
|
|
3
|
+
from configargparse import ArgumentParser
|
|
4
|
+
|
|
4
5
|
from toil.job import Job
|
|
5
6
|
|
|
6
7
|
|
|
@@ -9,7 +10,7 @@ def f0(job):
|
|
|
9
10
|
raise RuntimeError('failed on purpose')
|
|
10
11
|
|
|
11
12
|
if __name__ == '__main__':
|
|
12
|
-
parser =
|
|
13
|
+
parser = ArgumentParser()
|
|
13
14
|
Job.Runner.addToilOptions(parser)
|
|
14
15
|
options = parser.parse_args()
|
|
15
16
|
rootJob = Job.wrapJobFn(f0, cores=0.5, memory='50 M', disk='50 M')
|
toil/test/server/serverTest.py
CHANGED
toil/test/sort/restart_sort.py
CHANGED
toil/test/sort/sort.py
CHANGED
toil/test/sort/sortTest.py
CHANGED
|
@@ -23,10 +23,10 @@ from uuid import uuid4
|
|
|
23
23
|
from toil import resolveEntryPoint
|
|
24
24
|
from toil.batchSystems.mesos.test import MesosTestSupport
|
|
25
25
|
from toil.common import Toil
|
|
26
|
+
from toil.exceptions import FailedJobsException
|
|
26
27
|
from toil.job import Job
|
|
27
28
|
from toil.jobStores.abstractJobStore import (JobStoreExistsException,
|
|
28
29
|
NoSuchJobStoreException)
|
|
29
|
-
from toil.exceptions import FailedJobsException
|
|
30
30
|
from toil.lib.bioio import root_logger
|
|
31
31
|
from toil.test import (ToilTest,
|
|
32
32
|
needs_aws_ec2,
|
|
@@ -34,10 +34,8 @@ from toil.test import (ToilTest,
|
|
|
34
34
|
needs_google_storage,
|
|
35
35
|
needs_gridengine,
|
|
36
36
|
needs_mesos,
|
|
37
|
-
needs_parasol,
|
|
38
37
|
needs_torque,
|
|
39
38
|
slow)
|
|
40
|
-
from toil.test.batchSystems.parasolTestSupport import ParasolTestSupport
|
|
41
39
|
from toil.test.sort.sort import (copySubRangeOfFile,
|
|
42
40
|
getMidPoint,
|
|
43
41
|
main,
|
|
@@ -64,7 +62,7 @@ def runMain(options):
|
|
|
64
62
|
|
|
65
63
|
|
|
66
64
|
@slow
|
|
67
|
-
class SortTest(ToilTest, MesosTestSupport
|
|
65
|
+
class SortTest(ToilTest, MesosTestSupport):
|
|
68
66
|
"""
|
|
69
67
|
Tests Toil by sorting a file in parallel on various combinations of job stores and batch
|
|
70
68
|
systems.
|
|
@@ -238,15 +236,6 @@ class SortTest(ToilTest, MesosTestSupport, ParasolTestSupport):
|
|
|
238
236
|
def testFileTorqueEngine(self):
|
|
239
237
|
self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='torque')
|
|
240
238
|
|
|
241
|
-
@needs_parasol
|
|
242
|
-
@unittest.skip("skipping until parasol support is less flaky (see github issue #449")
|
|
243
|
-
def testFileParasol(self):
|
|
244
|
-
self._startParasol()
|
|
245
|
-
try:
|
|
246
|
-
self._toilSort(jobStoreLocator=self._getTestJobStorePath(), batchSystem='parasol')
|
|
247
|
-
finally:
|
|
248
|
-
self._stopParasol()
|
|
249
|
-
|
|
250
239
|
testNo = 5
|
|
251
240
|
|
|
252
241
|
def testSort(self):
|
|
@@ -3,13 +3,13 @@ import subprocess
|
|
|
3
3
|
import time
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
|
|
6
|
+
from toil.exceptions import FailedJobsException
|
|
6
7
|
from toil.lib.iterables import concat
|
|
7
8
|
from toil.test import (ApplianceTestSupport,
|
|
8
9
|
needs_local_appliance,
|
|
9
10
|
needs_mesos,
|
|
10
11
|
slow)
|
|
11
12
|
from toil.version import exactPython
|
|
12
|
-
from toil.exceptions import FailedJobsException
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
@@ -207,28 +207,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
|
|
|
207
207
|
|
|
208
208
|
def testDeferralWithConcurrentEncapsulation(self):
|
|
209
209
|
"""
|
|
210
|
-
Ensure that the following DAG succeeds
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
210
|
+
Ensure that the following DAG succeeds::
|
|
211
|
+
|
|
212
|
+
┌───────────┐
|
|
213
|
+
│ Root (W1) │
|
|
214
|
+
└───────────┘
|
|
215
|
+
│
|
|
216
|
+
┌──────────┴─────────┐
|
|
217
|
+
▼ ▼
|
|
218
|
+
┌────────────────┐ ┌────────────────────┐
|
|
219
|
+
│ Deferring (W2) │ │ Encapsulating (W3) │═══════════════╗
|
|
220
|
+
└────────────────┘ └────────────────────┘ ║
|
|
221
|
+
│ ║
|
|
222
|
+
▼ ▼
|
|
223
|
+
┌───────────────────┐ ┌────────────────┐
|
|
224
|
+
│ Encapsulated (W3) │ │ Follow-on (W6) │
|
|
225
|
+
└───────────────────┘ └────────────────┘
|
|
226
|
+
│ │
|
|
227
|
+
┌───────┴────────┐ │
|
|
228
|
+
▼ ▼ ▼
|
|
229
|
+
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
|
230
|
+
│ Dummy 1 (W4) │ │ Dummy 2 (W5) │ │ Last (W6) │
|
|
231
|
+
└──────────────┘ └──────────────┘ └──────────────┘
|
|
232
232
|
|
|
233
233
|
The Wn numbers denote the worker processes that a particular job is run in. `Deferring`
|
|
234
234
|
adds a deferred function and then runs for a long time. The deferred function will be
|
|
@@ -313,28 +313,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
|
|
|
313
313
|
|
|
314
314
|
def testDeferralWithFailureAndEncapsulation(self):
|
|
315
315
|
"""
|
|
316
|
-
Ensure that the following DAG succeeds
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
316
|
+
Ensure that the following DAG succeeds::
|
|
317
|
+
|
|
318
|
+
┌───────────┐
|
|
319
|
+
│ Root (W1) │
|
|
320
|
+
└───────────┘
|
|
321
|
+
│
|
|
322
|
+
┌──────────┴─────────┐
|
|
323
|
+
▼ ▼
|
|
324
|
+
┌────────────────┐ ┌────────────────────┐
|
|
325
|
+
│ Deferring (W2) │ │ Encapsulating (W3) │═══════════════════════╗
|
|
326
|
+
└────────────────┘ └────────────────────┘ ║
|
|
327
|
+
│ ║
|
|
328
|
+
▼ ▼
|
|
329
|
+
┌───────────────────┐ ┌────────────────┐
|
|
330
|
+
│ Encapsulated (W3) │════════════╗ │ Follow-on (W7) │
|
|
331
|
+
└───────────────────┘ ║ └────────────────┘
|
|
332
|
+
│ ║
|
|
333
|
+
┌──────┴──────┐ ║
|
|
334
|
+
▼ ▼ ▼
|
|
335
|
+
┌────────────┐┌────────────┐ ┌──────────────┐
|
|
336
|
+
│ Dummy (W4) ││ Dummy (W5) │ │ Trigger (W6) │
|
|
337
|
+
└────────────┘└────────────┘ └──────────────┘
|
|
338
338
|
|
|
339
339
|
`Trigger` causes `Deferring` to crash. `Follow-on` runs next, detects `Deferring`'s
|
|
340
340
|
left-overs and runs the deferred function. `Follow-on` is an instance of `Job` and the
|
toil/test/src/busTest.py
CHANGED
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
import logging
|
|
16
16
|
import os
|
|
17
17
|
from threading import Thread, current_thread
|
|
18
|
-
from typing import Optional
|
|
19
18
|
|
|
20
19
|
from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
|
|
21
|
-
from toil.bus import JobCompletedMessage,
|
|
20
|
+
from toil.bus import (JobCompletedMessage,
|
|
21
|
+
JobIssuedMessage,
|
|
22
|
+
MessageBus,
|
|
23
|
+
replay_message_bus)
|
|
22
24
|
from toil.common import Toil
|
|
23
|
-
from toil.job import Job
|
|
24
25
|
from toil.exceptions import FailedJobsException
|
|
26
|
+
from toil.job import Job
|
|
25
27
|
from toil.test import ToilTest, get_temp_file
|
|
26
28
|
|
|
27
|
-
|
|
28
|
-
|
|
29
29
|
logger = logging.getLogger(__name__)
|
|
30
30
|
|
|
31
31
|
class MessageBusTest(ToilTest):
|
toil/test/src/checkpointTest.py
CHANGED
|
@@ -12,9 +12,9 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from toil.exceptions import FailedJobsException
|
|
15
16
|
from toil.job import Job
|
|
16
17
|
from toil.jobStores.abstractJobStore import NoSuchFileException
|
|
17
|
-
from toil.exceptions import FailedJobsException
|
|
18
18
|
from toil.test import ToilTest, slow
|
|
19
19
|
|
|
20
20
|
|
|
@@ -82,7 +82,7 @@ class CheckRetryCount(Job):
|
|
|
82
82
|
|
|
83
83
|
def run(self, fileStore):
|
|
84
84
|
retryCount = self.getNumRetries(fileStore)
|
|
85
|
-
fileStore.
|
|
85
|
+
fileStore.log_to_leader(str(retryCount))
|
|
86
86
|
if retryCount < self.numFailuresBeforeSuccess:
|
|
87
87
|
self.addChild(AlwaysFail())
|
|
88
88
|
|
toil/test/src/fileStoreTest.py
CHANGED
|
@@ -29,14 +29,18 @@ from uuid import uuid4
|
|
|
29
29
|
import pytest
|
|
30
30
|
|
|
31
31
|
from toil.common import Toil
|
|
32
|
+
from toil.exceptions import FailedJobsException
|
|
32
33
|
from toil.fileStores import FileID
|
|
33
34
|
from toil.fileStores.cachingFileStore import (CacheUnbalancedError,
|
|
34
35
|
IllegalDeletionCacheError)
|
|
35
36
|
from toil.job import Job
|
|
36
37
|
from toil.jobStores.abstractJobStore import NoSuchFileException
|
|
37
|
-
from toil.exceptions import FailedJobsException
|
|
38
38
|
from toil.realtimeLogger import RealtimeLogger
|
|
39
|
-
from toil.test import ToilTest,
|
|
39
|
+
from toil.test import (ToilTest,
|
|
40
|
+
needs_aws_ec2,
|
|
41
|
+
needs_google_project,
|
|
42
|
+
needs_google_storage,
|
|
43
|
+
slow)
|
|
40
44
|
|
|
41
45
|
# Some tests take too long on the AWS jobstore and are unquitable for CI. They can be
|
|
42
46
|
# be run during manual tests by setting this to False.
|
|
@@ -397,6 +401,7 @@ class hidden:
|
|
|
397
401
|
self.options.caching = True
|
|
398
402
|
|
|
399
403
|
@slow
|
|
404
|
+
@pytest.mark.xfail(reason="Cannot succeed in time on small CI runners")
|
|
400
405
|
def testExtremeCacheSetup(self):
|
|
401
406
|
"""
|
|
402
407
|
Try to create the cache with bad worker active and then have 10 child jobs try to run in
|
|
@@ -472,7 +477,15 @@ class hidden:
|
|
|
472
477
|
# the cache hence this test is redundant (caching will be free).
|
|
473
478
|
if not self.options.jobStore.startswith(('aws', 'google')):
|
|
474
479
|
workDirDev = os.stat(self.options.workDir).st_dev
|
|
475
|
-
|
|
480
|
+
if self.options.jobStore.startswith("file:"):
|
|
481
|
+
# Before #4538, options.jobStore would have the raw path while the Config object would prepend the
|
|
482
|
+
# filesystem to the path (/path/to/file vs file:/path/to/file)
|
|
483
|
+
# The options namespace and the Config object now have the exact same behavior
|
|
484
|
+
# which means parse_jobstore will be called with argparse rather than with the config object
|
|
485
|
+
# so remove the prepended file: scheme
|
|
486
|
+
jobStoreDev = os.stat(os.path.dirname(self.options.jobStore[5:])).st_dev
|
|
487
|
+
else:
|
|
488
|
+
jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
|
|
476
489
|
if workDirDev == jobStoreDev:
|
|
477
490
|
self.skipTest('Job store and working directory are on the same filesystem.')
|
|
478
491
|
|
|
@@ -633,6 +646,8 @@ class hidden:
|
|
|
633
646
|
|
|
634
647
|
Attempting to get the file from the jobstore should not fail.
|
|
635
648
|
"""
|
|
649
|
+
print("Testing")
|
|
650
|
+
logger.debug("Testing testing 123")
|
|
636
651
|
self.options.retryCount = 0
|
|
637
652
|
self.options.logLevel = 'DEBUG'
|
|
638
653
|
A = Job.wrapJobFn(self._adjustCacheLimit, newTotalMB=1024, disk='1G')
|
|
@@ -655,20 +670,20 @@ class hidden:
|
|
|
655
670
|
:param fileMB: File Size
|
|
656
671
|
:return: Job store file ID for second written file
|
|
657
672
|
"""
|
|
658
|
-
job.fileStore.
|
|
673
|
+
job.fileStore.log_to_leader('Double writing a file into job store')
|
|
659
674
|
work_dir = job.fileStore.getLocalTempDir()
|
|
660
675
|
with open(os.path.join(work_dir, str(uuid4())), 'wb') as testFile:
|
|
661
676
|
testFile.write(os.urandom(fileMB * 1024 * 1024))
|
|
662
677
|
|
|
663
|
-
job.fileStore.
|
|
678
|
+
job.fileStore.log_to_leader('Writing copy 1 and discarding ID')
|
|
664
679
|
job.fileStore.writeGlobalFile(testFile.name)
|
|
665
|
-
job.fileStore.
|
|
680
|
+
job.fileStore.log_to_leader('Writing copy 2 and saving ID')
|
|
666
681
|
fsID = job.fileStore.writeGlobalFile(testFile.name)
|
|
667
|
-
job.fileStore.
|
|
682
|
+
job.fileStore.log_to_leader(f'Copy 2 ID: {fsID}')
|
|
668
683
|
|
|
669
684
|
hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAssertions(job, fsID)
|
|
670
685
|
|
|
671
|
-
job.fileStore.
|
|
686
|
+
job.fileStore.log_to_leader('Writing copy 3 and returning ID')
|
|
672
687
|
return job.fileStore.writeGlobalFile(testFile.name)
|
|
673
688
|
|
|
674
689
|
@staticmethod
|
|
@@ -680,7 +695,7 @@ class hidden:
|
|
|
680
695
|
:param fsID: Job store file ID for the read file
|
|
681
696
|
:return: None
|
|
682
697
|
"""
|
|
683
|
-
job.fileStore.
|
|
698
|
+
job.fileStore.log_to_leader('Reading the written file')
|
|
684
699
|
job.fileStore.readGlobalFile(fsID)
|
|
685
700
|
|
|
686
701
|
# writeGlobalFile tests
|
|
@@ -794,8 +809,7 @@ class hidden:
|
|
|
794
809
|
"""
|
|
795
810
|
Write a local file to the job store (hence adding a copy to cache), then have 10 jobs
|
|
796
811
|
read it. Assert cached file size never goes up, assert unused job
|
|
797
|
-
required disk space is always
|
|
798
|
-
(a multiple of job reqs) - (number of current file readers * filesize).
|
|
812
|
+
required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
|
|
799
813
|
At the end, assert the cache shows unused job-required space = 0.
|
|
800
814
|
"""
|
|
801
815
|
self._testMultipleJobsReadGlobalFileFunction(cacheHit=True)
|
|
@@ -805,8 +819,7 @@ class hidden:
|
|
|
805
819
|
"""
|
|
806
820
|
Write a non-local file to the job store(hence no cached copy), then have 10 jobs read
|
|
807
821
|
it. Assert cached file size never goes up, assert unused job
|
|
808
|
-
required disk space is always
|
|
809
|
-
(a multiple of job reqs) - (number of current file readers * filesize).
|
|
822
|
+
required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
|
|
810
823
|
At the end, assert the cache shows unused job-required space = 0.
|
|
811
824
|
"""
|
|
812
825
|
self._testMultipleJobsReadGlobalFileFunction(cacheHit=False)
|
|
@@ -834,7 +847,10 @@ class hidden:
|
|
|
834
847
|
jobs[i].addChild(B)
|
|
835
848
|
Job.Runner.startToil(A, self.options)
|
|
836
849
|
with open(x.name) as y:
|
|
837
|
-
|
|
850
|
+
# At least one job at a time should have been observed.
|
|
851
|
+
# We can't actually guarantee that any of our jobs will
|
|
852
|
+
# see each other currently running.
|
|
853
|
+
assert int(y.read()) > 1
|
|
838
854
|
|
|
839
855
|
@staticmethod
|
|
840
856
|
def _multipleFileReader(job, diskMB, fsID, maxWriteFile):
|
|
@@ -1121,7 +1137,7 @@ class hidden:
|
|
|
1121
1137
|
"""
|
|
1122
1138
|
|
|
1123
1139
|
# Make sure we actually have the disk size we are supposed to
|
|
1124
|
-
job.fileStore.
|
|
1140
|
+
job.fileStore.log_to_leader('Job is running with %d bytes of disk, %d requested' % (job.disk, jobDisk))
|
|
1125
1141
|
assert job.disk == jobDisk, 'Job was scheduled with %d bytes but requested %d' % (job.disk, jobDisk)
|
|
1126
1142
|
|
|
1127
1143
|
cls = hidden.AbstractCachingFileStoreTest
|
|
@@ -1187,7 +1203,7 @@ class hidden:
|
|
|
1187
1203
|
try:
|
|
1188
1204
|
job.fileStore.deleteLocalFile(fileToDelete)
|
|
1189
1205
|
except IllegalDeletionCacheError:
|
|
1190
|
-
job.fileStore.
|
|
1206
|
+
job.fileStore.log_to_leader('Detected a deleted file %s.' % fileToDelete)
|
|
1191
1207
|
os.rename(tempfile, outfile)
|
|
1192
1208
|
else:
|
|
1193
1209
|
# If we are processing the write test, or if we are testing the immutably read
|
toil/test/src/helloWorldTest.py
CHANGED
|
@@ -24,7 +24,7 @@ class HelloWorldTest(ToilTest):
|
|
|
24
24
|
|
|
25
25
|
class HelloWorld(Job):
|
|
26
26
|
def __init__(self):
|
|
27
|
-
Job.__init__(self, memory=100000, cores=
|
|
27
|
+
Job.__init__(self, memory=100000, cores=1, disk="3G")
|
|
28
28
|
|
|
29
29
|
def run(self, fileStore):
|
|
30
30
|
fileID = self.addChildJobFn(childFn, cores=1, memory="1M", disk="3G").rv()
|
|
@@ -18,9 +18,9 @@ import stat
|
|
|
18
18
|
import uuid
|
|
19
19
|
|
|
20
20
|
from toil.common import Toil
|
|
21
|
+
from toil.exceptions import FailedJobsException
|
|
21
22
|
from toil.fileStores import FileID
|
|
22
23
|
from toil.job import Job
|
|
23
|
-
from toil.exceptions import FailedJobsException
|
|
24
24
|
from toil.test import ToilTest, slow
|
|
25
25
|
|
|
26
26
|
|
toil/test/src/jobServiceTest.py
CHANGED
|
@@ -24,9 +24,9 @@ from unittest import skipIf
|
|
|
24
24
|
import pytest
|
|
25
25
|
|
|
26
26
|
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
|
|
27
|
+
from toil.exceptions import FailedJobsException
|
|
27
28
|
from toil.job import Job
|
|
28
29
|
from toil.leader import DeadlockException
|
|
29
|
-
from toil.exceptions import FailedJobsException
|
|
30
30
|
from toil.lib.retry import retry_flaky_test
|
|
31
31
|
from toil.test import ToilTest, get_temp_file, slow
|
|
32
32
|
|
toil/test/src/jobTest.py
CHANGED
|
@@ -20,8 +20,8 @@ import unittest
|
|
|
20
20
|
import pytest
|
|
21
21
|
|
|
22
22
|
from toil.common import Toil
|
|
23
|
-
from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
|
|
24
23
|
from toil.exceptions import FailedJobsException
|
|
24
|
+
from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
|
|
25
25
|
from toil.test import ToilTest, get_temp_file, slow
|
|
26
26
|
|
|
27
27
|
logger = logging.getLogger(__name__)
|
|
@@ -37,15 +37,15 @@ class JobTest(ToilTest):
|
|
|
37
37
|
@slow
|
|
38
38
|
def testStatic(self):
|
|
39
39
|
r"""
|
|
40
|
-
Create a DAG of jobs non-dynamically and run it. DAG is
|
|
40
|
+
Create a DAG of jobs non-dynamically and run it. DAG is::
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
42
|
+
A -> F
|
|
43
|
+
\-------
|
|
44
|
+
B -> D \
|
|
45
|
+
\ \
|
|
46
|
+
------- C -> E
|
|
47
47
|
|
|
48
|
-
Follow on is marked by
|
|
48
|
+
Follow on is marked by ``->``
|
|
49
49
|
"""
|
|
50
50
|
outFile = get_temp_file(rootDir=self._createTempDir())
|
|
51
51
|
try:
|
|
@@ -81,15 +81,15 @@ class JobTest(ToilTest):
|
|
|
81
81
|
|
|
82
82
|
def testStatic2(self):
|
|
83
83
|
r"""
|
|
84
|
-
Create a DAG of jobs non-dynamically and run it. DAG is
|
|
84
|
+
Create a DAG of jobs non-dynamically and run it. DAG is::
|
|
85
85
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
86
|
+
A -> F
|
|
87
|
+
\-------
|
|
88
|
+
B -> D \
|
|
89
|
+
\ \
|
|
90
|
+
------- C -> E
|
|
91
91
|
|
|
92
|
-
Follow on is marked by
|
|
92
|
+
Follow on is marked by ``->``
|
|
93
93
|
"""
|
|
94
94
|
outFile = get_temp_file(rootDir=self._createTempDir())
|
|
95
95
|
try:
|
|
@@ -269,7 +269,7 @@ class JobTest(ToilTest):
|
|
|
269
269
|
identifies leaf vertices incorrectly
|
|
270
270
|
|
|
271
271
|
Test verification of new checkpoint jobs being leaf verticies,
|
|
272
|
-
starting with the following baseline workflow
|
|
272
|
+
starting with the following baseline workflow::
|
|
273
273
|
|
|
274
274
|
Parent
|
|
275
275
|
|
|
|
@@ -291,7 +291,7 @@ class JobTest(ToilTest):
|
|
|
291
291
|
omits the workflow root job
|
|
292
292
|
|
|
293
293
|
Test verification of a new checkpoint job being leaf vertex,
|
|
294
|
-
starting with a baseline workflow of a single, root job
|
|
294
|
+
starting with a baseline workflow of a single, root job::
|
|
295
295
|
|
|
296
296
|
Root # Checkpoint=True
|
|
297
297
|
|
|
@@ -623,7 +623,7 @@ class JobTest(ToilTest):
|
|
|
623
623
|
return True
|
|
624
624
|
|
|
625
625
|
def simpleJobFn(job, value):
|
|
626
|
-
job.fileStore.
|
|
626
|
+
job.fileStore.log_to_leader(value)
|
|
627
627
|
|
|
628
628
|
def fn1Test(string, outputFile):
|
|
629
629
|
"""
|