toil 5.12.0__py3-none-any.whl → 6.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +18 -13
- toil/batchSystems/abstractBatchSystem.py +21 -10
- toil/batchSystems/abstractGridEngineBatchSystem.py +2 -2
- toil/batchSystems/awsBatch.py +14 -14
- toil/batchSystems/contained_executor.py +3 -3
- toil/batchSystems/htcondor.py +0 -1
- toil/batchSystems/kubernetes.py +34 -31
- toil/batchSystems/local_support.py +3 -1
- toil/batchSystems/mesos/batchSystem.py +7 -7
- toil/batchSystems/options.py +32 -83
- toil/batchSystems/registry.py +104 -23
- toil/batchSystems/singleMachine.py +16 -13
- toil/batchSystems/slurm.py +3 -3
- toil/batchSystems/torque.py +0 -1
- toil/bus.py +6 -8
- toil/common.py +532 -743
- toil/cwl/__init__.py +28 -32
- toil/cwl/cwltoil.py +523 -520
- toil/cwl/utils.py +55 -10
- toil/fileStores/__init__.py +2 -2
- toil/fileStores/abstractFileStore.py +36 -11
- toil/fileStores/cachingFileStore.py +607 -530
- toil/fileStores/nonCachingFileStore.py +43 -10
- toil/job.py +140 -75
- toil/jobStores/abstractJobStore.py +147 -79
- toil/jobStores/aws/jobStore.py +23 -9
- toil/jobStores/aws/utils.py +1 -2
- toil/jobStores/fileJobStore.py +117 -19
- toil/jobStores/googleJobStore.py +16 -7
- toil/jobStores/utils.py +5 -6
- toil/leader.py +71 -43
- toil/lib/accelerators.py +10 -5
- toil/lib/aws/__init__.py +3 -14
- toil/lib/aws/ami.py +22 -9
- toil/lib/aws/iam.py +21 -13
- toil/lib/aws/session.py +2 -16
- toil/lib/aws/utils.py +4 -5
- toil/lib/compatibility.py +1 -1
- toil/lib/conversions.py +7 -3
- toil/lib/docker.py +22 -23
- toil/lib/ec2.py +10 -6
- toil/lib/ec2nodes.py +106 -100
- toil/lib/encryption/_nacl.py +2 -1
- toil/lib/generatedEC2Lists.py +325 -18
- toil/lib/io.py +21 -0
- toil/lib/misc.py +1 -1
- toil/lib/resources.py +1 -1
- toil/lib/threading.py +74 -26
- toil/options/common.py +738 -0
- toil/options/cwl.py +336 -0
- toil/options/wdl.py +32 -0
- toil/provisioners/abstractProvisioner.py +1 -4
- toil/provisioners/aws/__init__.py +3 -6
- toil/provisioners/aws/awsProvisioner.py +6 -0
- toil/provisioners/clusterScaler.py +3 -2
- toil/provisioners/gceProvisioner.py +2 -2
- toil/realtimeLogger.py +2 -1
- toil/resource.py +24 -18
- toil/server/app.py +2 -3
- toil/server/cli/wes_cwl_runner.py +4 -4
- toil/server/utils.py +1 -1
- toil/server/wes/abstract_backend.py +3 -2
- toil/server/wes/amazon_wes_utils.py +5 -4
- toil/server/wes/tasks.py +2 -3
- toil/server/wes/toil_backend.py +2 -10
- toil/server/wsgi_app.py +2 -0
- toil/serviceManager.py +12 -10
- toil/statsAndLogging.py +5 -1
- toil/test/__init__.py +29 -54
- toil/test/batchSystems/batchSystemTest.py +11 -111
- toil/test/batchSystems/test_slurm.py +3 -2
- toil/test/cwl/cwlTest.py +213 -90
- toil/test/cwl/glob_dir.cwl +15 -0
- toil/test/cwl/preemptible.cwl +21 -0
- toil/test/cwl/preemptible_expression.cwl +28 -0
- toil/test/cwl/revsort.cwl +1 -1
- toil/test/cwl/revsort2.cwl +1 -1
- toil/test/docs/scriptsTest.py +0 -1
- toil/test/jobStores/jobStoreTest.py +27 -16
- toil/test/lib/aws/test_iam.py +4 -14
- toil/test/lib/aws/test_utils.py +0 -3
- toil/test/lib/dockerTest.py +4 -4
- toil/test/lib/test_ec2.py +11 -16
- toil/test/mesos/helloWorld.py +4 -5
- toil/test/mesos/stress.py +1 -1
- toil/test/provisioners/aws/awsProvisionerTest.py +9 -5
- toil/test/provisioners/clusterScalerTest.py +6 -4
- toil/test/provisioners/clusterTest.py +14 -3
- toil/test/provisioners/gceProvisionerTest.py +0 -6
- toil/test/provisioners/restartScript.py +3 -2
- toil/test/server/serverTest.py +1 -1
- toil/test/sort/restart_sort.py +2 -1
- toil/test/sort/sort.py +2 -1
- toil/test/sort/sortTest.py +2 -13
- toil/test/src/autoDeploymentTest.py +45 -45
- toil/test/src/busTest.py +5 -5
- toil/test/src/checkpointTest.py +2 -2
- toil/test/src/deferredFunctionTest.py +1 -1
- toil/test/src/fileStoreTest.py +32 -16
- toil/test/src/helloWorldTest.py +1 -1
- toil/test/src/importExportFileTest.py +1 -1
- toil/test/src/jobDescriptionTest.py +2 -1
- toil/test/src/jobServiceTest.py +1 -1
- toil/test/src/jobTest.py +18 -18
- toil/test/src/miscTests.py +5 -3
- toil/test/src/promisedRequirementTest.py +3 -3
- toil/test/src/realtimeLoggerTest.py +1 -1
- toil/test/src/resourceTest.py +2 -2
- toil/test/src/restartDAGTest.py +1 -1
- toil/test/src/resumabilityTest.py +36 -2
- toil/test/src/retainTempDirTest.py +1 -1
- toil/test/src/systemTest.py +2 -2
- toil/test/src/toilContextManagerTest.py +2 -2
- toil/test/src/userDefinedJobArgTypeTest.py +1 -1
- toil/test/utils/toilDebugTest.py +98 -32
- toil/test/utils/toilKillTest.py +2 -2
- toil/test/utils/utilsTest.py +20 -0
- toil/test/wdl/wdltoil_test.py +148 -45
- toil/toilState.py +7 -6
- toil/utils/toilClean.py +1 -1
- toil/utils/toilConfig.py +36 -0
- toil/utils/toilDebugFile.py +60 -33
- toil/utils/toilDebugJob.py +39 -12
- toil/utils/toilDestroyCluster.py +1 -1
- toil/utils/toilKill.py +1 -1
- toil/utils/toilLaunchCluster.py +13 -2
- toil/utils/toilMain.py +3 -2
- toil/utils/toilRsyncCluster.py +1 -1
- toil/utils/toilSshCluster.py +1 -1
- toil/utils/toilStats.py +240 -143
- toil/utils/toilStatus.py +1 -4
- toil/version.py +11 -11
- toil/wdl/utils.py +2 -122
- toil/wdl/wdltoil.py +999 -386
- toil/worker.py +25 -31
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/METADATA +60 -53
- toil-6.1.0a1.dist-info/RECORD +237 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/WHEEL +1 -1
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/entry_points.txt +0 -1
- toil/batchSystems/parasol.py +0 -379
- toil/batchSystems/tes.py +0 -459
- toil/test/batchSystems/parasolTestSupport.py +0 -117
- toil/test/wdl/builtinTest.py +0 -506
- toil/test/wdl/conftest.py +0 -23
- toil/test/wdl/toilwdlTest.py +0 -522
- toil/wdl/toilwdl.py +0 -141
- toil/wdl/versions/dev.py +0 -107
- toil/wdl/versions/draft2.py +0 -980
- toil/wdl/versions/v1.py +0 -794
- toil/wdl/wdl_analysis.py +0 -116
- toil/wdl/wdl_functions.py +0 -997
- toil/wdl/wdl_synthesis.py +0 -1011
- toil/wdl/wdl_types.py +0 -243
- toil-5.12.0.dist-info/RECORD +0 -244
- /toil/{wdl/versions → options}/__init__.py +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/LICENSE +0 -0
- {toil-5.12.0.dist-info → toil-6.1.0a1.dist-info}/top_level.txt +0 -0
|
@@ -3,13 +3,13 @@ import subprocess
|
|
|
3
3
|
import time
|
|
4
4
|
from contextlib import contextmanager
|
|
5
5
|
|
|
6
|
+
from toil.exceptions import FailedJobsException
|
|
6
7
|
from toil.lib.iterables import concat
|
|
7
8
|
from toil.test import (ApplianceTestSupport,
|
|
8
9
|
needs_local_appliance,
|
|
9
10
|
needs_mesos,
|
|
10
11
|
slow)
|
|
11
12
|
from toil.version import exactPython
|
|
12
|
-
from toil.exceptions import FailedJobsException
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
@@ -207,28 +207,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
|
|
|
207
207
|
|
|
208
208
|
def testDeferralWithConcurrentEncapsulation(self):
|
|
209
209
|
"""
|
|
210
|
-
Ensure that the following DAG succeeds
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
210
|
+
Ensure that the following DAG succeeds::
|
|
211
|
+
|
|
212
|
+
┌───────────┐
|
|
213
|
+
│ Root (W1) │
|
|
214
|
+
└───────────┘
|
|
215
|
+
│
|
|
216
|
+
┌──────────┴─────────┐
|
|
217
|
+
▼ ▼
|
|
218
|
+
┌────────────────┐ ┌────────────────────┐
|
|
219
|
+
│ Deferring (W2) │ │ Encapsulating (W3) │═══════════════╗
|
|
220
|
+
└────────────────┘ └────────────────────┘ ║
|
|
221
|
+
│ ║
|
|
222
|
+
▼ ▼
|
|
223
|
+
┌───────────────────┐ ┌────────────────┐
|
|
224
|
+
│ Encapsulated (W3) │ │ Follow-on (W6) │
|
|
225
|
+
└───────────────────┘ └────────────────┘
|
|
226
|
+
│ │
|
|
227
|
+
┌───────┴────────┐ │
|
|
228
|
+
▼ ▼ ▼
|
|
229
|
+
┌──────────────┐ ┌──────────────┐ ┌──────────────┐
|
|
230
|
+
│ Dummy 1 (W4) │ │ Dummy 2 (W5) │ │ Last (W6) │
|
|
231
|
+
└──────────────┘ └──────────────┘ └──────────────┘
|
|
232
232
|
|
|
233
233
|
The Wn numbers denote the worker processes that a particular job is run in. `Deferring`
|
|
234
234
|
adds a deferred function and then runs for a long time. The deferred function will be
|
|
@@ -313,28 +313,28 @@ class AutoDeploymentTest(ApplianceTestSupport):
|
|
|
313
313
|
|
|
314
314
|
def testDeferralWithFailureAndEncapsulation(self):
|
|
315
315
|
"""
|
|
316
|
-
Ensure that the following DAG succeeds
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
316
|
+
Ensure that the following DAG succeeds::
|
|
317
|
+
|
|
318
|
+
┌───────────┐
|
|
319
|
+
│ Root (W1) │
|
|
320
|
+
└───────────┘
|
|
321
|
+
│
|
|
322
|
+
┌──────────┴─────────┐
|
|
323
|
+
▼ ▼
|
|
324
|
+
┌────────────────┐ ┌────────────────────┐
|
|
325
|
+
│ Deferring (W2) │ │ Encapsulating (W3) │═══════════════════════╗
|
|
326
|
+
└────────────────┘ └────────────────────┘ ║
|
|
327
|
+
│ ║
|
|
328
|
+
▼ ▼
|
|
329
|
+
┌───────────────────┐ ┌────────────────┐
|
|
330
|
+
│ Encapsulated (W3) │════════════╗ │ Follow-on (W7) │
|
|
331
|
+
└───────────────────┘ ║ └────────────────┘
|
|
332
|
+
│ ║
|
|
333
|
+
┌──────┴──────┐ ║
|
|
334
|
+
▼ ▼ ▼
|
|
335
|
+
┌────────────┐┌────────────┐ ┌──────────────┐
|
|
336
|
+
│ Dummy (W4) ││ Dummy (W5) │ │ Trigger (W6) │
|
|
337
|
+
└────────────┘└────────────┘ └──────────────┘
|
|
338
338
|
|
|
339
339
|
`Trigger` causes `Deferring` to crash. `Follow-on` runs next, detects `Deferring`'s
|
|
340
340
|
left-overs and runs the deferred function. `Follow-on` is an instance of `Job` and the
|
toil/test/src/busTest.py
CHANGED
|
@@ -15,17 +15,17 @@
|
|
|
15
15
|
import logging
|
|
16
16
|
import os
|
|
17
17
|
from threading import Thread, current_thread
|
|
18
|
-
from typing import Optional
|
|
19
18
|
|
|
20
19
|
from toil.batchSystems.abstractBatchSystem import BatchJobExitReason
|
|
21
|
-
from toil.bus import JobCompletedMessage,
|
|
20
|
+
from toil.bus import (JobCompletedMessage,
|
|
21
|
+
JobIssuedMessage,
|
|
22
|
+
MessageBus,
|
|
23
|
+
replay_message_bus)
|
|
22
24
|
from toil.common import Toil
|
|
23
|
-
from toil.job import Job
|
|
24
25
|
from toil.exceptions import FailedJobsException
|
|
26
|
+
from toil.job import Job
|
|
25
27
|
from toil.test import ToilTest, get_temp_file
|
|
26
28
|
|
|
27
|
-
|
|
28
|
-
|
|
29
29
|
logger = logging.getLogger(__name__)
|
|
30
30
|
|
|
31
31
|
class MessageBusTest(ToilTest):
|
toil/test/src/checkpointTest.py
CHANGED
|
@@ -12,9 +12,9 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
from toil.exceptions import FailedJobsException
|
|
15
16
|
from toil.job import Job
|
|
16
17
|
from toil.jobStores.abstractJobStore import NoSuchFileException
|
|
17
|
-
from toil.exceptions import FailedJobsException
|
|
18
18
|
from toil.test import ToilTest, slow
|
|
19
19
|
|
|
20
20
|
|
|
@@ -82,7 +82,7 @@ class CheckRetryCount(Job):
|
|
|
82
82
|
|
|
83
83
|
def run(self, fileStore):
|
|
84
84
|
retryCount = self.getNumRetries(fileStore)
|
|
85
|
-
fileStore.
|
|
85
|
+
fileStore.log_to_leader(str(retryCount))
|
|
86
86
|
if retryCount < self.numFailuresBeforeSuccess:
|
|
87
87
|
self.addChild(AlwaysFail())
|
|
88
88
|
|
toil/test/src/fileStoreTest.py
CHANGED
|
@@ -29,14 +29,18 @@ from uuid import uuid4
|
|
|
29
29
|
import pytest
|
|
30
30
|
|
|
31
31
|
from toil.common import Toil
|
|
32
|
+
from toil.exceptions import FailedJobsException
|
|
32
33
|
from toil.fileStores import FileID
|
|
33
34
|
from toil.fileStores.cachingFileStore import (CacheUnbalancedError,
|
|
34
35
|
IllegalDeletionCacheError)
|
|
35
36
|
from toil.job import Job
|
|
36
37
|
from toil.jobStores.abstractJobStore import NoSuchFileException
|
|
37
|
-
from toil.exceptions import FailedJobsException
|
|
38
38
|
from toil.realtimeLogger import RealtimeLogger
|
|
39
|
-
from toil.test import ToilTest,
|
|
39
|
+
from toil.test import (ToilTest,
|
|
40
|
+
needs_aws_ec2,
|
|
41
|
+
needs_google_project,
|
|
42
|
+
needs_google_storage,
|
|
43
|
+
slow)
|
|
40
44
|
|
|
41
45
|
# Some tests take too long on the AWS jobstore and are unquitable for CI. They can be
|
|
42
46
|
# be run during manual tests by setting this to False.
|
|
@@ -397,6 +401,7 @@ class hidden:
|
|
|
397
401
|
self.options.caching = True
|
|
398
402
|
|
|
399
403
|
@slow
|
|
404
|
+
@pytest.mark.xfail(reason="Cannot succeed in time on small CI runners")
|
|
400
405
|
def testExtremeCacheSetup(self):
|
|
401
406
|
"""
|
|
402
407
|
Try to create the cache with bad worker active and then have 10 child jobs try to run in
|
|
@@ -472,7 +477,15 @@ class hidden:
|
|
|
472
477
|
# the cache hence this test is redundant (caching will be free).
|
|
473
478
|
if not self.options.jobStore.startswith(('aws', 'google')):
|
|
474
479
|
workDirDev = os.stat(self.options.workDir).st_dev
|
|
475
|
-
|
|
480
|
+
if self.options.jobStore.startswith("file:"):
|
|
481
|
+
# Before #4538, options.jobStore would have the raw path while the Config object would prepend the
|
|
482
|
+
# filesystem to the path (/path/to/file vs file:/path/to/file)
|
|
483
|
+
# The options namespace and the Config object now have the exact same behavior
|
|
484
|
+
# which means parse_jobstore will be called with argparse rather than with the config object
|
|
485
|
+
# so remove the prepended file: scheme
|
|
486
|
+
jobStoreDev = os.stat(os.path.dirname(self.options.jobStore[5:])).st_dev
|
|
487
|
+
else:
|
|
488
|
+
jobStoreDev = os.stat(os.path.dirname(self.options.jobStore)).st_dev
|
|
476
489
|
if workDirDev == jobStoreDev:
|
|
477
490
|
self.skipTest('Job store and working directory are on the same filesystem.')
|
|
478
491
|
|
|
@@ -633,6 +646,8 @@ class hidden:
|
|
|
633
646
|
|
|
634
647
|
Attempting to get the file from the jobstore should not fail.
|
|
635
648
|
"""
|
|
649
|
+
print("Testing")
|
|
650
|
+
logger.debug("Testing testing 123")
|
|
636
651
|
self.options.retryCount = 0
|
|
637
652
|
self.options.logLevel = 'DEBUG'
|
|
638
653
|
A = Job.wrapJobFn(self._adjustCacheLimit, newTotalMB=1024, disk='1G')
|
|
@@ -655,20 +670,20 @@ class hidden:
|
|
|
655
670
|
:param fileMB: File Size
|
|
656
671
|
:return: Job store file ID for second written file
|
|
657
672
|
"""
|
|
658
|
-
job.fileStore.
|
|
673
|
+
job.fileStore.log_to_leader('Double writing a file into job store')
|
|
659
674
|
work_dir = job.fileStore.getLocalTempDir()
|
|
660
675
|
with open(os.path.join(work_dir, str(uuid4())), 'wb') as testFile:
|
|
661
676
|
testFile.write(os.urandom(fileMB * 1024 * 1024))
|
|
662
677
|
|
|
663
|
-
job.fileStore.
|
|
678
|
+
job.fileStore.log_to_leader('Writing copy 1 and discarding ID')
|
|
664
679
|
job.fileStore.writeGlobalFile(testFile.name)
|
|
665
|
-
job.fileStore.
|
|
680
|
+
job.fileStore.log_to_leader('Writing copy 2 and saving ID')
|
|
666
681
|
fsID = job.fileStore.writeGlobalFile(testFile.name)
|
|
667
|
-
job.fileStore.
|
|
682
|
+
job.fileStore.log_to_leader(f'Copy 2 ID: {fsID}')
|
|
668
683
|
|
|
669
684
|
hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAssertions(job, fsID)
|
|
670
685
|
|
|
671
|
-
job.fileStore.
|
|
686
|
+
job.fileStore.log_to_leader('Writing copy 3 and returning ID')
|
|
672
687
|
return job.fileStore.writeGlobalFile(testFile.name)
|
|
673
688
|
|
|
674
689
|
@staticmethod
|
|
@@ -680,7 +695,7 @@ class hidden:
|
|
|
680
695
|
:param fsID: Job store file ID for the read file
|
|
681
696
|
:return: None
|
|
682
697
|
"""
|
|
683
|
-
job.fileStore.
|
|
698
|
+
job.fileStore.log_to_leader('Reading the written file')
|
|
684
699
|
job.fileStore.readGlobalFile(fsID)
|
|
685
700
|
|
|
686
701
|
# writeGlobalFile tests
|
|
@@ -794,8 +809,7 @@ class hidden:
|
|
|
794
809
|
"""
|
|
795
810
|
Write a local file to the job store (hence adding a copy to cache), then have 10 jobs
|
|
796
811
|
read it. Assert cached file size never goes up, assert unused job
|
|
797
|
-
required disk space is always
|
|
798
|
-
(a multiple of job reqs) - (number of current file readers * filesize).
|
|
812
|
+
required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
|
|
799
813
|
At the end, assert the cache shows unused job-required space = 0.
|
|
800
814
|
"""
|
|
801
815
|
self._testMultipleJobsReadGlobalFileFunction(cacheHit=True)
|
|
@@ -805,8 +819,7 @@ class hidden:
|
|
|
805
819
|
"""
|
|
806
820
|
Write a non-local file to the job store(hence no cached copy), then have 10 jobs read
|
|
807
821
|
it. Assert cached file size never goes up, assert unused job
|
|
808
|
-
required disk space is always
|
|
809
|
-
(a multiple of job reqs) - (number of current file readers * filesize).
|
|
822
|
+
required disk space is always ``(a multiple of job reqs) - (number of current file readers * filesize)``.
|
|
810
823
|
At the end, assert the cache shows unused job-required space = 0.
|
|
811
824
|
"""
|
|
812
825
|
self._testMultipleJobsReadGlobalFileFunction(cacheHit=False)
|
|
@@ -834,7 +847,10 @@ class hidden:
|
|
|
834
847
|
jobs[i].addChild(B)
|
|
835
848
|
Job.Runner.startToil(A, self.options)
|
|
836
849
|
with open(x.name) as y:
|
|
837
|
-
|
|
850
|
+
# At least one job at a time should have been observed.
|
|
851
|
+
# We can't actually guarantee that any of our jobs will
|
|
852
|
+
# see each other currently running.
|
|
853
|
+
assert int(y.read()) > 1
|
|
838
854
|
|
|
839
855
|
@staticmethod
|
|
840
856
|
def _multipleFileReader(job, diskMB, fsID, maxWriteFile):
|
|
@@ -1121,7 +1137,7 @@ class hidden:
|
|
|
1121
1137
|
"""
|
|
1122
1138
|
|
|
1123
1139
|
# Make sure we actually have the disk size we are supposed to
|
|
1124
|
-
job.fileStore.
|
|
1140
|
+
job.fileStore.log_to_leader('Job is running with %d bytes of disk, %d requested' % (job.disk, jobDisk))
|
|
1125
1141
|
assert job.disk == jobDisk, 'Job was scheduled with %d bytes but requested %d' % (job.disk, jobDisk)
|
|
1126
1142
|
|
|
1127
1143
|
cls = hidden.AbstractCachingFileStoreTest
|
|
@@ -1187,7 +1203,7 @@ class hidden:
|
|
|
1187
1203
|
try:
|
|
1188
1204
|
job.fileStore.deleteLocalFile(fileToDelete)
|
|
1189
1205
|
except IllegalDeletionCacheError:
|
|
1190
|
-
job.fileStore.
|
|
1206
|
+
job.fileStore.log_to_leader('Detected a deleted file %s.' % fileToDelete)
|
|
1191
1207
|
os.rename(tempfile, outfile)
|
|
1192
1208
|
else:
|
|
1193
1209
|
# If we are processing the write test, or if we are testing the immutably read
|
toil/test/src/helloWorldTest.py
CHANGED
|
@@ -24,7 +24,7 @@ class HelloWorldTest(ToilTest):
|
|
|
24
24
|
|
|
25
25
|
class HelloWorld(Job):
|
|
26
26
|
def __init__(self):
|
|
27
|
-
Job.__init__(self, memory=100000, cores=
|
|
27
|
+
Job.__init__(self, memory=100000, cores=1, disk="3G")
|
|
28
28
|
|
|
29
29
|
def run(self, fileStore):
|
|
30
30
|
fileID = self.addChildJobFn(childFn, cores=1, memory="1M", disk="3G").rv()
|
|
@@ -18,9 +18,9 @@ import stat
|
|
|
18
18
|
import uuid
|
|
19
19
|
|
|
20
20
|
from toil.common import Toil
|
|
21
|
+
from toil.exceptions import FailedJobsException
|
|
21
22
|
from toil.fileStores import FileID
|
|
22
23
|
from toil.job import Job
|
|
23
|
-
from toil.exceptions import FailedJobsException
|
|
24
24
|
from toil.test import ToilTest, slow
|
|
25
25
|
|
|
26
26
|
|
toil/test/src/jobServiceTest.py
CHANGED
|
@@ -24,9 +24,9 @@ from unittest import skipIf
|
|
|
24
24
|
import pytest
|
|
25
25
|
|
|
26
26
|
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
|
|
27
|
+
from toil.exceptions import FailedJobsException
|
|
27
28
|
from toil.job import Job
|
|
28
29
|
from toil.leader import DeadlockException
|
|
29
|
-
from toil.exceptions import FailedJobsException
|
|
30
30
|
from toil.lib.retry import retry_flaky_test
|
|
31
31
|
from toil.test import ToilTest, get_temp_file, slow
|
|
32
32
|
|
toil/test/src/jobTest.py
CHANGED
|
@@ -20,8 +20,8 @@ import unittest
|
|
|
20
20
|
import pytest
|
|
21
21
|
|
|
22
22
|
from toil.common import Toil
|
|
23
|
-
from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
|
|
24
23
|
from toil.exceptions import FailedJobsException
|
|
24
|
+
from toil.job import Job, JobFunctionWrappingJob, JobGraphDeadlockException
|
|
25
25
|
from toil.test import ToilTest, get_temp_file, slow
|
|
26
26
|
|
|
27
27
|
logger = logging.getLogger(__name__)
|
|
@@ -37,15 +37,15 @@ class JobTest(ToilTest):
|
|
|
37
37
|
@slow
|
|
38
38
|
def testStatic(self):
|
|
39
39
|
r"""
|
|
40
|
-
Create a DAG of jobs non-dynamically and run it. DAG is
|
|
40
|
+
Create a DAG of jobs non-dynamically and run it. DAG is::
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
42
|
+
A -> F
|
|
43
|
+
\-------
|
|
44
|
+
B -> D \
|
|
45
|
+
\ \
|
|
46
|
+
------- C -> E
|
|
47
47
|
|
|
48
|
-
Follow on is marked by
|
|
48
|
+
Follow on is marked by ``->``
|
|
49
49
|
"""
|
|
50
50
|
outFile = get_temp_file(rootDir=self._createTempDir())
|
|
51
51
|
try:
|
|
@@ -81,15 +81,15 @@ class JobTest(ToilTest):
|
|
|
81
81
|
|
|
82
82
|
def testStatic2(self):
|
|
83
83
|
r"""
|
|
84
|
-
Create a DAG of jobs non-dynamically and run it. DAG is
|
|
84
|
+
Create a DAG of jobs non-dynamically and run it. DAG is::
|
|
85
85
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
86
|
+
A -> F
|
|
87
|
+
\-------
|
|
88
|
+
B -> D \
|
|
89
|
+
\ \
|
|
90
|
+
------- C -> E
|
|
91
91
|
|
|
92
|
-
Follow on is marked by
|
|
92
|
+
Follow on is marked by ``->``
|
|
93
93
|
"""
|
|
94
94
|
outFile = get_temp_file(rootDir=self._createTempDir())
|
|
95
95
|
try:
|
|
@@ -269,7 +269,7 @@ class JobTest(ToilTest):
|
|
|
269
269
|
identifies leaf vertices incorrectly
|
|
270
270
|
|
|
271
271
|
Test verification of new checkpoint jobs being leaf verticies,
|
|
272
|
-
starting with the following baseline workflow
|
|
272
|
+
starting with the following baseline workflow::
|
|
273
273
|
|
|
274
274
|
Parent
|
|
275
275
|
|
|
|
@@ -291,7 +291,7 @@ class JobTest(ToilTest):
|
|
|
291
291
|
omits the workflow root job
|
|
292
292
|
|
|
293
293
|
Test verification of a new checkpoint job being leaf vertex,
|
|
294
|
-
starting with a baseline workflow of a single, root job
|
|
294
|
+
starting with a baseline workflow of a single, root job::
|
|
295
295
|
|
|
296
296
|
Root # Checkpoint=True
|
|
297
297
|
|
|
@@ -623,7 +623,7 @@ class JobTest(ToilTest):
|
|
|
623
623
|
return True
|
|
624
624
|
|
|
625
625
|
def simpleJobFn(job, value):
|
|
626
|
-
job.fileStore.
|
|
626
|
+
job.fileStore.log_to_leader(value)
|
|
627
627
|
|
|
628
628
|
def fn1Test(string, outputFile):
|
|
629
629
|
"""
|
toil/test/src/miscTests.py
CHANGED
|
@@ -16,12 +16,14 @@ import logging
|
|
|
16
16
|
import os
|
|
17
17
|
import random
|
|
18
18
|
import sys
|
|
19
|
-
import tempfile
|
|
20
19
|
from uuid import uuid4
|
|
21
20
|
|
|
22
21
|
from toil.common import getNodeID
|
|
23
22
|
from toil.lib.exceptions import panic, raise_
|
|
24
|
-
from toil.lib.io import AtomicFileCreate,
|
|
23
|
+
from toil.lib.io import (AtomicFileCreate,
|
|
24
|
+
atomic_install,
|
|
25
|
+
atomic_tmp_file,
|
|
26
|
+
mkdtemp)
|
|
25
27
|
from toil.lib.misc import CalledProcessErrorStderr, call_command
|
|
26
28
|
from toil.test import ToilTest, slow
|
|
27
29
|
|
|
@@ -63,7 +65,7 @@ class MiscTests(ToilTest):
|
|
|
63
65
|
files = {}
|
|
64
66
|
# Create a random directory structure
|
|
65
67
|
for i in range(0,10):
|
|
66
|
-
directories.append(
|
|
68
|
+
directories.append(mkdtemp(dir=random.choice(directories), prefix='test'))
|
|
67
69
|
# Create 50 random file entries in different locations in the directories. 75% of the time
|
|
68
70
|
# these are fresh files of size [1, 10] MB and 25% of the time they are hard links to old
|
|
69
71
|
# files.
|
|
@@ -48,7 +48,7 @@ class hidden:
|
|
|
48
48
|
cores=1, memory='1M', disk='1M')
|
|
49
49
|
values = Job.Runner.startToil(root, self.getOptions(tempDir))
|
|
50
50
|
maxValue = max(values)
|
|
51
|
-
self.
|
|
51
|
+
self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
|
|
52
52
|
|
|
53
53
|
@slow
|
|
54
54
|
@retry_flaky_test(prepare=[batchSystemTest.hidden.AbstractBatchSystemJobTest.tearDown,
|
|
@@ -74,7 +74,7 @@ class hidden:
|
|
|
74
74
|
disk='1M'))
|
|
75
75
|
Job.Runner.startToil(root, self.getOptions(tempDir))
|
|
76
76
|
_, maxValue = batchSystemTest.getCounters(counterPath)
|
|
77
|
-
self.
|
|
77
|
+
self.assertLessEqual(maxValue, self.cpuCount // coresPerJob)
|
|
78
78
|
|
|
79
79
|
def getOptions(self, tempDir, caching=True):
|
|
80
80
|
options = super().getOptions(tempDir)
|
|
@@ -194,7 +194,7 @@ def logDiskUsage(job, funcName, sleep=0):
|
|
|
194
194
|
:return: job function's disk usage
|
|
195
195
|
"""
|
|
196
196
|
diskUsage = job.disk
|
|
197
|
-
job.fileStore.
|
|
197
|
+
job.fileStore.log_to_leader(f'{funcName}: {diskUsage}')
|
|
198
198
|
time.sleep(sleep)
|
|
199
199
|
return diskUsage
|
|
200
200
|
|
|
@@ -57,7 +57,7 @@ class MessageDetector(logging.StreamHandler):
|
|
|
57
57
|
|
|
58
58
|
class LogTest(Job):
|
|
59
59
|
def __init__(self):
|
|
60
|
-
Job.__init__(self, memory=100000, cores=
|
|
60
|
+
Job.__init__(self, memory=100000, cores=1, disk='3G')
|
|
61
61
|
|
|
62
62
|
def run(self, fileStore):
|
|
63
63
|
RealtimeLogger.info('This should be logged at info level')
|
toil/test/src/resourceTest.py
CHANGED
|
@@ -209,7 +209,7 @@ class ResourceTest(ToilTest):
|
|
|
209
209
|
"""
|
|
210
210
|
|
|
211
211
|
def script():
|
|
212
|
-
import
|
|
212
|
+
from configargparse import ArgumentParser
|
|
213
213
|
|
|
214
214
|
from toil.common import Toil
|
|
215
215
|
from toil.job import Job
|
|
@@ -218,7 +218,7 @@ class ResourceTest(ToilTest):
|
|
|
218
218
|
pass
|
|
219
219
|
|
|
220
220
|
if __name__ == '__main__':
|
|
221
|
-
parser =
|
|
221
|
+
parser = ArgumentParser()
|
|
222
222
|
Job.Runner.addToilOptions(parser)
|
|
223
223
|
options = parser.parse_args()
|
|
224
224
|
job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
|
toil/test/src/restartDAGTest.py
CHANGED
|
@@ -14,17 +14,17 @@
|
|
|
14
14
|
|
|
15
15
|
import os
|
|
16
16
|
|
|
17
|
+
from toil.exceptions import FailedJobsException
|
|
17
18
|
from toil.job import Job
|
|
18
19
|
from toil.jobStores.abstractJobStore import NoSuchFileException
|
|
19
|
-
from toil.exceptions import FailedJobsException
|
|
20
20
|
from toil.test import ToilTest, slow
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
@slow
|
|
24
23
|
class ResumabilityTest(ToilTest):
|
|
25
24
|
"""
|
|
26
25
|
https://github.com/BD2KGenomics/toil/issues/808
|
|
27
26
|
"""
|
|
27
|
+
@slow
|
|
28
28
|
def test(self):
|
|
29
29
|
"""
|
|
30
30
|
Tests that a toil workflow that fails once can be resumed without a NoSuchJobException.
|
|
@@ -53,6 +53,34 @@ class ResumabilityTest(ToilTest):
|
|
|
53
53
|
# store ID: n/t/jobwbijqL failed with exit value 1"
|
|
54
54
|
self.assertTrue("failed with exit value" not in logString)
|
|
55
55
|
|
|
56
|
+
def test_chaining(self):
|
|
57
|
+
"""
|
|
58
|
+
Tests that a job which is chained to and fails can resume and succeed.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
|
|
62
|
+
options.logLevel = "DEBUG"
|
|
63
|
+
options.retryCount = 0
|
|
64
|
+
tempDir = self._createTempDir()
|
|
65
|
+
options.logFile = os.path.join(tempDir, "log.txt")
|
|
66
|
+
|
|
67
|
+
root = Job.wrapJobFn(chaining_parent)
|
|
68
|
+
|
|
69
|
+
with self.assertRaises(FailedJobsException):
|
|
70
|
+
# This one is intended to fail.
|
|
71
|
+
Job.Runner.startToil(root, options)
|
|
72
|
+
|
|
73
|
+
with open(options.logFile, 'r') as f:
|
|
74
|
+
log_content = f.read()
|
|
75
|
+
# Make sure we actually did do chaining
|
|
76
|
+
assert "Chaining from" in log_content
|
|
77
|
+
|
|
78
|
+
# Because of the chaining, the problem we are looking for is the job
|
|
79
|
+
# with the root ID not being able to load the body of a job with a
|
|
80
|
+
# different ID. That doesn't look like a job deleted despite failure.
|
|
81
|
+
options.restart = True
|
|
82
|
+
Job.Runner.startToil(root, options)
|
|
83
|
+
|
|
56
84
|
def parent(job):
|
|
57
85
|
"""
|
|
58
86
|
Set up a bunch of dummy child jobs, and a bad job that needs to be
|
|
@@ -62,6 +90,12 @@ def parent(job):
|
|
|
62
90
|
job.addChildJobFn(goodChild)
|
|
63
91
|
job.addFollowOnJobFn(badChild)
|
|
64
92
|
|
|
93
|
+
def chaining_parent(job):
|
|
94
|
+
"""
|
|
95
|
+
Set up a failing job to chain to.
|
|
96
|
+
"""
|
|
97
|
+
job.addFollowOnJobFn(badChild)
|
|
98
|
+
|
|
65
99
|
def goodChild(job):
|
|
66
100
|
"""
|
|
67
101
|
Does nothing.
|
toil/test/src/systemTest.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import errno
|
|
2
2
|
import multiprocessing
|
|
3
3
|
import os
|
|
4
|
-
import tempfile
|
|
5
4
|
from functools import partial
|
|
6
5
|
|
|
6
|
+
from toil.lib.io import mkdtemp
|
|
7
7
|
from toil.lib.threading import cpu_count
|
|
8
8
|
from toil.test import ToilTest
|
|
9
9
|
|
|
@@ -37,7 +37,7 @@ class SystemTest(ToilTest):
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
def _testAtomicityOfNonEmptyDirectoryRenamesTask(parent, child, _):
|
|
40
|
-
tmpChildDir =
|
|
40
|
+
tmpChildDir = mkdtemp(dir=parent, prefix='child', suffix='.tmp')
|
|
41
41
|
grandChild = os.path.join(tmpChildDir, 'grandChild')
|
|
42
42
|
open(grandChild, 'w').close()
|
|
43
43
|
grandChildId = os.stat(grandChild).st_ino
|
|
@@ -61,10 +61,10 @@ class ToilContextManagerTest(ToilTest):
|
|
|
61
61
|
|
|
62
62
|
class HelloWorld(Job):
|
|
63
63
|
def __init__(self):
|
|
64
|
-
Job.__init__(self, memory=100000,
|
|
64
|
+
Job.__init__(self, memory=100000, disk='1M')
|
|
65
65
|
|
|
66
66
|
def run(self, fileStore):
|
|
67
|
-
fileID = self.addChildJobFn(childFn,
|
|
67
|
+
fileID = self.addChildJobFn(childFn, memory='1M', disk='1M').rv()
|
|
68
68
|
return self.addFollowOn(FollowOn(fileID)).rv()
|
|
69
69
|
|
|
70
70
|
|
|
@@ -59,7 +59,7 @@ class UserDefinedJobArgTypeTest(ToilTest):
|
|
|
59
59
|
|
|
60
60
|
class JobClass(Job):
|
|
61
61
|
def __init__(self, level, foo):
|
|
62
|
-
Job.__init__(self, memory=100000, cores=
|
|
62
|
+
Job.__init__(self, memory=100000, cores=1, disk="300M")
|
|
63
63
|
self.level = level
|
|
64
64
|
self.foo = foo
|
|
65
65
|
|