toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +121 -83
- toil/batchSystems/__init__.py +1 -0
- toil/batchSystems/abstractBatchSystem.py +137 -77
- toil/batchSystems/abstractGridEngineBatchSystem.py +211 -101
- toil/batchSystems/awsBatch.py +237 -128
- toil/batchSystems/cleanup_support.py +22 -16
- toil/batchSystems/contained_executor.py +30 -26
- toil/batchSystems/gridengine.py +85 -49
- toil/batchSystems/htcondor.py +164 -87
- toil/batchSystems/kubernetes.py +622 -386
- toil/batchSystems/local_support.py +17 -12
- toil/batchSystems/lsf.py +132 -79
- toil/batchSystems/lsfHelper.py +13 -11
- toil/batchSystems/mesos/__init__.py +41 -29
- toil/batchSystems/mesos/batchSystem.py +288 -149
- toil/batchSystems/mesos/executor.py +77 -49
- toil/batchSystems/mesos/test/__init__.py +31 -23
- toil/batchSystems/options.py +38 -29
- toil/batchSystems/registry.py +53 -19
- toil/batchSystems/singleMachine.py +293 -123
- toil/batchSystems/slurm.py +489 -137
- toil/batchSystems/torque.py +46 -32
- toil/bus.py +141 -73
- toil/common.py +630 -359
- toil/cwl/__init__.py +1 -1
- toil/cwl/cwltoil.py +1114 -532
- toil/cwl/utils.py +17 -22
- toil/deferred.py +62 -41
- toil/exceptions.py +5 -3
- toil/fileStores/__init__.py +5 -5
- toil/fileStores/abstractFileStore.py +88 -57
- toil/fileStores/cachingFileStore.py +711 -247
- toil/fileStores/nonCachingFileStore.py +113 -75
- toil/job.py +988 -315
- toil/jobStores/abstractJobStore.py +387 -243
- toil/jobStores/aws/jobStore.py +727 -403
- toil/jobStores/aws/utils.py +161 -109
- toil/jobStores/conftest.py +1 -0
- toil/jobStores/fileJobStore.py +289 -151
- toil/jobStores/googleJobStore.py +137 -70
- toil/jobStores/utils.py +36 -15
- toil/leader.py +614 -269
- toil/lib/accelerators.py +115 -18
- toil/lib/aws/__init__.py +55 -28
- toil/lib/aws/ami.py +122 -87
- toil/lib/aws/iam.py +284 -108
- toil/lib/aws/s3.py +31 -0
- toil/lib/aws/session.py +193 -58
- toil/lib/aws/utils.py +238 -218
- toil/lib/bioio.py +13 -5
- toil/lib/compatibility.py +11 -6
- toil/lib/conversions.py +83 -49
- toil/lib/docker.py +131 -103
- toil/lib/ec2.py +322 -209
- toil/lib/ec2nodes.py +174 -106
- toil/lib/encryption/_dummy.py +5 -3
- toil/lib/encryption/_nacl.py +10 -6
- toil/lib/encryption/conftest.py +1 -0
- toil/lib/exceptions.py +26 -7
- toil/lib/expando.py +4 -2
- toil/lib/ftp_utils.py +217 -0
- toil/lib/generatedEC2Lists.py +127 -19
- toil/lib/humanize.py +6 -2
- toil/lib/integration.py +341 -0
- toil/lib/io.py +99 -11
- toil/lib/iterables.py +4 -2
- toil/lib/memoize.py +12 -8
- toil/lib/misc.py +65 -18
- toil/lib/objects.py +2 -2
- toil/lib/resources.py +19 -7
- toil/lib/retry.py +115 -77
- toil/lib/threading.py +282 -80
- toil/lib/throttle.py +15 -14
- toil/options/common.py +834 -401
- toil/options/cwl.py +175 -90
- toil/options/runner.py +50 -0
- toil/options/wdl.py +70 -19
- toil/provisioners/__init__.py +111 -46
- toil/provisioners/abstractProvisioner.py +322 -157
- toil/provisioners/aws/__init__.py +62 -30
- toil/provisioners/aws/awsProvisioner.py +980 -627
- toil/provisioners/clusterScaler.py +541 -279
- toil/provisioners/gceProvisioner.py +282 -179
- toil/provisioners/node.py +147 -79
- toil/realtimeLogger.py +34 -22
- toil/resource.py +137 -75
- toil/server/app.py +127 -61
- toil/server/celery_app.py +3 -1
- toil/server/cli/wes_cwl_runner.py +82 -53
- toil/server/utils.py +54 -28
- toil/server/wes/abstract_backend.py +64 -26
- toil/server/wes/amazon_wes_utils.py +21 -15
- toil/server/wes/tasks.py +121 -63
- toil/server/wes/toil_backend.py +142 -107
- toil/server/wsgi_app.py +4 -3
- toil/serviceManager.py +58 -22
- toil/statsAndLogging.py +148 -64
- toil/test/__init__.py +263 -179
- toil/test/batchSystems/batchSystemTest.py +438 -195
- toil/test/batchSystems/batch_system_plugin_test.py +18 -7
- toil/test/batchSystems/test_gridengine.py +173 -0
- toil/test/batchSystems/test_lsf_helper.py +67 -58
- toil/test/batchSystems/test_slurm.py +93 -47
- toil/test/cactus/test_cactus_integration.py +20 -22
- toil/test/cwl/cwlTest.py +271 -71
- toil/test/cwl/measure_default_memory.cwl +12 -0
- toil/test/cwl/not_run_required_input.cwl +29 -0
- toil/test/cwl/scatter_duplicate_outputs.cwl +40 -0
- toil/test/docs/scriptsTest.py +60 -34
- toil/test/jobStores/jobStoreTest.py +412 -235
- toil/test/lib/aws/test_iam.py +116 -48
- toil/test/lib/aws/test_s3.py +16 -9
- toil/test/lib/aws/test_utils.py +5 -6
- toil/test/lib/dockerTest.py +118 -141
- toil/test/lib/test_conversions.py +113 -115
- toil/test/lib/test_ec2.py +57 -49
- toil/test/lib/test_integration.py +104 -0
- toil/test/lib/test_misc.py +12 -5
- toil/test/mesos/MesosDataStructuresTest.py +23 -10
- toil/test/mesos/helloWorld.py +7 -6
- toil/test/mesos/stress.py +25 -20
- toil/test/options/options.py +7 -2
- toil/test/provisioners/aws/awsProvisionerTest.py +293 -140
- toil/test/provisioners/clusterScalerTest.py +440 -250
- toil/test/provisioners/clusterTest.py +81 -42
- toil/test/provisioners/gceProvisionerTest.py +174 -100
- toil/test/provisioners/provisionerTest.py +25 -13
- toil/test/provisioners/restartScript.py +5 -4
- toil/test/server/serverTest.py +188 -141
- toil/test/sort/restart_sort.py +137 -68
- toil/test/sort/sort.py +134 -66
- toil/test/sort/sortTest.py +91 -49
- toil/test/src/autoDeploymentTest.py +140 -100
- toil/test/src/busTest.py +20 -18
- toil/test/src/checkpointTest.py +8 -2
- toil/test/src/deferredFunctionTest.py +49 -35
- toil/test/src/dockerCheckTest.py +33 -26
- toil/test/src/environmentTest.py +20 -10
- toil/test/src/fileStoreTest.py +538 -271
- toil/test/src/helloWorldTest.py +7 -4
- toil/test/src/importExportFileTest.py +61 -31
- toil/test/src/jobDescriptionTest.py +32 -17
- toil/test/src/jobEncapsulationTest.py +2 -0
- toil/test/src/jobFileStoreTest.py +74 -50
- toil/test/src/jobServiceTest.py +187 -73
- toil/test/src/jobTest.py +120 -70
- toil/test/src/miscTests.py +19 -18
- toil/test/src/promisedRequirementTest.py +82 -36
- toil/test/src/promisesTest.py +7 -6
- toil/test/src/realtimeLoggerTest.py +6 -6
- toil/test/src/regularLogTest.py +71 -37
- toil/test/src/resourceTest.py +80 -49
- toil/test/src/restartDAGTest.py +36 -22
- toil/test/src/resumabilityTest.py +9 -2
- toil/test/src/retainTempDirTest.py +45 -14
- toil/test/src/systemTest.py +12 -8
- toil/test/src/threadingTest.py +44 -25
- toil/test/src/toilContextManagerTest.py +10 -7
- toil/test/src/userDefinedJobArgTypeTest.py +8 -5
- toil/test/src/workerTest.py +33 -16
- toil/test/utils/toilDebugTest.py +70 -58
- toil/test/utils/toilKillTest.py +4 -5
- toil/test/utils/utilsTest.py +239 -102
- toil/test/wdl/wdltoil_test.py +789 -148
- toil/test/wdl/wdltoil_test_kubernetes.py +37 -23
- toil/toilState.py +52 -26
- toil/utils/toilConfig.py +13 -4
- toil/utils/toilDebugFile.py +44 -27
- toil/utils/toilDebugJob.py +85 -25
- toil/utils/toilDestroyCluster.py +11 -6
- toil/utils/toilKill.py +8 -3
- toil/utils/toilLaunchCluster.py +251 -145
- toil/utils/toilMain.py +37 -16
- toil/utils/toilRsyncCluster.py +27 -14
- toil/utils/toilSshCluster.py +45 -22
- toil/utils/toilStats.py +75 -36
- toil/utils/toilStatus.py +226 -119
- toil/utils/toilUpdateEC2Instances.py +3 -1
- toil/version.py +11 -11
- toil/wdl/utils.py +5 -5
- toil/wdl/wdltoil.py +3513 -1052
- toil/worker.py +269 -128
- toil-8.0.0.dist-info/METADATA +173 -0
- toil-8.0.0.dist-info/RECORD +253 -0
- {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/WHEEL +1 -1
- toil-7.0.0.dist-info/METADATA +0 -158
- toil-7.0.0.dist-info/RECORD +0 -244
- {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/LICENSE +0 -0
- {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/entry_points.txt +0 -0
- {toil-7.0.0.dist-info → toil-8.0.0.dist-info}/top_level.txt +0 -0
toil/test/mesos/helloWorld.py
CHANGED
|
@@ -23,14 +23,15 @@ from toil.job import Job
|
|
|
23
23
|
childMessage = "The child job is now running!"
|
|
24
24
|
parentMessage = "The parent job is now running!"
|
|
25
25
|
|
|
26
|
+
|
|
26
27
|
def hello_world(job):
|
|
27
28
|
|
|
28
29
|
job.fileStore.log_to_leader(parentMessage)
|
|
29
|
-
with open(
|
|
30
|
-
handle.write(
|
|
30
|
+
with open("foo_bam.txt", "w") as handle:
|
|
31
|
+
handle.write("\nThis is a triumph...\n")
|
|
31
32
|
|
|
32
33
|
# Assign FileStoreID to a given file
|
|
33
|
-
foo_bam = job.fileStore.writeGlobalFile(
|
|
34
|
+
foo_bam = job.fileStore.writeGlobalFile("foo_bam.txt")
|
|
34
35
|
|
|
35
36
|
# Spawn child
|
|
36
37
|
job.addChildJobFn(hello_world_child, foo_bam, memory=100, cores=0.5, disk="3G")
|
|
@@ -43,13 +44,13 @@ def hello_world_child(job, hw):
|
|
|
43
44
|
# NOTE: path and the udpated file are stored to /tmp
|
|
44
45
|
# If we want to SAVE our changes to this tmp file, we must write it out.
|
|
45
46
|
with open(path) as r:
|
|
46
|
-
with open(
|
|
47
|
+
with open("bar_bam.txt", "w") as handle:
|
|
47
48
|
for line in r.readlines():
|
|
48
49
|
handle.write(line)
|
|
49
50
|
|
|
50
51
|
# Assign FileStoreID to a given file
|
|
51
52
|
# can also use: job.updateGlobalFile() given the FileStoreID instantiation.
|
|
52
|
-
job.fileStore.writeGlobalFile(
|
|
53
|
+
job.fileStore.writeGlobalFile("bar_bam.txt")
|
|
53
54
|
|
|
54
55
|
|
|
55
56
|
def main():
|
|
@@ -67,5 +68,5 @@ def main():
|
|
|
67
68
|
toil.start(i)
|
|
68
69
|
|
|
69
70
|
|
|
70
|
-
if __name__ ==
|
|
71
|
+
if __name__ == "__main__":
|
|
71
72
|
main()
|
toil/test/mesos/stress.py
CHANGED
|
@@ -17,58 +17,63 @@ from configargparse import ArgumentParser
|
|
|
17
17
|
from toil.job import Job
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
def touchFile(
|
|
20
|
+
def touchFile(fileStore):
|
|
21
21
|
with fileStore.writeGlobalFileStream() as (f, id):
|
|
22
|
-
f.write(
|
|
22
|
+
f.write("This is a triumph")
|
|
23
|
+
|
|
23
24
|
|
|
24
25
|
class LongTestJob(Job):
|
|
25
26
|
def __init__(self, numJobs):
|
|
26
|
-
Job.__init__(self,
|
|
27
|
+
Job.__init__(self, memory=100000, cores=0.01)
|
|
27
28
|
self.numJobs = numJobs
|
|
28
29
|
|
|
29
30
|
def run(self, fileStore):
|
|
30
|
-
for i in range(0,self.numJobs):
|
|
31
|
+
for i in range(0, self.numJobs):
|
|
31
32
|
self.addChild(HelloWorldJob(i))
|
|
32
33
|
self.addFollowOn(LongTestFollowOn())
|
|
33
34
|
|
|
35
|
+
|
|
34
36
|
class LongTestFollowOn(Job):
|
|
35
37
|
|
|
36
38
|
def __init__(self):
|
|
37
|
-
Job.__init__(self,
|
|
39
|
+
Job.__init__(self, memory=1000000, cores=0.01)
|
|
38
40
|
|
|
39
41
|
def run(self, fileStore):
|
|
40
|
-
touchFile(
|
|
42
|
+
touchFile(fileStore)
|
|
41
43
|
|
|
42
|
-
class HelloWorldJob(Job):
|
|
43
44
|
|
|
44
|
-
|
|
45
|
-
Job.__init__(self, memory=100000, cores=0.01)
|
|
46
|
-
self.i=i
|
|
45
|
+
class HelloWorldJob(Job):
|
|
47
46
|
|
|
47
|
+
def __init__(self, i):
|
|
48
|
+
Job.__init__(self, memory=100000, cores=0.01)
|
|
49
|
+
self.i = i
|
|
48
50
|
|
|
49
51
|
def run(self, fileStore):
|
|
50
|
-
touchFile(
|
|
52
|
+
touchFile(fileStore)
|
|
51
53
|
self.addFollowOn(HelloWorldFollowOn(self.i))
|
|
52
54
|
|
|
55
|
+
|
|
53
56
|
class HelloWorldFollowOn(Job):
|
|
54
57
|
|
|
55
|
-
def __init__(self,i):
|
|
56
|
-
Job.__init__(self,
|
|
58
|
+
def __init__(self, i):
|
|
59
|
+
Job.__init__(self, memory=200000, cores=0.01)
|
|
57
60
|
self.i = i
|
|
58
61
|
|
|
59
62
|
def run(self, fileStore):
|
|
60
|
-
touchFile(
|
|
63
|
+
touchFile(fileStore)
|
|
64
|
+
|
|
61
65
|
|
|
62
66
|
def main(numJobs):
|
|
63
67
|
# Boilerplate -- startToil requires options
|
|
64
68
|
parser = ArgumentParser()
|
|
65
69
|
Job.Runner.addToilOptions(parser)
|
|
66
|
-
options = parser.parse_args(
|
|
67
|
-
options.batchSystem="mesos"
|
|
68
|
-
options.mesos_endpoint="localhost:5050"
|
|
70
|
+
options = parser.parse_args(args=["./toilTest"])
|
|
71
|
+
options.batchSystem = "mesos"
|
|
72
|
+
options.mesos_endpoint = "localhost:5050"
|
|
69
73
|
# Launch first toil Job
|
|
70
|
-
i = LongTestJob(
|
|
71
|
-
Job.Runner.startToil(i,
|
|
74
|
+
i = LongTestJob(numJobs)
|
|
75
|
+
Job.Runner.startToil(i, options)
|
|
76
|
+
|
|
72
77
|
|
|
73
|
-
if __name__=="__main__":
|
|
78
|
+
if __name__ == "__main__":
|
|
74
79
|
main(numJobs=5)
|
toil/test/options/options.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from configargparse import ArgParser
|
|
2
2
|
|
|
3
|
-
from toil.common import
|
|
3
|
+
from toil.common import Toil, addOptions
|
|
4
4
|
from toil.test import ToilTest
|
|
5
5
|
|
|
6
6
|
|
|
@@ -8,6 +8,7 @@ class OptionsTest(ToilTest):
|
|
|
8
8
|
"""
|
|
9
9
|
Class to test functionality of all Toil options
|
|
10
10
|
"""
|
|
11
|
+
|
|
11
12
|
def test_default_caching_slurm(self):
|
|
12
13
|
"""
|
|
13
14
|
Test to ensure that caching will be set to false when running on Slurm
|
|
@@ -30,7 +31,11 @@ class OptionsTest(ToilTest):
|
|
|
30
31
|
addOptions(parser, jobstore_as_flag=True, wdl=False, cwl=False)
|
|
31
32
|
# the kubernetes batchsystem (and I think all batchsystems including singlemachine) return False
|
|
32
33
|
# for default_caching
|
|
33
|
-
test_args = [
|
|
34
|
+
test_args = [
|
|
35
|
+
"--jobstore=example-jobstore",
|
|
36
|
+
"--batchSystem=kubernetes",
|
|
37
|
+
"--caching=True",
|
|
38
|
+
]
|
|
34
39
|
options = parser.parse_args(test_args)
|
|
35
40
|
with Toil(options) as toil:
|
|
36
41
|
caching_value = toil.config.caching
|