toil 8.1.0b1__py3-none-any.whl → 9.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toil/__init__.py +0 -35
- toil/batchSystems/abstractBatchSystem.py +1 -1
- toil/batchSystems/abstractGridEngineBatchSystem.py +1 -1
- toil/batchSystems/awsBatch.py +1 -1
- toil/batchSystems/cleanup_support.py +1 -1
- toil/batchSystems/kubernetes.py +53 -7
- toil/batchSystems/local_support.py +1 -1
- toil/batchSystems/mesos/batchSystem.py +13 -8
- toil/batchSystems/mesos/test/__init__.py +3 -2
- toil/batchSystems/registry.py +15 -118
- toil/batchSystems/singleMachine.py +1 -1
- toil/batchSystems/slurm.py +27 -26
- toil/bus.py +5 -3
- toil/common.py +59 -12
- toil/cwl/cwltoil.py +81 -38
- toil/cwl/utils.py +103 -3
- toil/job.py +64 -49
- toil/jobStores/abstractJobStore.py +35 -239
- toil/jobStores/aws/jobStore.py +2 -1
- toil/jobStores/fileJobStore.py +27 -2
- toil/jobStores/googleJobStore.py +110 -33
- toil/leader.py +9 -0
- toil/lib/accelerators.py +4 -2
- toil/lib/aws/utils.py.orig +504 -0
- toil/lib/bioio.py +1 -1
- toil/lib/docker.py +252 -91
- toil/lib/dockstore.py +11 -3
- toil/lib/exceptions.py +5 -3
- toil/lib/generatedEC2Lists.py +81 -19
- toil/lib/history.py +87 -13
- toil/lib/history_submission.py +23 -9
- toil/lib/io.py +34 -22
- toil/lib/misc.py +8 -2
- toil/lib/plugins.py +106 -0
- toil/lib/resources.py +2 -1
- toil/lib/threading.py +11 -10
- toil/lib/url.py +320 -0
- toil/options/common.py +8 -0
- toil/options/cwl.py +13 -1
- toil/options/runner.py +17 -10
- toil/options/wdl.py +22 -0
- toil/provisioners/aws/awsProvisioner.py +25 -2
- toil/server/api_spec/LICENSE +201 -0
- toil/server/api_spec/README.rst +5 -0
- toil/server/app.py +12 -6
- toil/server/cli/wes_cwl_runner.py +3 -2
- toil/server/wes/abstract_backend.py +21 -43
- toil/server/wes/toil_backend.py +2 -2
- toil/test/__init__.py +275 -115
- toil/test/batchSystems/batchSystemTest.py +228 -213
- toil/test/batchSystems/batch_system_plugin_test.py +7 -0
- toil/test/batchSystems/test_slurm.py +27 -0
- toil/test/cactus/pestis.tar.gz +0 -0
- toil/test/conftest.py +7 -0
- toil/test/cwl/2.fasta +11 -0
- toil/test/cwl/2.fastq +12 -0
- toil/test/cwl/conftest.py +1 -1
- toil/test/cwl/cwlTest.py +1175 -870
- toil/test/cwl/directory/directory/file.txt +15 -0
- toil/test/cwl/download_directory_file.json +4 -0
- toil/test/cwl/download_directory_s3.json +4 -0
- toil/test/cwl/download_file.json +6 -0
- toil/test/cwl/download_http.json +6 -0
- toil/test/cwl/download_https.json +6 -0
- toil/test/cwl/download_s3.json +6 -0
- toil/test/cwl/download_subdirectory_file.json +5 -0
- toil/test/cwl/download_subdirectory_s3.json +5 -0
- toil/test/cwl/empty.json +1 -0
- toil/test/cwl/mock_mpi/fake_mpi.yml +8 -0
- toil/test/cwl/mock_mpi/fake_mpi_run.py +42 -0
- toil/test/cwl/optional-file-exists.json +6 -0
- toil/test/cwl/optional-file-missing.json +6 -0
- toil/test/cwl/preemptible_expression.json +1 -0
- toil/test/cwl/revsort-job-missing.json +6 -0
- toil/test/cwl/revsort-job.json +6 -0
- toil/test/cwl/s3_secondary_file.json +16 -0
- toil/test/cwl/seqtk_seq_job.json +6 -0
- toil/test/cwl/stream.json +6 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.dat +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f1 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f1i +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f2 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f2_TSM0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f3 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f3_TSM0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f4 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f4_TSM0 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.f5 +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.info +0 -0
- toil/test/cwl/test_filename_conflict_resolution.ms/table.lock +0 -0
- toil/test/cwl/whale.txt +16 -0
- toil/test/docs/scripts/example_alwaysfail.py +38 -0
- toil/test/docs/scripts/example_alwaysfail_with_files.wdl +33 -0
- toil/test/docs/scripts/example_cachingbenchmark.py +117 -0
- toil/test/docs/scripts/stagingExampleFiles/in.txt +1 -0
- toil/test/docs/scripts/stagingExampleFiles/out.txt +2 -0
- toil/test/docs/scripts/tutorial_arguments.py +23 -0
- toil/test/docs/scripts/tutorial_debugging.patch +12 -0
- toil/test/docs/scripts/tutorial_debugging_hangs.wdl +126 -0
- toil/test/docs/scripts/tutorial_debugging_works.wdl +129 -0
- toil/test/docs/scripts/tutorial_docker.py +20 -0
- toil/test/docs/scripts/tutorial_dynamic.py +24 -0
- toil/test/docs/scripts/tutorial_encapsulation.py +28 -0
- toil/test/docs/scripts/tutorial_encapsulation2.py +29 -0
- toil/test/docs/scripts/tutorial_helloworld.py +15 -0
- toil/test/docs/scripts/tutorial_invokeworkflow.py +27 -0
- toil/test/docs/scripts/tutorial_invokeworkflow2.py +30 -0
- toil/test/docs/scripts/tutorial_jobfunctions.py +22 -0
- toil/test/docs/scripts/tutorial_managing.py +29 -0
- toil/test/docs/scripts/tutorial_managing2.py +56 -0
- toil/test/docs/scripts/tutorial_multiplejobs.py +25 -0
- toil/test/docs/scripts/tutorial_multiplejobs2.py +21 -0
- toil/test/docs/scripts/tutorial_multiplejobs3.py +22 -0
- toil/test/docs/scripts/tutorial_promises.py +25 -0
- toil/test/docs/scripts/tutorial_promises2.py +30 -0
- toil/test/docs/scripts/tutorial_quickstart.py +22 -0
- toil/test/docs/scripts/tutorial_requirements.py +44 -0
- toil/test/docs/scripts/tutorial_services.py +45 -0
- toil/test/docs/scripts/tutorial_staging.py +45 -0
- toil/test/docs/scripts/tutorial_stats.py +64 -0
- toil/test/docs/scriptsTest.py +2 -1
- toil/test/lib/aws/test_iam.py +3 -1
- toil/test/lib/dockerTest.py +205 -122
- toil/test/lib/test_history.py +101 -77
- toil/test/lib/test_url.py +69 -0
- toil/test/lib/url_plugin_test.py +105 -0
- toil/test/provisioners/aws/awsProvisionerTest.py +13 -10
- toil/test/provisioners/clusterTest.py +17 -4
- toil/test/provisioners/gceProvisionerTest.py +17 -15
- toil/test/server/serverTest.py +78 -36
- toil/test/sort/sort.py +4 -1
- toil/test/src/busTest.py +17 -17
- toil/test/src/deferredFunctionTest.py +145 -132
- toil/test/src/importExportFileTest.py +71 -63
- toil/test/src/jobEncapsulationTest.py +27 -28
- toil/test/src/jobServiceTest.py +149 -133
- toil/test/src/jobTest.py +219 -211
- toil/test/src/miscTests.py +66 -60
- toil/test/src/promisedRequirementTest.py +163 -169
- toil/test/src/regularLogTest.py +24 -24
- toil/test/src/resourceTest.py +82 -76
- toil/test/src/restartDAGTest.py +51 -47
- toil/test/src/resumabilityTest.py +24 -19
- toil/test/src/retainTempDirTest.py +60 -57
- toil/test/src/systemTest.py +17 -13
- toil/test/src/threadingTest.py +29 -32
- toil/test/utils/ABCWorkflowDebug/B_file.txt +1 -0
- toil/test/utils/ABCWorkflowDebug/debugWorkflow.py +204 -0
- toil/test/utils/ABCWorkflowDebug/mkFile.py +16 -0
- toil/test/utils/ABCWorkflowDebug/sleep.cwl +12 -0
- toil/test/utils/ABCWorkflowDebug/sleep.yaml +1 -0
- toil/test/utils/toilDebugTest.py +117 -102
- toil/test/utils/toilKillTest.py +54 -53
- toil/test/utils/utilsTest.py +303 -229
- toil/test/wdl/lint_error.wdl +9 -0
- toil/test/wdl/md5sum/empty_file.json +1 -0
- toil/test/wdl/md5sum/md5sum-gs.json +1 -0
- toil/test/wdl/md5sum/md5sum.1.0.wdl +32 -0
- toil/test/wdl/md5sum/md5sum.input +1 -0
- toil/test/wdl/md5sum/md5sum.json +1 -0
- toil/test/wdl/md5sum/md5sum.wdl +25 -0
- toil/test/wdl/miniwdl_self_test/inputs-namespaced.json +1 -0
- toil/test/wdl/miniwdl_self_test/inputs.json +1 -0
- toil/test/wdl/miniwdl_self_test/self_test.wdl +40 -0
- toil/test/wdl/standard_library/as_map.json +16 -0
- toil/test/wdl/standard_library/as_map_as_input.wdl +23 -0
- toil/test/wdl/standard_library/as_pairs.json +7 -0
- toil/test/wdl/standard_library/as_pairs_as_input.wdl +23 -0
- toil/test/wdl/standard_library/ceil.json +3 -0
- toil/test/wdl/standard_library/ceil_as_command.wdl +16 -0
- toil/test/wdl/standard_library/ceil_as_input.wdl +16 -0
- toil/test/wdl/standard_library/collect_by_key.json +1 -0
- toil/test/wdl/standard_library/collect_by_key_as_input.wdl +23 -0
- toil/test/wdl/standard_library/cross.json +11 -0
- toil/test/wdl/standard_library/cross_as_input.wdl +19 -0
- toil/test/wdl/standard_library/flatten.json +7 -0
- toil/test/wdl/standard_library/flatten_as_input.wdl +18 -0
- toil/test/wdl/standard_library/floor.json +3 -0
- toil/test/wdl/standard_library/floor_as_command.wdl +16 -0
- toil/test/wdl/standard_library/floor_as_input.wdl +16 -0
- toil/test/wdl/standard_library/keys.json +8 -0
- toil/test/wdl/standard_library/keys_as_input.wdl +24 -0
- toil/test/wdl/standard_library/length.json +7 -0
- toil/test/wdl/standard_library/length_as_input.wdl +16 -0
- toil/test/wdl/standard_library/length_as_input_with_map.json +7 -0
- toil/test/wdl/standard_library/length_as_input_with_map.wdl +17 -0
- toil/test/wdl/standard_library/length_invalid.json +3 -0
- toil/test/wdl/standard_library/range.json +3 -0
- toil/test/wdl/standard_library/range_0.json +3 -0
- toil/test/wdl/standard_library/range_as_input.wdl +17 -0
- toil/test/wdl/standard_library/range_invalid.json +3 -0
- toil/test/wdl/standard_library/read_boolean.json +3 -0
- toil/test/wdl/standard_library/read_boolean_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_float.json +3 -0
- toil/test/wdl/standard_library/read_float_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_int.json +3 -0
- toil/test/wdl/standard_library/read_int_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_json.json +3 -0
- toil/test/wdl/standard_library/read_json_as_output.wdl +31 -0
- toil/test/wdl/standard_library/read_lines.json +3 -0
- toil/test/wdl/standard_library/read_lines_as_output.wdl +31 -0
- toil/test/wdl/standard_library/read_map.json +3 -0
- toil/test/wdl/standard_library/read_map_as_output.wdl +31 -0
- toil/test/wdl/standard_library/read_string.json +3 -0
- toil/test/wdl/standard_library/read_string_as_command.wdl +17 -0
- toil/test/wdl/standard_library/read_tsv.json +3 -0
- toil/test/wdl/standard_library/read_tsv_as_output.wdl +31 -0
- toil/test/wdl/standard_library/round.json +3 -0
- toil/test/wdl/standard_library/round_as_command.wdl +16 -0
- toil/test/wdl/standard_library/round_as_input.wdl +16 -0
- toil/test/wdl/standard_library/size.json +3 -0
- toil/test/wdl/standard_library/size_as_command.wdl +17 -0
- toil/test/wdl/standard_library/size_as_output.wdl +36 -0
- toil/test/wdl/standard_library/stderr.json +3 -0
- toil/test/wdl/standard_library/stderr_as_output.wdl +30 -0
- toil/test/wdl/standard_library/stdout.json +3 -0
- toil/test/wdl/standard_library/stdout_as_output.wdl +30 -0
- toil/test/wdl/standard_library/sub.json +3 -0
- toil/test/wdl/standard_library/sub_as_input.wdl +17 -0
- toil/test/wdl/standard_library/sub_as_input_with_file.wdl +17 -0
- toil/test/wdl/standard_library/transpose.json +6 -0
- toil/test/wdl/standard_library/transpose_as_input.wdl +18 -0
- toil/test/wdl/standard_library/write_json.json +6 -0
- toil/test/wdl/standard_library/write_json_as_command.wdl +17 -0
- toil/test/wdl/standard_library/write_lines.json +7 -0
- toil/test/wdl/standard_library/write_lines_as_command.wdl +17 -0
- toil/test/wdl/standard_library/write_map.json +6 -0
- toil/test/wdl/standard_library/write_map_as_command.wdl +17 -0
- toil/test/wdl/standard_library/write_tsv.json +6 -0
- toil/test/wdl/standard_library/write_tsv_as_command.wdl +17 -0
- toil/test/wdl/standard_library/zip.json +12 -0
- toil/test/wdl/standard_library/zip_as_input.wdl +19 -0
- toil/test/wdl/test.csv +3 -0
- toil/test/wdl/test.tsv +3 -0
- toil/test/wdl/testfiles/croo.wdl +38 -0
- toil/test/wdl/testfiles/drop_files.wdl +62 -0
- toil/test/wdl/testfiles/drop_files_subworkflow.wdl +13 -0
- toil/test/wdl/testfiles/empty.txt +0 -0
- toil/test/wdl/testfiles/not_enough_outputs.wdl +33 -0
- toil/test/wdl/testfiles/random.wdl +66 -0
- toil/test/wdl/testfiles/read_file.wdl +18 -0
- toil/test/wdl/testfiles/string_file_coercion.json +1 -0
- toil/test/wdl/testfiles/string_file_coercion.wdl +35 -0
- toil/test/wdl/testfiles/test.json +4 -0
- toil/test/wdl/testfiles/test_boolean.txt +1 -0
- toil/test/wdl/testfiles/test_float.txt +1 -0
- toil/test/wdl/testfiles/test_int.txt +1 -0
- toil/test/wdl/testfiles/test_lines.txt +5 -0
- toil/test/wdl/testfiles/test_map.txt +2 -0
- toil/test/wdl/testfiles/test_string.txt +1 -0
- toil/test/wdl/testfiles/url_to_file.wdl +13 -0
- toil/test/wdl/testfiles/url_to_optional_file.wdl +14 -0
- toil/test/wdl/testfiles/vocab.json +1 -0
- toil/test/wdl/testfiles/vocab.wdl +66 -0
- toil/test/wdl/testfiles/wait.wdl +34 -0
- toil/test/wdl/wdl_specification/type_pair.json +23 -0
- toil/test/wdl/wdl_specification/type_pair_basic.wdl +36 -0
- toil/test/wdl/wdl_specification/type_pair_with_files.wdl +36 -0
- toil/test/wdl/wdl_specification/v1_spec.json +1 -0
- toil/test/wdl/wdl_specification/v1_spec_declaration.wdl +39 -0
- toil/test/wdl/wdltoil_test.py +751 -529
- toil/test/wdl/wdltoil_test_kubernetes.py +2 -2
- toil/utils/toilSshCluster.py +23 -0
- toil/utils/toilUpdateEC2Instances.py +1 -0
- toil/version.py +5 -5
- toil/wdl/wdltoil.py +518 -437
- toil/worker.py +11 -6
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/METADATA +25 -24
- toil-9.0.0.dist-info/RECORD +444 -0
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/WHEEL +1 -1
- toil-8.1.0b1.dist-info/RECORD +0 -259
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/entry_points.txt +0 -0
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info/licenses}/LICENSE +0 -0
- {toil-8.1.0b1.dist-info → toil-9.0.0.dist-info}/top_level.txt +0 -0
|
@@ -26,6 +26,7 @@ from toil.batchSystems.registry import add_batch_system_factory
|
|
|
26
26
|
from toil.common import Toil, addOptions
|
|
27
27
|
from toil.job import JobDescription
|
|
28
28
|
from toil.test import ToilTest
|
|
29
|
+
from toil.lib.plugins import remove_plugin
|
|
29
30
|
|
|
30
31
|
logger = logging.getLogger(__name__)
|
|
31
32
|
|
|
@@ -68,6 +69,11 @@ class FakeBatchSystem(BatchSystemCleanupSupport):
|
|
|
68
69
|
|
|
69
70
|
|
|
70
71
|
class BatchSystemPluginTest(ToilTest):
|
|
72
|
+
def tearDown(self) -> None:
|
|
73
|
+
# Restore plugin state
|
|
74
|
+
remove_plugin("batch_system", "fake")
|
|
75
|
+
super().tearDown()
|
|
76
|
+
|
|
71
77
|
def test_batchsystem_plugin_installable(self):
|
|
72
78
|
"""
|
|
73
79
|
Test that installing a batch system plugin works.
|
|
@@ -76,6 +82,7 @@ class BatchSystemPluginTest(ToilTest):
|
|
|
76
82
|
|
|
77
83
|
def fake_batch_system_factory() -> type[AbstractBatchSystem]:
|
|
78
84
|
return FakeBatchSystem
|
|
85
|
+
|
|
79
86
|
|
|
80
87
|
add_batch_system_factory("fake", fake_batch_system_factory)
|
|
81
88
|
|
|
@@ -619,3 +619,30 @@ class SlurmTest(ToilTest):
|
|
|
619
619
|
command = self.worker.prepareSbatch(1, 100, 5, "job5", None, None)
|
|
620
620
|
assert "--export=ALL" not in command
|
|
621
621
|
|
|
622
|
+
def test_option_detector(self):
|
|
623
|
+
detector = toil.batchSystems.slurm.option_detector("foobar", "f")
|
|
624
|
+
|
|
625
|
+
self.assertTrue(detector("--foobar"))
|
|
626
|
+
self.assertTrue(detector("--foobar=1"))
|
|
627
|
+
self.assertTrue(detector("-f"))
|
|
628
|
+
self.assertFalse(detector("-F"))
|
|
629
|
+
self.assertFalse(detector("--foo-bar"))
|
|
630
|
+
self.assertFalse(detector("foobar"))
|
|
631
|
+
self.assertFalse(detector("-foobar"))
|
|
632
|
+
self.assertFalse(detector("f"))
|
|
633
|
+
|
|
634
|
+
def test_any_option_detector(self):
|
|
635
|
+
detector = toil.batchSystems.slurm.any_option_detector([])
|
|
636
|
+
self.assertFalse(detector("--anything"))
|
|
637
|
+
|
|
638
|
+
detector = toil.batchSystems.slurm.any_option_detector([("foobar", "f"), "many-bothans", ("bazz-only", "B")])
|
|
639
|
+
|
|
640
|
+
self.assertTrue(detector("--foobar"))
|
|
641
|
+
self.assertTrue(detector("-f"))
|
|
642
|
+
self.assertTrue(detector("--many-bothans=False"))
|
|
643
|
+
self.assertTrue(detector("--bazz-only"))
|
|
644
|
+
self.assertTrue(detector("-B"))
|
|
645
|
+
self.assertFalse(detector("--no-bazz"))
|
|
646
|
+
self.assertFalse(detector("--foo-bar=--bazz-only"))
|
|
647
|
+
|
|
648
|
+
|
|
Binary file
|
toil/test/conftest.py
ADDED
toil/test/cwl/2.fasta
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
>Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;
|
|
2
|
+
gttcgatgcc taaaatacct tcttttgtcc ctacacagac cacagttttc ctaatggctt
|
|
3
|
+
tacaccgact agaaattctt gtgcaagcac taattgaaag cggttggcct agagtgttac
|
|
4
|
+
cggtttgtat agctgagcgc gtctcttgcc ctgatcaaag gttcattttc tctactttgg
|
|
5
|
+
aagacgttgt ggaagaatac aacaagtacg agtctctccc ccctggtttg ctgattactg
|
|
6
|
+
gatacagttg taataccctt cgcaacaccg cgtaactatc tatatgaatt attttccctt
|
|
7
|
+
tattatatgt agtaggttcg tctttaatct tcctttagca agtcttttac tgttttcgac
|
|
8
|
+
ctcaatgttc atgttcttag gttgttttgg ataatatgcg gtcagtttaa tcttcgttgt
|
|
9
|
+
ttcttcttaa aatatttatt catggtttaa tttttggttt gtacttgttc aggggccagt
|
|
10
|
+
tcattattta ctctgtttgt atacagcagt tcttttattt ttagtatgat tttaatttaa
|
|
11
|
+
aacaattcta atggtcaaaa a
|
toil/test/cwl/2.fastq
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
@EAS54_6_R1_2_1_413_324
|
|
2
|
+
CCCTTCTTGTCTTCAGCGTTTCTCC
|
|
3
|
+
+
|
|
4
|
+
;;3;;;;;;;;;;;;7;;;;;;;88
|
|
5
|
+
@EAS54_6_R1_2_1_540_792
|
|
6
|
+
TTGGCAGGCCAAGGCCGATGGATCA
|
|
7
|
+
+
|
|
8
|
+
;;;;;;;;;;;7;;;;;-;;;3;83
|
|
9
|
+
@EAS54_6_R1_2_1_443_348
|
|
10
|
+
GTTGCTTCTGGCGTGGGTGGGGGGG
|
|
11
|
+
+EAS54_6_R1_2_1_443_348
|
|
12
|
+
;;;;;;;;;;;9;7;;.7;393333
|
toil/test/cwl/conftest.py
CHANGED
|
@@ -34,7 +34,7 @@ def pytest_cwl_execute_test(
|
|
|
34
34
|
processfile: str,
|
|
35
35
|
jobfile: Optional[str]
|
|
36
36
|
) -> Tuple[int, Optional[Dict[str, Any]]]:
|
|
37
|
-
"""Use
|
|
37
|
+
"""Use Toil to execute CWL tests (equivalent to running toil-cwl-runner)."""
|
|
38
38
|
from toil.cwl.cwltoil import main
|
|
39
39
|
|
|
40
40
|
stdout = StringIO()
|