toil 8.2.0__py3-none-any.whl → 9.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. toil/batchSystems/abstractBatchSystem.py +13 -5
  2. toil/batchSystems/abstractGridEngineBatchSystem.py +17 -5
  3. toil/batchSystems/kubernetes.py +13 -2
  4. toil/batchSystems/mesos/batchSystem.py +33 -2
  5. toil/batchSystems/registry.py +15 -118
  6. toil/batchSystems/slurm.py +191 -16
  7. toil/common.py +20 -1
  8. toil/cwl/cwltoil.py +97 -119
  9. toil/cwl/utils.py +103 -3
  10. toil/fileStores/__init__.py +1 -1
  11. toil/fileStores/abstractFileStore.py +5 -2
  12. toil/fileStores/cachingFileStore.py +1 -1
  13. toil/job.py +30 -14
  14. toil/jobStores/abstractJobStore.py +35 -255
  15. toil/jobStores/aws/jobStore.py +864 -1964
  16. toil/jobStores/aws/utils.py +24 -270
  17. toil/jobStores/fileJobStore.py +2 -1
  18. toil/jobStores/googleJobStore.py +32 -13
  19. toil/jobStores/utils.py +0 -327
  20. toil/leader.py +27 -22
  21. toil/lib/accelerators.py +1 -1
  22. toil/lib/aws/config.py +22 -0
  23. toil/lib/aws/s3.py +477 -9
  24. toil/lib/aws/utils.py +22 -33
  25. toil/lib/checksum.py +88 -0
  26. toil/lib/conversions.py +33 -31
  27. toil/lib/directory.py +217 -0
  28. toil/lib/ec2.py +97 -29
  29. toil/lib/exceptions.py +2 -1
  30. toil/lib/expando.py +2 -2
  31. toil/lib/generatedEC2Lists.py +138 -19
  32. toil/lib/io.py +33 -2
  33. toil/lib/memoize.py +21 -7
  34. toil/lib/misc.py +1 -1
  35. toil/lib/pipes.py +385 -0
  36. toil/lib/plugins.py +106 -0
  37. toil/lib/retry.py +1 -1
  38. toil/lib/threading.py +1 -1
  39. toil/lib/url.py +320 -0
  40. toil/lib/web.py +4 -5
  41. toil/options/cwl.py +13 -1
  42. toil/options/runner.py +17 -10
  43. toil/options/wdl.py +12 -1
  44. toil/provisioners/__init__.py +5 -2
  45. toil/provisioners/aws/__init__.py +43 -36
  46. toil/provisioners/aws/awsProvisioner.py +47 -15
  47. toil/provisioners/node.py +60 -12
  48. toil/resource.py +3 -13
  49. toil/server/app.py +12 -6
  50. toil/server/cli/wes_cwl_runner.py +2 -2
  51. toil/server/wes/abstract_backend.py +21 -43
  52. toil/server/wes/toil_backend.py +2 -2
  53. toil/test/__init__.py +16 -18
  54. toil/test/batchSystems/batchSystemTest.py +2 -9
  55. toil/test/batchSystems/batch_system_plugin_test.py +7 -0
  56. toil/test/batchSystems/test_slurm.py +103 -14
  57. toil/test/cwl/cwlTest.py +181 -8
  58. toil/test/cwl/staging_cat.cwl +27 -0
  59. toil/test/cwl/staging_make_file.cwl +25 -0
  60. toil/test/cwl/staging_workflow.cwl +43 -0
  61. toil/test/cwl/zero_default.cwl +61 -0
  62. toil/test/docs/scripts/tutorial_staging.py +17 -8
  63. toil/test/docs/scriptsTest.py +2 -1
  64. toil/test/jobStores/jobStoreTest.py +23 -133
  65. toil/test/lib/aws/test_iam.py +7 -7
  66. toil/test/lib/aws/test_s3.py +30 -33
  67. toil/test/lib/aws/test_utils.py +9 -9
  68. toil/test/lib/test_url.py +69 -0
  69. toil/test/lib/url_plugin_test.py +105 -0
  70. toil/test/provisioners/aws/awsProvisionerTest.py +60 -7
  71. toil/test/provisioners/clusterTest.py +15 -2
  72. toil/test/provisioners/gceProvisionerTest.py +1 -1
  73. toil/test/server/serverTest.py +78 -36
  74. toil/test/src/autoDeploymentTest.py +2 -3
  75. toil/test/src/fileStoreTest.py +89 -87
  76. toil/test/utils/ABCWorkflowDebug/ABC.txt +1 -0
  77. toil/test/utils/ABCWorkflowDebug/debugWorkflow.py +4 -4
  78. toil/test/utils/toilKillTest.py +35 -28
  79. toil/test/wdl/md5sum/md5sum-gs.json +1 -1
  80. toil/test/wdl/md5sum/md5sum.json +1 -1
  81. toil/test/wdl/testfiles/read_file.wdl +18 -0
  82. toil/test/wdl/testfiles/url_to_optional_file.wdl +2 -1
  83. toil/test/wdl/wdltoil_test.py +171 -162
  84. toil/test/wdl/wdltoil_test_kubernetes.py +9 -0
  85. toil/utils/toilDebugFile.py +6 -3
  86. toil/utils/toilSshCluster.py +23 -0
  87. toil/utils/toilStats.py +17 -2
  88. toil/utils/toilUpdateEC2Instances.py +1 -0
  89. toil/version.py +10 -10
  90. toil/wdl/wdltoil.py +1179 -825
  91. toil/worker.py +16 -8
  92. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/METADATA +32 -32
  93. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/RECORD +97 -85
  94. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/WHEEL +1 -1
  95. toil/lib/iterables.py +0 -112
  96. toil/test/docs/scripts/stagingExampleFiles/in.txt +0 -1
  97. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/entry_points.txt +0 -0
  98. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/licenses/LICENSE +0 -0
  99. {toil-8.2.0.dist-info → toil-9.1.0.dist-info}/top_level.txt +0 -0
toil/lib/iterables.py DELETED
@@ -1,112 +0,0 @@
1
- # Copyright (C) 2015-2021 Regents of the University of California
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
- from collections.abc import Iterable, Iterator
16
-
17
- # 5.14.2018: copied into Toil from https://github.com/BD2KGenomics/bd2k-python-lib
18
- from typing import Any, TypeVar
19
-
20
- IT = TypeVar("IT")
21
-
22
-
23
- def flatten(iterables: Iterable[IT]) -> Iterator[IT]:
24
- """Flatten an iterable, except for string elements."""
25
- for it in iterables:
26
- if isinstance(it, str):
27
- yield it
28
- else:
29
- yield from it
30
-
31
-
32
- # noinspection PyPep8Naming
33
- class concat:
34
- """
35
- A literal iterable to combine sequence literals (lists, set) with generators or list comprehensions.
36
-
37
- Instead of
38
-
39
- >>> [ -1 ] + [ x * 2 for x in range( 3 ) ] + [ -1 ]
40
- [-1, 0, 2, 4, -1]
41
-
42
- you can write
43
-
44
- >>> list( concat( -1, ( x * 2 for x in range( 3 ) ), -1 ) )
45
- [-1, 0, 2, 4, -1]
46
-
47
- This is slightly shorter (not counting the list constructor) and does not involve array
48
- construction or concatenation.
49
-
50
- Note that concat() flattens (or chains) all iterable arguments into a single
51
- result iterable:
52
-
53
- >>> list( concat( 1, range( 2, 4 ), 4 ) )
54
- [1, 2, 3, 4]
55
-
56
- It only does so one level deep. If you need to recursively flatten a data structure,
57
- check out crush().
58
-
59
- If you want to prevent that flattening for an iterable argument, wrap it in concat():
60
-
61
- >>> list( concat( 1, concat( range( 2, 4 ) ), 4 ) )
62
- [1, range(2, 4), 4]
63
-
64
- Some more example.
65
-
66
- >>> list( concat() ) # empty concat
67
- []
68
- >>> list( concat( 1 ) ) # non-iterable
69
- [1]
70
- >>> list( concat( concat() ) ) # empty iterable
71
- []
72
- >>> list( concat( concat( 1 ) ) ) # singleton iterable
73
- [1]
74
- >>> list( concat( 1, concat( 2 ), 3 ) ) # flattened iterable
75
- [1, 2, 3]
76
- >>> list( concat( 1, [2], 3 ) ) # flattened iterable
77
- [1, 2, 3]
78
- >>> list( concat( 1, concat( [2] ), 3 ) ) # protecting an iterable from being flattened
79
- [1, [2], 3]
80
- >>> list( concat( 1, concat( [2], 3 ), 4 ) ) # protection only works with a single argument
81
- [1, 2, 3, 4]
82
- >>> list( concat( 1, 2, concat( 3, 4 ), 5, 6 ) )
83
- [1, 2, 3, 4, 5, 6]
84
- >>> list( concat( 1, 2, concat( [ 3, 4 ] ), 5, 6 ) )
85
- [1, 2, [3, 4], 5, 6]
86
-
87
- Note that while strings are technically iterable, concat() does not flatten them.
88
-
89
- >>> list( concat( 'ab' ) )
90
- ['ab']
91
- >>> list( concat( concat( 'ab' ) ) )
92
- ['ab']
93
- """
94
-
95
- def __init__(self, *args: Any) -> None:
96
- super().__init__()
97
- self.args = args
98
-
99
- def __iter__(self) -> Iterator[Any]:
100
- def expand(x):
101
- if isinstance(x, concat) and len(x.args) == 1:
102
- i = x.args
103
- elif not isinstance(x, str):
104
- try:
105
- i = x.__iter__()
106
- except AttributeError:
107
- i = (x,)
108
- else:
109
- i = x
110
- return i
111
-
112
- return flatten(map(expand, self.args))
@@ -1 +0,0 @@
1
- Hello,