toil 7.0.0__py3-none-any.whl → 8.1.0b1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. toil/__init__.py +124 -86
  2. toil/batchSystems/__init__.py +1 -0
  3. toil/batchSystems/abstractBatchSystem.py +137 -77
  4. toil/batchSystems/abstractGridEngineBatchSystem.py +211 -101
  5. toil/batchSystems/awsBatch.py +237 -128
  6. toil/batchSystems/cleanup_support.py +22 -16
  7. toil/batchSystems/contained_executor.py +30 -26
  8. toil/batchSystems/gridengine.py +85 -49
  9. toil/batchSystems/htcondor.py +164 -87
  10. toil/batchSystems/kubernetes.py +622 -386
  11. toil/batchSystems/local_support.py +17 -12
  12. toil/batchSystems/lsf.py +132 -79
  13. toil/batchSystems/lsfHelper.py +13 -11
  14. toil/batchSystems/mesos/__init__.py +41 -29
  15. toil/batchSystems/mesos/batchSystem.py +288 -149
  16. toil/batchSystems/mesos/executor.py +77 -49
  17. toil/batchSystems/mesos/test/__init__.py +31 -23
  18. toil/batchSystems/options.py +39 -29
  19. toil/batchSystems/registry.py +53 -19
  20. toil/batchSystems/singleMachine.py +293 -123
  21. toil/batchSystems/slurm.py +651 -155
  22. toil/batchSystems/torque.py +46 -32
  23. toil/bus.py +141 -73
  24. toil/common.py +784 -397
  25. toil/cwl/__init__.py +1 -1
  26. toil/cwl/cwltoil.py +1137 -534
  27. toil/cwl/utils.py +17 -22
  28. toil/deferred.py +62 -41
  29. toil/exceptions.py +5 -3
  30. toil/fileStores/__init__.py +5 -5
  31. toil/fileStores/abstractFileStore.py +88 -57
  32. toil/fileStores/cachingFileStore.py +711 -247
  33. toil/fileStores/nonCachingFileStore.py +113 -75
  34. toil/job.py +1031 -349
  35. toil/jobStores/abstractJobStore.py +387 -243
  36. toil/jobStores/aws/jobStore.py +772 -412
  37. toil/jobStores/aws/utils.py +161 -109
  38. toil/jobStores/conftest.py +1 -0
  39. toil/jobStores/fileJobStore.py +289 -151
  40. toil/jobStores/googleJobStore.py +137 -70
  41. toil/jobStores/utils.py +36 -15
  42. toil/leader.py +614 -269
  43. toil/lib/accelerators.py +115 -18
  44. toil/lib/aws/__init__.py +55 -28
  45. toil/lib/aws/ami.py +122 -87
  46. toil/lib/aws/iam.py +284 -108
  47. toil/lib/aws/s3.py +31 -0
  48. toil/lib/aws/session.py +204 -58
  49. toil/lib/aws/utils.py +290 -213
  50. toil/lib/bioio.py +13 -5
  51. toil/lib/compatibility.py +11 -6
  52. toil/lib/conversions.py +83 -49
  53. toil/lib/docker.py +131 -103
  54. toil/lib/dockstore.py +379 -0
  55. toil/lib/ec2.py +322 -209
  56. toil/lib/ec2nodes.py +174 -105
  57. toil/lib/encryption/_dummy.py +5 -3
  58. toil/lib/encryption/_nacl.py +10 -6
  59. toil/lib/encryption/conftest.py +1 -0
  60. toil/lib/exceptions.py +26 -7
  61. toil/lib/expando.py +4 -2
  62. toil/lib/ftp_utils.py +217 -0
  63. toil/lib/generatedEC2Lists.py +127 -19
  64. toil/lib/history.py +1271 -0
  65. toil/lib/history_submission.py +681 -0
  66. toil/lib/humanize.py +6 -2
  67. toil/lib/io.py +121 -12
  68. toil/lib/iterables.py +4 -2
  69. toil/lib/memoize.py +12 -8
  70. toil/lib/misc.py +83 -18
  71. toil/lib/objects.py +2 -2
  72. toil/lib/resources.py +19 -7
  73. toil/lib/retry.py +125 -87
  74. toil/lib/threading.py +282 -80
  75. toil/lib/throttle.py +15 -14
  76. toil/lib/trs.py +390 -0
  77. toil/lib/web.py +38 -0
  78. toil/options/common.py +850 -402
  79. toil/options/cwl.py +185 -90
  80. toil/options/runner.py +50 -0
  81. toil/options/wdl.py +70 -19
  82. toil/provisioners/__init__.py +111 -46
  83. toil/provisioners/abstractProvisioner.py +322 -157
  84. toil/provisioners/aws/__init__.py +62 -30
  85. toil/provisioners/aws/awsProvisioner.py +980 -627
  86. toil/provisioners/clusterScaler.py +541 -279
  87. toil/provisioners/gceProvisioner.py +283 -180
  88. toil/provisioners/node.py +147 -79
  89. toil/realtimeLogger.py +34 -22
  90. toil/resource.py +137 -75
  91. toil/server/app.py +127 -61
  92. toil/server/celery_app.py +3 -1
  93. toil/server/cli/wes_cwl_runner.py +84 -55
  94. toil/server/utils.py +56 -31
  95. toil/server/wes/abstract_backend.py +64 -26
  96. toil/server/wes/amazon_wes_utils.py +21 -15
  97. toil/server/wes/tasks.py +121 -63
  98. toil/server/wes/toil_backend.py +142 -107
  99. toil/server/wsgi_app.py +4 -3
  100. toil/serviceManager.py +58 -22
  101. toil/statsAndLogging.py +183 -65
  102. toil/test/__init__.py +263 -179
  103. toil/test/batchSystems/batchSystemTest.py +438 -195
  104. toil/test/batchSystems/batch_system_plugin_test.py +18 -7
  105. toil/test/batchSystems/test_gridengine.py +173 -0
  106. toil/test/batchSystems/test_lsf_helper.py +67 -58
  107. toil/test/batchSystems/test_slurm.py +265 -49
  108. toil/test/cactus/test_cactus_integration.py +20 -22
  109. toil/test/cwl/conftest.py +39 -0
  110. toil/test/cwl/cwlTest.py +375 -72
  111. toil/test/cwl/measure_default_memory.cwl +12 -0
  112. toil/test/cwl/not_run_required_input.cwl +29 -0
  113. toil/test/cwl/optional-file.cwl +18 -0
  114. toil/test/cwl/scatter_duplicate_outputs.cwl +40 -0
  115. toil/test/docs/scriptsTest.py +60 -34
  116. toil/test/jobStores/jobStoreTest.py +412 -235
  117. toil/test/lib/aws/test_iam.py +116 -48
  118. toil/test/lib/aws/test_s3.py +16 -9
  119. toil/test/lib/aws/test_utils.py +5 -6
  120. toil/test/lib/dockerTest.py +118 -141
  121. toil/test/lib/test_conversions.py +113 -115
  122. toil/test/lib/test_ec2.py +57 -49
  123. toil/test/lib/test_history.py +212 -0
  124. toil/test/lib/test_misc.py +12 -5
  125. toil/test/lib/test_trs.py +161 -0
  126. toil/test/mesos/MesosDataStructuresTest.py +23 -10
  127. toil/test/mesos/helloWorld.py +7 -6
  128. toil/test/mesos/stress.py +25 -20
  129. toil/test/options/options.py +7 -2
  130. toil/test/provisioners/aws/awsProvisionerTest.py +293 -140
  131. toil/test/provisioners/clusterScalerTest.py +440 -250
  132. toil/test/provisioners/clusterTest.py +81 -42
  133. toil/test/provisioners/gceProvisionerTest.py +174 -100
  134. toil/test/provisioners/provisionerTest.py +25 -13
  135. toil/test/provisioners/restartScript.py +5 -4
  136. toil/test/server/serverTest.py +188 -141
  137. toil/test/sort/restart_sort.py +137 -68
  138. toil/test/sort/sort.py +134 -66
  139. toil/test/sort/sortTest.py +91 -49
  140. toil/test/src/autoDeploymentTest.py +140 -100
  141. toil/test/src/busTest.py +20 -18
  142. toil/test/src/checkpointTest.py +8 -2
  143. toil/test/src/deferredFunctionTest.py +49 -35
  144. toil/test/src/dockerCheckTest.py +33 -26
  145. toil/test/src/environmentTest.py +20 -10
  146. toil/test/src/fileStoreTest.py +538 -271
  147. toil/test/src/helloWorldTest.py +7 -4
  148. toil/test/src/importExportFileTest.py +61 -31
  149. toil/test/src/jobDescriptionTest.py +32 -17
  150. toil/test/src/jobEncapsulationTest.py +2 -0
  151. toil/test/src/jobFileStoreTest.py +74 -50
  152. toil/test/src/jobServiceTest.py +187 -73
  153. toil/test/src/jobTest.py +120 -70
  154. toil/test/src/miscTests.py +19 -18
  155. toil/test/src/promisedRequirementTest.py +82 -36
  156. toil/test/src/promisesTest.py +7 -6
  157. toil/test/src/realtimeLoggerTest.py +6 -6
  158. toil/test/src/regularLogTest.py +71 -37
  159. toil/test/src/resourceTest.py +80 -49
  160. toil/test/src/restartDAGTest.py +36 -22
  161. toil/test/src/resumabilityTest.py +9 -2
  162. toil/test/src/retainTempDirTest.py +45 -14
  163. toil/test/src/systemTest.py +12 -8
  164. toil/test/src/threadingTest.py +44 -25
  165. toil/test/src/toilContextManagerTest.py +10 -7
  166. toil/test/src/userDefinedJobArgTypeTest.py +8 -5
  167. toil/test/src/workerTest.py +33 -16
  168. toil/test/utils/toilDebugTest.py +70 -58
  169. toil/test/utils/toilKillTest.py +4 -5
  170. toil/test/utils/utilsTest.py +239 -102
  171. toil/test/wdl/wdltoil_test.py +789 -148
  172. toil/test/wdl/wdltoil_test_kubernetes.py +37 -23
  173. toil/toilState.py +52 -26
  174. toil/utils/toilConfig.py +13 -4
  175. toil/utils/toilDebugFile.py +44 -27
  176. toil/utils/toilDebugJob.py +85 -25
  177. toil/utils/toilDestroyCluster.py +11 -6
  178. toil/utils/toilKill.py +8 -3
  179. toil/utils/toilLaunchCluster.py +251 -145
  180. toil/utils/toilMain.py +37 -16
  181. toil/utils/toilRsyncCluster.py +27 -14
  182. toil/utils/toilSshCluster.py +45 -22
  183. toil/utils/toilStats.py +75 -36
  184. toil/utils/toilStatus.py +226 -119
  185. toil/utils/toilUpdateEC2Instances.py +3 -1
  186. toil/version.py +6 -6
  187. toil/wdl/utils.py +5 -5
  188. toil/wdl/wdltoil.py +3528 -1053
  189. toil/worker.py +370 -149
  190. toil-8.1.0b1.dist-info/METADATA +178 -0
  191. toil-8.1.0b1.dist-info/RECORD +259 -0
  192. {toil-7.0.0.dist-info → toil-8.1.0b1.dist-info}/WHEEL +1 -1
  193. toil-7.0.0.dist-info/METADATA +0 -158
  194. toil-7.0.0.dist-info/RECORD +0 -244
  195. {toil-7.0.0.dist-info → toil-8.1.0b1.dist-info}/LICENSE +0 -0
  196. {toil-7.0.0.dist-info → toil-8.1.0b1.dist-info}/entry_points.txt +0 -0
  197. {toil-7.0.0.dist-info → toil-8.1.0b1.dist-info}/top_level.txt +0 -0
toil/lib/aws/iam.py CHANGED
@@ -3,83 +3,204 @@ import json
3
3
  import logging
4
4
  from collections import defaultdict
5
5
  from functools import lru_cache
6
- from typing import Dict, List, Optional, Union, cast
6
+ from typing import TYPE_CHECKING, Any, Optional, Union
7
7
 
8
8
  import boto3
9
- from mypy_boto3_iam import IAMClient
10
- from mypy_boto3_iam.type_defs import (AttachedPolicyTypeDef,
11
- PolicyDocumentDictTypeDef)
12
- from mypy_boto3_sts import STSClient
9
+ from botocore.exceptions import ClientError
13
10
 
11
+ from toil.lib.aws import AWSServerErrors, session
14
12
  from toil.lib.aws.session import client as get_client
13
+ from toil.lib.misc import printq
14
+ from toil.lib.retry import get_error_code, get_error_status, retry
15
+
16
+ if TYPE_CHECKING:
17
+ from mypy_boto3_iam import IAMClient
18
+ from mypy_boto3_iam.type_defs import (
19
+ AttachedPolicyTypeDef,
20
+ PolicyDocumentDictTypeDef,
21
+ )
22
+ from mypy_boto3_sts import STSClient
15
23
 
16
24
  logger = logging.getLogger(__name__)
17
25
 
18
- #TODO Make this comprehensive
19
- CLUSTER_LAUNCHING_PERMISSIONS = ["iam:CreateRole",
20
- "iam:CreateInstanceProfile",
21
- "iam:TagInstanceProfile",
22
- "iam:DeleteRole",
23
- "iam:DeleteRoleProfile",
24
- "iam:ListAttatchedRolePolicies",
25
- "iam:ListPolicies",
26
- "iam:ListRoleTags",
27
- "iam:PassRole",
28
- "iam:PutRolePolicy",
29
- "iam:RemoveRoleFromInstanceProfile",
30
- "iam:TagRole",
31
- "ec2:AuthorizeSecurityGroupIngress",
32
- "ec2:CancelSpotInstanceRequests",
33
- "ec2:CreateSecurityGroup",
34
- "ec2:CreateTags",
35
- "ec2:DeleteSecurityGroup",
36
- "ec2:DescribeAvailabilityZones",
37
- "ec2:DescribeImages",
38
- "ec2:DescribeInstances",
39
- "ec2:DescribeInstanceStatus",
40
- "ec2:DescribeKeyPairs",
41
- "ec2:DescribeSecurityGroups",
42
- "ec2:DescribeSpotInstanceRequests",
43
- "ec2:DescribeSpotPriceHistory",
44
- "ec2:DescribeVolumes",
45
- "ec2:ModifyInstanceAttribute",
46
- "ec2:RequestSpotInstances",
47
- "ec2:RunInstances",
48
- "ec2:StartInstances",
49
- "ec2:StopInstances",
50
- "ec2:TerminateInstances",
51
- ]
52
-
53
- AllowedActionCollection = Dict[str, Dict[str, List[str]]]
26
+ # TODO Make this comprehensive
27
+ CLUSTER_LAUNCHING_PERMISSIONS = [
28
+ "iam:CreateRole",
29
+ "iam:CreateInstanceProfile",
30
+ "iam:TagInstanceProfile",
31
+ "iam:DeleteRole",
32
+ "iam:DeleteRoleProfile",
33
+ "iam:ListAttatchedRolePolicies",
34
+ "iam:ListPolicies",
35
+ "iam:ListRoleTags",
36
+ "iam:PassRole",
37
+ "iam:PutRolePolicy",
38
+ "iam:RemoveRoleFromInstanceProfile",
39
+ "iam:TagRole",
40
+ "ec2:AuthorizeSecurityGroupIngress",
41
+ "ec2:CancelSpotInstanceRequests",
42
+ "ec2:CreateSecurityGroup",
43
+ "ec2:CreateTags",
44
+ "ec2:DeleteSecurityGroup",
45
+ "ec2:DescribeAvailabilityZones",
46
+ "ec2:DescribeImages",
47
+ "ec2:DescribeInstances",
48
+ "ec2:DescribeInstanceStatus",
49
+ "ec2:DescribeKeyPairs",
50
+ "ec2:DescribeSecurityGroups",
51
+ "ec2:DescribeSpotInstanceRequests",
52
+ "ec2:DescribeSpotPriceHistory",
53
+ "ec2:DescribeVolumes",
54
+ "ec2:ModifyInstanceAttribute",
55
+ "ec2:RequestSpotInstances",
56
+ "ec2:RunInstances",
57
+ "ec2:StartInstances",
58
+ "ec2:StopInstances",
59
+ "ec2:TerminateInstances",
60
+ ]
61
+
62
+ AllowedActionCollection = dict[str, dict[str, list[str]]]
63
+
64
+
65
+ @retry(errors=[AWSServerErrors])
66
+ def delete_iam_instance_profile(
67
+ instance_profile_name: str, region: Optional[str] = None, quiet: bool = True
68
+ ) -> None:
69
+ iam_resource = session.resource("iam", region_name=region)
70
+ instance_profile = iam_resource.InstanceProfile(instance_profile_name)
71
+ if instance_profile.roles is not None:
72
+ for role in instance_profile.roles:
73
+ printq(
74
+ f"Now dissociating role: {role.name} from instance profile {instance_profile_name}",
75
+ quiet,
76
+ )
77
+ instance_profile.remove_role(RoleName=role.name)
78
+ instance_profile.delete()
79
+ printq(f'Instance profile "{instance_profile_name}" successfully deleted.', quiet)
80
+
81
+
82
+ @retry(errors=[AWSServerErrors])
83
+ def delete_iam_role(
84
+ role_name: str, region: Optional[str] = None, quiet: bool = True
85
+ ) -> None:
86
+ """
87
+ Deletes an AWS IAM role. Any separate policies are detached from the role, and any inline policies are deleted.
88
+
89
+ :param role_name: The name of the AWS IAM role.
90
+ :param region: The AWS region that the role_name is in.
91
+ :param quiet: Whether or not to print/log information about the deletion to stdout.
92
+ """
93
+ # TODO: This function could benefit from less complex Boto3 type hints
94
+ iam_client = session.client("iam", region_name=region)
95
+ iam_resource = session.resource("iam", region_name=region)
96
+ role = iam_resource.Role(role_name)
97
+ # normal policies
98
+ for attached_policy in role.attached_policies.all():
99
+ printq(
100
+ f"Now dissociating policy: {attached_policy.policy_name} from role {role.name}",
101
+ quiet,
102
+ )
103
+ role.detach_policy(PolicyArn=attached_policy.arn)
104
+ # inline policies
105
+ for inline_policy in role.policies.all():
106
+ printq(
107
+ f"Deleting inline policy: {inline_policy.policy_name} from role {role.name}",
108
+ quiet,
109
+ )
110
+ iam_client.delete_role_policy(
111
+ RoleName=role.name, PolicyName=inline_policy.policy_name
112
+ )
113
+ iam_client.delete_role(RoleName=role_name)
114
+ printq(f"Role {role_name} successfully deleted.", quiet)
115
+
116
+
117
+ # "PolicyDocumentDictTypeDef"
118
+ def create_iam_role(
119
+ role_name: str,
120
+ assume_role_policy_document: str,
121
+ policies: dict[str, Any],
122
+ region: Optional[str] = None,
123
+ ) -> str:
124
+ """
125
+ Creates an AWS IAM role. Any separate policies are detached from the role, and any inline policies are deleted.
126
+
127
+ :param role_name: The name of the AWS IAM role.
128
+ :param region: The AWS region that the role_name is in.
129
+ :param assume_role_policy_document: Policies to create inline with the role.
130
+ :param policies: Global policies to attach to the role.
131
+ """
132
+ iam_client = session.client("iam", region_name=region)
133
+ try:
134
+ # Make the role
135
+ logger.debug("Creating IAM role %s...", role_name)
136
+ iam_client.create_role(
137
+ RoleName=role_name, AssumeRolePolicyDocument=assume_role_policy_document
138
+ )
139
+ logger.debug("Created new IAM role")
140
+ except ClientError as e:
141
+ if get_error_status(e) == 409 and get_error_code(e) == "EntityAlreadyExists":
142
+ logger.debug("IAM role already exists. Reusing.")
143
+ else:
144
+ raise
145
+
146
+ # Delete superfluous policies
147
+ policy_names = set(iam_client.list_role_policies(RoleName=role_name)["PolicyNames"])
148
+ for policy_name in policy_names.difference(set(list(policies.keys()))):
149
+ iam_client.delete_role_policy(RoleName=role_name, PolicyName=policy_name)
150
+
151
+ # Create expected policies
152
+ for policy_name, policy in policies.items():
153
+ current_policy = None
154
+ try:
155
+ current_policy = iam_client.get_role_policy(
156
+ RoleName=role_name, PolicyName=policy_name
157
+ )["PolicyDocument"]
158
+ except iam_client.exceptions.NoSuchEntityException:
159
+ pass
160
+ if current_policy != policy:
161
+ iam_client.put_role_policy(
162
+ RoleName=role_name,
163
+ PolicyName=policy_name,
164
+ PolicyDocument=json.dumps(policy),
165
+ )
166
+
167
+ # Now the role has the right policies so it is ready.
168
+ return role_name
169
+
54
170
 
55
171
  def init_action_collection() -> AllowedActionCollection:
56
- '''
172
+ """
57
173
  Initialization of an action collection, an action collection contains allowed Actions and NotActions
58
174
  by resource, these are patterns containing wildcards, an Action explicitly allows a matched pattern,
59
175
  eg ec2:* will explicitly allow all ec2 permissions
60
176
 
61
177
  A NotAction will explicitly allow all actions that don't match a specific pattern
62
178
  eg iam:* allows all non iam actions
63
- '''
64
- return defaultdict(lambda: {'Action': [], 'NotAction': []})
179
+ """
180
+ return defaultdict(lambda: {"Action": [], "NotAction": []})
181
+
65
182
 
66
- def add_to_action_collection(a: AllowedActionCollection, b: AllowedActionCollection) -> AllowedActionCollection:
67
- '''
183
+ def add_to_action_collection(
184
+ a: AllowedActionCollection, b: AllowedActionCollection
185
+ ) -> AllowedActionCollection:
186
+ """
68
187
  Combines two action collections
69
- '''
188
+ """
70
189
  to_return = init_action_collection()
71
190
  for key in a.keys():
72
- to_return[key]['Action'] += a[key]['Action']
73
- to_return[key]['NotAction'] += a[key]['NotAction']
191
+ to_return[key]["Action"] += a[key]["Action"]
192
+ to_return[key]["NotAction"] += a[key]["NotAction"]
74
193
 
75
194
  for key in b.keys():
76
- to_return[key]['Action'] += b[key]['Action']
77
- to_return[key]['NotAction'] += b[key]['NotAction']
195
+ to_return[key]["Action"] += b[key]["Action"]
196
+ to_return[key]["NotAction"] += b[key]["NotAction"]
78
197
 
79
198
  return to_return
80
199
 
81
200
 
82
- def policy_permissions_allow(given_permissions: AllowedActionCollection, required_permissions: List[str] = []) -> bool:
201
+ def policy_permissions_allow(
202
+ given_permissions: AllowedActionCollection, required_permissions: list[str] = []
203
+ ) -> bool:
83
204
  """
84
205
  Check whether given set of actions are a subset of another given set of actions, returns true if they are
85
206
  otherwise false and prints a warning.
@@ -89,24 +210,31 @@ def policy_permissions_allow(given_permissions: AllowedActionCollection, require
89
210
  """
90
211
 
91
212
  # We only check actions explicitly allowed on all resources here,
92
- #TODO: Add a resource parameter to check for actions allowed by resource
213
+ # TODO: Add a resource parameter to check for actions allowed by resource
93
214
  resource = "*"
94
215
 
95
216
  missing_perms = []
96
217
 
97
218
  for permission in required_permissions:
98
- if not permission_matches_any(permission, given_permissions[resource]['Action']):
99
- if given_permissions[resource]['NotAction'] == [] or permission_matches_any(permission, given_permissions[resource]["NotAction"]):
219
+ if not permission_matches_any(
220
+ permission, given_permissions[resource]["Action"]
221
+ ):
222
+ if given_permissions[resource]["NotAction"] == [] or permission_matches_any(
223
+ permission, given_permissions[resource]["NotAction"]
224
+ ):
100
225
  missing_perms.append(permission)
101
226
 
102
227
  if missing_perms:
103
- logger.warning('You appear to lack the folowing AWS permissions: %s', ', '.join(missing_perms))
228
+ logger.warning(
229
+ "You appear to lack the folowing AWS permissions: %s",
230
+ ", ".join(missing_perms),
231
+ )
104
232
  return False
105
233
 
106
234
  return True
107
235
 
108
236
 
109
- def permission_matches_any(perm: str, list_perms: List[str]) -> bool:
237
+ def permission_matches_any(perm: str, list_perms: list[str]) -> bool:
110
238
  """
111
239
  Takes a permission and checks whether it's contained within a list of given permissions
112
240
  Returns True if it is otherwise False
@@ -120,13 +248,16 @@ def permission_matches_any(perm: str, list_perms: List[str]) -> bool:
120
248
  return True
121
249
  return False
122
250
 
123
- def get_actions_from_policy_document(policy_doc: PolicyDocumentDictTypeDef) -> AllowedActionCollection:
124
- '''
251
+
252
+ def get_actions_from_policy_document(
253
+ policy_doc: "PolicyDocumentDictTypeDef",
254
+ ) -> AllowedActionCollection:
255
+ """
125
256
  Given a policy document, go through each statement and create an AllowedActionCollection representing the
126
257
  permissions granted in the policy document.
127
258
 
128
259
  :param policy_doc: A policy document to examine
129
- '''
260
+ """
130
261
  allowed_actions: AllowedActionCollection = init_action_collection()
131
262
  # Policy document structured like so https://boto3.amazonaws.com/v1/documentation/api/latest/guide/iam-example-policies.html#example
132
263
  logger.debug(policy_doc)
@@ -145,11 +276,15 @@ def get_actions_from_policy_document(policy_doc: PolicyDocumentDictTypeDef) -> A
145
276
  if isinstance(statement[key], list): # type: ignore[literal-required]
146
277
  allowed_actions[resource][key] += statement[key] # type: ignore[literal-required]
147
278
  else:
148
- #Assumes that if it isn't a list it's probably a string
279
+ # Assumes that if it isn't a list it's probably a string
149
280
  allowed_actions[resource][key].append(statement[key]) # type: ignore[literal-required]
150
281
 
151
282
  return allowed_actions
152
- def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPolicyTypeDef]) -> AllowedActionCollection:
283
+
284
+
285
+ def allowed_actions_attached(
286
+ iam: "IAMClient", attached_policies: list["AttachedPolicyTypeDef"]
287
+ ) -> AllowedActionCollection:
153
288
  """
154
289
  Go through all attached policy documents and create an AllowedActionCollection representing granted permissions.
155
290
 
@@ -159,16 +294,21 @@ def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPol
159
294
 
160
295
  allowed_actions: AllowedActionCollection = init_action_collection()
161
296
  for policy in attached_policies:
162
- policy_desc = iam.get_policy(PolicyArn=policy['PolicyArn'])
163
- policy_ver = iam.get_policy_version(PolicyArn=policy_desc['Policy']['Arn'], VersionId=policy_desc['Policy']['DefaultVersionId'])
164
- policy_document = policy_ver['PolicyVersion']['Document']
165
- #TODO whenever boto fixes the typing, stop ignoring this line in typecheck
166
- allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_document)) # type: ignore
297
+ policy_desc = iam.get_policy(PolicyArn=policy["PolicyArn"])
298
+ policy_ver = iam.get_policy_version(
299
+ PolicyArn=policy_desc["Policy"]["Arn"],
300
+ VersionId=policy_desc["Policy"]["DefaultVersionId"],
301
+ )
302
+ policy_document = policy_ver["PolicyVersion"]["Document"]
303
+ # TODO whenever boto fixes the typing, stop ignoring this line in typecheck
304
+ allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_document)) # type: ignore
167
305
 
168
306
  return allowed_actions
169
307
 
170
308
 
171
- def allowed_actions_roles(iam: IAMClient, policy_names: List[str], role_name: str) -> AllowedActionCollection:
309
+ def allowed_actions_roles(
310
+ iam: "IAMClient", policy_names: list[str], role_name: str
311
+ ) -> AllowedActionCollection:
172
312
  """
173
313
  Returns a dictionary containing a list of all aws actions allowed for a given role.
174
314
  This dictionary is keyed by resource and gives a list of policies allowed on that resource.
@@ -180,10 +320,7 @@ def allowed_actions_roles(iam: IAMClient, policy_names: List[str], role_name: st
180
320
  allowed_actions: AllowedActionCollection = init_action_collection()
181
321
 
182
322
  for policy_name in policy_names:
183
- role_policy = iam.get_role_policy(
184
- RoleName=role_name,
185
- PolicyName=policy_name
186
- )
323
+ role_policy = iam.get_role_policy(RoleName=role_name, PolicyName=policy_name)
187
324
  logger.debug("Checking role policy")
188
325
  # PolicyDocument is now a TypedDict, but an instance of TypedDict is not an instance of dict?
189
326
  if isinstance(role_policy["PolicyDocument"], str):
@@ -191,12 +328,16 @@ def allowed_actions_roles(iam: IAMClient, policy_names: List[str], role_name: st
191
328
  else:
192
329
  policy_document = role_policy["PolicyDocument"]
193
330
 
194
- allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_document))
331
+ allowed_actions = add_to_action_collection(
332
+ allowed_actions, get_actions_from_policy_document(policy_document)
333
+ )
195
334
 
196
335
  return allowed_actions
197
336
 
198
337
 
199
- def collect_policy_actions(policy_documents: List[Union[str, PolicyDocumentDictTypeDef]]) -> AllowedActionCollection:
338
+ def collect_policy_actions(
339
+ policy_documents: list[Union[str, "PolicyDocumentDictTypeDef"]]
340
+ ) -> AllowedActionCollection:
200
341
  """
201
342
  Collect all of the actions allowed by the given policy documents into one AllowedActionCollection.
202
343
  """
@@ -207,11 +348,15 @@ def collect_policy_actions(policy_documents: List[Union[str, PolicyDocumentDictT
207
348
  policy_dict = json.loads(policy_str)
208
349
  else:
209
350
  policy_dict = policy_str
210
- allowed_actions = add_to_action_collection(allowed_actions, get_actions_from_policy_document(policy_dict))
351
+ allowed_actions = add_to_action_collection(
352
+ allowed_actions, get_actions_from_policy_document(policy_dict)
353
+ )
211
354
  return allowed_actions
212
355
 
213
356
 
214
- def allowed_actions_user(iam: IAMClient, policy_names: List[str], user_name: str) -> AllowedActionCollection:
357
+ def allowed_actions_user(
358
+ iam: "IAMClient", policy_names: list[str], user_name: str
359
+ ) -> AllowedActionCollection:
215
360
  """
216
361
  Gets all allowed actions for a user given by user_name, returns a dictionary, keyed by resource,
217
362
  with a list of permissions allowed for each given resource.
@@ -221,16 +366,17 @@ def allowed_actions_user(iam: IAMClient, policy_names: List[str], user_name: str
221
366
  :param user_name: Name of user to get associated policies
222
367
  """
223
368
  user_policies = [
224
- iam.get_user_policy(
225
- UserName=user_name,
226
- PolicyName=policy_name
227
- )["PolicyDocument"]
369
+ iam.get_user_policy(UserName=user_name, PolicyName=policy_name)[
370
+ "PolicyDocument"
371
+ ]
228
372
  for policy_name in policy_names
229
373
  ]
230
374
  return collect_policy_actions(user_policies)
231
375
 
232
376
 
233
- def allowed_actions_group(iam: IAMClient, policy_names: List[str], group_name: str) -> AllowedActionCollection:
377
+ def allowed_actions_group(
378
+ iam: "IAMClient", policy_names: list[str], group_name: str
379
+ ) -> AllowedActionCollection:
234
380
  """
235
381
  Gets all allowed actions for a group given by group_name, returns a dictionary, keyed by resource,
236
382
  with a list of permissions allowed for each given resource.
@@ -240,10 +386,9 @@ def allowed_actions_group(iam: IAMClient, policy_names: List[str], group_name: s
240
386
  :param group_name: Name of group to get associated policies
241
387
  """
242
388
  group_policies = [
243
- iam.get_group_policy(
244
- GroupName=group_name,
245
- PolicyName=policy_name
246
- )["PolicyDocument"]
389
+ iam.get_group_policy(GroupName=group_name, PolicyName=policy_name)[
390
+ "PolicyDocument"
391
+ ]
247
392
  for policy_name in policy_names
248
393
  ]
249
394
  return collect_policy_actions(group_policies)
@@ -257,29 +402,51 @@ def get_policy_permissions(region: str) -> AllowedActionCollection:
257
402
  :param zone: AWS zone to connect to
258
403
  """
259
404
 
260
- iam: IAMClient = get_client('iam', region)
261
- sts: STSClient = get_client('sts', region)
262
- #TODO Condider effect: deny at some point
263
- allowed_actions: AllowedActionCollection = defaultdict(lambda: {'Action': [], 'NotAction': []})
405
+ iam: "IAMClient" = get_client("iam", region)
406
+ sts: "STSClient" = get_client("sts", region)
407
+ # TODO Condider effect: deny at some point
408
+ allowed_actions: AllowedActionCollection = defaultdict(
409
+ lambda: {"Action": [], "NotAction": []}
410
+ )
264
411
  try:
265
412
  # If successful then we assume we are operating as a user, and grab the associated permissions
266
413
  user = iam.get_user()
267
- list_policies = iam.list_user_policies(UserName=user['User']['UserName'])
268
- attached_policies = iam.list_attached_user_policies(UserName=user['User']['UserName'])
269
- user_attached_policies = allowed_actions_attached(iam, attached_policies['AttachedPolicies'])
270
- allowed_actions = add_to_action_collection(allowed_actions, user_attached_policies)
271
- user_inline_policies = allowed_actions_user(iam, list_policies['PolicyNames'], user['User']['UserName'])
272
- allowed_actions = add_to_action_collection(allowed_actions, user_inline_policies)
414
+ list_policies = iam.list_user_policies(UserName=user["User"]["UserName"])
415
+ attached_policies = iam.list_attached_user_policies(
416
+ UserName=user["User"]["UserName"]
417
+ )
418
+ user_attached_policies = allowed_actions_attached(
419
+ iam, attached_policies["AttachedPolicies"]
420
+ )
421
+ allowed_actions = add_to_action_collection(
422
+ allowed_actions, user_attached_policies
423
+ )
424
+ user_inline_policies = allowed_actions_user(
425
+ iam, list_policies["PolicyNames"], user["User"]["UserName"]
426
+ )
427
+ allowed_actions = add_to_action_collection(
428
+ allowed_actions, user_inline_policies
429
+ )
273
430
 
274
431
  # grab group policies associated with the user
275
- groups = iam.list_groups_for_user(UserName=user['User']['UserName'])
432
+ groups = iam.list_groups_for_user(UserName=user["User"]["UserName"])
276
433
  for group in groups["Groups"]:
277
- list_policies = iam.list_group_policies(GroupName=group['GroupName'])
278
- attached_policies = iam.list_attached_group_policies(GroupName=group['GroupName'])
279
- group_attached_policies = allowed_actions_attached(iam, attached_policies['AttachedPolicies'])
280
- allowed_actions = add_to_action_collection(allowed_actions, group_attached_policies)
281
- group_inline_policies = allowed_actions_group(iam, list_policies['PolicyNames'], group['GroupName'])
282
- allowed_actions = add_to_action_collection(allowed_actions, group_inline_policies)
434
+ list_policies = iam.list_group_policies(GroupName=group["GroupName"])
435
+ attached_policies = iam.list_attached_group_policies(
436
+ GroupName=group["GroupName"]
437
+ )
438
+ group_attached_policies = allowed_actions_attached(
439
+ iam, attached_policies["AttachedPolicies"]
440
+ )
441
+ allowed_actions = add_to_action_collection(
442
+ allowed_actions, group_attached_policies
443
+ )
444
+ group_inline_policies = allowed_actions_group(
445
+ iam, list_policies["PolicyNames"], group["GroupName"]
446
+ )
447
+ allowed_actions = add_to_action_collection(
448
+ allowed_actions, group_inline_policies
449
+ )
283
450
 
284
451
  except:
285
452
  # If not successful, we check the role associated with an instance profile
@@ -291,19 +458,28 @@ def get_policy_permissions(region: str) -> AllowedActionCollection:
291
458
  role_name = role["Arn"].split("/")[1]
292
459
  list_policies = iam.list_role_policies(RoleName=role_name)
293
460
  attached_policies = iam.list_attached_role_policies(RoleName=role_name)
294
- role_attached_policies = allowed_actions_attached(iam, attached_policies['AttachedPolicies'])
295
- allowed_actions = add_to_action_collection(allowed_actions, role_attached_policies)
296
- role_inline_policies = allowed_actions_roles(iam, list_policies['PolicyNames'], role_name)
297
- allowed_actions = add_to_action_collection(allowed_actions, role_inline_policies)
461
+ role_attached_policies = allowed_actions_attached(
462
+ iam, attached_policies["AttachedPolicies"]
463
+ )
464
+ allowed_actions = add_to_action_collection(
465
+ allowed_actions, role_attached_policies
466
+ )
467
+ role_inline_policies = allowed_actions_roles(
468
+ iam, list_policies["PolicyNames"], role_name
469
+ )
470
+ allowed_actions = add_to_action_collection(
471
+ allowed_actions, role_inline_policies
472
+ )
298
473
 
299
474
  except:
300
475
  logger.exception("Exception when trying to get role policies")
301
476
  logger.debug("Allowed actions: %s", allowed_actions)
302
477
  return allowed_actions
303
478
 
304
- @lru_cache()
479
+
480
+ @lru_cache
305
481
  def get_aws_account_num() -> Optional[str]:
306
482
  """
307
483
  Returns AWS account num
308
484
  """
309
- return boto3.client('sts').get_caller_identity().get('Account')
485
+ return boto3.client("sts").get_caller_identity().get("Account")
toil/lib/aws/s3.py ADDED
@@ -0,0 +1,31 @@
1
+ # Copyright (C) 2015-2024 Regents of the University of California
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import logging
15
+
16
+ from mypy_boto3_s3.type_defs import ListMultipartUploadsOutputTypeDef
17
+
18
+ from toil.lib.aws import AWSServerErrors, session
19
+ from toil.lib.retry import retry
20
+
21
+ logger = logging.getLogger(__name__)
22
+
23
+
24
+ @retry(errors=[AWSServerErrors])
25
+ def list_multipart_uploads(
26
+ bucket: str, region: str, prefix: str, max_uploads: int = 1
27
+ ) -> ListMultipartUploadsOutputTypeDef:
28
+ s3_client = session.client("s3", region_name=region)
29
+ return s3_client.list_multipart_uploads(
30
+ Bucket=bucket, MaxUploads=max_uploads, Prefix=prefix
31
+ )