awscli 1.41.9__py3-none-any.whl → 1.41.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- awscli/__init__.py +1 -1
- awscli/customizations/emr/createcluster.py +508 -282
- awscli/customizations/emr/helptext.py +122 -74
- awscli/examples/emr/create-cluster-synopsis.txt +1 -0
- {awscli-1.41.9.dist-info → awscli-1.41.11.dist-info}/METADATA +2 -2
- {awscli-1.41.9.dist-info → awscli-1.41.11.dist-info}/RECORD +14 -14
- {awscli-1.41.9.data → awscli-1.41.11.data}/scripts/aws +0 -0
- {awscli-1.41.9.data → awscli-1.41.11.data}/scripts/aws.cmd +0 -0
- {awscli-1.41.9.data → awscli-1.41.11.data}/scripts/aws_bash_completer +0 -0
- {awscli-1.41.9.data → awscli-1.41.11.data}/scripts/aws_completer +0 -0
- {awscli-1.41.9.data → awscli-1.41.11.data}/scripts/aws_zsh_completer.sh +0 -0
- {awscli-1.41.9.dist-info → awscli-1.41.11.dist-info}/LICENSE.txt +0 -0
- {awscli-1.41.9.dist-info → awscli-1.41.11.dist-info}/WHEEL +0 -0
- {awscli-1.41.9.dist-info → awscli-1.41.11.dist-info}/top_level.txt +0 -0
|
@@ -12,136 +12,217 @@
|
|
|
12
12
|
# language governing permissions and limitations under the License.
|
|
13
13
|
|
|
14
14
|
import re
|
|
15
|
+
|
|
16
|
+
from botocore.compat import json
|
|
17
|
+
|
|
15
18
|
from awscli.customizations.commands import BasicCommand
|
|
16
|
-
from awscli.customizations.emr import
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
19
|
+
from awscli.customizations.emr import (
|
|
20
|
+
applicationutils,
|
|
21
|
+
argumentschema,
|
|
22
|
+
constants,
|
|
23
|
+
emrfsutils,
|
|
24
|
+
emrutils,
|
|
25
|
+
exceptions,
|
|
26
|
+
hbaseutils,
|
|
27
|
+
helptext,
|
|
28
|
+
instancefleetsutils,
|
|
29
|
+
instancegroupsutils,
|
|
30
|
+
steputils,
|
|
31
|
+
)
|
|
27
32
|
from awscli.customizations.emr.command import Command
|
|
28
|
-
from awscli.customizations.emr.constants import EC2_ROLE_NAME
|
|
29
|
-
from awscli.customizations.emr.constants import EMR_ROLE_NAME
|
|
30
|
-
from botocore.compat import json
|
|
33
|
+
from awscli.customizations.emr.constants import EC2_ROLE_NAME, EMR_ROLE_NAME
|
|
31
34
|
|
|
32
35
|
|
|
33
36
|
class CreateCluster(Command):
|
|
34
37
|
NAME = 'create-cluster'
|
|
35
38
|
DESCRIPTION = helptext.CREATE_CLUSTER_DESCRIPTION
|
|
36
39
|
ARG_TABLE = [
|
|
37
|
-
{'name': 'release-label',
|
|
38
|
-
|
|
39
|
-
{'name': '
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
{'name': 'instance-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
{
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
{
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
{
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
{
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
{'name': '
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
{
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
{
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
{
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
40
|
+
{'name': 'release-label', 'help_text': helptext.RELEASE_LABEL},
|
|
41
|
+
{'name': 'os-release-label', 'help_text': helptext.OS_RELEASE_LABEL},
|
|
42
|
+
{'name': 'ami-version', 'help_text': helptext.AMI_VERSION},
|
|
43
|
+
{
|
|
44
|
+
'name': 'instance-groups',
|
|
45
|
+
'schema': argumentschema.INSTANCE_GROUPS_SCHEMA,
|
|
46
|
+
'help_text': helptext.INSTANCE_GROUPS,
|
|
47
|
+
},
|
|
48
|
+
{'name': 'instance-type', 'help_text': helptext.INSTANCE_TYPE},
|
|
49
|
+
{'name': 'instance-count', 'help_text': helptext.INSTANCE_COUNT},
|
|
50
|
+
{
|
|
51
|
+
'name': 'auto-terminate',
|
|
52
|
+
'action': 'store_true',
|
|
53
|
+
'group_name': 'auto_terminate',
|
|
54
|
+
'help_text': helptext.AUTO_TERMINATE,
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
'name': 'no-auto-terminate',
|
|
58
|
+
'action': 'store_true',
|
|
59
|
+
'group_name': 'auto_terminate',
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
'name': 'instance-fleets',
|
|
63
|
+
'schema': argumentschema.INSTANCE_FLEETS_SCHEMA,
|
|
64
|
+
'help_text': helptext.INSTANCE_FLEETS,
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
'name': 'name',
|
|
68
|
+
'default': 'Development Cluster',
|
|
69
|
+
'help_text': helptext.CLUSTER_NAME,
|
|
70
|
+
},
|
|
71
|
+
{'name': 'log-uri', 'help_text': helptext.LOG_URI},
|
|
72
|
+
{
|
|
73
|
+
'name': 'log-encryption-kms-key-id',
|
|
74
|
+
'help_text': helptext.LOG_ENCRYPTION_KMS_KEY_ID,
|
|
75
|
+
},
|
|
76
|
+
{'name': 'service-role', 'help_text': helptext.SERVICE_ROLE},
|
|
77
|
+
{'name': 'auto-scaling-role', 'help_text': helptext.AUTOSCALING_ROLE},
|
|
78
|
+
{
|
|
79
|
+
'name': 'use-default-roles',
|
|
80
|
+
'action': 'store_true',
|
|
81
|
+
'help_text': helptext.USE_DEFAULT_ROLES,
|
|
82
|
+
},
|
|
83
|
+
{'name': 'configurations', 'help_text': helptext.CONFIGURATIONS},
|
|
84
|
+
{
|
|
85
|
+
'name': 'ec2-attributes',
|
|
86
|
+
'help_text': helptext.EC2_ATTRIBUTES,
|
|
87
|
+
'schema': argumentschema.EC2_ATTRIBUTES_SCHEMA,
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
'name': 'termination-protected',
|
|
91
|
+
'action': 'store_true',
|
|
92
|
+
'group_name': 'termination_protected',
|
|
93
|
+
'help_text': helptext.TERMINATION_PROTECTED,
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
'name': 'no-termination-protected',
|
|
97
|
+
'action': 'store_true',
|
|
98
|
+
'group_name': 'termination_protected',
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
'name': 'unhealthy-node-replacement',
|
|
102
|
+
'action': 'store_true',
|
|
103
|
+
'group_name': 'unhealthy_node_replacement',
|
|
104
|
+
'help_text': helptext.UNHEALTHY_NODE_REPLACEMENT,
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
'name': 'no-unhealthy-node-replacement',
|
|
108
|
+
'action': 'store_true',
|
|
109
|
+
'group_name': 'unhealthy_node_replacement',
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
'name': 'scale-down-behavior',
|
|
113
|
+
'help_text': helptext.SCALE_DOWN_BEHAVIOR,
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
'name': 'visible-to-all-users',
|
|
117
|
+
'action': 'store_true',
|
|
118
|
+
'group_name': 'visibility',
|
|
119
|
+
'help_text': helptext.VISIBILITY,
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
'name': 'no-visible-to-all-users',
|
|
123
|
+
'action': 'store_true',
|
|
124
|
+
'group_name': 'visibility',
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
'name': 'enable-debugging',
|
|
128
|
+
'action': 'store_true',
|
|
129
|
+
'group_name': 'debug',
|
|
130
|
+
'help_text': helptext.DEBUGGING,
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
'name': 'no-enable-debugging',
|
|
134
|
+
'action': 'store_true',
|
|
135
|
+
'group_name': 'debug',
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
'name': 'tags',
|
|
139
|
+
'nargs': '+',
|
|
140
|
+
'help_text': helptext.TAGS,
|
|
141
|
+
'schema': argumentschema.TAGS_SCHEMA,
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
'name': 'bootstrap-actions',
|
|
145
|
+
'help_text': helptext.BOOTSTRAP_ACTIONS,
|
|
146
|
+
'schema': argumentschema.BOOTSTRAP_ACTIONS_SCHEMA,
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
'name': 'applications',
|
|
150
|
+
'help_text': helptext.APPLICATIONS,
|
|
151
|
+
'schema': argumentschema.APPLICATIONS_SCHEMA,
|
|
152
|
+
},
|
|
153
|
+
{
|
|
154
|
+
'name': 'emrfs',
|
|
155
|
+
'help_text': helptext.EMR_FS,
|
|
156
|
+
'schema': argumentschema.EMR_FS_SCHEMA,
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
'name': 'steps',
|
|
160
|
+
'schema': argumentschema.STEPS_SCHEMA,
|
|
161
|
+
'help_text': helptext.STEPS,
|
|
162
|
+
},
|
|
163
|
+
{'name': 'additional-info', 'help_text': helptext.ADDITIONAL_INFO},
|
|
164
|
+
{
|
|
165
|
+
'name': 'restore-from-hbase-backup',
|
|
166
|
+
'schema': argumentschema.HBASE_RESTORE_FROM_BACKUP_SCHEMA,
|
|
167
|
+
'help_text': helptext.RESTORE_FROM_HBASE,
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
'name': 'security-configuration',
|
|
171
|
+
'help_text': helptext.SECURITY_CONFIG,
|
|
172
|
+
},
|
|
173
|
+
{'name': 'custom-ami-id', 'help_text': helptext.CUSTOM_AMI_ID},
|
|
174
|
+
{
|
|
175
|
+
'name': 'ebs-root-volume-size',
|
|
176
|
+
'help_text': helptext.EBS_ROOT_VOLUME_SIZE,
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
'name': 'ebs-root-volume-iops',
|
|
180
|
+
'help_text': helptext.EBS_ROOT_VOLUME_IOPS,
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
'name': 'ebs-root-volume-throughput',
|
|
184
|
+
'help_text': helptext.EBS_ROOT_VOLUME_THROUGHPUT,
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
'name': 'repo-upgrade-on-boot',
|
|
188
|
+
'help_text': helptext.REPO_UPGRADE_ON_BOOT,
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
'name': 'kerberos-attributes',
|
|
192
|
+
'schema': argumentschema.KERBEROS_ATTRIBUTES_SCHEMA,
|
|
193
|
+
'help_text': helptext.KERBEROS_ATTRIBUTES,
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
'name': 'step-concurrency-level',
|
|
197
|
+
'cli_type_name': 'integer',
|
|
198
|
+
'help_text': helptext.STEP_CONCURRENCY_LEVEL,
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
'name': 'managed-scaling-policy',
|
|
202
|
+
'schema': argumentschema.MANAGED_SCALING_POLICY_SCHEMA,
|
|
203
|
+
'help_text': helptext.MANAGED_SCALING_POLICY,
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
'name': 'placement-group-configs',
|
|
207
|
+
'schema': argumentschema.PLACEMENT_GROUP_CONFIGS_SCHEMA,
|
|
208
|
+
'help_text': helptext.PLACEMENT_GROUP_CONFIGS,
|
|
209
|
+
},
|
|
210
|
+
{
|
|
211
|
+
'name': 'auto-termination-policy',
|
|
212
|
+
'schema': argumentschema.AUTO_TERMINATION_POLICY_SCHEMA,
|
|
213
|
+
'help_text': helptext.AUTO_TERMINATION_POLICY,
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
'name': 'extended-support',
|
|
217
|
+
'action': 'store_true',
|
|
218
|
+
'group_name': 'extended-support',
|
|
219
|
+
'help_text': helptext.EXTENDED_SUPPORT,
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
'name': 'no-extended-support',
|
|
223
|
+
'action': 'store_true',
|
|
224
|
+
'group_name': 'extended-support',
|
|
225
|
+
},
|
|
145
226
|
]
|
|
146
227
|
SYNOPSIS = BasicCommand.FROM_FILE('emr', 'create-cluster-synopsis.txt')
|
|
147
228
|
EXAMPLES = BasicCommand.FROM_FILE('emr', 'create-cluster-examples.rst')
|
|
@@ -154,71 +235,95 @@ class CreateCluster(Command):
|
|
|
154
235
|
|
|
155
236
|
service_role_validation_message = (
|
|
156
237
|
" Either choose --use-default-roles or use both --service-role "
|
|
157
|
-
"<roleName> and --ec2-attributes InstanceProfile=<profileName>."
|
|
238
|
+
"<roleName> and --ec2-attributes InstanceProfile=<profileName>."
|
|
239
|
+
)
|
|
158
240
|
|
|
159
|
-
if
|
|
160
|
-
|
|
241
|
+
if (
|
|
242
|
+
parsed_args.use_default_roles is True
|
|
243
|
+
and parsed_args.service_role is not None
|
|
244
|
+
):
|
|
161
245
|
raise exceptions.MutualExclusiveOptionError(
|
|
162
246
|
option1="--use-default-roles",
|
|
163
247
|
option2="--service-role",
|
|
164
|
-
message=service_role_validation_message
|
|
248
|
+
message=service_role_validation_message,
|
|
249
|
+
)
|
|
165
250
|
|
|
166
|
-
if
|
|
167
|
-
|
|
168
|
-
|
|
251
|
+
if (
|
|
252
|
+
parsed_args.use_default_roles is True
|
|
253
|
+
and parsed_args.ec2_attributes is not None
|
|
254
|
+
and 'InstanceProfile' in parsed_args.ec2_attributes
|
|
255
|
+
):
|
|
169
256
|
raise exceptions.MutualExclusiveOptionError(
|
|
170
257
|
option1="--use-default-roles",
|
|
171
258
|
option2="--ec2-attributes InstanceProfile",
|
|
172
|
-
message=service_role_validation_message
|
|
259
|
+
message=service_role_validation_message,
|
|
260
|
+
)
|
|
173
261
|
|
|
174
|
-
if
|
|
175
|
-
|
|
262
|
+
if (
|
|
263
|
+
parsed_args.instance_groups is not None
|
|
264
|
+
and parsed_args.instance_fleets is not None
|
|
265
|
+
):
|
|
176
266
|
raise exceptions.MutualExclusiveOptionError(
|
|
177
|
-
option1="--instance-groups",
|
|
178
|
-
|
|
267
|
+
option1="--instance-groups", option2="--instance-fleets"
|
|
268
|
+
)
|
|
179
269
|
|
|
180
270
|
instances_config = {}
|
|
181
271
|
if parsed_args.instance_fleets is not None:
|
|
182
|
-
instances_config['InstanceFleets'] =
|
|
272
|
+
instances_config['InstanceFleets'] = (
|
|
183
273
|
instancefleetsutils.validate_and_build_instance_fleets(
|
|
184
|
-
parsed_args.instance_fleets
|
|
274
|
+
parsed_args.instance_fleets
|
|
275
|
+
)
|
|
276
|
+
)
|
|
185
277
|
else:
|
|
186
|
-
instances_config['InstanceGroups'] =
|
|
278
|
+
instances_config['InstanceGroups'] = (
|
|
187
279
|
instancegroupsutils.validate_and_build_instance_groups(
|
|
188
280
|
instance_groups=parsed_args.instance_groups,
|
|
189
281
|
instance_type=parsed_args.instance_type,
|
|
190
|
-
instance_count=parsed_args.instance_count
|
|
282
|
+
instance_count=parsed_args.instance_count,
|
|
283
|
+
)
|
|
284
|
+
)
|
|
191
285
|
|
|
192
286
|
if parsed_args.release_label is not None:
|
|
193
287
|
params["ReleaseLabel"] = parsed_args.release_label
|
|
194
288
|
if parsed_args.configurations is not None:
|
|
195
289
|
try:
|
|
196
290
|
params["Configurations"] = json.loads(
|
|
197
|
-
parsed_args.configurations
|
|
291
|
+
parsed_args.configurations
|
|
292
|
+
)
|
|
198
293
|
except ValueError:
|
|
199
|
-
raise ValueError(
|
|
200
|
-
|
|
294
|
+
raise ValueError(
|
|
295
|
+
'aws: error: invalid json argument for '
|
|
296
|
+
'option --configurations'
|
|
297
|
+
)
|
|
201
298
|
|
|
202
299
|
if (
|
|
203
300
|
parsed_args.release_label is None
|
|
204
301
|
and parsed_args.ami_version is not None
|
|
205
302
|
):
|
|
206
|
-
is_valid_ami_version = re.match(
|
|
303
|
+
is_valid_ami_version = re.match(
|
|
304
|
+
r'\d?\..*', parsed_args.ami_version
|
|
305
|
+
)
|
|
207
306
|
if is_valid_ami_version is None:
|
|
208
307
|
raise exceptions.InvalidAmiVersionError(
|
|
209
|
-
ami_version=parsed_args.ami_version
|
|
308
|
+
ami_version=parsed_args.ami_version
|
|
309
|
+
)
|
|
210
310
|
params['AmiVersion'] = parsed_args.ami_version
|
|
211
311
|
emrutils.apply_dict(
|
|
212
|
-
params, 'AdditionalInfo', parsed_args.additional_info
|
|
312
|
+
params, 'AdditionalInfo', parsed_args.additional_info
|
|
313
|
+
)
|
|
213
314
|
emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
|
|
214
315
|
|
|
215
316
|
if parsed_args.os_release_label is not None:
|
|
216
|
-
emrutils.apply_dict(
|
|
217
|
-
parsed_args.os_release_label
|
|
317
|
+
emrutils.apply_dict(
|
|
318
|
+
params, 'OSReleaseLabel', parsed_args.os_release_label
|
|
319
|
+
)
|
|
218
320
|
|
|
219
321
|
if parsed_args.log_encryption_kms_key_id is not None:
|
|
220
|
-
emrutils.apply_dict(
|
|
221
|
-
|
|
322
|
+
emrutils.apply_dict(
|
|
323
|
+
params,
|
|
324
|
+
'LogEncryptionKmsKeyId',
|
|
325
|
+
parsed_args.log_encryption_kms_key_id,
|
|
326
|
+
)
|
|
222
327
|
|
|
223
328
|
if parsed_args.use_default_roles is True:
|
|
224
329
|
parsed_args.service_role = EMR_ROLE_NAME
|
|
@@ -234,61 +339,79 @@ class CreateCluster(Command):
|
|
|
234
339
|
if parsed_args.auto_scaling_role is None:
|
|
235
340
|
raise exceptions.MissingAutoScalingRoleError()
|
|
236
341
|
|
|
237
|
-
emrutils.apply_dict(
|
|
342
|
+
emrutils.apply_dict(
|
|
343
|
+
params, 'AutoScalingRole', parsed_args.auto_scaling_role
|
|
344
|
+
)
|
|
238
345
|
|
|
239
346
|
if parsed_args.scale_down_behavior is not None:
|
|
240
|
-
emrutils.apply_dict(
|
|
347
|
+
emrutils.apply_dict(
|
|
348
|
+
params, 'ScaleDownBehavior', parsed_args.scale_down_behavior
|
|
349
|
+
)
|
|
241
350
|
|
|
242
351
|
if (
|
|
243
|
-
|
|
244
|
-
|
|
352
|
+
parsed_args.no_auto_terminate is False
|
|
353
|
+
and parsed_args.auto_terminate is False
|
|
354
|
+
):
|
|
245
355
|
parsed_args.no_auto_terminate = True
|
|
246
356
|
|
|
247
|
-
instances_config['KeepJobFlowAliveWhenNoSteps'] =
|
|
357
|
+
instances_config['KeepJobFlowAliveWhenNoSteps'] = (
|
|
248
358
|
emrutils.apply_boolean_options(
|
|
249
359
|
parsed_args.no_auto_terminate,
|
|
250
360
|
'--no-auto-terminate',
|
|
251
361
|
parsed_args.auto_terminate,
|
|
252
|
-
'--auto-terminate'
|
|
362
|
+
'--auto-terminate',
|
|
363
|
+
)
|
|
364
|
+
)
|
|
253
365
|
|
|
254
|
-
instances_config['TerminationProtected'] =
|
|
366
|
+
instances_config['TerminationProtected'] = (
|
|
255
367
|
emrutils.apply_boolean_options(
|
|
256
368
|
parsed_args.termination_protected,
|
|
257
369
|
'--termination-protected',
|
|
258
370
|
parsed_args.no_termination_protected,
|
|
259
|
-
'--no-termination-protected'
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
instances_config['UnhealthyNodeReplacement'] = \
|
|
263
|
-
emrutils.apply_boolean_options(
|
|
264
|
-
parsed_args.unhealthy_node_replacement,
|
|
265
|
-
'--unhealthy-node-replacement',
|
|
266
|
-
parsed_args.no_unhealthy_node_replacement,
|
|
267
|
-
'--no-unhealthy-node-replacement')
|
|
371
|
+
'--no-termination-protected',
|
|
372
|
+
)
|
|
373
|
+
)
|
|
268
374
|
|
|
269
|
-
if (
|
|
270
|
-
|
|
375
|
+
if (
|
|
376
|
+
parsed_args.unhealthy_node_replacement
|
|
377
|
+
or parsed_args.no_unhealthy_node_replacement
|
|
378
|
+
):
|
|
379
|
+
instances_config['UnhealthyNodeReplacement'] = (
|
|
380
|
+
emrutils.apply_boolean_options(
|
|
381
|
+
parsed_args.unhealthy_node_replacement,
|
|
382
|
+
'--unhealthy-node-replacement',
|
|
383
|
+
parsed_args.no_unhealthy_node_replacement,
|
|
384
|
+
'--no-unhealthy-node-replacement',
|
|
385
|
+
)
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
if (
|
|
389
|
+
parsed_args.visible_to_all_users is False
|
|
390
|
+
and parsed_args.no_visible_to_all_users is False
|
|
391
|
+
):
|
|
271
392
|
parsed_args.visible_to_all_users = True
|
|
272
393
|
|
|
273
|
-
params['VisibleToAllUsers'] =
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
394
|
+
params['VisibleToAllUsers'] = emrutils.apply_boolean_options(
|
|
395
|
+
parsed_args.visible_to_all_users,
|
|
396
|
+
'--visible-to-all-users',
|
|
397
|
+
parsed_args.no_visible_to_all_users,
|
|
398
|
+
'--no-visible-to-all-users',
|
|
399
|
+
)
|
|
279
400
|
|
|
280
401
|
params['Tags'] = emrutils.parse_tags(parsed_args.tags)
|
|
281
402
|
params['Instances'] = instances_config
|
|
282
403
|
|
|
283
404
|
if parsed_args.ec2_attributes is not None:
|
|
284
405
|
self._build_ec2_attributes(
|
|
285
|
-
cluster=params, parsed_attrs=parsed_args.ec2_attributes
|
|
406
|
+
cluster=params, parsed_attrs=parsed_args.ec2_attributes
|
|
407
|
+
)
|
|
286
408
|
|
|
287
409
|
debugging_enabled = emrutils.apply_boolean_options(
|
|
288
410
|
parsed_args.enable_debugging,
|
|
289
411
|
'--enable-debugging',
|
|
290
412
|
parsed_args.no_enable_debugging,
|
|
291
|
-
'--no-enable-debugging'
|
|
413
|
+
'--no-enable-debugging',
|
|
414
|
+
)
|
|
292
415
|
|
|
293
416
|
if parsed_args.log_uri is None and debugging_enabled is True:
|
|
294
417
|
raise exceptions.LogUriError
|
|
@@ -298,21 +421,24 @@ class CreateCluster(Command):
|
|
|
298
421
|
cluster=params,
|
|
299
422
|
key='Steps',
|
|
300
423
|
value=[
|
|
301
|
-
self._build_enable_debugging(parsed_args, parsed_globals)
|
|
424
|
+
self._build_enable_debugging(parsed_args, parsed_globals)
|
|
425
|
+
],
|
|
426
|
+
)
|
|
302
427
|
|
|
303
428
|
if parsed_args.applications is not None:
|
|
304
429
|
if parsed_args.release_label is None:
|
|
305
|
-
app_list, ba_list, step_list =
|
|
430
|
+
app_list, ba_list, step_list = (
|
|
306
431
|
applicationutils.build_applications(
|
|
307
432
|
region=self.region,
|
|
308
433
|
parsed_applications=parsed_args.applications,
|
|
309
|
-
ami_version=params['AmiVersion']
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
self._update_cluster_dict(
|
|
313
|
-
params, 'BootstrapActions', ba_list)
|
|
434
|
+
ami_version=params['AmiVersion'],
|
|
435
|
+
)
|
|
436
|
+
)
|
|
314
437
|
self._update_cluster_dict(
|
|
315
|
-
params, '
|
|
438
|
+
params, 'NewSupportedProducts', app_list
|
|
439
|
+
)
|
|
440
|
+
self._update_cluster_dict(params, 'BootstrapActions', ba_list)
|
|
441
|
+
self._update_cluster_dict(params, 'Steps', step_list)
|
|
316
442
|
else:
|
|
317
443
|
params["Applications"] = []
|
|
318
444
|
for application in parsed_args.applications:
|
|
@@ -322,37 +448,45 @@ class CreateCluster(Command):
|
|
|
322
448
|
if hbase_restore_config is not None:
|
|
323
449
|
args = hbaseutils.build_hbase_restore_from_backup_args(
|
|
324
450
|
dir=hbase_restore_config.get('Dir'),
|
|
325
|
-
backup_version=hbase_restore_config.get('BackupVersion')
|
|
451
|
+
backup_version=hbase_restore_config.get('BackupVersion'),
|
|
452
|
+
)
|
|
326
453
|
step_config = emrutils.build_step(
|
|
327
454
|
jar=constants.HBASE_JAR_PATH,
|
|
328
455
|
name=constants.HBASE_RESTORE_STEP_NAME,
|
|
329
456
|
action_on_failure=constants.CANCEL_AND_WAIT,
|
|
330
|
-
args=args
|
|
331
|
-
|
|
332
|
-
|
|
457
|
+
args=args,
|
|
458
|
+
)
|
|
459
|
+
self._update_cluster_dict(params, 'Steps', [step_config])
|
|
333
460
|
|
|
334
461
|
if parsed_args.bootstrap_actions is not None:
|
|
335
462
|
self._build_bootstrap_actions(
|
|
336
463
|
cluster=params,
|
|
337
|
-
parsed_boostrap_actions=parsed_args.bootstrap_actions
|
|
464
|
+
parsed_boostrap_actions=parsed_args.bootstrap_actions,
|
|
465
|
+
)
|
|
338
466
|
|
|
339
467
|
if parsed_args.emrfs is not None:
|
|
340
468
|
self._handle_emrfs_parameters(
|
|
341
469
|
cluster=params,
|
|
342
470
|
emrfs_args=parsed_args.emrfs,
|
|
343
|
-
release_label=parsed_args.release_label
|
|
471
|
+
release_label=parsed_args.release_label,
|
|
472
|
+
)
|
|
344
473
|
|
|
345
474
|
if parsed_args.steps is not None:
|
|
346
475
|
steps_list = steputils.build_step_config_list(
|
|
347
476
|
parsed_step_list=parsed_args.steps,
|
|
348
477
|
region=self.region,
|
|
349
|
-
release_label=parsed_args.release_label
|
|
478
|
+
release_label=parsed_args.release_label,
|
|
479
|
+
)
|
|
350
480
|
self._update_cluster_dict(
|
|
351
|
-
cluster=params, key='Steps', value=steps_list
|
|
481
|
+
cluster=params, key='Steps', value=steps_list
|
|
482
|
+
)
|
|
352
483
|
|
|
353
484
|
if parsed_args.security_configuration is not None:
|
|
354
485
|
emrutils.apply_dict(
|
|
355
|
-
params,
|
|
486
|
+
params,
|
|
487
|
+
'SecurityConfiguration',
|
|
488
|
+
parsed_args.security_configuration,
|
|
489
|
+
)
|
|
356
490
|
|
|
357
491
|
if parsed_args.custom_ami_id is not None:
|
|
358
492
|
emrutils.apply_dict(
|
|
@@ -360,15 +494,21 @@ class CreateCluster(Command):
|
|
|
360
494
|
)
|
|
361
495
|
if parsed_args.ebs_root_volume_size is not None:
|
|
362
496
|
emrutils.apply_dict(
|
|
363
|
-
params,
|
|
497
|
+
params,
|
|
498
|
+
'EbsRootVolumeSize',
|
|
499
|
+
int(parsed_args.ebs_root_volume_size),
|
|
364
500
|
)
|
|
365
501
|
if parsed_args.ebs_root_volume_iops is not None:
|
|
366
502
|
emrutils.apply_dict(
|
|
367
|
-
params,
|
|
503
|
+
params,
|
|
504
|
+
'EbsRootVolumeIops',
|
|
505
|
+
int(parsed_args.ebs_root_volume_iops),
|
|
368
506
|
)
|
|
369
507
|
if parsed_args.ebs_root_volume_throughput is not None:
|
|
370
508
|
emrutils.apply_dict(
|
|
371
|
-
params,
|
|
509
|
+
params,
|
|
510
|
+
'EbsRootVolumeThroughput',
|
|
511
|
+
int(parsed_args.ebs_root_volume_throughput),
|
|
372
512
|
)
|
|
373
513
|
|
|
374
514
|
if parsed_args.repo_upgrade_on_boot is not None:
|
|
@@ -378,34 +518,56 @@ class CreateCluster(Command):
|
|
|
378
518
|
|
|
379
519
|
if parsed_args.kerberos_attributes is not None:
|
|
380
520
|
emrutils.apply_dict(
|
|
381
|
-
params, 'KerberosAttributes', parsed_args.kerberos_attributes
|
|
521
|
+
params, 'KerberosAttributes', parsed_args.kerberos_attributes
|
|
522
|
+
)
|
|
382
523
|
|
|
383
524
|
if parsed_args.step_concurrency_level is not None:
|
|
384
525
|
params['StepConcurrencyLevel'] = parsed_args.step_concurrency_level
|
|
385
526
|
|
|
527
|
+
if parsed_args.extended_support or parsed_args.no_extended_support:
|
|
528
|
+
params['ExtendedSupport'] = emrutils.apply_boolean_options(
|
|
529
|
+
parsed_args.extended_support,
|
|
530
|
+
'--extended-support',
|
|
531
|
+
parsed_args.no_extended_support,
|
|
532
|
+
'--no-extended-support',
|
|
533
|
+
)
|
|
534
|
+
|
|
386
535
|
if parsed_args.managed_scaling_policy is not None:
|
|
387
536
|
emrutils.apply_dict(
|
|
388
|
-
params,
|
|
537
|
+
params,
|
|
538
|
+
'ManagedScalingPolicy',
|
|
539
|
+
parsed_args.managed_scaling_policy,
|
|
540
|
+
)
|
|
389
541
|
|
|
390
542
|
if parsed_args.placement_group_configs is not None:
|
|
391
543
|
emrutils.apply_dict(
|
|
392
|
-
params,
|
|
393
|
-
|
|
544
|
+
params,
|
|
545
|
+
'PlacementGroupConfigs',
|
|
546
|
+
parsed_args.placement_group_configs,
|
|
547
|
+
)
|
|
394
548
|
|
|
395
549
|
if parsed_args.auto_termination_policy is not None:
|
|
396
550
|
emrutils.apply_dict(
|
|
397
|
-
params,
|
|
398
|
-
|
|
551
|
+
params,
|
|
552
|
+
'AutoTerminationPolicy',
|
|
553
|
+
parsed_args.auto_termination_policy,
|
|
554
|
+
)
|
|
399
555
|
|
|
400
556
|
self._validate_required_applications(parsed_args)
|
|
401
557
|
|
|
402
558
|
run_job_flow_response = emrutils.call(
|
|
403
|
-
self._session,
|
|
404
|
-
|
|
559
|
+
self._session,
|
|
560
|
+
'run_job_flow',
|
|
561
|
+
params,
|
|
562
|
+
self.region,
|
|
563
|
+
parsed_globals.endpoint_url,
|
|
564
|
+
parsed_globals.verify_ssl,
|
|
565
|
+
)
|
|
405
566
|
|
|
406
567
|
constructed_result = self._construct_result(run_job_flow_response)
|
|
407
|
-
emrutils.display_response(
|
|
408
|
-
|
|
568
|
+
emrutils.display_response(
|
|
569
|
+
self._session, 'run_job_flow', constructed_result, parsed_globals
|
|
570
|
+
)
|
|
409
571
|
|
|
410
572
|
return 0
|
|
411
573
|
|
|
@@ -417,8 +579,7 @@ class CreateCluster(Command):
|
|
|
417
579
|
clusterArn = run_job_flow_result.get('ClusterArn')
|
|
418
580
|
|
|
419
581
|
if jobFlowId is not None:
|
|
420
|
-
return {'ClusterId': jobFlowId,
|
|
421
|
-
'ClusterArn': clusterArn }
|
|
582
|
+
return {'ClusterId': jobFlowId, 'ClusterArn': clusterArn}
|
|
422
583
|
else:
|
|
423
584
|
return {}
|
|
424
585
|
|
|
@@ -426,83 +587,118 @@ class CreateCluster(Command):
|
|
|
426
587
|
keys = parsed_attrs.keys()
|
|
427
588
|
instances = cluster['Instances']
|
|
428
589
|
|
|
429
|
-
if
|
|
590
|
+
if 'SubnetId' in keys and 'SubnetIds' in keys:
|
|
430
591
|
raise exceptions.MutualExclusiveOptionError(
|
|
431
|
-
option1="SubnetId",
|
|
432
|
-
|
|
592
|
+
option1="SubnetId", option2="SubnetIds"
|
|
593
|
+
)
|
|
433
594
|
|
|
434
|
-
if
|
|
595
|
+
if 'AvailabilityZone' in keys and 'AvailabilityZones' in keys:
|
|
435
596
|
raise exceptions.MutualExclusiveOptionError(
|
|
436
|
-
option1="AvailabilityZone",
|
|
437
|
-
|
|
597
|
+
option1="AvailabilityZone", option2="AvailabilityZones"
|
|
598
|
+
)
|
|
438
599
|
|
|
439
|
-
if ('SubnetId' in keys or 'SubnetIds' in keys)
|
|
440
|
-
|
|
600
|
+
if ('SubnetId' in keys or 'SubnetIds' in keys) and (
|
|
601
|
+
'AvailabilityZone' in keys or 'AvailabilityZones' in keys
|
|
602
|
+
):
|
|
441
603
|
raise exceptions.SubnetAndAzValidationError
|
|
442
604
|
|
|
443
605
|
emrutils.apply_params(
|
|
444
|
-
src_params=parsed_attrs,
|
|
445
|
-
|
|
606
|
+
src_params=parsed_attrs,
|
|
607
|
+
src_key='KeyName',
|
|
608
|
+
dest_params=instances,
|
|
609
|
+
dest_key='Ec2KeyName',
|
|
610
|
+
)
|
|
446
611
|
emrutils.apply_params(
|
|
447
|
-
src_params=parsed_attrs,
|
|
448
|
-
|
|
612
|
+
src_params=parsed_attrs,
|
|
613
|
+
src_key='SubnetId',
|
|
614
|
+
dest_params=instances,
|
|
615
|
+
dest_key='Ec2SubnetId',
|
|
616
|
+
)
|
|
449
617
|
emrutils.apply_params(
|
|
450
|
-
src_params=parsed_attrs,
|
|
451
|
-
|
|
618
|
+
src_params=parsed_attrs,
|
|
619
|
+
src_key='SubnetIds',
|
|
620
|
+
dest_params=instances,
|
|
621
|
+
dest_key='Ec2SubnetIds',
|
|
622
|
+
)
|
|
452
623
|
|
|
453
624
|
if 'AvailabilityZone' in keys:
|
|
454
625
|
instances['Placement'] = dict()
|
|
455
626
|
emrutils.apply_params(
|
|
456
|
-
src_params=parsed_attrs,
|
|
627
|
+
src_params=parsed_attrs,
|
|
628
|
+
src_key='AvailabilityZone',
|
|
457
629
|
dest_params=instances['Placement'],
|
|
458
|
-
dest_key='AvailabilityZone'
|
|
630
|
+
dest_key='AvailabilityZone',
|
|
631
|
+
)
|
|
459
632
|
|
|
460
633
|
if 'AvailabilityZones' in keys:
|
|
461
634
|
instances['Placement'] = dict()
|
|
462
635
|
emrutils.apply_params(
|
|
463
|
-
src_params=parsed_attrs,
|
|
636
|
+
src_params=parsed_attrs,
|
|
637
|
+
src_key='AvailabilityZones',
|
|
464
638
|
dest_params=instances['Placement'],
|
|
465
|
-
dest_key='AvailabilityZones'
|
|
639
|
+
dest_key='AvailabilityZones',
|
|
640
|
+
)
|
|
466
641
|
|
|
467
642
|
emrutils.apply_params(
|
|
468
|
-
src_params=parsed_attrs,
|
|
469
|
-
|
|
643
|
+
src_params=parsed_attrs,
|
|
644
|
+
src_key='InstanceProfile',
|
|
645
|
+
dest_params=cluster,
|
|
646
|
+
dest_key='JobFlowRole',
|
|
647
|
+
)
|
|
470
648
|
|
|
471
649
|
emrutils.apply_params(
|
|
472
|
-
src_params=parsed_attrs,
|
|
473
|
-
|
|
650
|
+
src_params=parsed_attrs,
|
|
651
|
+
src_key='EmrManagedMasterSecurityGroup',
|
|
652
|
+
dest_params=instances,
|
|
653
|
+
dest_key='EmrManagedMasterSecurityGroup',
|
|
654
|
+
)
|
|
474
655
|
|
|
475
656
|
emrutils.apply_params(
|
|
476
|
-
src_params=parsed_attrs,
|
|
477
|
-
|
|
657
|
+
src_params=parsed_attrs,
|
|
658
|
+
src_key='EmrManagedSlaveSecurityGroup',
|
|
659
|
+
dest_params=instances,
|
|
660
|
+
dest_key='EmrManagedSlaveSecurityGroup',
|
|
661
|
+
)
|
|
478
662
|
|
|
479
663
|
emrutils.apply_params(
|
|
480
|
-
src_params=parsed_attrs,
|
|
481
|
-
|
|
664
|
+
src_params=parsed_attrs,
|
|
665
|
+
src_key='ServiceAccessSecurityGroup',
|
|
666
|
+
dest_params=instances,
|
|
667
|
+
dest_key='ServiceAccessSecurityGroup',
|
|
668
|
+
)
|
|
482
669
|
|
|
483
670
|
emrutils.apply_params(
|
|
484
|
-
src_params=parsed_attrs,
|
|
485
|
-
|
|
671
|
+
src_params=parsed_attrs,
|
|
672
|
+
src_key='AdditionalMasterSecurityGroups',
|
|
673
|
+
dest_params=instances,
|
|
674
|
+
dest_key='AdditionalMasterSecurityGroups',
|
|
675
|
+
)
|
|
486
676
|
|
|
487
677
|
emrutils.apply_params(
|
|
488
|
-
src_params=parsed_attrs,
|
|
489
|
-
|
|
678
|
+
src_params=parsed_attrs,
|
|
679
|
+
src_key='AdditionalSlaveSecurityGroups',
|
|
680
|
+
dest_params=instances,
|
|
681
|
+
dest_key='AdditionalSlaveSecurityGroups',
|
|
682
|
+
)
|
|
490
683
|
|
|
491
684
|
emrutils.apply(params=cluster, key='Instances', value=instances)
|
|
492
685
|
|
|
493
686
|
return cluster
|
|
494
687
|
|
|
495
|
-
def _build_bootstrap_actions(
|
|
496
|
-
self, cluster, parsed_boostrap_actions):
|
|
688
|
+
def _build_bootstrap_actions(self, cluster, parsed_boostrap_actions):
|
|
497
689
|
cluster_ba_list = cluster.get('BootstrapActions')
|
|
498
690
|
if cluster_ba_list is None:
|
|
499
691
|
cluster_ba_list = []
|
|
500
692
|
|
|
501
693
|
bootstrap_actions = []
|
|
502
|
-
if
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
694
|
+
if (
|
|
695
|
+
len(cluster_ba_list) + len(parsed_boostrap_actions)
|
|
696
|
+
> constants.MAX_BOOTSTRAP_ACTION_NUMBER
|
|
697
|
+
):
|
|
698
|
+
raise ValueError(
|
|
699
|
+
'aws: error: maximum number of '
|
|
700
|
+
'bootstrap actions for a cluster exceeded.'
|
|
701
|
+
)
|
|
506
702
|
|
|
507
703
|
for ba in parsed_boostrap_actions:
|
|
508
704
|
ba_config = {}
|
|
@@ -512,15 +708,22 @@ class CreateCluster(Command):
|
|
|
512
708
|
ba_config['Name'] = constants.BOOTSTRAP_ACTION_NAME
|
|
513
709
|
script_arg_config = {}
|
|
514
710
|
emrutils.apply_params(
|
|
515
|
-
src_params=ba,
|
|
516
|
-
|
|
711
|
+
src_params=ba,
|
|
712
|
+
src_key='Path',
|
|
713
|
+
dest_params=script_arg_config,
|
|
714
|
+
dest_key='Path',
|
|
715
|
+
)
|
|
517
716
|
emrutils.apply_params(
|
|
518
|
-
src_params=ba,
|
|
519
|
-
|
|
717
|
+
src_params=ba,
|
|
718
|
+
src_key='Args',
|
|
719
|
+
dest_params=script_arg_config,
|
|
720
|
+
dest_key='Args',
|
|
721
|
+
)
|
|
520
722
|
emrutils.apply(
|
|
521
723
|
params=ba_config,
|
|
522
724
|
key='ScriptBootstrapAction',
|
|
523
|
-
value=script_arg_config
|
|
725
|
+
value=script_arg_config,
|
|
726
|
+
)
|
|
524
727
|
bootstrap_actions.append(ba_config)
|
|
525
728
|
|
|
526
729
|
result = cluster_ba_list + bootstrap_actions
|
|
@@ -535,15 +738,18 @@ class CreateCluster(Command):
|
|
|
535
738
|
args = [constants.DEBUGGING_COMMAND]
|
|
536
739
|
else:
|
|
537
740
|
jar = emrutils.get_script_runner(self.region)
|
|
538
|
-
args = [
|
|
539
|
-
|
|
540
|
-
|
|
741
|
+
args = [
|
|
742
|
+
emrutils.build_s3_link(
|
|
743
|
+
relative_path=constants.DEBUGGING_PATH, region=self.region
|
|
744
|
+
)
|
|
745
|
+
]
|
|
541
746
|
|
|
542
747
|
return emrutils.build_step(
|
|
543
748
|
name=constants.DEBUGGING_NAME,
|
|
544
749
|
action_on_failure=constants.TERMINATE_CLUSTER,
|
|
545
750
|
jar=jar,
|
|
546
|
-
args=args
|
|
751
|
+
args=args,
|
|
752
|
+
)
|
|
547
753
|
|
|
548
754
|
def _update_cluster_dict(self, cluster, key, value):
|
|
549
755
|
if key in cluster:
|
|
@@ -553,29 +759,34 @@ class CreateCluster(Command):
|
|
|
553
759
|
return cluster
|
|
554
760
|
|
|
555
761
|
def _validate_release_label_ami_version(self, parsed_args):
|
|
556
|
-
if
|
|
557
|
-
|
|
762
|
+
if (
|
|
763
|
+
parsed_args.ami_version is not None
|
|
764
|
+
and parsed_args.release_label is not None
|
|
765
|
+
):
|
|
558
766
|
raise exceptions.MutualExclusiveOptionError(
|
|
559
|
-
option1="--ami-version",
|
|
560
|
-
|
|
767
|
+
option1="--ami-version", option2="--release-label"
|
|
768
|
+
)
|
|
561
769
|
|
|
562
|
-
if
|
|
563
|
-
|
|
770
|
+
if (
|
|
771
|
+
parsed_args.ami_version is None
|
|
772
|
+
and parsed_args.release_label is None
|
|
773
|
+
):
|
|
564
774
|
raise exceptions.RequiredOptionsError(
|
|
565
|
-
option1="--ami-version",
|
|
566
|
-
|
|
775
|
+
option1="--ami-version", option2="--release-label"
|
|
776
|
+
)
|
|
567
777
|
|
|
568
778
|
# Checks if the applications required by steps are specified
|
|
569
779
|
# using the --applications option.
|
|
570
780
|
def _validate_required_applications(self, parsed_args):
|
|
571
|
-
|
|
572
781
|
specified_apps = set([])
|
|
573
782
|
if parsed_args.applications is not None:
|
|
574
|
-
specified_apps =
|
|
575
|
-
|
|
783
|
+
specified_apps = set(
|
|
784
|
+
[app['Name'].lower() for app in parsed_args.applications]
|
|
785
|
+
)
|
|
576
786
|
|
|
577
|
-
missing_apps = self._get_missing_applications_for_steps(
|
|
578
|
-
|
|
787
|
+
missing_apps = self._get_missing_applications_for_steps(
|
|
788
|
+
specified_apps, parsed_args
|
|
789
|
+
)
|
|
579
790
|
# Check for HBase.
|
|
580
791
|
if parsed_args.restore_from_hbase_backup is not None:
|
|
581
792
|
if constants.HBASE not in specified_apps:
|
|
@@ -583,11 +794,13 @@ class CreateCluster(Command):
|
|
|
583
794
|
|
|
584
795
|
if missing_apps:
|
|
585
796
|
raise exceptions.MissingApplicationsError(
|
|
586
|
-
applications=missing_apps
|
|
797
|
+
applications=missing_apps
|
|
798
|
+
)
|
|
587
799
|
|
|
588
800
|
def _get_missing_applications_for_steps(self, specified_apps, parsed_args):
|
|
589
|
-
allowed_app_steps = set(
|
|
590
|
-
|
|
801
|
+
allowed_app_steps = set(
|
|
802
|
+
[constants.HIVE, constants.PIG, constants.IMPALA]
|
|
803
|
+
)
|
|
591
804
|
missing_apps = set()
|
|
592
805
|
if parsed_args.steps is not None:
|
|
593
806
|
for step in parsed_args.steps:
|
|
@@ -597,38 +810,51 @@ class CreateCluster(Command):
|
|
|
597
810
|
|
|
598
811
|
if step_type is not None:
|
|
599
812
|
step_type = step_type.lower()
|
|
600
|
-
if
|
|
601
|
-
|
|
813
|
+
if (
|
|
814
|
+
step_type in allowed_app_steps
|
|
815
|
+
and step_type not in specified_apps
|
|
816
|
+
):
|
|
602
817
|
missing_apps.add(step['Type'].title())
|
|
603
818
|
return missing_apps
|
|
604
819
|
|
|
605
|
-
def _filter_configurations_in_special_cases(
|
|
606
|
-
|
|
820
|
+
def _filter_configurations_in_special_cases(
|
|
821
|
+
self, configurations, parsed_args, parsed_configs
|
|
822
|
+
):
|
|
607
823
|
if parsed_args.use_default_roles:
|
|
608
|
-
configurations = [
|
|
609
|
-
|
|
610
|
-
|
|
824
|
+
configurations = [
|
|
825
|
+
x
|
|
826
|
+
for x in configurations
|
|
827
|
+
if x.name != 'service_role' and x.name != 'instance_profile'
|
|
828
|
+
]
|
|
611
829
|
return configurations
|
|
612
830
|
|
|
613
831
|
def _handle_emrfs_parameters(self, cluster, emrfs_args, release_label):
|
|
614
832
|
if release_label:
|
|
615
833
|
self.validate_no_emrfs_configuration(cluster)
|
|
616
834
|
emrfs_configuration = emrfsutils.build_emrfs_confiuration(
|
|
617
|
-
emrfs_args
|
|
835
|
+
emrfs_args
|
|
836
|
+
)
|
|
618
837
|
|
|
619
838
|
self._update_cluster_dict(
|
|
620
|
-
cluster=cluster,
|
|
621
|
-
|
|
839
|
+
cluster=cluster,
|
|
840
|
+
key='Configurations',
|
|
841
|
+
value=[emrfs_configuration],
|
|
842
|
+
)
|
|
622
843
|
else:
|
|
623
844
|
emrfs_ba_config_list = emrfsutils.build_bootstrap_action_configs(
|
|
624
|
-
self.region, emrfs_args
|
|
845
|
+
self.region, emrfs_args
|
|
846
|
+
)
|
|
625
847
|
self._update_cluster_dict(
|
|
626
|
-
cluster=cluster,
|
|
627
|
-
|
|
848
|
+
cluster=cluster,
|
|
849
|
+
key='BootstrapActions',
|
|
850
|
+
value=emrfs_ba_config_list,
|
|
851
|
+
)
|
|
628
852
|
|
|
629
853
|
def validate_no_emrfs_configuration(self, cluster):
|
|
630
854
|
if 'Configurations' in cluster:
|
|
631
855
|
for config in cluster['Configurations']:
|
|
632
|
-
if
|
|
633
|
-
|
|
856
|
+
if (
|
|
857
|
+
config is not None
|
|
858
|
+
and config.get('Classification') == constants.EMRFS_SITE
|
|
859
|
+
):
|
|
634
860
|
raise exceptions.DuplicateEmrFsConfigurationError
|