awscli 1.42.69__py3-none-any.whl → 1.44.6__py3-none-any.whl

awscli/customizations/emr/createcluster.py

@@ -212,6 +212,11 @@ class CreateCluster(Command):
             'schema': argumentschema.AUTO_TERMINATION_POLICY_SCHEMA,
             'help_text': helptext.AUTO_TERMINATION_POLICY,
         },
+        {
+            'name': 'monitoring-configuration',
+            'schema': argumentschema.MONITORING_CONFIGURATION_SCHEMA,
+            'help_text': helptext.MONITORING_CONFIGURATION,
+        },
         {
             'name': 'extended-support',
             'action': 'store_true',
@@ -553,6 +558,13 @@ class CreateCluster(Command):
                 parsed_args.auto_termination_policy,
             )
 
+        if parsed_args.monitoring_configuration is not None:
+            emrutils.apply_dict(
+                params,
+                'MonitoringConfiguration',
+                parsed_args.monitoring_configuration,
+            )
+
         self._validate_required_applications(parsed_args)
 
         run_job_flow_response = emrutils.call(

awscli/customizations/emr/emrutils.py

@@ -15,12 +15,10 @@ import json
 import logging
 import os
 
+from botocore.exceptions import NoCredentialsError
 
 from awscli.clidriver import CLIOperationCaller
-from awscli.customizations.emr import constants
-from awscli.customizations.emr import exceptions
-from botocore.exceptions import WaiterError, NoCredentialsError
-from botocore import xform_name
+from awscli.customizations.emr import constants, exceptions
 
 LOG = logging.getLogger(__name__)
 
@@ -56,11 +54,16 @@ def parse_key_value_string(key_value_string):
 
 
 def apply_boolean_options(
-        true_option, true_option_name, false_option, false_option_name):
+    true_option, true_option_name, false_option, false_option_name
+):
     if true_option and false_option:
-        error_message = \
-            'aws: error: cannot use both ' + true_option_name + \
-            ' and ' + false_option_name + ' options together.'
+        error_message = (
+            'aws: error: cannot use both '
+            + true_option_name
+            + ' and '
+            + false_option_name
+            + ' options together.'
+        )
         raise ValueError(error_message)
     elif true_option:
         return True
@@ -91,13 +94,16 @@ def apply_params(src_params, src_key, dest_params, dest_key):
 
 
 def build_step(
-        jar, name='Step',
-        action_on_failure=constants.DEFAULT_FAILURE_ACTION,
-        args=None,
-        main_class=None,
-        properties=None):
-    check_required_field(
-        structure='HadoopJarStep', name='Jar', value=jar)
+    jar,
+    name='Step',
+    action_on_failure=constants.DEFAULT_FAILURE_ACTION,
+    args=None,
+    main_class=None,
+    properties=None,
+    log_uri=None,
+    encryption_key_arn=None,
+):
+    check_required_field(structure='HadoopJarStep', name='Jar', value=jar)
 
     step = {}
     apply_dict(step, 'Name', name)
@@ -108,17 +114,26 @@ def build_step(
     apply_dict(jar_config, 'MainClass', main_class)
     apply_dict(jar_config, 'Properties', properties)
     step['HadoopJarStep'] = jar_config
+    step_monitoring_config = {}
+    s3_monitoring_configuration = {}
+    apply_dict(s3_monitoring_configuration, 'LogUri', log_uri)
+    apply_dict(
+        s3_monitoring_configuration, 'EncryptionKeyArn', encryption_key_arn
+    )
+    if s3_monitoring_configuration:
+        step_monitoring_config['S3MonitoringConfiguration'] = (
+            s3_monitoring_configuration
+        )
+        step['StepMonitoringConfiguration'] = step_monitoring_config
 
     return step
 
 
-def build_bootstrap_action(
-        path,
-        name='Bootstrap Action',
-        args=None):
+def build_bootstrap_action(path, name='Bootstrap Action', args=None):
     if path is None:
         raise exceptions.MissingParametersError(
-            object_name='ScriptBootstrapActionConfig', missing='Path')
+            object_name='ScriptBootstrapActionConfig', missing='Path'
+        )
     ba_config = {}
     apply_dict(ba_config, 'Name', name)
     script_config = {}
@@ -132,20 +147,22 @@ def build_bootstrap_action(
 def build_s3_link(relative_path='', region='us-east-1'):
     if region is None:
         region = 'us-east-1'
-    return 's3://{0}.elasticmapreduce{1}'.format(region, relative_path)
+    return f's3://{region}.elasticmapreduce{relative_path}'
 
 
 def get_script_runner(region='us-east-1'):
     if region is None:
         region = 'us-east-1'
     return build_s3_link(
-        relative_path=constants.SCRIPT_RUNNER_PATH, region=region)
+        relative_path=constants.SCRIPT_RUNNER_PATH, region=region
+    )
 
 
 def check_required_field(structure, name, value):
     if not value:
         raise exceptions.MissingParametersError(
-            object_name=structure, missing=name)
+            object_name=structure, missing=name
+        )
 
 
 def check_empty_string_list(name, value):
@@ -153,8 +170,14 @@ def check_empty_string_list(name, value):
         raise exceptions.EmptyListError(param=name)
 
 
-def call(session, operation_name, parameters, region_name=None,
-         endpoint_url=None, verify=None):
+def call(
+    session,
+    operation_name,
+    parameters,
+    region_name=None,
+    endpoint_url=None,
+    verify=None,
+):
     # We could get an error from get_endpoint() about not having
     # a region configured.  Before this happens we want to check
     # for credentials so we can give a good error message.
@@ -162,8 +185,11 @@ def call(session, operation_name, parameters, region_name=None,
         raise NoCredentialsError()
 
     client = session.create_client(
-        'emr', region_name=region_name, endpoint_url=endpoint_url,
-        verify=verify)
+        'emr',
+        region_name=region_name,
+        endpoint_url=endpoint_url,
+        verify=verify,
+    )
     LOG.debug('Calling ' + str(operation_name))
     return getattr(client, operation_name)(**parameters)
 
@@ -181,7 +207,8 @@ def get_client(session, parsed_globals):
         'emr',
         region_name=get_region(session, parsed_globals),
         endpoint_url=parsed_globals.endpoint_url,
-        verify=parsed_globals.verify_ssl)
+        verify=parsed_globals.verify_ssl,
+    )
 
 
 def get_cluster_state(session, parsed_globals, cluster_id):
@@ -209,12 +236,13 @@ def which(program):
     return None
 
 
-def call_and_display_response(session, operation_name, parameters,
-                              parsed_globals):
+def call_and_display_response(
+    session, operation_name, parameters, parsed_globals
+):
     cli_operation_caller = CLIOperationCaller(session)
     cli_operation_caller.invoke(
-        'emr', operation_name,
-        parameters, parsed_globals)
+        'emr', operation_name, parameters, parsed_globals
+    )
 
 
 def display_response(session, operation_name, result, parsed_globals):
@@ -222,7 +250,8 @@ def display_response(session, operation_name, result, parsed_globals):
     # Calling a private method. Should be changed after the functionality
     # is moved outside CliOperationCaller.
     cli_operation_caller._display_response(
-        operation_name, result, parsed_globals)
+        operation_name, result, parsed_globals
+    )
 
 
 def get_region(session, parsed_globals):
@@ -244,8 +273,9 @@ def join(values, separator=',', lastSeparator='and'):
         return values[0]
     else:
         separator = '%s ' % separator
-        return ' '.join([separator.join(values[:-1]),
-                         lastSeparator, values[-1]])
+        return ' '.join(
+            [separator.join(values[:-1]), lastSeparator, values[-1]]
+        )
 
 
 def split_to_key_value(string):
@@ -255,21 +285,24 @@ def split_to_key_value(string):
         return string.split('=', 1)
 
 
-def get_cluster(cluster_id, session, region,
-                endpoint_url, verify_ssl):
-        describe_cluster_params = {'ClusterId': cluster_id}
-        describe_cluster_response = call(
-            session, 'describe_cluster', describe_cluster_params,
-            region, endpoint_url,
-            verify_ssl)
+def get_cluster(cluster_id, session, region, endpoint_url, verify_ssl):
+    describe_cluster_params = {'ClusterId': cluster_id}
+    describe_cluster_response = call(
+        session,
+        'describe_cluster',
+        describe_cluster_params,
+        region,
+        endpoint_url,
+        verify_ssl,
+    )
 
-        if describe_cluster_response is not None:
-            return describe_cluster_response.get('Cluster')
+    if describe_cluster_response is not None:
+        return describe_cluster_response.get('Cluster')
 
 
-def get_release_label(cluster_id, session, region,
-                      endpoint_url, verify_ssl):
-        cluster = get_cluster(cluster_id, session, region,
-                              endpoint_url, verify_ssl)
-        if cluster is not None:
-            return cluster.get('ReleaseLabel')
+def get_release_label(cluster_id, session, region, endpoint_url, verify_ssl):
+    cluster = get_cluster(
+        cluster_id, session, region, endpoint_url, verify_ssl
+    )
+    if cluster is not None:
+        return cluster.get('ReleaseLabel')

awscli/customizations/emr/helptext.py

@@ -570,3 +570,13 @@ UNHEALTHY_NODE_REPLACEMENT = (
 )
 
 EXTENDED_SUPPORT = '<p>Reserved.</p> '
+
+MONITORING_CONFIGURATION = (
+    '<p>Monitoring configuration for an Amazon EMR cluster. '
+    'The configuration specifies CloudWatch logging settings for the cluster. '
+    'You can configure the CloudWatchLogConfiguration which includes '
+    'the Enabled flag (required), LogGroupName, LogStreamNamePrefix, '
+    'EncryptionKeyArn, and LogTypes. The LogTypes parameter is a map '
+    'of log type categories (e.g., "STEP_LOGS", "SPARK_DRIVER", '
+    '"SPARK_EXECUTOR") to a list of file names (e.g., "STDOUT", "STDERR").</p>'
+)

awscli/customizations/emr/steputils.py

@@ -11,9 +11,7 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
-from awscli.customizations.emr import emrutils
-from awscli.customizations.emr import constants
-from awscli.customizations.emr import exceptions
+from awscli.customizations.emr import constants, emrutils, exceptions
 
 
 def build_step_config_list(parsed_step_list, region, release_label):
@@ -29,23 +27,24 @@ def build_step_config_list(parsed_step_list, region, release_label):
             step_config = build_custom_jar_step(parsed_step=step)
         elif step_type == constants.STREAMING:
             step_config = build_streaming_step(
-                parsed_step=step, release_label=release_label)
+                parsed_step=step, release_label=release_label
+            )
         elif step_type == constants.HIVE:
             step_config = build_hive_step(
-                parsed_step=step, region=region,
-                release_label=release_label)
+                parsed_step=step, region=region, release_label=release_label
+            )
         elif step_type == constants.PIG:
             step_config = build_pig_step(
-                parsed_step=step, region=region,
-                release_label=release_label)
+                parsed_step=step, region=region, release_label=release_label
+            )
         elif step_type == constants.IMPALA:
             step_config = build_impala_step(
-                parsed_step=step, region=region,
-                release_label=release_label)
+                parsed_step=step, region=region, release_label=release_label
+            )
         elif step_type == constants.SPARK:
             step_config = build_spark_step(
-                parsed_step=step, region=region,
-                release_label=release_label)
+                parsed_step=step, region=region, release_label=release_label
+            )
         else:
             raise exceptions.UnknownStepTypeError(step_type=step_type)
 
@@ -57,14 +56,17 @@ def build_step_config_list(parsed_step_list, region, release_label):
 def build_custom_jar_step(parsed_step):
     name = _apply_default_value(
         arg=parsed_step.get('Name'),
-        value=constants.DEFAULT_CUSTOM_JAR_STEP_NAME)
+        value=constants.DEFAULT_CUSTOM_JAR_STEP_NAME,
+    )
     action_on_failure = _apply_default_value(
         arg=parsed_step.get('ActionOnFailure'),
-        value=constants.DEFAULT_FAILURE_ACTION)
+        value=constants.DEFAULT_FAILURE_ACTION,
+    )
     emrutils.check_required_field(
         structure=constants.CUSTOM_JAR_STEP_CONFIG,
         name='Jar',
-        value=parsed_step.get('Jar'))
+        value=parsed_step.get('Jar'),
+    )
     return emrutils.build_step(
         jar=parsed_step.get('Jar'),
         args=parsed_step.get('Args'),
@@ -72,22 +74,27 @@ def build_custom_jar_step(parsed_step):
         action_on_failure=action_on_failure,
         main_class=parsed_step.get('MainClass'),
         properties=emrutils.parse_key_value_string(
-            parsed_step.get('Properties')))
+            parsed_step.get('Properties')
+        ),
+        log_uri=parsed_step.get('LogUri'),
+        encryption_key_arn=parsed_step.get('EncryptionKeyArn'),
+    )
 
 
 def build_streaming_step(parsed_step, release_label):
     name = _apply_default_value(
         arg=parsed_step.get('Name'),
-        value=constants.DEFAULT_STREAMING_STEP_NAME)
+        value=constants.DEFAULT_STREAMING_STEP_NAME,
+    )
     action_on_failure = _apply_default_value(
         arg=parsed_step.get('ActionOnFailure'),
-        value=constants.DEFAULT_FAILURE_ACTION)
+        value=constants.DEFAULT_FAILURE_ACTION,
+    )
 
     args = parsed_step.get('Args')
     emrutils.check_required_field(
-        structure=constants.STREAMING_STEP_CONFIG,
-        name='Args',
-        value=args)
+        structure=constants.STREAMING_STEP_CONFIG, name='Args', value=args
+    )
     emrutils.check_empty_string_list(name='Args', value=args)
     args_list = []
 
@@ -103,27 +110,34 @@ def build_streaming_step(parsed_step, release_label):
         jar=jar,
         args=args_list,
         name=name,
-        action_on_failure=action_on_failure)
+        action_on_failure=action_on_failure,
+        log_uri=parsed_step.get('LogUri'),
+        encryption_key_arn=parsed_step.get('EncryptionKeyArn'),
+    )
 
 
 def build_hive_step(parsed_step, release_label, region=None):
     args = parsed_step.get('Args')
     emrutils.check_required_field(
-        structure=constants.HIVE_STEP_CONFIG, name='Args', value=args)
+        structure=constants.HIVE_STEP_CONFIG, name='Args', value=args
+    )
     emrutils.check_empty_string_list(name='Args', value=args)
     name = _apply_default_value(
-        arg=parsed_step.get('Name'),
-        value=constants.DEFAULT_HIVE_STEP_NAME)
-    action_on_failure = \
-        _apply_default_value(
-            arg=parsed_step.get('ActionOnFailure'),
-            value=constants.DEFAULT_FAILURE_ACTION)
+        arg=parsed_step.get('Name'), value=constants.DEFAULT_HIVE_STEP_NAME
+    )
+    action_on_failure = _apply_default_value(
+        arg=parsed_step.get('ActionOnFailure'),
+        value=constants.DEFAULT_FAILURE_ACTION,
+    )
 
     return emrutils.build_step(
         jar=_get_runner_jar(release_label, region),
         args=_build_hive_args(args, release_label, region),
         name=name,
-        action_on_failure=action_on_failure)
+        action_on_failure=action_on_failure,
+        log_uri=parsed_step.get('LogUri'),
+        encryption_key_arn=parsed_step.get('EncryptionKeyArn'),
+    )
 
 
 def _build_hive_args(args, release_label, region):
@@ -131,8 +145,11 @@ def _build_hive_args(args, release_label, region):
     if release_label:
         args_list.append(constants.HIVE_SCRIPT_COMMAND)
     else:
-        args_list.append(emrutils.build_s3_link(
-            relative_path=constants.HIVE_SCRIPT_PATH, region=region))
+        args_list.append(
+            emrutils.build_s3_link(
+                relative_path=constants.HIVE_SCRIPT_PATH, region=region
+            )
+        )
 
     args_list.append(constants.RUN_HIVE_SCRIPT)
 
@@ -149,20 +166,25 @@ def _build_hive_args(args, release_label, region):
 def build_pig_step(parsed_step, release_label, region=None):
     args = parsed_step.get('Args')
     emrutils.check_required_field(
-        structure=constants.PIG_STEP_CONFIG, name='Args', value=args)
+        structure=constants.PIG_STEP_CONFIG, name='Args', value=args
+    )
     emrutils.check_empty_string_list(name='Args', value=args)
     name = _apply_default_value(
-        arg=parsed_step.get('Name'),
-        value=constants.DEFAULT_PIG_STEP_NAME)
+        arg=parsed_step.get('Name'), value=constants.DEFAULT_PIG_STEP_NAME
+    )
     action_on_failure = _apply_default_value(
         arg=parsed_step.get('ActionOnFailure'),
-        value=constants.DEFAULT_FAILURE_ACTION)
+        value=constants.DEFAULT_FAILURE_ACTION,
+    )
 
     return emrutils.build_step(
         jar=_get_runner_jar(release_label, region),
         args=_build_pig_args(args, release_label, region),
         name=name,
-        action_on_failure=action_on_failure)
+        action_on_failure=action_on_failure,
+        log_uri=parsed_step.get('LogUri'),
+        encryption_key_arn=parsed_step.get('EncryptionKeyArn'),
+    )
 
 
 def _build_pig_args(args, release_label, region):
@@ -170,8 +192,11 @@ def _build_pig_args(args, release_label, region):
     if release_label:
         args_list.append(constants.PIG_SCRIPT_COMMAND)
     else:
-        args_list.append(emrutils.build_s3_link(
-            relative_path=constants.PIG_SCRIPT_PATH, region=region))
+        args_list.append(
+            emrutils.build_s3_link(
+                relative_path=constants.PIG_SCRIPT_PATH, region=region
+            )
+        )
 
     args_list.append(constants.RUN_PIG_SCRIPT)
 
@@ -189,43 +214,55 @@ def build_impala_step(parsed_step, release_label, region=None):
     if release_label:
         raise exceptions.UnknownStepTypeError(step_type=constants.IMPALA)
     name = _apply_default_value(
-        arg=parsed_step.get('Name'),
-        value=constants.DEFAULT_IMPALA_STEP_NAME)
+        arg=parsed_step.get('Name'), value=constants.DEFAULT_IMPALA_STEP_NAME
+    )
     action_on_failure = _apply_default_value(
         arg=parsed_step.get('ActionOnFailure'),
-        value=constants.DEFAULT_FAILURE_ACTION)
+        value=constants.DEFAULT_FAILURE_ACTION,
+    )
     args_list = [
         emrutils.build_s3_link(
-            relative_path=constants.IMPALA_INSTALL_PATH, region=region),
-        constants.RUN_IMPALA_SCRIPT]
+            relative_path=constants.IMPALA_INSTALL_PATH, region=region
+        ),
+        constants.RUN_IMPALA_SCRIPT,
+    ]
     args = parsed_step.get('Args')
     emrutils.check_required_field(
-        structure=constants.IMPALA_STEP_CONFIG, name='Args', value=args)
+        structure=constants.IMPALA_STEP_CONFIG, name='Args', value=args
+    )
     args_list += args
 
     return emrutils.build_step(
         jar=emrutils.get_script_runner(region),
         args=args_list,
         name=name,
-        action_on_failure=action_on_failure)
+        action_on_failure=action_on_failure,
+        log_uri=parsed_step.get('LogUri'),
+        encryption_key_arn=parsed_step.get('EncryptionKeyArn'),
+    )
 
 
 def build_spark_step(parsed_step, release_label, region=None):
     name = _apply_default_value(
-        arg=parsed_step.get('Name'),
-        value=constants.DEFAULT_SPARK_STEP_NAME)
+        arg=parsed_step.get('Name'), value=constants.DEFAULT_SPARK_STEP_NAME
+    )
     action_on_failure = _apply_default_value(
         arg=parsed_step.get('ActionOnFailure'),
-        value=constants.DEFAULT_FAILURE_ACTION)
+        value=constants.DEFAULT_FAILURE_ACTION,
+    )
     args = parsed_step.get('Args')
     emrutils.check_required_field(
-        structure=constants.SPARK_STEP_CONFIG, name='Args', value=args)
+        structure=constants.SPARK_STEP_CONFIG, name='Args', value=args
+    )
 
     return emrutils.build_step(
         jar=_get_runner_jar(release_label, region),
         args=_build_spark_args(args, release_label, region),
         name=name,
-        action_on_failure=action_on_failure)
+        action_on_failure=action_on_failure,
+        log_uri=parsed_step.get('LogUri'),
+        encryption_key_arn=parsed_step.get('EncryptionKeyArn'),
+    )
 
 
 def _build_spark_args(args, release_label, region):
@@ -247,5 +284,8 @@ def _apply_default_value(arg, value):
 
 
 def _get_runner_jar(release_label, region):
-    return constants.COMMAND_RUNNER if release_label \
+    return (
+        constants.COMMAND_RUNNER
+        if release_label
         else emrutils.get_script_runner(region)
+    )