awscli 1.42.69__py3-none-any.whl → 1.44.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of awscli might be problematic. Click here for more details.

Files changed (59) hide show
  1. awscli/__init__.py +1 -1
  2. awscli/alias.py +3 -3
  3. awscli/argprocess.py +2 -1
  4. awscli/bcdoc/docevents.py +3 -0
  5. awscli/clidocs.py +19 -2
  6. awscli/clidriver.py +55 -9
  7. awscli/customizations/argrename.py +1 -0
  8. awscli/customizations/cliinputjson.py +4 -0
  9. awscli/customizations/cloudformation/deploy.py +19 -3
  10. awscli/customizations/cloudtrail/validation.py +22 -2
  11. awscli/customizations/commands.py +2 -1
  12. awscli/customizations/emr/argumentschema.py +355 -344
  13. awscli/customizations/emr/createcluster.py +12 -0
  14. awscli/customizations/emr/emrutils.py +83 -50
  15. awscli/customizations/emr/helptext.py +10 -0
  16. awscli/customizations/emr/steputils.py +92 -52
  17. awscli/customizations/globalargs.py +169 -0
  18. awscli/customizations/paginate.py +66 -1
  19. awscli/customizations/s3/filegenerator.py +4 -1
  20. awscli/customizations/s3/fileinfo.py +4 -1
  21. awscli/customizations/s3/fileinfobuilder.py +6 -0
  22. awscli/customizations/s3/s3handler.py +9 -0
  23. awscli/customizations/s3/subcommands.py +147 -6
  24. awscli/customizations/s3/syncstrategy/base.py +9 -0
  25. awscli/customizations/s3/syncstrategy/caseconflict.py +92 -0
  26. awscli/customizations/s3/utils.py +14 -0
  27. awscli/customizations/scalarparse.py +42 -5
  28. awscli/data/cli.json +5 -0
  29. awscli/examples/cloudformation/create-generated-template.rst +50 -0
  30. awscli/examples/cloudformation/create-stack-refactor.rst +16 -0
  31. awscli/examples/cloudformation/delete-generated-template.rst +10 -0
  32. awscli/examples/cloudformation/describe-generated-template.rst +62 -0
  33. awscli/examples/cloudformation/describe-resource-scan.rst +38 -0
  34. awscli/examples/cloudformation/describe-stack-refactor.rst +20 -0
  35. awscli/examples/cloudformation/execute-stack-refactor.rst +10 -0
  36. awscli/examples/cloudformation/list-generated-templates.rst +41 -0
  37. awscli/examples/cloudformation/list-resource-scan-related-resources.rst +47 -0
  38. awscli/examples/cloudformation/list-resource-scan-resources.rst +30 -0
  39. awscli/examples/cloudformation/list-stack-refactor-actions.rst +71 -0
  40. awscli/examples/cloudformation/start-resource-scan.rst +14 -0
  41. awscli/examples/global_options.rst +4 -0
  42. awscli/examples/global_synopsis.rst +1 -0
  43. awscli/examples/medical-imaging/create-datastore.rst +19 -2
  44. awscli/examples/medical-imaging/get-datastore.rst +24 -1
  45. awscli/paramfile.py +21 -4
  46. awscli/testutils.py +17 -0
  47. awscli/topics/s3-case-insensitivity.rst +105 -0
  48. awscli/topics/topic-tags.json +16 -0
  49. awscli/utils.py +19 -2
  50. {awscli-1.42.69.dist-info → awscli-1.44.6.dist-info}/METADATA +3 -3
  51. {awscli-1.42.69.dist-info → awscli-1.44.6.dist-info}/RECORD +59 -45
  52. {awscli-1.42.69.data → awscli-1.44.6.data}/scripts/aws +0 -0
  53. {awscli-1.42.69.data → awscli-1.44.6.data}/scripts/aws.cmd +0 -0
  54. {awscli-1.42.69.data → awscli-1.44.6.data}/scripts/aws_bash_completer +0 -0
  55. {awscli-1.42.69.data → awscli-1.44.6.data}/scripts/aws_completer +0 -0
  56. {awscli-1.42.69.data → awscli-1.44.6.data}/scripts/aws_zsh_completer.sh +0 -0
  57. {awscli-1.42.69.dist-info → awscli-1.44.6.dist-info}/LICENSE.txt +0 -0
  58. {awscli-1.42.69.dist-info → awscli-1.44.6.dist-info}/WHEEL +0 -0
  59. {awscli-1.42.69.dist-info → awscli-1.44.6.dist-info}/top_level.txt +0 -0
@@ -12,12 +12,18 @@
12
12
  # language governing permissions and limitations under the License.
13
13
  import sys
14
14
  import os
15
+
16
+ from awscli.customizations.argrename import HIDDEN_ALIASES
17
+ from awscli.customizations.utils import uni_print
15
18
  from botocore.client import Config
16
19
  from botocore import UNSIGNED
17
20
  from botocore.endpoint import DEFAULT_TIMEOUT
21
+ from botocore.useragent import register_feature_id
18
22
  import jmespath
19
23
 
20
24
  from awscli.compat import urlparse
25
+ from awscli.utils import resolve_v2_debug_mode
26
+
21
27
 
22
28
  def register_parse_global_args(cli):
23
29
  cli.register('top-level-args-parsed', resolve_types,
@@ -30,6 +36,8 @@ def register_parse_global_args(cli):
30
36
  unique_id='resolve-cli-read-timeout')
31
37
  cli.register('top-level-args-parsed', resolve_cli_connect_timeout,
32
38
  unique_id='resolve-cli-connect-timeout')
39
+ cli.register('top-level-args-parsed', detect_migration_breakage,
40
+ unique_id='detect-migration-breakage')
33
41
 
34
42
 
35
43
  def resolve_types(parsed_args, **kwargs):
@@ -90,6 +98,167 @@ def resolve_cli_connect_timeout(parsed_args, session, **kwargs):
90
98
  arg_name = 'connect_timeout'
91
99
  _resolve_timeout(session, parsed_args, arg_name)
92
100
 
101
+ def detect_migration_breakage(parsed_args, session, remaining_args, **kwargs):
102
+ if not resolve_v2_debug_mode(parsed_args):
103
+ return
104
+ region = parsed_args.region or session.get_config_variable('region')
105
+ s3_config = session.get_config_variable('s3')
106
+ if (
107
+ not session.get_scoped_config().get('cli_pager', None)
108
+ == '' and 'AWS_PAGER' not in os.environ
109
+ ):
110
+ uni_print(
111
+ '\nAWS CLI v2 UPGRADE WARNING: By default, the AWS CLI v2 returns '
112
+ 'all output through your operating system’s default pager '
113
+ 'program. This is different from v1 behavior, where the system '
114
+ 'pager is not used by default. To retain AWS CLI v1 behavior in '
115
+ 'AWS CLI v2, set the `cli_pager` configuration setting, or the '
116
+ '`AWS_PAGER` environment variable, to the empty string. See '
117
+ 'https://docs.aws.amazon.com/cli/latest/userguide/'
118
+ 'cliv2-migration-changes.html#cliv2-migration-output-pager.\n',
119
+ out_file=sys.stderr
120
+ )
121
+ if 'PYTHONUTF8' in os.environ or 'PYTHONIOENCODING' in os.environ:
122
+ if 'AWS_CLI_FILE_ENCODING' not in os.environ:
123
+ uni_print(
124
+ '\nThe AWS CLI v2 does not support The `PYTHONUTF8` and '
125
+ '`PYTHONIOENCODING` environment variables, and instead uses '
126
+ 'the `AWS_CLI_FILE_ENCODING` variable. This is different from '
127
+ 'v1 behavior, where the former two variables are used '
128
+ 'instead. To retain AWS CLI v1 behavior in AWS CLI v2, set '
129
+ 'the `AWS_CLI_FILE_ENCODING` environment variable instead. '
130
+ 'See https://docs.aws.amazon.com/cli/latest/userguide/'
131
+ 'cliv2-migration-changes.html'
132
+ '#cliv2-migration-encodingenvvar.\n',
133
+ out_file=sys.stderr
134
+ )
135
+ if (
136
+ (
137
+ s3_config is None
138
+ or s3_config.get('us_east_1_regional_endpoint', 'legacy')
139
+ == 'legacy'
140
+ )
141
+ and region in ('us-east-1', None)
142
+ ):
143
+ session.register(
144
+ 'request-created.s3.*',
145
+ warn_if_east_configured_global_endpoint
146
+ )
147
+ session.register(
148
+ 'request-created.s3api.*',
149
+ warn_if_east_configured_global_endpoint
150
+ )
151
+ if session.get_config_variable('api_versions'):
152
+ uni_print(
153
+ '\nAWS CLI v2 UPGRADE WARNING: AWS CLI v2 UPGRADE WARNING: '
154
+ 'The AWS CLI v2 does not support calling older versions of AWS '
155
+ 'service APIs via the `api_versions` configuration file setting. This '
156
+ 'is different from v1 behavior, where this configuration setting '
157
+ 'can be used to pin older API versions. To migrate to v2 '
158
+ 'behavior, remove the `api_versions` configuration setting, and '
159
+ 'test against the latest service API versions. See '
160
+ 'https://docs.aws.amazon.com/cli/latest/userguide/'
161
+ 'cliv2-migration-changes.html#cliv2-migration-api-versions.\n',
162
+ out_file = sys.stderr
163
+ )
164
+ if session.full_config.get('plugins', {}):
165
+ uni_print(
166
+ '\nAWS CLI v2 UPGRADE WARNING: In AWS CLI v2, plugins are '
167
+ 'disabled by default, and support for plugins is provisional. '
168
+ 'This is different from v1 behavior, where plugin support is URL '
169
+ 'below to update your configuration to enable plugins in AWS CLI '
170
+ 'v2. Also, be sure to lock into a particular version of the AWS '
171
+ 'CLI and test the functionality of your plugins every time AWS '
172
+ 'CLI v2 is upgraded. See https://docs.aws.amazon.com/cli/latest/'
173
+ 'userguide/cliv2-migration-changes.html'
174
+ '#cliv2-migration-profile-plugins.\n',
175
+ out_file=sys.stderr
176
+ )
177
+ if (
178
+ parsed_args.command == 'ecr' and
179
+ remaining_args is not None and
180
+ remaining_args[0] == 'get-login'
181
+ ):
182
+ uni_print(
183
+ '\nAWS CLI v2 UPGRADE WARNING: The `ecr get-login` command has '
184
+ 'been removed in AWS CLI v2. You must use `ecr get-login-password` '
185
+ 'instead. See https://docs.aws.amazon.com/cli/latest/userguide/'
186
+ 'cliv2-migration-changes.html#cliv2-migration-ecr-get-login.\n',
187
+ out_file=sys.stderr
188
+ )
189
+ for working, obsolete in HIDDEN_ALIASES.items():
190
+ working_split = working.split('.')
191
+ working_service = working_split[0]
192
+ working_cmd = working_split[1]
193
+ working_param = working_split[2]
194
+ if (
195
+ parsed_args.command == working_service
196
+ and remaining_args is not None
197
+ and remaining_args[0] == working_cmd
198
+ and f"--{working_param}" in remaining_args
199
+ ):
200
+ uni_print(
201
+ '\nAWS CLI v2 UPGRADE WARNING: You have entered command '
202
+ 'arguments that use at least 1 of 21 built-in ("hidden") '
203
+ 'aliases that were removed in AWS CLI v2. For this command '
204
+ 'to work in AWS CLI v2, you must replace usage of the alias '
205
+ 'with the corresponding parameter in AWS CLI v2. See '
206
+ 'https://docs.aws.amazon.com/cli/latest/userguide/'
207
+ 'cliv2-migration-changes.html#cliv2-migration-aliases.\n',
208
+ out_file=sys.stderr
209
+ )
210
+ # Register against the provide-client-params event to ensure that the
211
+ # feature ID is registered before any API requests are made. We
212
+ # cannot register the feature ID in this function because no
213
+ # botocore context is created at this point.
214
+ session.register(
215
+ 'provide-client-params.*.*',
216
+ _register_v2_debug_feature_id
217
+ )
218
+ session.register('choose-signer.s3.*', warn_if_sigv2)
219
+
220
+
221
+ def _register_v2_debug_feature_id(params, model, **kwargs):
222
+ register_feature_id('CLI_V1_TO_V2_MIGRATION_DEBUG_MODE')
223
+
224
+ def warn_if_east_configured_global_endpoint(request, operation_name, **kwargs):
225
+ # The regional us-east-1 endpoint is used in certain cases (e.g.
226
+ # FIPS/Dual-Stack is enabled). Rather than duplicating this logic
227
+ # from botocore, we check the endpoint URL directly.
228
+ parsed_url = urlparse.urlparse(request.url)
229
+ if parsed_url.hostname.endswith('s3.amazonaws.com'):
230
+ uni_print(
231
+ '\nAWS CLI v2 UPGRADE WARNING: When you configure AWS CLI v2 to '
232
+ 'use the `us-east-1` region, it uses the true regional endpoint '
233
+ 'rather than the global endpoint. This is different from v1 '
234
+ 'behavior, where the global endpoint would be used when the '
235
+ 'region is `us-east-1`. To retain AWS CLI v1 behavior in AWS '
236
+ 'CLI v2, configure the region setting to `aws-global`. See '
237
+ 'https://docs.aws.amazon.com/cli/latest/userguide/'
238
+ 'cliv2-migration-changes.html'
239
+ '#cliv2-migration-s3-regional-endpoint.\n',
240
+ out_file=sys.stderr
241
+ )
242
+
243
+ def warn_if_sigv2(
244
+ signing_name,
245
+ region_name,
246
+ signature_version,
247
+ context,
248
+ **kwargs
249
+ ):
250
+ if context.get('auth_type', None) == 'v2':
251
+ uni_print(
252
+ '\nAWS CLI v2 UPGRADE WARNING: The AWS CLI v2 only uses Signature '
253
+ 'v4 to authenticate Amazon S3 requests. This is different from '
254
+ 'v1 behavior, where the signature used for Amazon S3 requests may '
255
+ 'vary depending on configuration settings, region, and the '
256
+ 'bucket being used. To migrate to AWS CLI v2 behavior, configure '
257
+ 'the Signature Version S3 setting to version 4. See '
258
+ 'https://docs.aws.amazon.com/cli/latest/userguide/'
259
+ 'cliv2-migration-changes.html#cliv2-migration-sigv4.\n',
260
+ out_file=sys.stderr
261
+ )
93
262
 
94
263
  def resolve_cli_read_timeout(parsed_args, session, **kwargs):
95
264
  arg_name = 'read_timeout'
@@ -33,7 +33,7 @@ from botocore.exceptions import DataNotFoundError, PaginationError
33
33
  from botocore import model
34
34
 
35
35
  from awscli.arguments import BaseCLIArgument
36
-
36
+ from awscli.utils import resolve_v2_debug_mode
37
37
 
38
38
  logger = logging.getLogger(__name__)
39
39
 
@@ -135,6 +135,9 @@ def unify_paging_params(argument_table, operation_model, event_name,
135
135
  _remove_existing_paging_arguments(argument_table, paginator_config)
136
136
  parsed_args_event = event_name.replace('building-argument-table.',
137
137
  'operation-args-parsed.')
138
+ call_parameters_event = event_name.replace(
139
+ 'building-argument-table', 'calling-command'
140
+ )
138
141
  shadowed_args = {}
139
142
  add_paging_argument(argument_table, 'starting-token',
140
143
  PageArgument('starting-token', STARTING_TOKEN_HELP,
@@ -168,6 +171,14 @@ def unify_paging_params(argument_table, operation_model, event_name,
168
171
  partial(check_should_enable_pagination,
169
172
  list(_get_all_cli_input_tokens(paginator_config)),
170
173
  shadowed_args, argument_table))
174
+ session.register(
175
+ call_parameters_event,
176
+ partial(
177
+ check_should_enable_pagination_call_parameters,
178
+ session,
179
+ list(_get_all_input_tokens(paginator_config)),
180
+ ),
181
+ )
171
182
 
172
183
 
173
184
  def add_paging_argument(argument_table, arg_name, argument, shadowed_args):
@@ -240,6 +251,18 @@ def _get_all_cli_input_tokens(pagination_config):
240
251
  yield cli_name
241
252
 
242
253
 
254
+ # Get all tokens but return them in API namespace rather than CLI namespace
255
+ def _get_all_input_tokens(pagination_config):
256
+ # Get all input tokens including the limit_key
257
+ # if it exists.
258
+ tokens = _get_input_tokens(pagination_config)
259
+ for token_name in tokens:
260
+ yield token_name
261
+ if 'limit_key' in pagination_config:
262
+ key_name = pagination_config['limit_key']
263
+ yield key_name
264
+
265
+
243
266
  def _get_input_tokens(pagination_config):
244
267
  tokens = pagination_config['input_token']
245
268
  if not isinstance(tokens, list):
@@ -253,6 +276,48 @@ def _get_cli_name(param_objects, token_name):
253
276
  return param.cli_name.lstrip('-')
254
277
 
255
278
 
279
+ def check_should_enable_pagination_call_parameters(
280
+ session,
281
+ input_tokens,
282
+ call_parameters,
283
+ parsed_args,
284
+ parsed_globals,
285
+ **kwargs
286
+ ):
287
+ """
288
+ Check for pagination args in the actual calling arguments passed to
289
+ the function.
290
+
291
+ If the user is using the --cli-input-json parameter to provide JSON
292
+ parameters they are all in the API naming space rather than the CLI
293
+ naming space and would be missed by the processing above. This function
294
+ gets called on the calling-command event.
295
+ """
296
+ if resolve_v2_debug_mode(parsed_globals):
297
+ cli_input_json_data = session.emit_first_non_none_response(
298
+ f"get-cli-input-json-data",
299
+ )
300
+ if cli_input_json_data is None:
301
+ cli_input_json_data = {}
302
+ pagination_params_in_input_tokens = [
303
+ param for param in cli_input_json_data if param in input_tokens
304
+ ]
305
+ if pagination_params_in_input_tokens:
306
+ uni_print(
307
+ '\nAWS CLI v2 UPGRADE WARNING: In AWS CLI v2, if you specify '
308
+ 'pagination parameters by using a file with the '
309
+ '`--cli-input-json` parameter, automatic pagination will be '
310
+ 'turned off. This is different from v1 behavior, where '
311
+ 'pagination parameters specified via the `--cli-input-json` '
312
+ 'parameter are ignored. To retain AWS CLI v1 behavior in '
313
+ 'AWS CLI v2, remove all pagination parameters from the input '
314
+ 'JSON. See https://docs.aws.amazon.com/cli/latest/userguide/'
315
+ 'cliv2-migration-changes.html'
316
+ '#cliv2-migration-skeleton-paging.\n',
317
+ out_file=sys.stderr
318
+ )
319
+
320
+
256
321
  class PageArgument(BaseCLIArgument):
257
322
  type_map = {
258
323
  'string': str,
@@ -94,7 +94,8 @@ class FileDecodingError(Exception):
94
94
  class FileStat(object):
95
95
  def __init__(self, src, dest=None, compare_key=None, size=None,
96
96
  last_update=None, src_type=None, dest_type=None,
97
- operation_name=None, response_data=None, etag=None):
97
+ operation_name=None, response_data=None, etag=None,
98
+ case_conflict_submitted=None, case_conflict_key=None,):
98
99
  self.src = src
99
100
  self.dest = dest
100
101
  self.compare_key = compare_key
@@ -105,6 +106,8 @@ class FileStat(object):
105
106
  self.operation_name = operation_name
106
107
  self.response_data = response_data
107
108
  self.etag = etag
109
+ self.case_conflict_submitted = case_conflict_submitted
110
+ self.case_conflict_key = case_conflict_key
108
111
 
109
112
 
110
113
  class FileGenerator(object):
@@ -42,7 +42,8 @@ class FileInfo(object):
42
42
  last_update=None, src_type=None, dest_type=None,
43
43
  operation_name=None, client=None, parameters=None,
44
44
  source_client=None, is_stream=False,
45
- associated_response_data=None, etag=None):
45
+ associated_response_data=None, etag=None,
46
+ case_conflict_submitted=None, case_conflict_key=None,):
46
47
  self.src = src
47
48
  self.src_type = src_type
48
49
  self.operation_name = operation_name
@@ -60,6 +61,8 @@ class FileInfo(object):
60
61
  self.is_stream = is_stream
61
62
  self.associated_response_data = associated_response_data
62
63
  self.etag = etag
64
+ self.case_conflict_submitted = case_conflict_submitted
65
+ self.case_conflict_key = case_conflict_key
63
66
 
64
67
  def is_glacier_compatible(self):
65
68
  """Determines if a file info object is glacier compatible
@@ -46,6 +46,12 @@ class FileInfoBuilder(object):
46
46
  file_info_attr['is_stream'] = self._is_stream
47
47
  file_info_attr['associated_response_data'] = file_base.response_data
48
48
  file_info_attr['etag'] = file_base.etag
49
+ file_info_attr['case_conflict_submitted'] = getattr(
50
+ file_base, 'case_conflict_submitted', None
51
+ )
52
+ file_info_attr['case_conflict_key'] = getattr(
53
+ file_base, 'case_conflict_key', None
54
+ )
49
55
 
50
56
  # This is a bit quirky. The below conditional hinges on the --delete
51
57
  # flag being set, which only occurs during a sync command. The source
@@ -47,6 +47,7 @@ from awscli.customizations.s3.utils import DirectoryCreatorSubscriber
47
47
  from awscli.customizations.s3.utils import DeleteSourceFileSubscriber
48
48
  from awscli.customizations.s3.utils import DeleteSourceObjectSubscriber
49
49
  from awscli.customizations.s3.utils import DeleteCopySourceObjectSubscriber
50
+ from awscli.customizations.s3.utils import CaseConflictCleanupSubscriber
50
51
  from awscli.compat import get_binary_stdin
51
52
 
52
53
 
@@ -403,6 +404,13 @@ class DownloadRequestSubmitter(BaseTransferRequestSubmitter):
403
404
  if self._cli_params.get('is_move', False):
404
405
  subscribers.append(DeleteSourceObjectSubscriber(
405
406
  fileinfo.source_client))
407
+ if fileinfo.case_conflict_submitted is not None:
408
+ subscribers.append(
409
+ CaseConflictCleanupSubscriber(
410
+ fileinfo.case_conflict_submitted,
411
+ fileinfo.case_conflict_key,
412
+ )
413
+ )
406
414
 
407
415
  def _submit_transfer_request(self, fileinfo, extra_args, subscribers):
408
416
  bucket, key = find_bucket_key(fileinfo.src)
@@ -433,6 +441,7 @@ class CopyRequestSubmitter(BaseTransferRequestSubmitter):
433
441
 
434
442
  def _add_additional_subscribers(self, subscribers, fileinfo):
435
443
  subscribers.append(ProvideSizeSubscriber(fileinfo.size))
444
+ subscribers.append(ProvideETagSubscriber(fileinfo.etag))
436
445
  if self._should_inject_content_type():
437
446
  subscribers.append(ProvideCopyContentTypeSubscriber())
438
447
  if self._cli_params.get('is_move', False):
@@ -34,9 +34,10 @@ from awscli.customizations.s3.utils import find_bucket_key, AppendFilter, \
34
34
  S3PathResolver
35
35
  from awscli.customizations.utils import uni_print
36
36
  from awscli.customizations.s3.syncstrategy.base import MissingFileSync, \
37
- SizeAndLastModifiedSync, NeverSync
37
+ SizeAndLastModifiedSync, NeverSync, AlwaysSync
38
+ from awscli.customizations.s3.syncstrategy.caseconflict import CaseConflictSync
38
39
  from awscli.customizations.s3 import transferconfig
39
-
40
+ from awscli.utils import resolve_v2_debug_mode
40
41
 
41
42
  LOGGER = logging.getLogger(__name__)
42
43
 
@@ -482,6 +483,33 @@ BUCKET_REGION = {
482
483
  )
483
484
  }
484
485
 
486
+ CASE_CONFLICT = {
487
+ 'name': 'case-conflict',
488
+ 'choices': [
489
+ 'ignore',
490
+ 'skip',
491
+ 'warn',
492
+ 'error',
493
+ ],
494
+ 'default': 'warn',
495
+ 'help_text': (
496
+ "Configures behavior when attempting to download multiple objects "
497
+ "whose keys differ only by case, which can cause undefined behavior "
498
+ "on case-insensitive filesystems. "
499
+ "This parameter only applies for commands that perform multiple S3 "
500
+ "to local downloads. "
501
+ f"See <a href='{CaseConflictSync.DOC_URI}'>Handling case "
502
+ "conflicts</a> for details. Valid values are: "
503
+ "<ul>"
504
+ "<li>``error`` - Raise an error and abort downloads.</li>"
505
+ "<li>``warn`` - The default value. Emit a warning and download "
506
+ "the object.</li>"
507
+ "<li>``skip`` - Skip downloading the object.</li>"
508
+ "<li>``ignore`` - Ignore the conflict and download the object.</li>"
509
+ "</ul>"
510
+ ),
511
+ }
512
+
485
513
  TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL,
486
514
  FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
487
515
  SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE,
@@ -767,6 +795,7 @@ class S3TransferCommand(S3Command):
767
795
  cmd_params.add_verify_ssl(parsed_globals)
768
796
  cmd_params.add_page_size(parsed_args)
769
797
  cmd_params.add_paths(parsed_args.paths)
798
+ cmd_params.add_v2_debug(parsed_globals)
770
799
 
771
800
  runtime_config = transferconfig.RuntimeConfig().build_config(
772
801
  **self._session.get_scoped_config().get('s3', {}))
@@ -806,7 +835,8 @@ class CpCommand(S3TransferCommand):
806
835
  "or <S3Uri> <S3Uri>"
807
836
  ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
808
837
  'synopsis': USAGE}] + TRANSFER_ARGS + \
809
- [METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE]
838
+ [METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE,
839
+ CASE_CONFLICT]
810
840
 
811
841
 
812
842
  class MvCommand(S3TransferCommand):
@@ -816,7 +846,8 @@ class MvCommand(S3TransferCommand):
816
846
  "or <S3Uri> <S3Uri>"
817
847
  ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
818
848
  'synopsis': USAGE}] + TRANSFER_ARGS +\
819
- [METADATA, METADATA_DIRECTIVE, RECURSIVE, VALIDATE_SAME_S3_PATHS]
849
+ [METADATA, METADATA_DIRECTIVE, RECURSIVE, VALIDATE_SAME_S3_PATHS,
850
+ CASE_CONFLICT]
820
851
 
821
852
 
822
853
  class RmCommand(S3TransferCommand):
@@ -838,7 +869,7 @@ class SyncCommand(S3TransferCommand):
838
869
  "<LocalPath> or <S3Uri> <S3Uri>"
839
870
  ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
840
871
  'synopsis': USAGE}] + TRANSFER_ARGS + \
841
- [METADATA, METADATA_DIRECTIVE]
872
+ [METADATA, METADATA_DIRECTIVE, CASE_CONFLICT]
842
873
 
843
874
 
844
875
  class MbCommand(S3Command):
@@ -1003,7 +1034,16 @@ class CommandArchitecture(object):
1003
1034
  # Set the default strategies.
1004
1035
  sync_strategies['file_at_src_and_dest_sync_strategy'] = \
1005
1036
  SizeAndLastModifiedSync()
1006
- sync_strategies['file_not_at_dest_sync_strategy'] = MissingFileSync()
1037
+ if self._should_handle_case_conflicts():
1038
+ sync_strategies['file_not_at_dest_sync_strategy'] = (
1039
+ CaseConflictSync(
1040
+ on_case_conflict=self.parameters['case_conflict']
1041
+ )
1042
+ )
1043
+ else:
1044
+ sync_strategies['file_not_at_dest_sync_strategy'] = (
1045
+ MissingFileSync()
1046
+ )
1007
1047
  sync_strategies['file_not_at_src_sync_strategy'] = NeverSync()
1008
1048
 
1009
1049
  # Determine what strategies to override if any.
@@ -1056,6 +1096,24 @@ class CommandArchitecture(object):
1056
1096
  result_queue = queue.Queue()
1057
1097
  operation_name = cmd_translation[paths_type]
1058
1098
 
1099
+ if self.parameters['v2_debug']:
1100
+ if operation_name == 'copy':
1101
+ uni_print(
1102
+ '\nAWS CLI v2 UPGRADE WARNING: In AWS CLI v2, object '
1103
+ 'properties will be copied from the source in multipart '
1104
+ 'copies between S3 buckets initiated via `aws s3` '
1105
+ 'commands, resulting in additional S3 API calls to '
1106
+ 'transfer the metadata. Note that the principal must '
1107
+ 'have permission to call these APIs, or the command may '
1108
+ 'fail. This is different from v1 behavior, where metadata '
1109
+ 'is not copied. For guidance on retaining v1 behavior in '
1110
+ 'AWS CLI v2, or for more details, see '
1111
+ 'https://docs.aws.amazon.com/cli/latest/userguide/'
1112
+ 'cliv2-migration-changes.html'
1113
+ '#cliv2-migration-s3-copy-metadata.\n\n',
1114
+ out_file=sys.stderr
1115
+ )
1116
+
1059
1117
  fgen_kwargs = {
1060
1118
  'client': self._source_client, 'operation_name': operation_name,
1061
1119
  'follow_symlinks': self.parameters['follow_symlinks'],
@@ -1119,6 +1177,12 @@ class CommandArchitecture(object):
1119
1177
  'filters': [create_filter(self.parameters)],
1120
1178
  'file_info_builder': [file_info_builder],
1121
1179
  's3_handler': [s3_transfer_handler]}
1180
+ if self._should_handle_case_conflicts():
1181
+ self._handle_case_conflicts(
1182
+ command_dict,
1183
+ rev_files,
1184
+ rev_generator,
1185
+ )
1122
1186
  elif self.cmd == 'rm':
1123
1187
  command_dict = {'setup': [files],
1124
1188
  'file_generator': [file_generator],
@@ -1131,6 +1195,12 @@ class CommandArchitecture(object):
1131
1195
  'filters': [create_filter(self.parameters)],
1132
1196
  'file_info_builder': [file_info_builder],
1133
1197
  's3_handler': [s3_transfer_handler]}
1198
+ if self._should_handle_case_conflicts():
1199
+ self._handle_case_conflicts(
1200
+ command_dict,
1201
+ rev_files,
1202
+ rev_generator,
1203
+ )
1134
1204
 
1135
1205
  files = command_dict['setup']
1136
1206
  while self.instructions:
@@ -1196,6 +1266,74 @@ class CommandArchitecture(object):
1196
1266
  }
1197
1267
  )
1198
1268
 
1269
+ def _should_handle_case_conflicts(self):
1270
+ return (
1271
+ self.cmd in {'sync', 'cp', 'mv'}
1272
+ and self.parameters.get('paths_type') == 's3local'
1273
+ and self.parameters['case_conflict'] != 'ignore'
1274
+ and self.parameters.get('dir_op')
1275
+ )
1276
+
1277
+ def _handle_case_conflicts(self, command_dict, rev_files, rev_generator):
1278
+ # Objects are not returned in lexicographical order when
1279
+ # operated on S3 Express directory buckets. This is required
1280
+ # for sync operations to behave correctly, which is what
1281
+ # recursive copies and moves fall back to so potential case
1282
+ # conflicts can be detected and handled.
1283
+ if not is_s3express_bucket(
1284
+ split_s3_bucket_key(self.parameters['src'])[0]
1285
+ ):
1286
+ self._modify_instructions_for_case_conflicts(
1287
+ command_dict, rev_files, rev_generator
1288
+ )
1289
+ return
1290
+ # `skip` and `error` are not valid choices in this case because
1291
+ # it's not possible to detect case conflicts.
1292
+ if self.parameters['case_conflict'] not in {'ignore', 'warn'}:
1293
+ raise ValueError(
1294
+ f"`{self.parameters['case_conflict']}` is not a valid value "
1295
+ "for `--case-conflict` when operating on S3 Express "
1296
+ "directory buckets. Valid values: `warn`, `ignore`."
1297
+ )
1298
+ msg = (
1299
+ "warning: Recursive copies/moves from an S3 Express "
1300
+ "directory bucket to a case-insensitive local filesystem "
1301
+ "may result in undefined behavior if there are "
1302
+ "S3 object key names that differ only by case. To disable "
1303
+ "this warning, set the `--case-conflict` parameter to `ignore`. "
1304
+ f"For more information, see {CaseConflictSync.DOC_URI}."
1305
+ )
1306
+ uni_print(msg, sys.stderr)
1307
+
1308
+ def _modify_instructions_for_case_conflicts(
1309
+ self, command_dict, rev_files, rev_generator
1310
+ ):
1311
+ # Command will perform recursive S3 to local downloads.
1312
+ # Checking for potential case conflicts requires knowledge
1313
+ # of local files. Instead of writing a separate validation
1314
+ # mechanism for recursive downloads, we modify the instructions
1315
+ # to mimic a sync command.
1316
+ sync_strategies = {
1317
+ # Local filename exists with exact case match. Always sync
1318
+ # because it's a copy operation.
1319
+ 'file_at_src_and_dest_sync_strategy': AlwaysSync(),
1320
+ # Local filename either doesn't exist or differs only by case.
1321
+ # Let `CaseConflictSync` determine which it is and handle it
1322
+ # according to configured `--case-conflict` parameter.
1323
+ 'file_not_at_dest_sync_strategy': CaseConflictSync(
1324
+ on_case_conflict=self.parameters['case_conflict']
1325
+ ),
1326
+ # Copy is one-way so never sync if not at source.
1327
+ 'file_not_at_src_sync_strategy': NeverSync(),
1328
+ }
1329
+ command_dict['setup'].append(rev_files)
1330
+ command_dict['file_generator'].append(rev_generator)
1331
+ command_dict['filters'].append(create_filter(self.parameters))
1332
+ command_dict['comparator'] = [Comparator(**sync_strategies)]
1333
+ self.instructions.insert(
1334
+ self.instructions.index('file_info_builder'), 'comparator'
1335
+ )
1336
+
1199
1337
 
1200
1338
  class CommandParameters(object):
1201
1339
  """
@@ -1447,6 +1585,9 @@ class CommandParameters(object):
1447
1585
  def add_page_size(self, parsed_args):
1448
1586
  self.parameters['page_size'] = getattr(parsed_args, 'page_size', None)
1449
1587
 
1588
+ def add_v2_debug(self, parsed_globals):
1589
+ self.parameters['v2_debug'] = resolve_v2_debug_mode(parsed_globals)
1590
+
1450
1591
  def _validate_sse_c_args(self):
1451
1592
  self._validate_sse_c_arg()
1452
1593
  self._validate_sse_c_arg('sse_c_copy_source')
@@ -254,3 +254,12 @@ class MissingFileSync(BaseSync):
254
254
  LOG.debug("syncing: %s -> %s, file does not exist at destination",
255
255
  src_file.src, src_file.dest)
256
256
  return True
257
+
258
+
259
+ class AlwaysSync(BaseSync):
260
+ def __init__(self, sync_type='file_at_src_and_dest'):
261
+ super(AlwaysSync, self).__init__(sync_type)
262
+
263
+ def determine_should_sync(self, src_file, dest_file):
264
+ LOG.debug(f"syncing: {src_file.src} -> {src_file.dest}")
265
+ return True