awscli 1.44.5__py3-none-any.whl → 1.44.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- awscli/__init__.py +1 -1
- awscli/customizations/s3/filegenerator.py +4 -1
- awscli/customizations/s3/fileinfo.py +4 -1
- awscli/customizations/s3/fileinfobuilder.py +6 -0
- awscli/customizations/s3/s3handler.py +8 -0
- awscli/customizations/s3/subcommands.py +124 -5
- awscli/customizations/s3/syncstrategy/base.py +9 -0
- awscli/customizations/s3/syncstrategy/caseconflict.py +92 -0
- awscli/customizations/s3/utils.py +14 -0
- awscli/testutils.py +17 -0
- awscli/topics/s3-case-insensitivity.rst +105 -0
- awscli/topics/topic-tags.json +16 -0
- {awscli-1.44.5.dist-info → awscli-1.44.7.dist-info}/METADATA +2 -2
- {awscli-1.44.5.dist-info → awscli-1.44.7.dist-info}/RECORD +22 -20
- {awscli-1.44.5.data → awscli-1.44.7.data}/scripts/aws +0 -0
- {awscli-1.44.5.data → awscli-1.44.7.data}/scripts/aws.cmd +0 -0
- {awscli-1.44.5.data → awscli-1.44.7.data}/scripts/aws_bash_completer +0 -0
- {awscli-1.44.5.data → awscli-1.44.7.data}/scripts/aws_completer +0 -0
- {awscli-1.44.5.data → awscli-1.44.7.data}/scripts/aws_zsh_completer.sh +0 -0
- {awscli-1.44.5.dist-info → awscli-1.44.7.dist-info}/LICENSE.txt +0 -0
- {awscli-1.44.5.dist-info → awscli-1.44.7.dist-info}/WHEEL +0 -0
- {awscli-1.44.5.dist-info → awscli-1.44.7.dist-info}/top_level.txt +0 -0
awscli/__init__.py
CHANGED
|
@@ -94,7 +94,8 @@ class FileDecodingError(Exception):
|
|
|
94
94
|
class FileStat(object):
|
|
95
95
|
def __init__(self, src, dest=None, compare_key=None, size=None,
|
|
96
96
|
last_update=None, src_type=None, dest_type=None,
|
|
97
|
-
operation_name=None, response_data=None, etag=None
|
|
97
|
+
operation_name=None, response_data=None, etag=None,
|
|
98
|
+
case_conflict_submitted=None, case_conflict_key=None,):
|
|
98
99
|
self.src = src
|
|
99
100
|
self.dest = dest
|
|
100
101
|
self.compare_key = compare_key
|
|
@@ -105,6 +106,8 @@ class FileStat(object):
|
|
|
105
106
|
self.operation_name = operation_name
|
|
106
107
|
self.response_data = response_data
|
|
107
108
|
self.etag = etag
|
|
109
|
+
self.case_conflict_submitted = case_conflict_submitted
|
|
110
|
+
self.case_conflict_key = case_conflict_key
|
|
108
111
|
|
|
109
112
|
|
|
110
113
|
class FileGenerator(object):
|
|
@@ -42,7 +42,8 @@ class FileInfo(object):
|
|
|
42
42
|
last_update=None, src_type=None, dest_type=None,
|
|
43
43
|
operation_name=None, client=None, parameters=None,
|
|
44
44
|
source_client=None, is_stream=False,
|
|
45
|
-
associated_response_data=None, etag=None
|
|
45
|
+
associated_response_data=None, etag=None,
|
|
46
|
+
case_conflict_submitted=None, case_conflict_key=None,):
|
|
46
47
|
self.src = src
|
|
47
48
|
self.src_type = src_type
|
|
48
49
|
self.operation_name = operation_name
|
|
@@ -60,6 +61,8 @@ class FileInfo(object):
|
|
|
60
61
|
self.is_stream = is_stream
|
|
61
62
|
self.associated_response_data = associated_response_data
|
|
62
63
|
self.etag = etag
|
|
64
|
+
self.case_conflict_submitted = case_conflict_submitted
|
|
65
|
+
self.case_conflict_key = case_conflict_key
|
|
63
66
|
|
|
64
67
|
def is_glacier_compatible(self):
|
|
65
68
|
"""Determines if a file info object is glacier compatible
|
|
@@ -46,6 +46,12 @@ class FileInfoBuilder(object):
|
|
|
46
46
|
file_info_attr['is_stream'] = self._is_stream
|
|
47
47
|
file_info_attr['associated_response_data'] = file_base.response_data
|
|
48
48
|
file_info_attr['etag'] = file_base.etag
|
|
49
|
+
file_info_attr['case_conflict_submitted'] = getattr(
|
|
50
|
+
file_base, 'case_conflict_submitted', None
|
|
51
|
+
)
|
|
52
|
+
file_info_attr['case_conflict_key'] = getattr(
|
|
53
|
+
file_base, 'case_conflict_key', None
|
|
54
|
+
)
|
|
49
55
|
|
|
50
56
|
# This is a bit quirky. The below conditional hinges on the --delete
|
|
51
57
|
# flag being set, which only occurs during a sync command. The source
|
|
@@ -47,6 +47,7 @@ from awscli.customizations.s3.utils import DirectoryCreatorSubscriber
|
|
|
47
47
|
from awscli.customizations.s3.utils import DeleteSourceFileSubscriber
|
|
48
48
|
from awscli.customizations.s3.utils import DeleteSourceObjectSubscriber
|
|
49
49
|
from awscli.customizations.s3.utils import DeleteCopySourceObjectSubscriber
|
|
50
|
+
from awscli.customizations.s3.utils import CaseConflictCleanupSubscriber
|
|
50
51
|
from awscli.compat import get_binary_stdin
|
|
51
52
|
|
|
52
53
|
|
|
@@ -403,6 +404,13 @@ class DownloadRequestSubmitter(BaseTransferRequestSubmitter):
|
|
|
403
404
|
if self._cli_params.get('is_move', False):
|
|
404
405
|
subscribers.append(DeleteSourceObjectSubscriber(
|
|
405
406
|
fileinfo.source_client))
|
|
407
|
+
if fileinfo.case_conflict_submitted is not None:
|
|
408
|
+
subscribers.append(
|
|
409
|
+
CaseConflictCleanupSubscriber(
|
|
410
|
+
fileinfo.case_conflict_submitted,
|
|
411
|
+
fileinfo.case_conflict_key,
|
|
412
|
+
)
|
|
413
|
+
)
|
|
406
414
|
|
|
407
415
|
def _submit_transfer_request(self, fileinfo, extra_args, subscribers):
|
|
408
416
|
bucket, key = find_bucket_key(fileinfo.src)
|
|
@@ -34,7 +34,8 @@ from awscli.customizations.s3.utils import find_bucket_key, AppendFilter, \
|
|
|
34
34
|
S3PathResolver
|
|
35
35
|
from awscli.customizations.utils import uni_print
|
|
36
36
|
from awscli.customizations.s3.syncstrategy.base import MissingFileSync, \
|
|
37
|
-
SizeAndLastModifiedSync, NeverSync
|
|
37
|
+
SizeAndLastModifiedSync, NeverSync, AlwaysSync
|
|
38
|
+
from awscli.customizations.s3.syncstrategy.caseconflict import CaseConflictSync
|
|
38
39
|
from awscli.customizations.s3 import transferconfig
|
|
39
40
|
from awscli.utils import resolve_v2_debug_mode
|
|
40
41
|
|
|
@@ -482,6 +483,33 @@ BUCKET_REGION = {
|
|
|
482
483
|
)
|
|
483
484
|
}
|
|
484
485
|
|
|
486
|
+
CASE_CONFLICT = {
|
|
487
|
+
'name': 'case-conflict',
|
|
488
|
+
'choices': [
|
|
489
|
+
'ignore',
|
|
490
|
+
'skip',
|
|
491
|
+
'warn',
|
|
492
|
+
'error',
|
|
493
|
+
],
|
|
494
|
+
'default': 'warn',
|
|
495
|
+
'help_text': (
|
|
496
|
+
"Configures behavior when attempting to download multiple objects "
|
|
497
|
+
"whose keys differ only by case, which can cause undefined behavior "
|
|
498
|
+
"on case-insensitive filesystems. "
|
|
499
|
+
"This parameter only applies for commands that perform multiple S3 "
|
|
500
|
+
"to local downloads. "
|
|
501
|
+
f"See <a href='{CaseConflictSync.DOC_URI}'>Handling case "
|
|
502
|
+
"conflicts</a> for details. Valid values are: "
|
|
503
|
+
"<ul>"
|
|
504
|
+
"<li>``error`` - Raise an error and abort downloads.</li>"
|
|
505
|
+
"<li>``warn`` - The default value. Emit a warning and download "
|
|
506
|
+
"the object.</li>"
|
|
507
|
+
"<li>``skip`` - Skip downloading the object.</li>"
|
|
508
|
+
"<li>``ignore`` - Ignore the conflict and download the object.</li>"
|
|
509
|
+
"</ul>"
|
|
510
|
+
),
|
|
511
|
+
}
|
|
512
|
+
|
|
485
513
|
TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL,
|
|
486
514
|
FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
|
|
487
515
|
SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE,
|
|
@@ -807,7 +835,8 @@ class CpCommand(S3TransferCommand):
|
|
|
807
835
|
"or <S3Uri> <S3Uri>"
|
|
808
836
|
ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
|
|
809
837
|
'synopsis': USAGE}] + TRANSFER_ARGS + \
|
|
810
|
-
[METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE
|
|
838
|
+
[METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE,
|
|
839
|
+
CASE_CONFLICT]
|
|
811
840
|
|
|
812
841
|
|
|
813
842
|
class MvCommand(S3TransferCommand):
|
|
@@ -817,7 +846,8 @@ class MvCommand(S3TransferCommand):
|
|
|
817
846
|
"or <S3Uri> <S3Uri>"
|
|
818
847
|
ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
|
|
819
848
|
'synopsis': USAGE}] + TRANSFER_ARGS +\
|
|
820
|
-
[METADATA, METADATA_DIRECTIVE, RECURSIVE, VALIDATE_SAME_S3_PATHS
|
|
849
|
+
[METADATA, METADATA_DIRECTIVE, RECURSIVE, VALIDATE_SAME_S3_PATHS,
|
|
850
|
+
CASE_CONFLICT]
|
|
821
851
|
|
|
822
852
|
|
|
823
853
|
class RmCommand(S3TransferCommand):
|
|
@@ -839,7 +869,7 @@ class SyncCommand(S3TransferCommand):
|
|
|
839
869
|
"<LocalPath> or <S3Uri> <S3Uri>"
|
|
840
870
|
ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
|
|
841
871
|
'synopsis': USAGE}] + TRANSFER_ARGS + \
|
|
842
|
-
[METADATA, METADATA_DIRECTIVE]
|
|
872
|
+
[METADATA, METADATA_DIRECTIVE, CASE_CONFLICT]
|
|
843
873
|
|
|
844
874
|
|
|
845
875
|
class MbCommand(S3Command):
|
|
@@ -1004,7 +1034,16 @@ class CommandArchitecture(object):
|
|
|
1004
1034
|
# Set the default strategies.
|
|
1005
1035
|
sync_strategies['file_at_src_and_dest_sync_strategy'] = \
|
|
1006
1036
|
SizeAndLastModifiedSync()
|
|
1007
|
-
|
|
1037
|
+
if self._should_handle_case_conflicts():
|
|
1038
|
+
sync_strategies['file_not_at_dest_sync_strategy'] = (
|
|
1039
|
+
CaseConflictSync(
|
|
1040
|
+
on_case_conflict=self.parameters['case_conflict']
|
|
1041
|
+
)
|
|
1042
|
+
)
|
|
1043
|
+
else:
|
|
1044
|
+
sync_strategies['file_not_at_dest_sync_strategy'] = (
|
|
1045
|
+
MissingFileSync()
|
|
1046
|
+
)
|
|
1008
1047
|
sync_strategies['file_not_at_src_sync_strategy'] = NeverSync()
|
|
1009
1048
|
|
|
1010
1049
|
# Determine what strategies to override if any.
|
|
@@ -1138,6 +1177,12 @@ class CommandArchitecture(object):
|
|
|
1138
1177
|
'filters': [create_filter(self.parameters)],
|
|
1139
1178
|
'file_info_builder': [file_info_builder],
|
|
1140
1179
|
's3_handler': [s3_transfer_handler]}
|
|
1180
|
+
if self._should_handle_case_conflicts():
|
|
1181
|
+
self._handle_case_conflicts(
|
|
1182
|
+
command_dict,
|
|
1183
|
+
rev_files,
|
|
1184
|
+
rev_generator,
|
|
1185
|
+
)
|
|
1141
1186
|
elif self.cmd == 'rm':
|
|
1142
1187
|
command_dict = {'setup': [files],
|
|
1143
1188
|
'file_generator': [file_generator],
|
|
@@ -1150,6 +1195,12 @@ class CommandArchitecture(object):
|
|
|
1150
1195
|
'filters': [create_filter(self.parameters)],
|
|
1151
1196
|
'file_info_builder': [file_info_builder],
|
|
1152
1197
|
's3_handler': [s3_transfer_handler]}
|
|
1198
|
+
if self._should_handle_case_conflicts():
|
|
1199
|
+
self._handle_case_conflicts(
|
|
1200
|
+
command_dict,
|
|
1201
|
+
rev_files,
|
|
1202
|
+
rev_generator,
|
|
1203
|
+
)
|
|
1153
1204
|
|
|
1154
1205
|
files = command_dict['setup']
|
|
1155
1206
|
while self.instructions:
|
|
@@ -1215,6 +1266,74 @@ class CommandArchitecture(object):
|
|
|
1215
1266
|
}
|
|
1216
1267
|
)
|
|
1217
1268
|
|
|
1269
|
+
def _should_handle_case_conflicts(self):
|
|
1270
|
+
return (
|
|
1271
|
+
self.cmd in {'sync', 'cp', 'mv'}
|
|
1272
|
+
and self.parameters.get('paths_type') == 's3local'
|
|
1273
|
+
and self.parameters['case_conflict'] != 'ignore'
|
|
1274
|
+
and self.parameters.get('dir_op')
|
|
1275
|
+
)
|
|
1276
|
+
|
|
1277
|
+
def _handle_case_conflicts(self, command_dict, rev_files, rev_generator):
|
|
1278
|
+
# Objects are not returned in lexicographical order when
|
|
1279
|
+
# operated on S3 Express directory buckets. This is required
|
|
1280
|
+
# for sync operations to behave correctly, which is what
|
|
1281
|
+
# recursive copies and moves fall back to so potential case
|
|
1282
|
+
# conflicts can be detected and handled.
|
|
1283
|
+
if not is_s3express_bucket(
|
|
1284
|
+
split_s3_bucket_key(self.parameters['src'])[0]
|
|
1285
|
+
):
|
|
1286
|
+
self._modify_instructions_for_case_conflicts(
|
|
1287
|
+
command_dict, rev_files, rev_generator
|
|
1288
|
+
)
|
|
1289
|
+
return
|
|
1290
|
+
# `skip` and `error` are not valid choices in this case because
|
|
1291
|
+
# it's not possible to detect case conflicts.
|
|
1292
|
+
if self.parameters['case_conflict'] not in {'ignore', 'warn'}:
|
|
1293
|
+
raise ValueError(
|
|
1294
|
+
f"`{self.parameters['case_conflict']}` is not a valid value "
|
|
1295
|
+
"for `--case-conflict` when operating on S3 Express "
|
|
1296
|
+
"directory buckets. Valid values: `warn`, `ignore`."
|
|
1297
|
+
)
|
|
1298
|
+
msg = (
|
|
1299
|
+
"warning: Recursive copies/moves from an S3 Express "
|
|
1300
|
+
"directory bucket to a case-insensitive local filesystem "
|
|
1301
|
+
"may result in undefined behavior if there are "
|
|
1302
|
+
"S3 object key names that differ only by case. To disable "
|
|
1303
|
+
"this warning, set the `--case-conflict` parameter to `ignore`. "
|
|
1304
|
+
f"For more information, see {CaseConflictSync.DOC_URI}."
|
|
1305
|
+
)
|
|
1306
|
+
uni_print(msg, sys.stderr)
|
|
1307
|
+
|
|
1308
|
+
def _modify_instructions_for_case_conflicts(
|
|
1309
|
+
self, command_dict, rev_files, rev_generator
|
|
1310
|
+
):
|
|
1311
|
+
# Command will perform recursive S3 to local downloads.
|
|
1312
|
+
# Checking for potential case conflicts requires knowledge
|
|
1313
|
+
# of local files. Instead of writing a separate validation
|
|
1314
|
+
# mechanism for recursive downloads, we modify the instructions
|
|
1315
|
+
# to mimic a sync command.
|
|
1316
|
+
sync_strategies = {
|
|
1317
|
+
# Local filename exists with exact case match. Always sync
|
|
1318
|
+
# because it's a copy operation.
|
|
1319
|
+
'file_at_src_and_dest_sync_strategy': AlwaysSync(),
|
|
1320
|
+
# Local filename either doesn't exist or differs only by case.
|
|
1321
|
+
# Let `CaseConflictSync` determine which it is and handle it
|
|
1322
|
+
# according to configured `--case-conflict` parameter.
|
|
1323
|
+
'file_not_at_dest_sync_strategy': CaseConflictSync(
|
|
1324
|
+
on_case_conflict=self.parameters['case_conflict']
|
|
1325
|
+
),
|
|
1326
|
+
# Copy is one-way so never sync if not at source.
|
|
1327
|
+
'file_not_at_src_sync_strategy': NeverSync(),
|
|
1328
|
+
}
|
|
1329
|
+
command_dict['setup'].append(rev_files)
|
|
1330
|
+
command_dict['file_generator'].append(rev_generator)
|
|
1331
|
+
command_dict['filters'].append(create_filter(self.parameters))
|
|
1332
|
+
command_dict['comparator'] = [Comparator(**sync_strategies)]
|
|
1333
|
+
self.instructions.insert(
|
|
1334
|
+
self.instructions.index('file_info_builder'), 'comparator'
|
|
1335
|
+
)
|
|
1336
|
+
|
|
1218
1337
|
|
|
1219
1338
|
class CommandParameters(object):
|
|
1220
1339
|
"""
|
|
@@ -254,3 +254,12 @@ class MissingFileSync(BaseSync):
|
|
|
254
254
|
LOG.debug("syncing: %s -> %s, file does not exist at destination",
|
|
255
255
|
src_file.src, src_file.dest)
|
|
256
256
|
return True
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class AlwaysSync(BaseSync):
|
|
260
|
+
def __init__(self, sync_type='file_at_src_and_dest'):
|
|
261
|
+
super(AlwaysSync, self).__init__(sync_type)
|
|
262
|
+
|
|
263
|
+
def determine_should_sync(self, src_file, dest_file):
|
|
264
|
+
LOG.debug(f"syncing: {src_file.src} -> {src_file.dest}")
|
|
265
|
+
return True
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
from awscli.customizations.s3.syncstrategy.base import BaseSync
|
|
8
|
+
from awscli.customizations.utils import uni_print
|
|
9
|
+
|
|
10
|
+
LOG = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class CaseConflictException(Exception):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CaseConflictSync(BaseSync):
|
|
18
|
+
DOC_URI = (
|
|
19
|
+
"https://docs.aws.amazon.com/cli/v1/topic/"
|
|
20
|
+
"s3-case-insensitivity.html"
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
def __init__(
|
|
24
|
+
self,
|
|
25
|
+
sync_type='file_not_at_dest',
|
|
26
|
+
on_case_conflict='warn',
|
|
27
|
+
submitted=None,
|
|
28
|
+
):
|
|
29
|
+
super().__init__(sync_type)
|
|
30
|
+
self._on_case_conflict = on_case_conflict
|
|
31
|
+
if submitted is None:
|
|
32
|
+
submitted = set()
|
|
33
|
+
self._submitted = submitted
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def submitted(self):
|
|
37
|
+
return self._submitted
|
|
38
|
+
|
|
39
|
+
def determine_should_sync(self, src_file, dest_file):
|
|
40
|
+
# `src_file.compare_key` and `dest_file.compare_key` are not equal.
|
|
41
|
+
# This could mean that they're completely different or differ
|
|
42
|
+
# only by case. eg, `/tmp/a` and `/tmp/b` versus `/tmp/a` and `/tmp/A`.
|
|
43
|
+
# If the source file's destination already exists, that means it
|
|
44
|
+
# differs only by case and the conflict needs to be handled.
|
|
45
|
+
should_sync = True
|
|
46
|
+
# Normalize compare key for case sensitivity.
|
|
47
|
+
lower_compare_key = src_file.compare_key.lower()
|
|
48
|
+
if lower_compare_key in self._submitted or os.path.exists(
|
|
49
|
+
src_file.dest
|
|
50
|
+
):
|
|
51
|
+
handler = getattr(self, f"_handle_{self._on_case_conflict}")
|
|
52
|
+
should_sync = handler(src_file)
|
|
53
|
+
if should_sync:
|
|
54
|
+
LOG.debug(f"syncing: {src_file.src} -> {src_file.dest}")
|
|
55
|
+
self._submitted.add(lower_compare_key)
|
|
56
|
+
# Set properties so that a subscriber can be created
|
|
57
|
+
# that removes the key from the set after download finishes.
|
|
58
|
+
src_file.case_conflict_submitted = self._submitted
|
|
59
|
+
src_file.case_conflict_key = lower_compare_key
|
|
60
|
+
return should_sync
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def _handle_skip(src_file):
|
|
64
|
+
msg = (
|
|
65
|
+
f"warning: Skipping {src_file.src} -> {src_file.dest} "
|
|
66
|
+
"because a file whose name differs only by case either exists "
|
|
67
|
+
"or is being downloaded.\n"
|
|
68
|
+
)
|
|
69
|
+
uni_print(msg, sys.stderr)
|
|
70
|
+
return False
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def _handle_warn(src_file):
|
|
74
|
+
msg = (
|
|
75
|
+
f"warning: Downloading {src_file.src} -> {src_file.dest} "
|
|
76
|
+
"despite a file whose name differs only by case either existing "
|
|
77
|
+
"or being downloaded. This behavior is not defined on "
|
|
78
|
+
"case-insensitive filesystems and may result in overwriting "
|
|
79
|
+
"existing files or race conditions between concurrent downloads. "
|
|
80
|
+
f"For more information, see {CaseConflictSync.DOC_URI}.\n"
|
|
81
|
+
)
|
|
82
|
+
uni_print(msg, sys.stderr)
|
|
83
|
+
return True
|
|
84
|
+
|
|
85
|
+
@staticmethod
|
|
86
|
+
def _handle_error(src_file):
|
|
87
|
+
msg = (
|
|
88
|
+
f"Failed to download {src_file.src} -> {src_file.dest} "
|
|
89
|
+
"because a file whose name differs only by case either exists "
|
|
90
|
+
"or is being downloaded."
|
|
91
|
+
)
|
|
92
|
+
raise CaseConflictException(msg)
|
|
@@ -690,6 +690,20 @@ class OnDoneFilteredSubscriber(BaseSubscriber):
|
|
|
690
690
|
pass
|
|
691
691
|
|
|
692
692
|
|
|
693
|
+
class CaseConflictCleanupSubscriber(BaseSubscriber):
|
|
694
|
+
"""
|
|
695
|
+
A subscriber which removes object compare key from case conflict set
|
|
696
|
+
when download finishes.
|
|
697
|
+
"""
|
|
698
|
+
|
|
699
|
+
def __init__(self, submitted, case_conflict_key):
|
|
700
|
+
self._submitted = submitted
|
|
701
|
+
self._key = case_conflict_key
|
|
702
|
+
|
|
703
|
+
def on_done(self, future, **kwargs):
|
|
704
|
+
self._submitted.discard(self._key)
|
|
705
|
+
|
|
706
|
+
|
|
693
707
|
class DeleteSourceSubscriber(OnDoneFilteredSubscriber):
|
|
694
708
|
"""A subscriber which deletes the source of the transfer."""
|
|
695
709
|
def _on_success(self, future):
|
awscli/testutils.py
CHANGED
|
@@ -27,7 +27,9 @@ import json
|
|
|
27
27
|
import logging
|
|
28
28
|
import os
|
|
29
29
|
import platform
|
|
30
|
+
import random
|
|
30
31
|
import shutil
|
|
32
|
+
import string
|
|
31
33
|
import sys
|
|
32
34
|
import tempfile
|
|
33
35
|
import time
|
|
@@ -36,6 +38,7 @@ import uuid
|
|
|
36
38
|
from pprint import pformat
|
|
37
39
|
from subprocess import PIPE, Popen
|
|
38
40
|
from unittest import mock
|
|
41
|
+
from pathlib import Path
|
|
39
42
|
|
|
40
43
|
import botocore.loaders
|
|
41
44
|
from botocore.awsrequest import AWSResponse
|
|
@@ -49,6 +52,11 @@ _LOADER = botocore.loaders.Loader()
|
|
|
49
52
|
INTEG_LOG = logging.getLogger('awscli.tests.integration')
|
|
50
53
|
AWS_CMD = None
|
|
51
54
|
|
|
55
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
56
|
+
with open(Path(tmpdir) / 'aws-cli-tmp-file', 'w') as f:
|
|
57
|
+
pass
|
|
58
|
+
CASE_INSENSITIVE = (Path(tmpdir) / 'AWS-CLI-TMP-FILE').exists()
|
|
59
|
+
|
|
52
60
|
|
|
53
61
|
def skip_if_windows(reason):
|
|
54
62
|
"""Decorator to skip tests that should not be run on windows.
|
|
@@ -69,6 +77,15 @@ def skip_if_windows(reason):
|
|
|
69
77
|
return decorator
|
|
70
78
|
|
|
71
79
|
|
|
80
|
+
def skip_if_case_sensitive():
|
|
81
|
+
def decorator(func):
|
|
82
|
+
return unittest.skipIf(
|
|
83
|
+
not CASE_INSENSITIVE,
|
|
84
|
+
"This test requires a case-insensitive filesystem."
|
|
85
|
+
)(func)
|
|
86
|
+
return decorator
|
|
87
|
+
|
|
88
|
+
|
|
72
89
|
def create_clidriver():
|
|
73
90
|
driver = awscli.clidriver.create_clidriver()
|
|
74
91
|
session = driver.session
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
:title: AWS CLI S3 Case-Insensitivity
|
|
2
|
+
:description: Using 'aws s3' commands on case-insensitive filesystems
|
|
3
|
+
:category: S3
|
|
4
|
+
:related command: s3 cp, s3 sync, s3 mv
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
This page explains how to detect and handle potential case conflicts when
|
|
8
|
+
downloading multiple objects from S3 to a local case-insensitive filesystem
|
|
9
|
+
using a single AWS CLI command.
|
|
10
|
+
|
|
11
|
+
Case conflicts
|
|
12
|
+
==============
|
|
13
|
+
S3 object keys are case-sensitive meaning that a bucket can have a set of
|
|
14
|
+
key names that differ only by case, for example, ``a.txt`` and ``A.txt``.
|
|
15
|
+
|
|
16
|
+
The AWS CLI offers high-level S3 commands that manage transfers of
|
|
17
|
+
multiple S3 objects using a single command:
|
|
18
|
+
|
|
19
|
+
* ``aws s3 sync``
|
|
20
|
+
* ``aws s3 cp --recursive``
|
|
21
|
+
* ``aws s3 mv --recursive``
|
|
22
|
+
|
|
23
|
+
Case conflicts can occur on case-insensitive filesystems when an S3 bucket
|
|
24
|
+
has multiple objects whose keys differ only by case and a single AWS CLI
|
|
25
|
+
command is called to download multiple S3 objects **OR** a local file
|
|
26
|
+
already exists whose name differs only by case.
|
|
27
|
+
|
|
28
|
+
For example, consider an S3 bucket with the following stored objects:
|
|
29
|
+
|
|
30
|
+
* ``a.txt``
|
|
31
|
+
* ``A.txt``
|
|
32
|
+
|
|
33
|
+
When the following AWS CLI command is called, the AWS CLI will submit
|
|
34
|
+
requests to download ``a.txt`` and ``A.txt``. Since only
|
|
35
|
+
one can exist on a case-insensitive filesystem, the last download to finish
|
|
36
|
+
will be the file that's locally available.
|
|
37
|
+
|
|
38
|
+
.. code-block::
|
|
39
|
+
|
|
40
|
+
aws s3 sync s3://examplebucket ./mylocaldir
|
|
41
|
+
|
|
42
|
+
Detecting and handling case conflicts
|
|
43
|
+
=====================================
|
|
44
|
+
To detect and handle case conflicts, you can specify the ``--case-conflict``
|
|
45
|
+
parameter. The following values are valid options:
|
|
46
|
+
|
|
47
|
+
* ``error`` - When a case conflict is detected, the command will immediately
|
|
48
|
+
fail and abort in-progress downloads.
|
|
49
|
+
* ``warn`` - (Default) When a case conflict is detected, the AWS CLI will
|
|
50
|
+
display a warning.
|
|
51
|
+
* ``skip`` - When a case conflict is detected, the command will skip
|
|
52
|
+
downloading the object and continue and display a warning.
|
|
53
|
+
* ``ignore`` - Case conflicts will not be detected or handled.
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
Continuing the prior example, the following describes what happens when
|
|
57
|
+
appending the ``--case-conflict`` parameter with possible values:
|
|
58
|
+
|
|
59
|
+
``--case-conflict error``
|
|
60
|
+
|
|
61
|
+
1. Submit a download request for ``A.txt``.
|
|
62
|
+
2. Detect that ``a.txt`` conflicts with an object that's been submitted for download.
|
|
63
|
+
3. Throw an error. If ``A.txt`` finished downloading, it will be locally available. Otherwise, the download request for ``A.txt`` will be aborted.
|
|
64
|
+
|
|
65
|
+
``--case-conflict warn``
|
|
66
|
+
|
|
67
|
+
1. Submit a download request for ``A.txt``.
|
|
68
|
+
2. Detect that ``a.txt`` conflicts with an object that's been submitted for download.
|
|
69
|
+
3. Display a warning.
|
|
70
|
+
4. Submit a download request for ``a.txt``, downloading ``A.txt`` and ``a.txt`` in parallel.
|
|
71
|
+
|
|
72
|
+
``--case-conflict skip``
|
|
73
|
+
|
|
74
|
+
1. Submit a download request for ``A.txt``.
|
|
75
|
+
2. Detect that ``a.txt`` conflicts with an object that's been submitted for download.
|
|
76
|
+
3. Skip downloading ``a.txt`` and continue.
|
|
77
|
+
|
|
78
|
+
``--case-conflict ignore``
|
|
79
|
+
|
|
80
|
+
1. Submit a download request for ``A.txt``.
|
|
81
|
+
2. Submit a download request for ``a.txt``, downloading ``A.txt`` and ``a.txt`` in parallel.
|
|
82
|
+
|
|
83
|
+
If your local filesystem is case-sensitive, there's no need to detect and
|
|
84
|
+
handle case conflicts. We recommend setting ``--case-conflict ignore``
|
|
85
|
+
in this case.
|
|
86
|
+
|
|
87
|
+
S3 Express directory buckets
|
|
88
|
+
============================
|
|
89
|
+
Detecting case conflicts is **NOT** supported when the source is an S3 Express
|
|
90
|
+
directory bucket. When operating on directory buckets, valid values for the
|
|
91
|
+
``--case-conflict`` parameter are:
|
|
92
|
+
|
|
93
|
+
* ``warn``
|
|
94
|
+
* ``ignore``
|
|
95
|
+
|
|
96
|
+
The following values are invalid when operating on directory buckets:
|
|
97
|
+
|
|
98
|
+
* ``error``
|
|
99
|
+
* ``skip``
|
|
100
|
+
|
|
101
|
+
For example, calling the following command will fail:
|
|
102
|
+
|
|
103
|
+
.. code-block::
|
|
104
|
+
|
|
105
|
+
aws s3 cp s3://mydirbucket--usw2-az1--x-s3 ./mylocaldir --recursive --case-conflict error
|
awscli/topics/topic-tags.json
CHANGED
|
@@ -69,5 +69,21 @@
|
|
|
69
69
|
"title": [
|
|
70
70
|
"AWS CLI S3 FAQ"
|
|
71
71
|
]
|
|
72
|
+
},
|
|
73
|
+
"s3-case-insensitivity": {
|
|
74
|
+
"category": [
|
|
75
|
+
"S3"
|
|
76
|
+
],
|
|
77
|
+
"description": [
|
|
78
|
+
"Using 'aws s3' commands on case-insensitive filesystems"
|
|
79
|
+
],
|
|
80
|
+
"related command": [
|
|
81
|
+
"s3 cp",
|
|
82
|
+
"s3 sync",
|
|
83
|
+
"s3 mv"
|
|
84
|
+
],
|
|
85
|
+
"title": [
|
|
86
|
+
"AWS CLI S3 Case-Insensitivity"
|
|
87
|
+
]
|
|
72
88
|
}
|
|
73
89
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: awscli
|
|
3
|
-
Version: 1.44.
|
|
3
|
+
Version: 1.44.7
|
|
4
4
|
Summary: Universal Command Line Environment for AWS.
|
|
5
5
|
Home-page: http://aws.amazon.com/cli/
|
|
6
6
|
Author: Amazon Web Services
|
|
@@ -24,7 +24,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
24
24
|
Classifier: Programming Language :: Python :: 3.14
|
|
25
25
|
Requires-Python: >= 3.9
|
|
26
26
|
License-File: LICENSE.txt
|
|
27
|
-
Requires-Dist: botocore (==1.42.
|
|
27
|
+
Requires-Dist: botocore (==1.42.17)
|
|
28
28
|
Requires-Dist: docutils (<=0.19,>=0.18.1)
|
|
29
29
|
Requires-Dist: s3transfer (<0.17.0,>=0.16.0)
|
|
30
30
|
Requires-Dist: PyYAML (<6.1,>=3.10)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
awscli/__init__.py,sha256=
|
|
1
|
+
awscli/__init__.py,sha256=TZfbYgX_NwzovkrDiF34qVDusgeoauFGIUNF2H64iog,1533
|
|
2
2
|
awscli/__main__.py,sha256=iBjOg0tBxNlhzTi_tyc1G0SMGBvHMVvBJzX3JqYaooY,662
|
|
3
3
|
awscli/alias.py,sha256=gB5jGInOl97UbgwGuKCjR_a_VudK0lAV1W6U_zeetZA,11348
|
|
4
4
|
awscli/argparser.py,sha256=3Pxx-vWytdV985Y6MIl9DeutUXyehIvACIs_PDby8GI,7650
|
|
@@ -18,7 +18,7 @@ awscli/plugin.py,sha256=B3wgRerhgFK1v-QQeASYv2VNLSa9oMuD4X_hcLSescI,2265
|
|
|
18
18
|
awscli/schema.py,sha256=vNFkuE2BsgeqesRTWRhuSU341eqiKuMF5QAJvClyaKw,6394
|
|
19
19
|
awscli/shorthand.py,sha256=ziXUkFyJGcDG8dMQzDMgdUSpZpe88gGBRk5Skkg60W4,18379
|
|
20
20
|
awscli/table.py,sha256=VCyPjNHK4wO_-KKRz7zvuuNp2RLHb6DUPVbPdueji3w,15459
|
|
21
|
-
awscli/testutils.py,sha256=
|
|
21
|
+
awscli/testutils.py,sha256=R_URfOci08TTar0mjfCk6u5BovWYjd-__6nRu2Tw74w,37444
|
|
22
22
|
awscli/text.py,sha256=pr40cSMkGWZ5n-VXMcEKCo1xO5bK3nUbDK3WLwy2HFE,4177
|
|
23
23
|
awscli/topictags.py,sha256=A1HDK4jE2ZxReOVM1sftjInQXVWL1DRz8DLS5JIGMag,12635
|
|
24
24
|
awscli/utils.py,sha256=29k14fL0jIHFTEErNgxIvMlIV3zYwjGDFH7uM9C7vdU,10594
|
|
@@ -190,18 +190,19 @@ awscli/customizations/logs/startlivetail.py,sha256=NwN5wp8DKKgRfoAtZySlLxgHHuSMz
|
|
|
190
190
|
awscli/customizations/s3/__init__.py,sha256=BFJ0dbdWkUBgJ2qav1Jhz06SddTMIeahxSaW_H0vop0,565
|
|
191
191
|
awscli/customizations/s3/comparator.py,sha256=bg9ewFtjOGUHmTSK37J9oP1OD9lWfdmIF7cDGMblFoc,6146
|
|
192
192
|
awscli/customizations/s3/fileformat.py,sha256=m4-tpR3_4AZOYfLxnlaGmIXG_vqTxyD4jNC8HtZRn2k,6027
|
|
193
|
-
awscli/customizations/s3/filegenerator.py,sha256=
|
|
194
|
-
awscli/customizations/s3/fileinfo.py,sha256=
|
|
195
|
-
awscli/customizations/s3/fileinfobuilder.py,sha256=
|
|
193
|
+
awscli/customizations/s3/filegenerator.py,sha256=nJRVmCz7wyPkUAdCZ5xGUSrbTVdjQSv1pH59Dzwpyrg,16373
|
|
194
|
+
awscli/customizations/s3/fileinfo.py,sha256=yT_aeDGVAFdO8sZU2r9TDMG7Vu9sXCEJC9VUG-2yoy8,4481
|
|
195
|
+
awscli/customizations/s3/fileinfobuilder.py,sha256=66XiO305bYwnikq9PN9OAJIvw42vqO-xGlkRyX12viw,3470
|
|
196
196
|
awscli/customizations/s3/filters.py,sha256=OuQUr6XAMkS1i6GO65_L7NbL7kwgmKT2ghWhv1YdkXo,6489
|
|
197
197
|
awscli/customizations/s3/results.py,sha256=L19gi3CtJYeHqWIWuIVcbj82Dshw2g5jNX8PFidcC2U,26625
|
|
198
198
|
awscli/customizations/s3/s3.py,sha256=Igwsn89G7i9M4nShjzcFmwlCVXvpfgmoyEf9wpNLXdk,2739
|
|
199
|
-
awscli/customizations/s3/s3handler.py,sha256=
|
|
200
|
-
awscli/customizations/s3/subcommands.py,sha256=
|
|
199
|
+
awscli/customizations/s3/s3handler.py,sha256=P6GNUUTu6KVH443tA2JO39GnHWyk2cruWHHNVlEWBsg,23837
|
|
200
|
+
awscli/customizations/s3/subcommands.py,sha256=EaVPdgC5X3HNxT5GM7rxh7cDiOmwQZBZvzaVWRF51Kk,69792
|
|
201
201
|
awscli/customizations/s3/transferconfig.py,sha256=7MW4hi90N5mLeQBlmVxs9J5ixRjejp1F4uPmGGF3TME,4472
|
|
202
|
-
awscli/customizations/s3/utils.py,sha256=
|
|
202
|
+
awscli/customizations/s3/utils.py,sha256=O6k3W3jJthNZ0bj-y3G6ZIFbDwVCxxexbPs0OZk_l64,34880
|
|
203
203
|
awscli/customizations/s3/syncstrategy/__init__.py,sha256=BFJ0dbdWkUBgJ2qav1Jhz06SddTMIeahxSaW_H0vop0,565
|
|
204
|
-
awscli/customizations/s3/syncstrategy/base.py,sha256=
|
|
204
|
+
awscli/customizations/s3/syncstrategy/base.py,sha256=HqJSzTgOX88YhJb-_UJOmSzfk6wAcctsgO59s7rsX88,10397
|
|
205
|
+
awscli/customizations/s3/syncstrategy/caseconflict.py,sha256=3E5XJFFqi-nTrgJ04sUXPOVo0JX3yUxlKcKWsNsUXf4,3308
|
|
205
206
|
awscli/customizations/s3/syncstrategy/delete.py,sha256=y-KSRQE14bZY9jQfJJx0WbZ8UyX6juKkDhS8lwC9ed8,1313
|
|
206
207
|
awscli/customizations/s3/syncstrategy/exacttimestamps.py,sha256=Bi_t4pbVfYbxKt4sKe3XJVYSbocfMTgg-HAx903Ts2o,1686
|
|
207
208
|
awscli/customizations/s3/syncstrategy/register.py,sha256=2jsuidA4sxf3v1_9rpmp3tJRa2pGX35zYxzRluAmggs,2001
|
|
@@ -6070,16 +6071,17 @@ awscli/examples/xray/update-group.rst,sha256=ThP94mCvPeSXQanVJjI13PSArn2rqnzX7Q-
|
|
|
6070
6071
|
awscli/examples/xray/update-sampling-rule.rst,sha256=a1THv2Q8ELvSSlZbnbUVN5YWopGFLiBU4yqliQLF8Q8,1521
|
|
6071
6072
|
awscli/topics/config-vars.rst,sha256=rGwpLEHEG355W6a4xmNjD-FXl2O3GRfkNBjQuW1ZaKY,22751
|
|
6072
6073
|
awscli/topics/return-codes.rst,sha256=d9lpNFZwD75IiYcDEADQzu-4QiR8P28UPHkrNwPV5J8,1996
|
|
6074
|
+
awscli/topics/s3-case-insensitivity.rst,sha256=xUI86tyD3QrkS7s_Ry7QnBmnsq8P6QSr6Q0TQAS8fRY,3778
|
|
6073
6075
|
awscli/topics/s3-config.rst,sha256=5EIVd4ggLBHtzjtHFvQp9hY415yMGZiG7S_rO9qy2t0,11663
|
|
6074
6076
|
awscli/topics/s3-faq.rst,sha256=9qO0HFI6F9hx1wVEUDl8Jy6yoCUd9zbtv-Z0Re4dsiw,2934
|
|
6075
|
-
awscli/topics/topic-tags.json,sha256=
|
|
6076
|
-
awscli-1.44.
|
|
6077
|
-
awscli-1.44.
|
|
6078
|
-
awscli-1.44.
|
|
6079
|
-
awscli-1.44.
|
|
6080
|
-
awscli-1.44.
|
|
6081
|
-
awscli-1.44.
|
|
6082
|
-
awscli-1.44.
|
|
6083
|
-
awscli-1.44.
|
|
6084
|
-
awscli-1.44.
|
|
6085
|
-
awscli-1.44.
|
|
6077
|
+
awscli/topics/topic-tags.json,sha256=wjVD-b-g_XUNYUffoUXyoUy8-ul1___xd2chd1MUv6U,1951
|
|
6078
|
+
awscli-1.44.7.data/scripts/aws,sha256=r24FExgs0-JjILTQ3XZAqXBYE4SV6UMTtALkLGAj86g,805
|
|
6079
|
+
awscli-1.44.7.data/scripts/aws.cmd,sha256=s46DkC6LNgX63CIkzxxbPnFMJ6DRDBkvc88GnWa8Pvg,1432
|
|
6080
|
+
awscli-1.44.7.data/scripts/aws_bash_completer,sha256=RRpoEGJRagRzyHZKZZOwpltuVYv2EoiZsdXhmyWPZ54,204
|
|
6081
|
+
awscli-1.44.7.data/scripts/aws_completer,sha256=oC9kuMDlWE47dWk_4xjPde2PQvN-M0vND0J4YSLabVQ,1126
|
|
6082
|
+
awscli-1.44.7.data/scripts/aws_zsh_completer.sh,sha256=Qm6Z8ejNAMzpJjaT0pzqxbSDT2zxdmzVe5haRA7qLoc,1808
|
|
6083
|
+
awscli-1.44.7.dist-info/LICENSE.txt,sha256=o5XhFlwu0OK_BBrijlKCRa7dQAm36UrUB3gCV_cEr8E,549
|
|
6084
|
+
awscli-1.44.7.dist-info/METADATA,sha256=lIhOM9eVulnCObSCj9ZhheD_vJd8LtfuMsTljg36oJE,11200
|
|
6085
|
+
awscli-1.44.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
|
|
6086
|
+
awscli-1.44.7.dist-info/top_level.txt,sha256=vt9wXFr1_nGYK6abhJgt6zY3fULe4JSZedm_5XOM9S0,7
|
|
6087
|
+
awscli-1.44.7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|