awscli 1.42.7__py3-none-any.whl → 1.44.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- awscli/__init__.py +1 -1
- awscli/alias.py +3 -3
- awscli/argprocess.py +7 -2
- awscli/arguments.py +1 -1
- awscli/bcdoc/docevents.py +3 -0
- awscli/clidocs.py +19 -2
- awscli/clidriver.py +55 -9
- awscli/customizations/argrename.py +1 -3
- awscli/customizations/cliinputjson.py +4 -0
- awscli/customizations/cloudformation/deploy.py +19 -3
- awscli/customizations/cloudtrail/validation.py +22 -2
- awscli/customizations/commands.py +2 -1
- awscli/customizations/eks/update_kubeconfig.py +12 -1
- awscli/customizations/emr/argumentschema.py +355 -344
- awscli/customizations/emr/createcluster.py +12 -0
- awscli/customizations/emr/emrutils.py +83 -50
- awscli/customizations/emr/helptext.py +10 -0
- awscli/customizations/emr/steputils.py +92 -52
- awscli/customizations/gamelift/uploadbuild.py +24 -11
- awscli/customizations/globalargs.py +169 -0
- awscli/customizations/paginate.py +66 -1
- awscli/customizations/s3/filegenerator.py +4 -1
- awscli/customizations/s3/fileinfo.py +4 -1
- awscli/customizations/s3/fileinfobuilder.py +6 -0
- awscli/customizations/s3/s3handler.py +9 -0
- awscli/customizations/s3/subcommands.py +147 -6
- awscli/customizations/s3/syncstrategy/base.py +9 -0
- awscli/customizations/s3/syncstrategy/caseconflict.py +96 -0
- awscli/customizations/s3/utils.py +14 -0
- awscli/customizations/scalarparse.py +48 -9
- awscli/data/cli.json +5 -0
- awscli/examples/apigateway/update-vpc-link.rst +47 -0
- awscli/examples/application-signals/batch-get-service-level-objective-budget-report.rst +99 -0
- awscli/examples/application-signals/create-service-level-objective.rst +88 -0
- awscli/examples/application-signals/delete-service-level-objective.rst +10 -0
- awscli/examples/application-signals/get-service-level-objective.rst +53 -0
- awscli/examples/application-signals/get-service.rst +72 -0
- awscli/examples/application-signals/list-service-dependencies.rst +96 -0
- awscli/examples/application-signals/list-service-dependents.rst +36 -0
- awscli/examples/application-signals/list-service-level-objectives.rst +17 -0
- awscli/examples/application-signals/list-service-operations.rst +63 -0
- awscli/examples/application-signals/list-services.rst +61 -0
- awscli/examples/application-signals/list-tags-for-resource.rst +17 -0
- awscli/examples/application-signals/start-discovery.rst +9 -0
- awscli/examples/application-signals/tag-resource.rst +11 -0
- awscli/examples/application-signals/untag-resource.rst +11 -0
- awscli/examples/application-signals/update-service-level-objective.rst +69 -0
- awscli/examples/cloudformation/create-generated-template.rst +50 -0
- awscli/examples/cloudformation/create-stack-refactor.rst +16 -0
- awscli/examples/cloudformation/delete-generated-template.rst +10 -0
- awscli/examples/cloudformation/describe-generated-template.rst +62 -0
- awscli/examples/cloudformation/describe-resource-scan.rst +38 -0
- awscli/examples/cloudformation/describe-stack-refactor.rst +20 -0
- awscli/examples/cloudformation/execute-stack-refactor.rst +10 -0
- awscli/examples/cloudformation/list-generated-templates.rst +41 -0
- awscli/examples/cloudformation/list-resource-scan-related-resources.rst +47 -0
- awscli/examples/cloudformation/list-resource-scan-resources.rst +30 -0
- awscli/examples/cloudformation/list-stack-refactor-actions.rst +71 -0
- awscli/examples/cloudformation/start-resource-scan.rst +14 -0
- awscli/examples/ec2/authorize-security-group-ingress.rst +1 -1
- awscli/examples/global_options.rst +4 -0
- awscli/examples/global_synopsis.rst +1 -0
- awscli/examples/guardduty/update-detector.rst +23 -0
- awscli/examples/ivs-realtime/get-composition.rst +7 -4
- awscli/examples/ivs-realtime/start-composition.rst +88 -3
- awscli/examples/lambda/create-function.rst +4 -4
- awscli/examples/lambda/get-function.rst +3 -3
- awscli/examples/lambda/list-functions.rst +6 -6
- awscli/examples/medical-imaging/create-datastore.rst +19 -2
- awscli/examples/medical-imaging/get-datastore.rst +24 -1
- awscli/examples/networkmanager/get-vpc-attachment.rst +1 -1
- awscli/examples/securityhub/describe-hub.rst +6 -4
- awscli/examples/servicediscovery/create-service.rst +50 -10
- awscli/examples/servicediscovery/delete-namespace.rst +18 -4
- awscli/examples/servicediscovery/delete-service-attributes.rst +15 -3
- awscli/examples/servicediscovery/delete-service.rst +13 -3
- awscli/examples/servicediscovery/deregister-instance.rst +18 -2
- awscli/examples/servicediscovery/discover-instances-revision.rst +18 -1
- awscli/examples/servicediscovery/discover-instances.rst +32 -2
- awscli/examples/servicediscovery/get-instance.rst +30 -4
- awscli/examples/servicediscovery/get-instances-health-status.rst +19 -1
- awscli/examples/servicediscovery/get-namespace.rst +40 -9
- awscli/examples/servicediscovery/get-operation.rst +32 -6
- awscli/examples/servicediscovery/get-service-attributes.rst +25 -3
- awscli/examples/servicediscovery/get-service.rst +35 -7
- awscli/examples/servicediscovery/list-instances.rst +38 -3
- awscli/examples/servicediscovery/list-namespaces.rst +45 -22
- awscli/examples/servicediscovery/list-services.rst +30 -2
- awscli/examples/servicediscovery/register-instance.rst +18 -2
- awscli/examples/servicediscovery/update-http-namespace.rst +22 -5
- awscli/examples/servicediscovery/update-instance-custom-health-status.rst +14 -1
- awscli/examples/servicediscovery/update-private-dns-namespace.rst +22 -5
- awscli/examples/servicediscovery/update-public-dns-namespace.rst +22 -5
- awscli/examples/servicediscovery/update-service-attributes.rst +14 -2
- awscli/examples/servicediscovery/update-service.rst +20 -4
- awscli/handlers.py +0 -4
- awscli/paramfile.py +21 -4
- awscli/testutils.py +17 -0
- awscli/topics/config-vars.rst +1 -1
- awscli/topics/s3-case-insensitivity.rst +105 -0
- awscli/topics/topic-tags.json +16 -0
- awscli/utils.py +19 -2
- {awscli-1.42.7.dist-info → awscli-1.44.26.dist-info}/METADATA +16 -3
- {awscli-1.42.7.dist-info → awscli-1.44.26.dist-info}/RECORD +112 -231
- awscli/customizations/opsworks.py +0 -543
- awscli/customizations/opsworkscm.py +0 -21
- awscli/examples/elastictranscoder/cancel-job.rst +0 -8
- awscli/examples/elastictranscoder/create-job.rst +0 -94
- awscli/examples/elastictranscoder/create-pipeline.rst +0 -94
- awscli/examples/elastictranscoder/create-preset.rst +0 -141
- awscli/examples/elastictranscoder/delete-pipeline.rst +0 -13
- awscli/examples/elastictranscoder/delete-preset.rst +0 -8
- awscli/examples/elastictranscoder/list-jobs-by-pipeline.rst +0 -13
- awscli/examples/elastictranscoder/list-jobs-by-status.rst +0 -14
- awscli/examples/elastictranscoder/list-pipelines.rst +0 -84
- awscli/examples/elastictranscoder/list-presets.rst +0 -95
- awscli/examples/elastictranscoder/read-job.rst +0 -65
- awscli/examples/elastictranscoder/read-pipeline.rst +0 -59
- awscli/examples/elastictranscoder/read-preset.rst +0 -100
- awscli/examples/elastictranscoder/update-pipeline-notifications.rst +0 -52
- awscli/examples/elastictranscoder/update-pipeline-status.rst +0 -53
- awscli/examples/elastictranscoder/update-pipeline.rst +0 -95
- awscli/examples/opsworks/assign-instance.rst +0 -14
- awscli/examples/opsworks/assign-volume.rst +0 -17
- awscli/examples/opsworks/associate-elastic-ip.rst +0 -14
- awscli/examples/opsworks/attach-elastic-load-balancer.rst +0 -14
- awscli/examples/opsworks/create-app.rst +0 -64
- awscli/examples/opsworks/create-deployment.rst +0 -66
- awscli/examples/opsworks/create-instance.rst +0 -25
- awscli/examples/opsworks/create-layer.rst +0 -17
- awscli/examples/opsworks/create-server.rst +0 -43
- awscli/examples/opsworks/create-stack.rst +0 -25
- awscli/examples/opsworks/create-user-profile.rst +0 -24
- awscli/examples/opsworks/delete-app.rst +0 -17
- awscli/examples/opsworks/delete-instance.rst +0 -15
- awscli/examples/opsworks/delete-layer.rst +0 -17
- awscli/examples/opsworks/delete-stack.rst +0 -18
- awscli/examples/opsworks/delete-user-profile.rst +0 -17
- awscli/examples/opsworks/deregister-elastic-ip.rst +0 -13
- awscli/examples/opsworks/deregister-instance.rst +0 -14
- awscli/examples/opsworks/deregister-rds-db-instance.rst +0 -20
- awscli/examples/opsworks/deregister-volume.rst +0 -15
- awscli/examples/opsworks/describe-apps.rst +0 -38
- awscli/examples/opsworks/describe-commands.rst +0 -43
- awscli/examples/opsworks/describe-deployments.rst +0 -52
- awscli/examples/opsworks/describe-elastic-ips.rst +0 -24
- awscli/examples/opsworks/describe-elastic-load-balancers.rst +0 -37
- awscli/examples/opsworks/describe-instances.rst +0 -95
- awscli/examples/opsworks/describe-layers.rst +0 -171
- awscli/examples/opsworks/describe-load-based-auto-scaling.rst +0 -37
- awscli/examples/opsworks/describe-my-user-profile.rst +0 -24
- awscli/examples/opsworks/describe-permissions.rst +0 -26
- awscli/examples/opsworks/describe-raid-arrays.rst +0 -31
- awscli/examples/opsworks/describe-rds-db-instances.rst +0 -29
- awscli/examples/opsworks/describe-stack-provisioning-parameters.rst +0 -32
- awscli/examples/opsworks/describe-stack-summary.rst +0 -27
- awscli/examples/opsworks/describe-stacks.rst +0 -65
- awscli/examples/opsworks/describe-timebased-auto-scaling.rst +0 -39
- awscli/examples/opsworks/describe-user-profiles.rst +0 -32
- awscli/examples/opsworks/describe-volumes.rst +0 -31
- awscli/examples/opsworks/detach-elastic-load-balancer.rst +0 -14
- awscli/examples/opsworks/disassociate-elastic-ip.rst +0 -14
- awscli/examples/opsworks/get-hostname-suggestion.rst +0 -21
- awscli/examples/opsworks/reboot-instance.rst +0 -14
- awscli/examples/opsworks/register-elastic-ip.rst +0 -19
- awscli/examples/opsworks/register-rds-db-instance.rst +0 -15
- awscli/examples/opsworks/register-volume.rst +0 -18
- awscli/examples/opsworks/register.rst +0 -105
- awscli/examples/opsworks/set-load-based-auto-scaling.rst +0 -38
- awscli/examples/opsworks/set-permission.rst +0 -23
- awscli/examples/opsworks/set-time-based-auto-scaling.rst +0 -33
- awscli/examples/opsworks/start-instance.rst +0 -20
- awscli/examples/opsworks/start-stack.rst +0 -15
- awscli/examples/opsworks/stop-instance.rst +0 -20
- awscli/examples/opsworks/stop-stack.rst +0 -15
- awscli/examples/opsworks/unassign-instance.rst +0 -14
- awscli/examples/opsworks/unassign-volume.rst +0 -16
- awscli/examples/opsworks/update-app.rst +0 -14
- awscli/examples/opsworks/update-elastic-ip.rst +0 -14
- awscli/examples/opsworks/update-instance.rst +0 -14
- awscli/examples/opsworks/update-layer.rst +0 -14
- awscli/examples/opsworks/update-my-user-profile.rst +0 -16
- awscli/examples/opsworks/update-rds-db-instance.rst +0 -18
- awscli/examples/opsworks/update-volume.rst +0 -16
- awscli/examples/opsworkscm/associate-node.rst +0 -22
- awscli/examples/opsworkscm/create-backup.rst +0 -46
- awscli/examples/opsworkscm/create-server.rst +0 -48
- awscli/examples/opsworkscm/delete-backup.rst +0 -17
- awscli/examples/opsworkscm/delete-server.rst +0 -16
- awscli/examples/opsworkscm/describe-account-attributes.rst +0 -26
- awscli/examples/opsworkscm/describe-backups.rst +0 -44
- awscli/examples/opsworkscm/describe-events.rst +0 -21
- awscli/examples/opsworkscm/describe-node-association-status.rst +0 -20
- awscli/examples/opsworkscm/describe-servers.rst +0 -48
- awscli/examples/opsworkscm/disassociate-node.rst +0 -19
- awscli/examples/opsworkscm/restore-server.rst +0 -20
- awscli/examples/opsworkscm/start-maintenance.rst +0 -39
- awscli/examples/opsworkscm/update-server-engine-attributes.rst +0 -43
- awscli/examples/opsworkscm/update-server.rst +0 -42
- awscli/examples/qldb/cancel-journal-kinesis-stream.rst +0 -15
- awscli/examples/qldb/create-ledger.rst +0 -43
- awscli/examples/qldb/delete-ledger.rst +0 -10
- awscli/examples/qldb/describe-journal-kinesis-stream.rst +0 -29
- awscli/examples/qldb/describe-journal-s3-export.rst +0 -30
- awscli/examples/qldb/describe-ledger.rst +0 -23
- awscli/examples/qldb/export-journal-to-s3.rst +0 -28
- awscli/examples/qldb/get-block.rst +0 -55
- awscli/examples/qldb/get-digest.rst +0 -17
- awscli/examples/qldb/get-revision.rst +0 -57
- awscli/examples/qldb/list-journal-kinesis-streams-for-ledger.rst +0 -30
- awscli/examples/qldb/list-journal-s3-exports-for-ledger.rst +0 -31
- awscli/examples/qldb/list-journal-s3-exports.rst +0 -46
- awscli/examples/qldb/list-ledgers.rst +0 -24
- awscli/examples/qldb/list-tags-for-resource.rst +0 -17
- awscli/examples/qldb/stream-journal-to-kinesis.rst +0 -46
- awscli/examples/qldb/tag-resource.rst +0 -11
- awscli/examples/qldb/untag-resource.rst +0 -11
- awscli/examples/qldb/update-ledger-permissions-mode.rst +0 -34
- awscli/examples/qldb/update-ledger.rst +0 -63
- awscli/examples/robomaker/batch-describe-simulation-job.rst +0 -150
- awscli/examples/robomaker/cancel-simulation-job.rst +0 -6
- awscli/examples/robomaker/create-deployment-job.rst +0 -37
- awscli/examples/robomaker/create-fleet.rst +0 -18
- awscli/examples/robomaker/create-robot-application-version.rst +0 -31
- awscli/examples/robomaker/create-robot-application.rst +0 -29
- awscli/examples/robomaker/create-robot.rst +0 -20
- awscli/examples/robomaker/create-simulation-application-version.rst +0 -39
- awscli/examples/robomaker/create-simulation-application.rst +0 -38
- awscli/examples/robomaker/create-simulation-job.rst +0 -43
- awscli/examples/robomaker/delete-fleet.rst +0 -7
- awscli/examples/robomaker/delete-robot-application.rst +0 -7
- awscli/examples/robomaker/delete-robot.rst +0 -7
- awscli/examples/robomaker/delete-simulation-application.rst +0 -7
- awscli/examples/robomaker/deregister-robot.rst +0 -14
- awscli/examples/robomaker/describe-deployment-job.rst +0 -38
- awscli/examples/robomaker/describe-fleet.rst +0 -28
- awscli/examples/robomaker/describe-robot-application.rst +0 -29
- awscli/examples/robomaker/describe-robot.rst +0 -21
- awscli/examples/robomaker/describe-simulation-application.rst +0 -37
- awscli/examples/robomaker/describe-simulation-job.rst +0 -45
- awscli/examples/robomaker/list-deployment-jobs.rst +0 -57
- awscli/examples/robomaker/list-fleets.rst +0 -22
- awscli/examples/robomaker/list-robot-applications.rst +0 -32
- awscli/examples/robomaker/list-robots.rst +0 -45
- awscli/examples/robomaker/list-simulation-applications.rst +0 -50
- awscli/examples/robomaker/list-simulation-jobs.rst +0 -80
- awscli/examples/robomaker/list-tags-for-resource.rst +0 -16
- awscli/examples/robomaker/register-robot.rst +0 -14
- awscli/examples/robomaker/restart-simulation-job.rst +0 -7
- awscli/examples/robomaker/sync-deployment-job.rst +0 -30
- awscli/examples/robomaker/tag-resource.rst +0 -7
- awscli/examples/robomaker/untag-resource.rst +0 -7
- awscli/examples/robomaker/update-robot-application.rst +0 -28
- awscli/examples/robomaker/update-simulation-application.rst +0 -36
- {awscli-1.42.7.data → awscli-1.44.26.data}/scripts/aws +0 -0
- {awscli-1.42.7.data → awscli-1.44.26.data}/scripts/aws.cmd +0 -0
- {awscli-1.42.7.data → awscli-1.44.26.data}/scripts/aws_bash_completer +0 -0
- {awscli-1.42.7.data → awscli-1.44.26.data}/scripts/aws_completer +0 -0
- {awscli-1.42.7.data → awscli-1.44.26.data}/scripts/aws_zsh_completer.sh +0 -0
- {awscli-1.42.7.dist-info → awscli-1.44.26.dist-info}/LICENSE.txt +0 -0
- {awscli-1.42.7.dist-info → awscli-1.44.26.dist-info}/WHEEL +0 -0
- {awscli-1.42.7.dist-info → awscli-1.44.26.dist-info}/top_level.txt +0 -0
|
@@ -12,12 +12,18 @@
|
|
|
12
12
|
# language governing permissions and limitations under the License.
|
|
13
13
|
import sys
|
|
14
14
|
import os
|
|
15
|
+
|
|
16
|
+
from awscli.customizations.argrename import HIDDEN_ALIASES
|
|
17
|
+
from awscli.customizations.utils import uni_print
|
|
15
18
|
from botocore.client import Config
|
|
16
19
|
from botocore import UNSIGNED
|
|
17
20
|
from botocore.endpoint import DEFAULT_TIMEOUT
|
|
21
|
+
from botocore.useragent import register_feature_id
|
|
18
22
|
import jmespath
|
|
19
23
|
|
|
20
24
|
from awscli.compat import urlparse
|
|
25
|
+
from awscli.utils import resolve_v2_debug_mode
|
|
26
|
+
|
|
21
27
|
|
|
22
28
|
def register_parse_global_args(cli):
|
|
23
29
|
cli.register('top-level-args-parsed', resolve_types,
|
|
@@ -30,6 +36,8 @@ def register_parse_global_args(cli):
|
|
|
30
36
|
unique_id='resolve-cli-read-timeout')
|
|
31
37
|
cli.register('top-level-args-parsed', resolve_cli_connect_timeout,
|
|
32
38
|
unique_id='resolve-cli-connect-timeout')
|
|
39
|
+
cli.register('top-level-args-parsed', detect_migration_breakage,
|
|
40
|
+
unique_id='detect-migration-breakage')
|
|
33
41
|
|
|
34
42
|
|
|
35
43
|
def resolve_types(parsed_args, **kwargs):
|
|
@@ -90,6 +98,167 @@ def resolve_cli_connect_timeout(parsed_args, session, **kwargs):
|
|
|
90
98
|
arg_name = 'connect_timeout'
|
|
91
99
|
_resolve_timeout(session, parsed_args, arg_name)
|
|
92
100
|
|
|
101
|
+
def detect_migration_breakage(parsed_args, session, remaining_args, **kwargs):
|
|
102
|
+
if not resolve_v2_debug_mode(parsed_args):
|
|
103
|
+
return
|
|
104
|
+
region = parsed_args.region or session.get_config_variable('region')
|
|
105
|
+
s3_config = session.get_config_variable('s3')
|
|
106
|
+
if (
|
|
107
|
+
not session.get_scoped_config().get('cli_pager', None)
|
|
108
|
+
== '' and 'AWS_PAGER' not in os.environ
|
|
109
|
+
):
|
|
110
|
+
uni_print(
|
|
111
|
+
'\nAWS CLI v2 UPGRADE WARNING: By default, the AWS CLI v2 returns '
|
|
112
|
+
'all output through your operating system’s default pager '
|
|
113
|
+
'program. This is different from v1 behavior, where the system '
|
|
114
|
+
'pager is not used by default. To retain AWS CLI v1 behavior in '
|
|
115
|
+
'AWS CLI v2, set the `cli_pager` configuration setting, or the '
|
|
116
|
+
'`AWS_PAGER` environment variable, to the empty string. See '
|
|
117
|
+
'https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
118
|
+
'cliv2-migration-changes.html#cliv2-migration-output-pager.\n',
|
|
119
|
+
out_file=sys.stderr
|
|
120
|
+
)
|
|
121
|
+
if 'PYTHONUTF8' in os.environ or 'PYTHONIOENCODING' in os.environ:
|
|
122
|
+
if 'AWS_CLI_FILE_ENCODING' not in os.environ:
|
|
123
|
+
uni_print(
|
|
124
|
+
'\nThe AWS CLI v2 does not support The `PYTHONUTF8` and '
|
|
125
|
+
'`PYTHONIOENCODING` environment variables, and instead uses '
|
|
126
|
+
'the `AWS_CLI_FILE_ENCODING` variable. This is different from '
|
|
127
|
+
'v1 behavior, where the former two variables are used '
|
|
128
|
+
'instead. To retain AWS CLI v1 behavior in AWS CLI v2, set '
|
|
129
|
+
'the `AWS_CLI_FILE_ENCODING` environment variable instead. '
|
|
130
|
+
'See https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
131
|
+
'cliv2-migration-changes.html'
|
|
132
|
+
'#cliv2-migration-encodingenvvar.\n',
|
|
133
|
+
out_file=sys.stderr
|
|
134
|
+
)
|
|
135
|
+
if (
|
|
136
|
+
(
|
|
137
|
+
s3_config is None
|
|
138
|
+
or s3_config.get('us_east_1_regional_endpoint', 'legacy')
|
|
139
|
+
== 'legacy'
|
|
140
|
+
)
|
|
141
|
+
and region in ('us-east-1', None)
|
|
142
|
+
):
|
|
143
|
+
session.register(
|
|
144
|
+
'request-created.s3.*',
|
|
145
|
+
warn_if_east_configured_global_endpoint
|
|
146
|
+
)
|
|
147
|
+
session.register(
|
|
148
|
+
'request-created.s3api.*',
|
|
149
|
+
warn_if_east_configured_global_endpoint
|
|
150
|
+
)
|
|
151
|
+
if session.get_config_variable('api_versions'):
|
|
152
|
+
uni_print(
|
|
153
|
+
'\nAWS CLI v2 UPGRADE WARNING: AWS CLI v2 UPGRADE WARNING: '
|
|
154
|
+
'The AWS CLI v2 does not support calling older versions of AWS '
|
|
155
|
+
'service APIs via the `api_versions` configuration file setting. This '
|
|
156
|
+
'is different from v1 behavior, where this configuration setting '
|
|
157
|
+
'can be used to pin older API versions. To migrate to v2 '
|
|
158
|
+
'behavior, remove the `api_versions` configuration setting, and '
|
|
159
|
+
'test against the latest service API versions. See '
|
|
160
|
+
'https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
161
|
+
'cliv2-migration-changes.html#cliv2-migration-api-versions.\n',
|
|
162
|
+
out_file = sys.stderr
|
|
163
|
+
)
|
|
164
|
+
if session.full_config.get('plugins', {}):
|
|
165
|
+
uni_print(
|
|
166
|
+
'\nAWS CLI v2 UPGRADE WARNING: In AWS CLI v2, plugins are '
|
|
167
|
+
'disabled by default, and support for plugins is provisional. '
|
|
168
|
+
'This is different from v1 behavior, where plugin support is URL '
|
|
169
|
+
'below to update your configuration to enable plugins in AWS CLI '
|
|
170
|
+
'v2. Also, be sure to lock into a particular version of the AWS '
|
|
171
|
+
'CLI and test the functionality of your plugins every time AWS '
|
|
172
|
+
'CLI v2 is upgraded. See https://docs.aws.amazon.com/cli/latest/'
|
|
173
|
+
'userguide/cliv2-migration-changes.html'
|
|
174
|
+
'#cliv2-migration-profile-plugins.\n',
|
|
175
|
+
out_file=sys.stderr
|
|
176
|
+
)
|
|
177
|
+
if (
|
|
178
|
+
parsed_args.command == 'ecr' and
|
|
179
|
+
remaining_args is not None and
|
|
180
|
+
remaining_args[0] == 'get-login'
|
|
181
|
+
):
|
|
182
|
+
uni_print(
|
|
183
|
+
'\nAWS CLI v2 UPGRADE WARNING: The `ecr get-login` command has '
|
|
184
|
+
'been removed in AWS CLI v2. You must use `ecr get-login-password` '
|
|
185
|
+
'instead. See https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
186
|
+
'cliv2-migration-changes.html#cliv2-migration-ecr-get-login.\n',
|
|
187
|
+
out_file=sys.stderr
|
|
188
|
+
)
|
|
189
|
+
for working, obsolete in HIDDEN_ALIASES.items():
|
|
190
|
+
working_split = working.split('.')
|
|
191
|
+
working_service = working_split[0]
|
|
192
|
+
working_cmd = working_split[1]
|
|
193
|
+
working_param = working_split[2]
|
|
194
|
+
if (
|
|
195
|
+
parsed_args.command == working_service
|
|
196
|
+
and remaining_args is not None
|
|
197
|
+
and remaining_args[0] == working_cmd
|
|
198
|
+
and f"--{working_param}" in remaining_args
|
|
199
|
+
):
|
|
200
|
+
uni_print(
|
|
201
|
+
'\nAWS CLI v2 UPGRADE WARNING: You have entered command '
|
|
202
|
+
'arguments that use at least 1 of 21 built-in ("hidden") '
|
|
203
|
+
'aliases that were removed in AWS CLI v2. For this command '
|
|
204
|
+
'to work in AWS CLI v2, you must replace usage of the alias '
|
|
205
|
+
'with the corresponding parameter in AWS CLI v2. See '
|
|
206
|
+
'https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
207
|
+
'cliv2-migration-changes.html#cliv2-migration-aliases.\n',
|
|
208
|
+
out_file=sys.stderr
|
|
209
|
+
)
|
|
210
|
+
# Register against the provide-client-params event to ensure that the
|
|
211
|
+
# feature ID is registered before any API requests are made. We
|
|
212
|
+
# cannot register the feature ID in this function because no
|
|
213
|
+
# botocore context is created at this point.
|
|
214
|
+
session.register(
|
|
215
|
+
'provide-client-params.*.*',
|
|
216
|
+
_register_v2_debug_feature_id
|
|
217
|
+
)
|
|
218
|
+
session.register('choose-signer.s3.*', warn_if_sigv2)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _register_v2_debug_feature_id(params, model, **kwargs):
|
|
222
|
+
register_feature_id('CLI_V1_TO_V2_MIGRATION_DEBUG_MODE')
|
|
223
|
+
|
|
224
|
+
def warn_if_east_configured_global_endpoint(request, operation_name, **kwargs):
|
|
225
|
+
# The regional us-east-1 endpoint is used in certain cases (e.g.
|
|
226
|
+
# FIPS/Dual-Stack is enabled). Rather than duplicating this logic
|
|
227
|
+
# from botocore, we check the endpoint URL directly.
|
|
228
|
+
parsed_url = urlparse.urlparse(request.url)
|
|
229
|
+
if parsed_url.hostname.endswith('s3.amazonaws.com'):
|
|
230
|
+
uni_print(
|
|
231
|
+
'\nAWS CLI v2 UPGRADE WARNING: When you configure AWS CLI v2 to '
|
|
232
|
+
'use the `us-east-1` region, it uses the true regional endpoint '
|
|
233
|
+
'rather than the global endpoint. This is different from v1 '
|
|
234
|
+
'behavior, where the global endpoint would be used when the '
|
|
235
|
+
'region is `us-east-1`. To retain AWS CLI v1 behavior in AWS '
|
|
236
|
+
'CLI v2, configure the region setting to `aws-global`. See '
|
|
237
|
+
'https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
238
|
+
'cliv2-migration-changes.html'
|
|
239
|
+
'#cliv2-migration-s3-regional-endpoint.\n',
|
|
240
|
+
out_file=sys.stderr
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
def warn_if_sigv2(
|
|
244
|
+
signing_name,
|
|
245
|
+
region_name,
|
|
246
|
+
signature_version,
|
|
247
|
+
context,
|
|
248
|
+
**kwargs
|
|
249
|
+
):
|
|
250
|
+
if context.get('auth_type', None) == 'v2':
|
|
251
|
+
uni_print(
|
|
252
|
+
'\nAWS CLI v2 UPGRADE WARNING: The AWS CLI v2 only uses Signature '
|
|
253
|
+
'v4 to authenticate Amazon S3 requests. This is different from '
|
|
254
|
+
'v1 behavior, where the signature used for Amazon S3 requests may '
|
|
255
|
+
'vary depending on configuration settings, region, and the '
|
|
256
|
+
'bucket being used. To migrate to AWS CLI v2 behavior, configure '
|
|
257
|
+
'the Signature Version S3 setting to version 4. See '
|
|
258
|
+
'https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
259
|
+
'cliv2-migration-changes.html#cliv2-migration-sigv4.\n',
|
|
260
|
+
out_file=sys.stderr
|
|
261
|
+
)
|
|
93
262
|
|
|
94
263
|
def resolve_cli_read_timeout(parsed_args, session, **kwargs):
|
|
95
264
|
arg_name = 'read_timeout'
|
|
@@ -33,7 +33,7 @@ from botocore.exceptions import DataNotFoundError, PaginationError
|
|
|
33
33
|
from botocore import model
|
|
34
34
|
|
|
35
35
|
from awscli.arguments import BaseCLIArgument
|
|
36
|
-
|
|
36
|
+
from awscli.utils import resolve_v2_debug_mode
|
|
37
37
|
|
|
38
38
|
logger = logging.getLogger(__name__)
|
|
39
39
|
|
|
@@ -135,6 +135,9 @@ def unify_paging_params(argument_table, operation_model, event_name,
|
|
|
135
135
|
_remove_existing_paging_arguments(argument_table, paginator_config)
|
|
136
136
|
parsed_args_event = event_name.replace('building-argument-table.',
|
|
137
137
|
'operation-args-parsed.')
|
|
138
|
+
call_parameters_event = event_name.replace(
|
|
139
|
+
'building-argument-table', 'calling-command'
|
|
140
|
+
)
|
|
138
141
|
shadowed_args = {}
|
|
139
142
|
add_paging_argument(argument_table, 'starting-token',
|
|
140
143
|
PageArgument('starting-token', STARTING_TOKEN_HELP,
|
|
@@ -168,6 +171,14 @@ def unify_paging_params(argument_table, operation_model, event_name,
|
|
|
168
171
|
partial(check_should_enable_pagination,
|
|
169
172
|
list(_get_all_cli_input_tokens(paginator_config)),
|
|
170
173
|
shadowed_args, argument_table))
|
|
174
|
+
session.register(
|
|
175
|
+
call_parameters_event,
|
|
176
|
+
partial(
|
|
177
|
+
check_should_enable_pagination_call_parameters,
|
|
178
|
+
session,
|
|
179
|
+
list(_get_all_input_tokens(paginator_config)),
|
|
180
|
+
),
|
|
181
|
+
)
|
|
171
182
|
|
|
172
183
|
|
|
173
184
|
def add_paging_argument(argument_table, arg_name, argument, shadowed_args):
|
|
@@ -240,6 +251,18 @@ def _get_all_cli_input_tokens(pagination_config):
|
|
|
240
251
|
yield cli_name
|
|
241
252
|
|
|
242
253
|
|
|
254
|
+
# Get all tokens but return them in API namespace rather than CLI namespace
|
|
255
|
+
def _get_all_input_tokens(pagination_config):
|
|
256
|
+
# Get all input tokens including the limit_key
|
|
257
|
+
# if it exists.
|
|
258
|
+
tokens = _get_input_tokens(pagination_config)
|
|
259
|
+
for token_name in tokens:
|
|
260
|
+
yield token_name
|
|
261
|
+
if 'limit_key' in pagination_config:
|
|
262
|
+
key_name = pagination_config['limit_key']
|
|
263
|
+
yield key_name
|
|
264
|
+
|
|
265
|
+
|
|
243
266
|
def _get_input_tokens(pagination_config):
|
|
244
267
|
tokens = pagination_config['input_token']
|
|
245
268
|
if not isinstance(tokens, list):
|
|
@@ -253,6 +276,48 @@ def _get_cli_name(param_objects, token_name):
|
|
|
253
276
|
return param.cli_name.lstrip('-')
|
|
254
277
|
|
|
255
278
|
|
|
279
|
+
def check_should_enable_pagination_call_parameters(
|
|
280
|
+
session,
|
|
281
|
+
input_tokens,
|
|
282
|
+
call_parameters,
|
|
283
|
+
parsed_args,
|
|
284
|
+
parsed_globals,
|
|
285
|
+
**kwargs
|
|
286
|
+
):
|
|
287
|
+
"""
|
|
288
|
+
Check for pagination args in the actual calling arguments passed to
|
|
289
|
+
the function.
|
|
290
|
+
|
|
291
|
+
If the user is using the --cli-input-json parameter to provide JSON
|
|
292
|
+
parameters they are all in the API naming space rather than the CLI
|
|
293
|
+
naming space and would be missed by the processing above. This function
|
|
294
|
+
gets called on the calling-command event.
|
|
295
|
+
"""
|
|
296
|
+
if resolve_v2_debug_mode(parsed_globals):
|
|
297
|
+
cli_input_json_data = session.emit_first_non_none_response(
|
|
298
|
+
f"get-cli-input-json-data",
|
|
299
|
+
)
|
|
300
|
+
if cli_input_json_data is None:
|
|
301
|
+
cli_input_json_data = {}
|
|
302
|
+
pagination_params_in_input_tokens = [
|
|
303
|
+
param for param in cli_input_json_data if param in input_tokens
|
|
304
|
+
]
|
|
305
|
+
if pagination_params_in_input_tokens:
|
|
306
|
+
uni_print(
|
|
307
|
+
'\nAWS CLI v2 UPGRADE WARNING: In AWS CLI v2, if you specify '
|
|
308
|
+
'pagination parameters by using a file with the '
|
|
309
|
+
'`--cli-input-json` parameter, automatic pagination will be '
|
|
310
|
+
'turned off. This is different from v1 behavior, where '
|
|
311
|
+
'pagination parameters specified via the `--cli-input-json` '
|
|
312
|
+
'parameter are ignored. To retain AWS CLI v1 behavior in '
|
|
313
|
+
'AWS CLI v2, remove all pagination parameters from the input '
|
|
314
|
+
'JSON. See https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
315
|
+
'cliv2-migration-changes.html'
|
|
316
|
+
'#cliv2-migration-skeleton-paging.\n',
|
|
317
|
+
out_file=sys.stderr
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
|
|
256
321
|
class PageArgument(BaseCLIArgument):
|
|
257
322
|
type_map = {
|
|
258
323
|
'string': str,
|
|
@@ -94,7 +94,8 @@ class FileDecodingError(Exception):
|
|
|
94
94
|
class FileStat(object):
|
|
95
95
|
def __init__(self, src, dest=None, compare_key=None, size=None,
|
|
96
96
|
last_update=None, src_type=None, dest_type=None,
|
|
97
|
-
operation_name=None, response_data=None, etag=None
|
|
97
|
+
operation_name=None, response_data=None, etag=None,
|
|
98
|
+
case_conflict_submitted=None, case_conflict_key=None,):
|
|
98
99
|
self.src = src
|
|
99
100
|
self.dest = dest
|
|
100
101
|
self.compare_key = compare_key
|
|
@@ -105,6 +106,8 @@ class FileStat(object):
|
|
|
105
106
|
self.operation_name = operation_name
|
|
106
107
|
self.response_data = response_data
|
|
107
108
|
self.etag = etag
|
|
109
|
+
self.case_conflict_submitted = case_conflict_submitted
|
|
110
|
+
self.case_conflict_key = case_conflict_key
|
|
108
111
|
|
|
109
112
|
|
|
110
113
|
class FileGenerator(object):
|
|
@@ -42,7 +42,8 @@ class FileInfo(object):
|
|
|
42
42
|
last_update=None, src_type=None, dest_type=None,
|
|
43
43
|
operation_name=None, client=None, parameters=None,
|
|
44
44
|
source_client=None, is_stream=False,
|
|
45
|
-
associated_response_data=None, etag=None
|
|
45
|
+
associated_response_data=None, etag=None,
|
|
46
|
+
case_conflict_submitted=None, case_conflict_key=None,):
|
|
46
47
|
self.src = src
|
|
47
48
|
self.src_type = src_type
|
|
48
49
|
self.operation_name = operation_name
|
|
@@ -60,6 +61,8 @@ class FileInfo(object):
|
|
|
60
61
|
self.is_stream = is_stream
|
|
61
62
|
self.associated_response_data = associated_response_data
|
|
62
63
|
self.etag = etag
|
|
64
|
+
self.case_conflict_submitted = case_conflict_submitted
|
|
65
|
+
self.case_conflict_key = case_conflict_key
|
|
63
66
|
|
|
64
67
|
def is_glacier_compatible(self):
|
|
65
68
|
"""Determines if a file info object is glacier compatible
|
|
@@ -46,6 +46,12 @@ class FileInfoBuilder(object):
|
|
|
46
46
|
file_info_attr['is_stream'] = self._is_stream
|
|
47
47
|
file_info_attr['associated_response_data'] = file_base.response_data
|
|
48
48
|
file_info_attr['etag'] = file_base.etag
|
|
49
|
+
file_info_attr['case_conflict_submitted'] = getattr(
|
|
50
|
+
file_base, 'case_conflict_submitted', None
|
|
51
|
+
)
|
|
52
|
+
file_info_attr['case_conflict_key'] = getattr(
|
|
53
|
+
file_base, 'case_conflict_key', None
|
|
54
|
+
)
|
|
49
55
|
|
|
50
56
|
# This is a bit quirky. The below conditional hinges on the --delete
|
|
51
57
|
# flag being set, which only occurs during a sync command. The source
|
|
@@ -47,6 +47,7 @@ from awscli.customizations.s3.utils import DirectoryCreatorSubscriber
|
|
|
47
47
|
from awscli.customizations.s3.utils import DeleteSourceFileSubscriber
|
|
48
48
|
from awscli.customizations.s3.utils import DeleteSourceObjectSubscriber
|
|
49
49
|
from awscli.customizations.s3.utils import DeleteCopySourceObjectSubscriber
|
|
50
|
+
from awscli.customizations.s3.utils import CaseConflictCleanupSubscriber
|
|
50
51
|
from awscli.compat import get_binary_stdin
|
|
51
52
|
|
|
52
53
|
|
|
@@ -403,6 +404,13 @@ class DownloadRequestSubmitter(BaseTransferRequestSubmitter):
|
|
|
403
404
|
if self._cli_params.get('is_move', False):
|
|
404
405
|
subscribers.append(DeleteSourceObjectSubscriber(
|
|
405
406
|
fileinfo.source_client))
|
|
407
|
+
if fileinfo.case_conflict_submitted is not None:
|
|
408
|
+
subscribers.append(
|
|
409
|
+
CaseConflictCleanupSubscriber(
|
|
410
|
+
fileinfo.case_conflict_submitted,
|
|
411
|
+
fileinfo.case_conflict_key,
|
|
412
|
+
)
|
|
413
|
+
)
|
|
406
414
|
|
|
407
415
|
def _submit_transfer_request(self, fileinfo, extra_args, subscribers):
|
|
408
416
|
bucket, key = find_bucket_key(fileinfo.src)
|
|
@@ -433,6 +441,7 @@ class CopyRequestSubmitter(BaseTransferRequestSubmitter):
|
|
|
433
441
|
|
|
434
442
|
def _add_additional_subscribers(self, subscribers, fileinfo):
|
|
435
443
|
subscribers.append(ProvideSizeSubscriber(fileinfo.size))
|
|
444
|
+
subscribers.append(ProvideETagSubscriber(fileinfo.etag))
|
|
436
445
|
if self._should_inject_content_type():
|
|
437
446
|
subscribers.append(ProvideCopyContentTypeSubscriber())
|
|
438
447
|
if self._cli_params.get('is_move', False):
|
|
@@ -34,9 +34,10 @@ from awscli.customizations.s3.utils import find_bucket_key, AppendFilter, \
|
|
|
34
34
|
S3PathResolver
|
|
35
35
|
from awscli.customizations.utils import uni_print
|
|
36
36
|
from awscli.customizations.s3.syncstrategy.base import MissingFileSync, \
|
|
37
|
-
SizeAndLastModifiedSync, NeverSync
|
|
37
|
+
SizeAndLastModifiedSync, NeverSync, AlwaysSync
|
|
38
|
+
from awscli.customizations.s3.syncstrategy.caseconflict import CaseConflictSync
|
|
38
39
|
from awscli.customizations.s3 import transferconfig
|
|
39
|
-
|
|
40
|
+
from awscli.utils import resolve_v2_debug_mode
|
|
40
41
|
|
|
41
42
|
LOGGER = logging.getLogger(__name__)
|
|
42
43
|
|
|
@@ -482,6 +483,33 @@ BUCKET_REGION = {
|
|
|
482
483
|
)
|
|
483
484
|
}
|
|
484
485
|
|
|
486
|
+
CASE_CONFLICT = {
|
|
487
|
+
'name': 'case-conflict',
|
|
488
|
+
'choices': [
|
|
489
|
+
'ignore',
|
|
490
|
+
'skip',
|
|
491
|
+
'warn',
|
|
492
|
+
'error',
|
|
493
|
+
],
|
|
494
|
+
'default': 'ignore',
|
|
495
|
+
'help_text': (
|
|
496
|
+
"Configures behavior when attempting to download multiple objects "
|
|
497
|
+
"whose keys differ only by case, which can cause undefined behavior "
|
|
498
|
+
"on case-insensitive filesystems. "
|
|
499
|
+
"This parameter only applies for commands that perform multiple S3 "
|
|
500
|
+
"to local downloads. "
|
|
501
|
+
f"See <a href='{CaseConflictSync.DOC_URI}'>Handling case "
|
|
502
|
+
"conflicts</a> for details. Valid values are: "
|
|
503
|
+
"<ul>"
|
|
504
|
+
"<li>``error`` - Raise an error and abort downloads.</li>"
|
|
505
|
+
"<li>``warn`` - Emit a warning and download the object.</li>"
|
|
506
|
+
"<li>``skip`` - Skip downloading the object.</li>"
|
|
507
|
+
"<li>``ignore`` - The default value. Ignore the conflict and "
|
|
508
|
+
"download the object.</li>"
|
|
509
|
+
"</ul>"
|
|
510
|
+
),
|
|
511
|
+
}
|
|
512
|
+
|
|
485
513
|
TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL,
|
|
486
514
|
FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
|
|
487
515
|
SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE,
|
|
@@ -767,6 +795,7 @@ class S3TransferCommand(S3Command):
|
|
|
767
795
|
cmd_params.add_verify_ssl(parsed_globals)
|
|
768
796
|
cmd_params.add_page_size(parsed_args)
|
|
769
797
|
cmd_params.add_paths(parsed_args.paths)
|
|
798
|
+
cmd_params.add_v2_debug(parsed_globals)
|
|
770
799
|
|
|
771
800
|
runtime_config = transferconfig.RuntimeConfig().build_config(
|
|
772
801
|
**self._session.get_scoped_config().get('s3', {}))
|
|
@@ -806,7 +835,8 @@ class CpCommand(S3TransferCommand):
|
|
|
806
835
|
"or <S3Uri> <S3Uri>"
|
|
807
836
|
ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
|
|
808
837
|
'synopsis': USAGE}] + TRANSFER_ARGS + \
|
|
809
|
-
[METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE
|
|
838
|
+
[METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE,
|
|
839
|
+
CASE_CONFLICT]
|
|
810
840
|
|
|
811
841
|
|
|
812
842
|
class MvCommand(S3TransferCommand):
|
|
@@ -816,7 +846,8 @@ class MvCommand(S3TransferCommand):
|
|
|
816
846
|
"or <S3Uri> <S3Uri>"
|
|
817
847
|
ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
|
|
818
848
|
'synopsis': USAGE}] + TRANSFER_ARGS +\
|
|
819
|
-
[METADATA, METADATA_DIRECTIVE, RECURSIVE, VALIDATE_SAME_S3_PATHS
|
|
849
|
+
[METADATA, METADATA_DIRECTIVE, RECURSIVE, VALIDATE_SAME_S3_PATHS,
|
|
850
|
+
CASE_CONFLICT]
|
|
820
851
|
|
|
821
852
|
|
|
822
853
|
class RmCommand(S3TransferCommand):
|
|
@@ -838,7 +869,7 @@ class SyncCommand(S3TransferCommand):
|
|
|
838
869
|
"<LocalPath> or <S3Uri> <S3Uri>"
|
|
839
870
|
ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True,
|
|
840
871
|
'synopsis': USAGE}] + TRANSFER_ARGS + \
|
|
841
|
-
[METADATA, METADATA_DIRECTIVE]
|
|
872
|
+
[METADATA, METADATA_DIRECTIVE, CASE_CONFLICT]
|
|
842
873
|
|
|
843
874
|
|
|
844
875
|
class MbCommand(S3Command):
|
|
@@ -1003,7 +1034,16 @@ class CommandArchitecture(object):
|
|
|
1003
1034
|
# Set the default strategies.
|
|
1004
1035
|
sync_strategies['file_at_src_and_dest_sync_strategy'] = \
|
|
1005
1036
|
SizeAndLastModifiedSync()
|
|
1006
|
-
|
|
1037
|
+
if self._should_handle_case_conflicts():
|
|
1038
|
+
sync_strategies['file_not_at_dest_sync_strategy'] = (
|
|
1039
|
+
CaseConflictSync(
|
|
1040
|
+
on_case_conflict=self.parameters['case_conflict']
|
|
1041
|
+
)
|
|
1042
|
+
)
|
|
1043
|
+
else:
|
|
1044
|
+
sync_strategies['file_not_at_dest_sync_strategy'] = (
|
|
1045
|
+
MissingFileSync()
|
|
1046
|
+
)
|
|
1007
1047
|
sync_strategies['file_not_at_src_sync_strategy'] = NeverSync()
|
|
1008
1048
|
|
|
1009
1049
|
# Determine what strategies to override if any.
|
|
@@ -1056,6 +1096,24 @@ class CommandArchitecture(object):
|
|
|
1056
1096
|
result_queue = queue.Queue()
|
|
1057
1097
|
operation_name = cmd_translation[paths_type]
|
|
1058
1098
|
|
|
1099
|
+
if self.parameters['v2_debug']:
|
|
1100
|
+
if operation_name == 'copy':
|
|
1101
|
+
uni_print(
|
|
1102
|
+
'\nAWS CLI v2 UPGRADE WARNING: In AWS CLI v2, object '
|
|
1103
|
+
'properties will be copied from the source in multipart '
|
|
1104
|
+
'copies between S3 buckets initiated via `aws s3` '
|
|
1105
|
+
'commands, resulting in additional S3 API calls to '
|
|
1106
|
+
'transfer the metadata. Note that the principal must '
|
|
1107
|
+
'have permission to call these APIs, or the command may '
|
|
1108
|
+
'fail. This is different from v1 behavior, where metadata '
|
|
1109
|
+
'is not copied. For guidance on retaining v1 behavior in '
|
|
1110
|
+
'AWS CLI v2, or for more details, see '
|
|
1111
|
+
'https://docs.aws.amazon.com/cli/latest/userguide/'
|
|
1112
|
+
'cliv2-migration-changes.html'
|
|
1113
|
+
'#cliv2-migration-s3-copy-metadata.\n\n',
|
|
1114
|
+
out_file=sys.stderr
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1059
1117
|
fgen_kwargs = {
|
|
1060
1118
|
'client': self._source_client, 'operation_name': operation_name,
|
|
1061
1119
|
'follow_symlinks': self.parameters['follow_symlinks'],
|
|
@@ -1119,6 +1177,12 @@ class CommandArchitecture(object):
|
|
|
1119
1177
|
'filters': [create_filter(self.parameters)],
|
|
1120
1178
|
'file_info_builder': [file_info_builder],
|
|
1121
1179
|
's3_handler': [s3_transfer_handler]}
|
|
1180
|
+
if self._should_handle_case_conflicts():
|
|
1181
|
+
self._handle_case_conflicts(
|
|
1182
|
+
command_dict,
|
|
1183
|
+
rev_files,
|
|
1184
|
+
rev_generator,
|
|
1185
|
+
)
|
|
1122
1186
|
elif self.cmd == 'rm':
|
|
1123
1187
|
command_dict = {'setup': [files],
|
|
1124
1188
|
'file_generator': [file_generator],
|
|
@@ -1131,6 +1195,12 @@ class CommandArchitecture(object):
|
|
|
1131
1195
|
'filters': [create_filter(self.parameters)],
|
|
1132
1196
|
'file_info_builder': [file_info_builder],
|
|
1133
1197
|
's3_handler': [s3_transfer_handler]}
|
|
1198
|
+
if self._should_handle_case_conflicts():
|
|
1199
|
+
self._handle_case_conflicts(
|
|
1200
|
+
command_dict,
|
|
1201
|
+
rev_files,
|
|
1202
|
+
rev_generator,
|
|
1203
|
+
)
|
|
1134
1204
|
|
|
1135
1205
|
files = command_dict['setup']
|
|
1136
1206
|
while self.instructions:
|
|
@@ -1196,6 +1266,74 @@ class CommandArchitecture(object):
|
|
|
1196
1266
|
}
|
|
1197
1267
|
)
|
|
1198
1268
|
|
|
1269
|
+
def _should_handle_case_conflicts(self):
|
|
1270
|
+
return (
|
|
1271
|
+
self.cmd in {'sync', 'cp', 'mv'}
|
|
1272
|
+
and self.parameters.get('paths_type') == 's3local'
|
|
1273
|
+
and self.parameters['case_conflict'] != 'ignore'
|
|
1274
|
+
and self.parameters.get('dir_op')
|
|
1275
|
+
)
|
|
1276
|
+
|
|
1277
|
+
def _handle_case_conflicts(self, command_dict, rev_files, rev_generator):
|
|
1278
|
+
# Objects are not returned in lexicographical order when
|
|
1279
|
+
# operated on S3 Express directory buckets. This is required
|
|
1280
|
+
# for sync operations to behave correctly, which is what
|
|
1281
|
+
# recursive copies and moves fall back to so potential case
|
|
1282
|
+
# conflicts can be detected and handled.
|
|
1283
|
+
if not is_s3express_bucket(
|
|
1284
|
+
split_s3_bucket_key(self.parameters['src'])[0]
|
|
1285
|
+
):
|
|
1286
|
+
self._modify_instructions_for_case_conflicts(
|
|
1287
|
+
command_dict, rev_files, rev_generator
|
|
1288
|
+
)
|
|
1289
|
+
return
|
|
1290
|
+
# `skip` and `error` are not valid choices in this case because
|
|
1291
|
+
# it's not possible to detect case conflicts.
|
|
1292
|
+
if self.parameters['case_conflict'] not in {'ignore', 'warn'}:
|
|
1293
|
+
raise ValueError(
|
|
1294
|
+
f"`{self.parameters['case_conflict']}` is not a valid value "
|
|
1295
|
+
"for `--case-conflict` when operating on S3 Express "
|
|
1296
|
+
"directory buckets. Valid values: `warn`, `ignore`."
|
|
1297
|
+
)
|
|
1298
|
+
msg = (
|
|
1299
|
+
"warning: Recursive copies/moves from an S3 Express "
|
|
1300
|
+
"directory bucket to a case-insensitive local filesystem "
|
|
1301
|
+
"may result in undefined behavior if there are "
|
|
1302
|
+
"S3 object key names that differ only by case. To disable "
|
|
1303
|
+
"this warning, set the `--case-conflict` parameter to `ignore`. "
|
|
1304
|
+
f"For more information, see {CaseConflictSync.DOC_URI}."
|
|
1305
|
+
)
|
|
1306
|
+
uni_print(msg, sys.stderr)
|
|
1307
|
+
|
|
1308
|
+
def _modify_instructions_for_case_conflicts(
|
|
1309
|
+
self, command_dict, rev_files, rev_generator
|
|
1310
|
+
):
|
|
1311
|
+
# Command will perform recursive S3 to local downloads.
|
|
1312
|
+
# Checking for potential case conflicts requires knowledge
|
|
1313
|
+
# of local files. Instead of writing a separate validation
|
|
1314
|
+
# mechanism for recursive downloads, we modify the instructions
|
|
1315
|
+
# to mimic a sync command.
|
|
1316
|
+
sync_strategies = {
|
|
1317
|
+
# Local filename exists with exact case match. Always sync
|
|
1318
|
+
# because it's a copy operation.
|
|
1319
|
+
'file_at_src_and_dest_sync_strategy': AlwaysSync(),
|
|
1320
|
+
# Local filename either doesn't exist or differs only by case.
|
|
1321
|
+
# Let `CaseConflictSync` determine which it is and handle it
|
|
1322
|
+
# according to configured `--case-conflict` parameter.
|
|
1323
|
+
'file_not_at_dest_sync_strategy': CaseConflictSync(
|
|
1324
|
+
on_case_conflict=self.parameters['case_conflict']
|
|
1325
|
+
),
|
|
1326
|
+
# Copy is one-way so never sync if not at source.
|
|
1327
|
+
'file_not_at_src_sync_strategy': NeverSync(),
|
|
1328
|
+
}
|
|
1329
|
+
command_dict['setup'].append(rev_files)
|
|
1330
|
+
command_dict['file_generator'].append(rev_generator)
|
|
1331
|
+
command_dict['filters'].append(create_filter(self.parameters))
|
|
1332
|
+
command_dict['comparator'] = [Comparator(**sync_strategies)]
|
|
1333
|
+
self.instructions.insert(
|
|
1334
|
+
self.instructions.index('file_info_builder'), 'comparator'
|
|
1335
|
+
)
|
|
1336
|
+
|
|
1199
1337
|
|
|
1200
1338
|
class CommandParameters(object):
|
|
1201
1339
|
"""
|
|
@@ -1447,6 +1585,9 @@ class CommandParameters(object):
|
|
|
1447
1585
|
def add_page_size(self, parsed_args):
|
|
1448
1586
|
self.parameters['page_size'] = getattr(parsed_args, 'page_size', None)
|
|
1449
1587
|
|
|
1588
|
+
def add_v2_debug(self, parsed_globals):
|
|
1589
|
+
self.parameters['v2_debug'] = resolve_v2_debug_mode(parsed_globals)
|
|
1590
|
+
|
|
1450
1591
|
def _validate_sse_c_args(self):
|
|
1451
1592
|
self._validate_sse_c_arg()
|
|
1452
1593
|
self._validate_sse_c_arg('sse_c_copy_source')
|
|
@@ -254,3 +254,12 @@ class MissingFileSync(BaseSync):
|
|
|
254
254
|
LOG.debug("syncing: %s -> %s, file does not exist at destination",
|
|
255
255
|
src_file.src, src_file.dest)
|
|
256
256
|
return True
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class AlwaysSync(BaseSync):
|
|
260
|
+
def __init__(self, sync_type='file_at_src_and_dest'):
|
|
261
|
+
super(AlwaysSync, self).__init__(sync_type)
|
|
262
|
+
|
|
263
|
+
def determine_should_sync(self, src_file, dest_file):
|
|
264
|
+
LOG.debug(f"syncing: {src_file.src} -> {src_file.dest}")
|
|
265
|
+
return True
|