boto3 1.36.7__tar.gz → 1.42.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {boto3-1.36.7 → boto3-1.42.1}/PKG-INFO +16 -11
  2. {boto3-1.36.7 → boto3-1.42.1}/README.rst +10 -4
  3. {boto3-1.36.7 → boto3-1.42.1}/boto3/__init__.py +3 -7
  4. {boto3-1.36.7 → boto3-1.42.1}/boto3/compat.py +19 -4
  5. {boto3-1.36.7 → boto3-1.42.1}/boto3/crt.py +53 -3
  6. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/__init__.py +1 -1
  7. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/collection.py +2 -3
  8. {boto3-1.36.7 → boto3-1.42.1}/boto3/dynamodb/table.py +1 -1
  9. {boto3-1.36.7 → boto3-1.42.1}/boto3/exceptions.py +4 -0
  10. {boto3-1.36.7 → boto3-1.42.1}/boto3/s3/constants.py +1 -0
  11. {boto3-1.36.7 → boto3-1.42.1}/boto3/s3/inject.py +65 -7
  12. {boto3-1.36.7 → boto3-1.42.1}/boto3/s3/transfer.py +102 -28
  13. {boto3-1.36.7 → boto3-1.42.1}/boto3/session.py +60 -15
  14. {boto3-1.36.7 → boto3-1.42.1}/boto3.egg-info/PKG-INFO +16 -11
  15. {boto3-1.36.7 → boto3-1.42.1}/boto3.egg-info/SOURCES.txt +0 -1
  16. {boto3-1.36.7 → boto3-1.42.1}/boto3.egg-info/requires.txt +2 -2
  17. {boto3-1.36.7 → boto3-1.42.1}/pyproject.toml +1 -1
  18. {boto3-1.36.7 → boto3-1.42.1}/setup.cfg +2 -2
  19. {boto3-1.36.7 → boto3-1.42.1}/setup.py +5 -6
  20. boto3-1.36.7/boto3/data/opsworks/2013-02-18/resources-1.json +0 -173
  21. {boto3-1.36.7 → boto3-1.42.1}/CONTRIBUTING.rst +0 -0
  22. {boto3-1.36.7 → boto3-1.42.1}/LICENSE +0 -0
  23. {boto3-1.36.7 → boto3-1.42.1}/MANIFEST.in +0 -0
  24. {boto3-1.36.7 → boto3-1.42.1}/NOTICE +0 -0
  25. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/cloudformation/2010-05-15/resources-1.json +0 -0
  26. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/cloudwatch/2010-08-01/resources-1.json +0 -0
  27. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/dynamodb/2012-08-10/resources-1.json +0 -0
  28. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2014-10-01/resources-1.json +0 -0
  29. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2015-03-01/resources-1.json +0 -0
  30. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2015-04-15/resources-1.json +0 -0
  31. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2015-10-01/resources-1.json +0 -0
  32. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2016-04-01/resources-1.json +0 -0
  33. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2016-09-15/resources-1.json +0 -0
  34. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/ec2/2016-11-15/resources-1.json +0 -0
  35. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/glacier/2012-06-01/resources-1.json +0 -0
  36. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/iam/2010-05-08/resources-1.json +0 -0
  37. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/s3/2006-03-01/resources-1.json +0 -0
  38. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/sns/2010-03-31/resources-1.json +0 -0
  39. {boto3-1.36.7 → boto3-1.42.1}/boto3/data/sqs/2012-11-05/resources-1.json +0 -0
  40. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/action.py +0 -0
  41. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/attr.py +0 -0
  42. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/base.py +0 -0
  43. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/client.py +0 -0
  44. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/docstring.py +0 -0
  45. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/method.py +0 -0
  46. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/resource.py +0 -0
  47. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/service.py +0 -0
  48. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/subresource.py +0 -0
  49. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/utils.py +0 -0
  50. {boto3-1.36.7 → boto3-1.42.1}/boto3/docs/waiter.py +0 -0
  51. {boto3-1.36.7 → boto3-1.42.1}/boto3/dynamodb/__init__.py +0 -0
  52. {boto3-1.36.7 → boto3-1.42.1}/boto3/dynamodb/conditions.py +0 -0
  53. {boto3-1.36.7 → boto3-1.42.1}/boto3/dynamodb/transform.py +0 -0
  54. {boto3-1.36.7 → boto3-1.42.1}/boto3/dynamodb/types.py +0 -0
  55. {boto3-1.36.7 → boto3-1.42.1}/boto3/ec2/__init__.py +0 -0
  56. {boto3-1.36.7 → boto3-1.42.1}/boto3/ec2/createtags.py +0 -0
  57. {boto3-1.36.7 → boto3-1.42.1}/boto3/ec2/deletetags.py +0 -0
  58. {boto3-1.36.7 → boto3-1.42.1}/boto3/examples/cloudfront.rst +0 -0
  59. {boto3-1.36.7 → boto3-1.42.1}/boto3/examples/s3.rst +0 -0
  60. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/__init__.py +0 -0
  61. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/action.py +0 -0
  62. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/base.py +0 -0
  63. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/collection.py +0 -0
  64. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/factory.py +0 -0
  65. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/model.py +0 -0
  66. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/params.py +0 -0
  67. {boto3-1.36.7 → boto3-1.42.1}/boto3/resources/response.py +0 -0
  68. {boto3-1.36.7 → boto3-1.42.1}/boto3/s3/__init__.py +0 -0
  69. {boto3-1.36.7 → boto3-1.42.1}/boto3/utils.py +0 -0
  70. {boto3-1.36.7 → boto3-1.42.1}/boto3.egg-info/dependency_links.txt +0 -0
  71. {boto3-1.36.7 → boto3-1.42.1}/boto3.egg-info/top_level.txt +0 -0
  72. {boto3-1.36.7 → boto3-1.42.1}/requirements.txt +0 -0
@@ -1,31 +1,30 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: boto3
3
- Version: 1.36.7
3
+ Version: 1.42.1
4
4
  Summary: The AWS SDK for Python
5
5
  Home-page: https://github.com/boto/boto3
6
6
  Author: Amazon Web Services
7
- License: Apache License 2.0
7
+ License: Apache-2.0
8
8
  Project-URL: Documentation, https://boto3.amazonaws.com/v1/documentation/api/latest/index.html
9
9
  Project-URL: Source, https://github.com/boto/boto3
10
10
  Classifier: Development Status :: 5 - Production/Stable
11
11
  Classifier: Intended Audience :: Developers
12
12
  Classifier: Natural Language :: English
13
- Classifier: License :: OSI Approved :: Apache Software License
14
13
  Classifier: Programming Language :: Python
15
14
  Classifier: Programming Language :: Python :: 3
16
15
  Classifier: Programming Language :: Python :: 3 :: Only
17
- Classifier: Programming Language :: Python :: 3.8
18
16
  Classifier: Programming Language :: Python :: 3.9
19
17
  Classifier: Programming Language :: Python :: 3.10
20
18
  Classifier: Programming Language :: Python :: 3.11
21
19
  Classifier: Programming Language :: Python :: 3.12
22
20
  Classifier: Programming Language :: Python :: 3.13
23
- Requires-Python: >= 3.8
21
+ Classifier: Programming Language :: Python :: 3.14
22
+ Requires-Python: >= 3.9
24
23
  License-File: LICENSE
25
24
  License-File: NOTICE
26
- Requires-Dist: botocore<1.37.0,>=1.36.7
25
+ Requires-Dist: botocore<1.43.0,>=1.42.0
27
26
  Requires-Dist: jmespath<2.0.0,>=0.7.1
28
- Requires-Dist: s3transfer<0.12.0,>=0.11.0
27
+ Requires-Dist: s3transfer<0.17.0,>=0.16.0
29
28
  Provides-Extra: crt
30
29
  Requires-Dist: botocore[crt]<2.0a0,>=1.21.0; extra == "crt"
31
30
 
@@ -48,10 +47,16 @@ Boto (pronounced boh-toh) was named after the fresh water dolphin native to the
48
47
  Notices
49
48
  -------
50
49
 
51
- On 2023-12-13, support for Python 3.7 ended for Boto3. This follows the
52
- Python Software Foundation `end of support <https://peps.python.org/pep-0537/#lifespan>`__
53
- for the runtime which occurred on 2023-06-27.
54
- For more information, see this `blog post <https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/>`__.
50
+ On 2026-04-29, support for Python 3.9 will end for Boto3. This follows the
51
+ Python Software Foundation `end of support <https://peps.python.org/pep-0596/#lifespan>`__
52
+ for the runtime which occurred on 2025-10-31.
53
+
54
+ On 2025-04-22, support for Python 3.8 ended for Boto3. This follows the
55
+ Python Software Foundation `end of support <https://peps.python.org/pep-0569/#lifespan>`__
56
+ for the runtime which occurred on 2024-10-07.
57
+
58
+ For more information on deprecations, see this
59
+ `blog post <https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/>`__.
55
60
 
56
61
  .. _boto: https://docs.pythonboto.org/
57
62
  .. _`doc site`: https://boto3.amazonaws.com/v1/documentation/api/latest/index.html
@@ -17,10 +17,16 @@ Boto (pronounced boh-toh) was named after the fresh water dolphin native to the
17
17
  Notices
18
18
  -------
19
19
 
20
- On 2023-12-13, support for Python 3.7 ended for Boto3. This follows the
21
- Python Software Foundation `end of support <https://peps.python.org/pep-0537/#lifespan>`__
22
- for the runtime which occurred on 2023-06-27.
23
- For more information, see this `blog post <https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/>`__.
20
+ On 2026-04-29, support for Python 3.9 will end for Boto3. This follows the
21
+ Python Software Foundation `end of support <https://peps.python.org/pep-0596/#lifespan>`__
22
+ for the runtime which occurred on 2025-10-31.
23
+
24
+ On 2025-04-22, support for Python 3.8 ended for Boto3. This follows the
25
+ Python Software Foundation `end of support <https://peps.python.org/pep-0569/#lifespan>`__
26
+ for the runtime which occurred on 2024-10-07.
27
+
28
+ For more information on deprecations, see this
29
+ `blog post <https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/>`__.
24
30
 
25
31
  .. _boto: https://docs.pythonboto.org/
26
32
  .. _`doc site`: https://boto3.amazonaws.com/v1/documentation/api/latest/index.html
@@ -12,12 +12,13 @@
12
12
  # language governing permissions and limitations under the License.
13
13
 
14
14
  import logging
15
+ from logging import NullHandler
15
16
 
16
17
  from boto3.compat import _warn_deprecated_python
17
18
  from boto3.session import Session
18
19
 
19
20
  __author__ = 'Amazon Web Services'
20
- __version__ = '1.36.7'
21
+ __version__ = '1.42.1'
21
22
 
22
23
 
23
24
  # The default Boto3 session; autoloaded when needed.
@@ -101,11 +102,6 @@ def resource(*args, **kwargs):
101
102
  return _get_default_session().resource(*args, **kwargs)
102
103
 
103
104
 
104
- # Set up logging to ``/dev/null`` like a library is supposed to.
105
+ # Set up do-nothing logging like a library is supposed to.
105
106
  # https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
106
- class NullHandler(logging.Handler):
107
- def emit(self, record):
108
- pass
109
-
110
-
111
107
  logging.getLogger('boto3').addHandler(NullHandler())
@@ -18,15 +18,22 @@ import warnings
18
18
 
19
19
  from boto3.exceptions import PythonDeprecationWarning
20
20
 
21
+ from s3transfer.manager import TransferConfig
22
+
21
23
  # In python3, socket.error is OSError, which is too general
22
24
  # for what we want (i.e FileNotFoundError is a subclass of OSError).
23
25
  # In py3 all the socket related errors are in a newly created
24
26
  # ConnectionError
25
27
  SOCKET_ERROR = ConnectionError
26
28
 
29
+ _APPEND_MODE_CHAR = 'a'
30
+
27
31
  import collections.abc as collections_abc
28
32
 
29
33
 
34
+ TRANSFER_CONFIG_SUPPORTS_CRT = hasattr(TransferConfig, 'UNSET_DEFAULT')
35
+
36
+
30
37
  if sys.platform.startswith('win'):
31
38
  def rename_file(current_filename, new_filename):
32
39
  try:
@@ -58,8 +65,8 @@ def filter_python_deprecation_warnings():
58
65
 
59
66
  def _warn_deprecated_python():
60
67
  """Use this template for future deprecation campaigns as needed."""
61
- py_37_params = {
62
- 'date': 'December 13, 2023',
68
+ py_39_params = {
69
+ 'date': 'April 29, 2026',
63
70
  'blog_link': (
64
71
  'https://aws.amazon.com/blogs/developer/'
65
72
  'python-support-policy-updates-for-aws-sdks-and-tools/'
@@ -67,7 +74,7 @@ def _warn_deprecated_python():
67
74
  }
68
75
  deprecated_versions = {
69
76
  # Example template for future deprecations
70
- (3, 7): py_37_params,
77
+ (3, 9): py_39_params,
71
78
  }
72
79
  py_version = sys.version_info[:2]
73
80
 
@@ -76,7 +83,15 @@ def _warn_deprecated_python():
76
83
  warning = (
77
84
  "Boto3 will no longer support Python {}.{} "
78
85
  "starting {}. To continue receiving service updates, "
79
- "bug fixes, and security updates please upgrade to Python 3.8 or "
86
+ "bug fixes, and security updates please upgrade to Python 3.10 or "
80
87
  "later. More information can be found here: {}"
81
88
  ).format(py_version[0], py_version[1], params['date'], params['blog_link'])
82
89
  warnings.warn(warning, PythonDeprecationWarning)
90
+
91
+
92
+ def is_append_mode(fileobj):
93
+ return (
94
+ hasattr(fileobj, 'mode') and
95
+ isinstance(fileobj.mode, str) and
96
+ _APPEND_MODE_CHAR in fileobj.mode
97
+ )
@@ -19,6 +19,7 @@ project and is not intended for external consumption. All interfaces
19
19
  contained within are subject to abrupt breaking changes.
20
20
  """
21
21
 
22
+ import logging
22
23
  import threading
23
24
 
24
25
  import botocore.exceptions
@@ -31,6 +32,12 @@ from s3transfer.crt import (
31
32
  create_s3_crt_client,
32
33
  )
33
34
 
35
+ from boto3.compat import TRANSFER_CONFIG_SUPPORTS_CRT
36
+ from boto3.exceptions import InvalidCrtTransferConfigError
37
+ from boto3.s3.constants import CRT_TRANSFER_CLIENT
38
+
39
+ logger = logging.getLogger(__name__)
40
+
34
41
  # Singletons for CRT-backed transfers
35
42
  CRT_S3_CLIENT = None
36
43
  BOTOCORE_CRT_SERIALIZER = None
@@ -39,6 +46,15 @@ CLIENT_CREATION_LOCK = threading.Lock()
39
46
  PROCESS_LOCK_NAME = 'boto3'
40
47
 
41
48
 
49
+ _ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS = {
50
+ 'multipart_threshold',
51
+ 'max_concurrency',
52
+ 'max_request_concurrency',
53
+ 'multipart_chunksize',
54
+ 'preferred_transfer_client',
55
+ }
56
+
57
+
42
58
  def _create_crt_client(session, config, region_name, cred_provider):
43
59
  """Create a CRT S3 Client for file transfer.
44
60
 
@@ -157,11 +173,45 @@ def compare_identity(boto3_creds, crt_s3_creds):
157
173
  return is_matching_identity
158
174
 
159
175
 
176
+ def _validate_crt_transfer_config(config):
177
+ if config is None:
178
+ return
179
+ # CRT client can also be configured via `AUTO_RESOLVE_TRANSFER_CLIENT`
180
+ # but it predates this validation. We only validate against CRT client
181
+ # configured via `CRT_TRANSFER_CLIENT` to preserve compatibility.
182
+ if config.preferred_transfer_client != CRT_TRANSFER_CLIENT:
183
+ return
184
+ invalid_crt_args = []
185
+ for param in config.DEFAULTS.keys():
186
+ val = config.get_deep_attr(param)
187
+ if (
188
+ param not in _ALLOWED_CRT_TRANSFER_CONFIG_OPTIONS
189
+ and val is not config.UNSET_DEFAULT
190
+ ):
191
+ invalid_crt_args.append(param)
192
+ if len(invalid_crt_args) > 0:
193
+ raise InvalidCrtTransferConfigError(
194
+ "The following transfer config options are invalid "
195
+ "when preferred_transfer_client is set to crt: "
196
+ f"{', '.join(invalid_crt_args)}`"
197
+ )
198
+
199
+
160
200
  def create_crt_transfer_manager(client, config):
161
201
  """Create a CRTTransferManager for optimized data transfer."""
162
202
  crt_s3_client = get_crt_s3_client(client, config)
163
203
  if is_crt_compatible_request(client, crt_s3_client):
164
- return CRTTransferManager(
165
- crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER
166
- )
204
+ crt_transfer_manager_kwargs = {
205
+ 'crt_s3_client': crt_s3_client.crt_client,
206
+ 'crt_request_serializer': BOTOCORE_CRT_SERIALIZER,
207
+ }
208
+ if TRANSFER_CONFIG_SUPPORTS_CRT:
209
+ _validate_crt_transfer_config(config)
210
+ crt_transfer_manager_kwargs['config'] = config
211
+ if not TRANSFER_CONFIG_SUPPORTS_CRT and config:
212
+ logger.warning(
213
+ 'Using TransferConfig with CRT client requires '
214
+ 's3transfer >= 0.16.0, configured values will be ignored.'
215
+ )
216
+ return CRTTransferManager(**crt_transfer_manager_kwargs)
167
217
  return None
@@ -24,7 +24,7 @@ def generate_docs(root_dir, session):
24
24
  text files documenting each service.
25
25
 
26
26
  :param root_dir: The directory to write the reference files to. Each
27
- service's reference documentation is loacated at
27
+ service's reference documentation is located at
28
28
  root_dir/reference/services/service-name.rst
29
29
 
30
30
  :param session: The boto3 session
@@ -252,8 +252,7 @@ def document_collection_method(
252
252
  name='count',
253
253
  type_name='integer',
254
254
  documentation=(
255
- 'The limit to the number of resources '
256
- 'in the iterable.'
255
+ 'The limit to the number of resources in the iterable.'
257
256
  ),
258
257
  )
259
258
  ],
@@ -271,7 +270,7 @@ def document_collection_method(
271
270
  name='count',
272
271
  type_name='integer',
273
272
  documentation=(
274
- 'The number of items returned by each ' 'service call'
273
+ 'The number of items returned by each service call'
275
274
  ),
276
275
  )
277
276
  ],
@@ -117,7 +117,7 @@ class BatchWriter:
117
117
  if self._extract_pkey_values(item) == pkey_values_new:
118
118
  self._items_buffer.remove(item)
119
119
  logger.debug(
120
- "With overwrite_by_pkeys enabled, skipping " "request:%s",
120
+ "With overwrite_by_pkeys enabled, skipping request:%s",
121
121
  item,
122
122
  )
123
123
 
@@ -124,3 +124,7 @@ class PythonDeprecationWarning(Warning):
124
124
  """
125
125
 
126
126
  pass
127
+
128
+
129
+ class InvalidCrtTransferConfigError(Boto3Error):
130
+ pass
@@ -14,4 +14,5 @@
14
14
 
15
15
  # TransferConfig preferred_transfer_client settings
16
16
  CLASSIC_TRANSFER_CLIENT = "classic"
17
+ CRT_TRANSFER_CLIENT = "crt"
17
18
  AUTO_RESOLVE_TRANSFER_CLIENT = "auto"
@@ -11,10 +11,13 @@
11
11
  # ANY KIND, either express or implied. See the License for the specific
12
12
  # language governing permissions and limitations under the License.
13
13
  import copy as python_copy
14
+ import logging
15
+ from functools import partial
14
16
 
15
17
  from botocore.exceptions import ClientError
16
18
 
17
19
  from boto3 import utils
20
+ from boto3.compat import is_append_mode
18
21
  from boto3.s3.transfer import (
19
22
  ProgressCallbackInvoker,
20
23
  S3Transfer,
@@ -22,6 +25,32 @@ from boto3.s3.transfer import (
22
25
  create_transfer_manager,
23
26
  )
24
27
 
28
+ try:
29
+ from botocore.context import with_current_context
30
+ except ImportError:
31
+ from functools import wraps
32
+
33
+ def with_current_context(hook=None):
34
+ def decorator(func):
35
+ @wraps(func)
36
+ def wrapper(*args, **kwargs):
37
+ return func(*args, **kwargs)
38
+
39
+ return wrapper
40
+
41
+ return decorator
42
+
43
+
44
+ try:
45
+ from botocore.useragent import register_feature_id
46
+ except ImportError:
47
+
48
+ def register_feature_id(feature_id):
49
+ pass
50
+
51
+
52
+ logger = logging.getLogger(__name__)
53
+
25
54
 
26
55
  def inject_s3_transfer_methods(class_attributes, **kwargs):
27
56
  utils.inject_attribute(class_attributes, 'upload_file', upload_file)
@@ -104,6 +133,7 @@ def object_summary_load(self, *args, **kwargs):
104
133
  self.meta.data = response
105
134
 
106
135
 
136
+ @with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
107
137
  def upload_file(
108
138
  self, Filename, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
109
139
  ):
@@ -151,6 +181,7 @@ def upload_file(
151
181
  )
152
182
 
153
183
 
184
+ @with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
154
185
  def download_file(
155
186
  self, Bucket, Key, Filename, ExtraArgs=None, Callback=None, Config=None
156
187
  ):
@@ -368,6 +399,7 @@ def object_download_file(
368
399
  )
369
400
 
370
401
 
402
+ @with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
371
403
  def copy(
372
404
  self,
373
405
  CopySource,
@@ -408,8 +440,8 @@ def copy(
408
440
 
409
441
  :type ExtraArgs: dict
410
442
  :param ExtraArgs: Extra arguments that may be passed to the
411
- client operation. For allowed download arguments see
412
- :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
443
+ client operation. For allowed copy arguments see
444
+ :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
413
445
 
414
446
  :type Callback: function
415
447
  :param Callback: A method which takes a number of bytes transferred to
@@ -487,8 +519,8 @@ def bucket_copy(
487
519
 
488
520
  :type ExtraArgs: dict
489
521
  :param ExtraArgs: Extra arguments that may be passed to the
490
- client operation. For allowed download arguments see
491
- :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
522
+ client operation. For allowed copy arguments see
523
+ :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
492
524
 
493
525
  :type Callback: function
494
526
  :param Callback: A method which takes a number of bytes transferred to
@@ -550,8 +582,8 @@ def object_copy(
550
582
 
551
583
  :type ExtraArgs: dict
552
584
  :param ExtraArgs: Extra arguments that may be passed to the
553
- client operation. For allowed download arguments see
554
- :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_DOWNLOAD_ARGS`.
585
+ client operation. For allowed copy arguments see
586
+ :py:attr:`boto3.s3.transfer.S3Transfer.ALLOWED_COPY_ARGS`.
555
587
 
556
588
  :type Callback: function
557
589
  :param Callback: A method which takes a number of bytes transferred to
@@ -579,6 +611,7 @@ def object_copy(
579
611
  )
580
612
 
581
613
 
614
+ @with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
582
615
  def upload_fileobj(
583
616
  self, Fileobj, Bucket, Key, ExtraArgs=None, Callback=None, Config=None
584
617
  ):
@@ -738,6 +771,28 @@ def object_upload_fileobj(
738
771
  )
739
772
 
740
773
 
774
+ def disable_threading_if_append_mode(config, fileobj):
775
+ """Set `TransferConfig.use_threads` to `False` if file-like
776
+ object is in append mode.
777
+
778
+ :type config: boto3.s3.transfer.TransferConfig
779
+ :param config: The transfer configuration to be used when performing the
780
+ download.
781
+
782
+ :type fileobj: A file-like object
783
+ :param fileobj: A file-like object to inspect for append mode.
784
+ """
785
+ if is_append_mode(fileobj):
786
+ config.use_threads = False
787
+ logger.warning(
788
+ 'A single thread will be used because the provided file object '
789
+ 'is in append mode. Writes may always be appended to the end of '
790
+ 'the file regardless of seek position, so a single thread must be '
791
+ 'used to ensure sequential writes.'
792
+ )
793
+
794
+
795
+ @with_current_context(partial(register_feature_id, 'S3_TRANSFER'))
741
796
  def download_fileobj(
742
797
  self, Bucket, Key, Fileobj, ExtraArgs=None, Callback=None, Config=None
743
798
  ):
@@ -790,7 +845,10 @@ def download_fileobj(
790
845
  if config is None:
791
846
  config = TransferConfig()
792
847
 
793
- with create_transfer_manager(self, config) as manager:
848
+ new_config = python_copy.copy(config)
849
+ disable_threading_if_append_mode(new_config, Fileobj)
850
+
851
+ with create_transfer_manager(self, new_config) as manager:
794
852
  future = manager.download(
795
853
  bucket=Bucket,
796
854
  key=Key,
@@ -128,7 +128,7 @@ import threading
128
128
  from os import PathLike, fspath, getpid
129
129
 
130
130
  from botocore.compat import HAS_CRT
131
- from botocore.exceptions import ClientError
131
+ from botocore.exceptions import ClientError, MissingDependencyException
132
132
  from s3transfer.exceptions import (
133
133
  RetriesExceededError as S3TransferRetriesExceededError,
134
134
  )
@@ -139,7 +139,11 @@ from s3transfer.subscribers import BaseSubscriber
139
139
  from s3transfer.utils import OSUtils
140
140
 
141
141
  import boto3.s3.constants as constants
142
- from boto3.exceptions import RetriesExceededError, S3UploadFailedError
142
+ from boto3.compat import TRANSFER_CONFIG_SUPPORTS_CRT
143
+ from boto3.exceptions import (
144
+ RetriesExceededError,
145
+ S3UploadFailedError,
146
+ )
143
147
 
144
148
  if HAS_CRT:
145
149
  import awscrt.s3
@@ -184,16 +188,26 @@ def create_transfer_manager(client, config, osutil=None):
184
188
 
185
189
  def _should_use_crt(config):
186
190
  # This feature requires awscrt>=0.19.18
187
- if HAS_CRT and has_minimum_crt_version((0, 19, 18)):
188
- is_optimized_instance = awscrt.s3.is_optimized_for_system()
189
- else:
190
- is_optimized_instance = False
191
+ has_min_crt = HAS_CRT and has_minimum_crt_version((0, 19, 18))
192
+ is_optimized_instance = has_min_crt and awscrt.s3.is_optimized_for_system()
191
193
  pref_transfer_client = config.preferred_transfer_client.lower()
192
194
 
195
+ if (
196
+ pref_transfer_client == constants.CRT_TRANSFER_CLIENT
197
+ and not has_min_crt
198
+ ):
199
+ msg = (
200
+ "CRT transfer client is configured but is missing minimum CRT "
201
+ f"version. CRT installed: {HAS_CRT}"
202
+ )
203
+ if HAS_CRT:
204
+ msg += f", with version: {awscrt.__version__}"
205
+ raise MissingDependencyException(msg=msg)
206
+
193
207
  if (
194
208
  is_optimized_instance
195
209
  and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT
196
- ):
210
+ ) or pref_transfer_client == constants.CRT_TRANSFER_CLIENT:
197
211
  logger.debug(
198
212
  "Attempting to use CRTTransferManager. Config settings may be ignored."
199
213
  )
@@ -235,18 +249,31 @@ class TransferConfig(S3TransferConfig):
235
249
  'max_concurrency': 'max_request_concurrency',
236
250
  'max_io_queue': 'max_io_queue_size',
237
251
  }
252
+ DEFAULTS = {
253
+ 'multipart_threshold': 8 * MB,
254
+ 'max_concurrency': 10,
255
+ 'max_request_concurrency': 10,
256
+ 'multipart_chunksize': 8 * MB,
257
+ 'num_download_attempts': 5,
258
+ 'max_io_queue': 100,
259
+ 'max_io_queue_size': 100,
260
+ 'io_chunksize': 256 * KB,
261
+ 'use_threads': True,
262
+ 'max_bandwidth': None,
263
+ 'preferred_transfer_client': constants.AUTO_RESOLVE_TRANSFER_CLIENT,
264
+ }
238
265
 
239
266
  def __init__(
240
267
  self,
241
- multipart_threshold=8 * MB,
242
- max_concurrency=10,
243
- multipart_chunksize=8 * MB,
244
- num_download_attempts=5,
245
- max_io_queue=100,
246
- io_chunksize=256 * KB,
247
- use_threads=True,
268
+ multipart_threshold=None,
269
+ max_concurrency=None,
270
+ multipart_chunksize=None,
271
+ num_download_attempts=None,
272
+ max_io_queue=None,
273
+ io_chunksize=None,
274
+ use_threads=None,
248
275
  max_bandwidth=None,
249
- preferred_transfer_client=constants.AUTO_RESOLVE_TRANSFER_CLIENT,
276
+ preferred_transfer_client=None,
250
277
  ):
251
278
  """Configuration object for managed S3 transfers
252
279
 
@@ -270,23 +297,30 @@ class TransferConfig(S3TransferConfig):
270
297
  Other retryable exceptions such as throttling errors and 5xx
271
298
  errors are already retried by botocore (this default is 5). This
272
299
  does not take into account the number of exceptions retried by
273
- botocore.
300
+ botocore. Note: This value is ignored when resolved transfer
301
+ manager type is CRTTransferManager.
274
302
 
275
303
  :param max_io_queue: The maximum amount of read parts that can be
276
304
  queued in memory to be written for a download. The size of each
277
305
  of these read parts is at most the size of ``io_chunksize``.
306
+ Note: This value is ignored when resolved transfer manager type
307
+ is CRTTransferManager.
278
308
 
279
309
  :param io_chunksize: The max size of each chunk in the io queue.
280
310
  Currently, this is size used when ``read`` is called on the
281
- downloaded stream as well.
311
+ downloaded stream as well. Note: This value is ignored when
312
+ resolved transfer manager type is CRTTransferManager.
282
313
 
283
314
  :param use_threads: If True, threads will be used when performing
284
315
  S3 transfers. If False, no threads will be used in
285
316
  performing transfers; all logic will be run in the current thread.
317
+ Note: This value is ignored when resolved transfer manager type is
318
+ CRTTransferManager.
286
319
 
287
320
  :param max_bandwidth: The maximum bandwidth that will be consumed
288
321
  in uploading and downloading file content. The value is an integer
289
- in terms of bytes per second.
322
+ in terms of bytes per second. Note: This value is ignored when
323
+ resolved transfer manager type is CRTTransferManager.
290
324
 
291
325
  :param preferred_transfer_client: String specifying preferred transfer
292
326
  client for transfer operations.
@@ -296,23 +330,40 @@ class TransferConfig(S3TransferConfig):
296
330
  are made with supported environment and settings.
297
331
  * classic - Only use the origin S3TransferManager with
298
332
  requests. Disables possible CRT upgrade on requests.
333
+ * crt - Only use the CRTTransferManager with requests.
299
334
  """
335
+ init_args = {
336
+ 'multipart_threshold': multipart_threshold,
337
+ 'max_concurrency': max_concurrency,
338
+ 'multipart_chunksize': multipart_chunksize,
339
+ 'num_download_attempts': num_download_attempts,
340
+ 'max_io_queue': max_io_queue,
341
+ 'io_chunksize': io_chunksize,
342
+ 'use_threads': use_threads,
343
+ 'max_bandwidth': max_bandwidth,
344
+ 'preferred_transfer_client': preferred_transfer_client,
345
+ }
346
+ resolved = self._resolve_init_args(init_args)
300
347
  super().__init__(
301
- multipart_threshold=multipart_threshold,
302
- max_request_concurrency=max_concurrency,
303
- multipart_chunksize=multipart_chunksize,
304
- num_download_attempts=num_download_attempts,
305
- max_io_queue_size=max_io_queue,
306
- io_chunksize=io_chunksize,
307
- max_bandwidth=max_bandwidth,
348
+ multipart_threshold=resolved['multipart_threshold'],
349
+ max_request_concurrency=resolved['max_concurrency'],
350
+ multipart_chunksize=resolved['multipart_chunksize'],
351
+ num_download_attempts=resolved['num_download_attempts'],
352
+ max_io_queue_size=resolved['max_io_queue'],
353
+ io_chunksize=resolved['io_chunksize'],
354
+ max_bandwidth=resolved['max_bandwidth'],
308
355
  )
309
356
  # Some of the argument names are not the same as the inherited
310
357
  # S3TransferConfig so we add aliases so you can still access the
311
358
  # old version of the names.
312
359
  for alias in self.ALIAS:
313
- setattr(self, alias, getattr(self, self.ALIAS[alias]))
314
- self.use_threads = use_threads
315
- self.preferred_transfer_client = preferred_transfer_client
360
+ setattr(
361
+ self,
362
+ alias,
363
+ object.__getattribute__(self, self.ALIAS[alias]),
364
+ )
365
+ self.use_threads = resolved['use_threads']
366
+ self.preferred_transfer_client = resolved['preferred_transfer_client']
316
367
 
317
368
  def __setattr__(self, name, value):
318
369
  # If the alias name is used, make sure we set the name that it points
@@ -322,10 +373,33 @@ class TransferConfig(S3TransferConfig):
322
373
  # Always set the value of the actual name provided.
323
374
  super().__setattr__(name, value)
324
375
 
376
+ def __getattribute__(self, item):
377
+ value = object.__getattribute__(self, item)
378
+ if not TRANSFER_CONFIG_SUPPORTS_CRT:
379
+ return value
380
+ defaults = object.__getattribute__(self, 'DEFAULTS')
381
+ if item not in defaults:
382
+ return value
383
+ if value is self.UNSET_DEFAULT:
384
+ return defaults[item]
385
+ return value
386
+
387
+ def _resolve_init_args(self, init_args):
388
+ resolved = {}
389
+ for init_arg, val in init_args.items():
390
+ if val is not None:
391
+ resolved[init_arg] = val
392
+ elif TRANSFER_CONFIG_SUPPORTS_CRT:
393
+ resolved[init_arg] = self.UNSET_DEFAULT
394
+ else:
395
+ resolved[init_arg] = self.DEFAULTS[init_arg]
396
+ return resolved
397
+
325
398
 
326
399
  class S3Transfer:
327
400
  ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS
328
401
  ALLOWED_UPLOAD_ARGS = TransferManager.ALLOWED_UPLOAD_ARGS
402
+ ALLOWED_COPY_ARGS = TransferManager.ALLOWED_COPY_ARGS
329
403
 
330
404
  def __init__(self, client=None, config=None, osutil=None, manager=None):
331
405
  if not client and not manager: