botocore 1.35.99__py3-none-any.whl → 1.36.1__py3-none-any.whl

botocore/httpchecksum.py

@@ -25,17 +25,15 @@ import logging
 from binascii import crc32
 from hashlib import sha1, sha256
 
-from botocore.compat import HAS_CRT
+from botocore.compat import HAS_CRT, urlparse
 from botocore.exceptions import (
     AwsChunkedWrapperError,
     FlexibleChecksumError,
     MissingDependencyException,
 )
+from botocore.model import StructureShape
 from botocore.response import StreamingBody
-from botocore.utils import (
-    conditionally_calculate_md5,
-    determine_content_length,
-)
+from botocore.utils import determine_content_length, has_checksum_header
 
 if HAS_CRT:
     from awscrt import checksums as crt_checksums
@@ -44,6 +42,8 @@ else:
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_CHECKSUM_ALGORITHM = "CRC32"
+
 
 class BaseChecksum:
     _CHUNK_SIZE = 1024 * 1024
@@ -109,6 +109,19 @@ class CrtCrc32cChecksum(BaseChecksum):
         return self._int_crc32c.to_bytes(4, byteorder="big")
 
 
+class CrtCrc64NvmeChecksum(BaseChecksum):
+    # Note: This class is only used if the CRT is available
+    def __init__(self):
+        self._int_crc64nvme = 0
+
+    def update(self, chunk):
+        new_checksum = crt_checksums.crc64nvme(chunk, self._int_crc64nvme)
+        self._int_crc64nvme = new_checksum & 0xFFFFFFFFFFFFFFFF
+
+    def digest(self):
+        return self._int_crc64nvme.to_bytes(8, byteorder="big")
+
+
 class Sha1Checksum(BaseChecksum):
     def __init__(self):
         self._checksum = sha1()
@@ -246,7 +259,19 @@ def resolve_request_checksum_algorithm(
     params,
     supported_algorithms=None,
 ):
+    # If the header is already set by the customer, skip calculation
+    if has_checksum_header(request):
+        return
+
+    checksum_context = request["context"].get("checksum", {})
+    request_checksum_calculation = request["context"][
+        "client_config"
+    ].request_checksum_calculation
     http_checksum = operation_model.http_checksum
+    request_checksum_required = (
+        operation_model.http_checksum_required
+        or http_checksum.get("requestChecksumRequired")
+    )
     algorithm_member = http_checksum.get("requestAlgorithmMember")
     if algorithm_member and algorithm_member in params:
         # If the client has opted into using flexible checksums and the
@@ -267,35 +292,59 @@ def resolve_request_checksum_algorithm(
             raise FlexibleChecksumError(
                 error_msg=f"Unsupported checksum algorithm: {algorithm_name}"
             )
+    elif request_checksum_required or (
+        algorithm_member and request_checksum_calculation == "when_supported"
+    ):
+        # Don't use a default checksum for presigned requests.
+        if request["context"].get("is_presign_request"):
+            return
+        algorithm_name = DEFAULT_CHECKSUM_ALGORITHM.lower()
+        algorithm_member_header = _get_request_algorithm_member_header(
+            operation_model, request, algorithm_member
+        )
+        if algorithm_member_header is not None:
+            checksum_context["request_algorithm_header"] = {
+                "name": algorithm_member_header,
+                "value": DEFAULT_CHECKSUM_ALGORITHM,
+            }
+    else:
+        return
 
-        location_type = "header"
-        if operation_model.has_streaming_input:
+    location_type = "header"
+    if (
+        operation_model.has_streaming_input
+        and urlparse(request["url"]).scheme == "https"
+    ):
+        if request["context"]["client_config"].signature_version != 's3':
             # Operations with streaming input must support trailers.
-            if request["url"].startswith("https:"):
-                # We only support unsigned trailer checksums currently. As this
-                # disables payload signing we'll only use trailers over TLS.
-                location_type = "trailer"
-
-        algorithm = {
-            "algorithm": algorithm_name,
-            "in": location_type,
-            "name": f"x-amz-checksum-{algorithm_name}",
-        }
+            # We only support unsigned trailer checksums currently. As this
+            # disables payload signing we'll only use trailers over TLS.
+            location_type = "trailer"
+
+    algorithm = {
+        "algorithm": algorithm_name,
+        "in": location_type,
+        "name": f"x-amz-checksum-{algorithm_name}",
+    }
 
-        if algorithm["name"] in request["headers"]:
-            # If the header is already set by the customer, skip calculation
-            return
+    checksum_context["request_algorithm"] = algorithm
+    request["context"]["checksum"] = checksum_context
 
-        checksum_context = request["context"].get("checksum", {})
-        checksum_context["request_algorithm"] = algorithm
-        request["context"]["checksum"] = checksum_context
-    elif operation_model.http_checksum_required or http_checksum.get(
-        "requestChecksumRequired"
-    ):
-        # Otherwise apply the old http checksum behavior via Content-MD5
-        checksum_context = request["context"].get("checksum", {})
-        checksum_context["request_algorithm"] = "conditional-md5"
-        request["context"]["checksum"] = checksum_context
+
+def _get_request_algorithm_member_header(
+    operation_model, request, algorithm_member
+):
+    """Get the name of the header targeted by the "requestAlgorithmMember"."""
+    operation_input_shape = operation_model.input_shape
+    if not isinstance(operation_input_shape, StructureShape):
+        return
+
+    algorithm_member_shape = operation_input_shape.members.get(
+        algorithm_member
+    )
+
+    if algorithm_member_shape:
+        return algorithm_member_shape.serialization.get("name")
 
 
 def apply_request_checksum(request):
@@ -305,10 +354,7 @@ def apply_request_checksum(request):
     if not algorithm:
         return
 
-    if algorithm == "conditional-md5":
-        # Special case to handle the http checksum required trait
-        conditionally_calculate_md5(request)
-    elif algorithm["in"] == "header":
+    if algorithm["in"] == "header":
         _apply_request_header_checksum(request)
     elif algorithm["in"] == "trailer":
         _apply_request_trailer_checksum(request)
@@ -316,6 +362,11 @@ def apply_request_checksum(request):
         raise FlexibleChecksumError(
             error_msg="Unknown checksum variant: {}".format(algorithm["in"])
         )
+    if "request_algorithm_header" in checksum_context:
+        request_algorithm_header = checksum_context["request_algorithm_header"]
+        request["headers"][request_algorithm_header["name"]] = (
+            request_algorithm_header["value"]
+        )
 
 
 def _apply_request_header_checksum(request):
@@ -465,12 +516,13 @@ _CHECKSUM_CLS = {
     "sha1": Sha1Checksum,
     "sha256": Sha256Checksum,
 }
-_CRT_CHECKSUM_ALGORITHMS = ["crc32", "crc32c"]
+_CRT_CHECKSUM_ALGORITHMS = ["crc32", "crc32c", "crc64nvme"]
 if HAS_CRT:
     # Use CRT checksum implementations if available
     _CRT_CHECKSUM_CLS = {
         "crc32": CrtCrc32Checksum,
         "crc32c": CrtCrc32cChecksum,
+        "crc64nvme": CrtCrc64NvmeChecksum,
     }
     _CHECKSUM_CLS.update(_CRT_CHECKSUM_CLS)
     # Validate this list isn't out of sync with _CRT_CHECKSUM_CLS keys
@@ -478,4 +530,4 @@ if HAS_CRT:
         name in _CRT_CHECKSUM_ALGORITHMS for name in _CRT_CHECKSUM_CLS.keys()
     )
 _SUPPORTED_CHECKSUM_ALGORITHMS = list(_CHECKSUM_CLS.keys())
-_ALGORITHMS_PRIORITY_LIST = ['crc32c', 'crc32', 'sha1', 'sha256']
+_ALGORITHMS_PRIORITY_LIST = ['crc64nvme', 'crc32c', 'crc32', 'sha1', 'sha256']

botocore/utils.py

@@ -3228,6 +3228,7 @@ def get_encoding_from_headers(headers, default='ISO-8859-1'):
 
 
 def calculate_md5(body, **kwargs):
+    """This function has been deprecated, but is kept for backwards compatibility."""
     if isinstance(body, (bytes, bytearray)):
         binary_md5 = _calculate_md5_from_bytes(body)
     else:
@@ -3236,11 +3237,13 @@ def calculate_md5(body, **kwargs):
 
 
 def _calculate_md5_from_bytes(body_bytes):
+    """This function has been deprecated, but is kept for backwards compatibility."""
     md5 = get_md5(body_bytes)
     return md5.digest()
 
 
 def _calculate_md5_from_file(fileobj):
+    """This function has been deprecated, but is kept for backwards compatibility."""
     start_position = fileobj.tell()
     md5 = get_md5()
     for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
@@ -3256,15 +3259,17 @@ def _is_s3express_request(params):
     return endpoint_properties.get('backend') == 'S3Express'
 
 
-def _has_checksum_header(params):
+def has_checksum_header(params):
+    """
+    Checks if a header starting with "x-amz-checksum-" is provided in a request.
+
+    This function is considered private and subject to abrupt breaking changes or
+    removal without prior announcement. Please do not use it directly.
+    """
     headers = params['headers']
-    # If a user provided Content-MD5 is present,
-    # don't try to compute a new one.
-    if 'Content-MD5' in headers:
-        return True
 
     # If a header matching the x-amz-checksum-* pattern is present, we
-    # assume a checksum has already been provided and an md5 is not needed
+    # assume a checksum has already been provided by the user.
     for header in headers:
         if CHECKSUM_HEADER_PATTERN.match(header):
             return True
@@ -3273,12 +3278,14 @@ def _has_checksum_header(params):
 
 
 def conditionally_calculate_checksum(params, **kwargs):
-    if not _has_checksum_header(params):
+    """This function has been deprecated, but is kept for backwards compatibility."""
+    if not has_checksum_header(params):
         conditionally_calculate_md5(params, **kwargs)
         conditionally_enable_crc32(params, **kwargs)
 
 
 def conditionally_enable_crc32(params, **kwargs):
+    """This function has been deprecated, but is kept for backwards compatibility."""
     checksum_context = params.get('context', {}).get('checksum', {})
     checksum_algorithm = checksum_context.get('request_algorithm')
     if (
@@ -3296,7 +3303,10 @@ def conditionally_enable_crc32(params, **kwargs):
 
 
 def conditionally_calculate_md5(params, **kwargs):
-    """Only add a Content-MD5 if the system supports it."""
+    """Only add a Content-MD5 if the system supports it.
+
+    This function has been deprecated, but is kept for backwards compatibility.
+    """
     body = params['body']
     checksum_context = params.get('context', {}).get('checksum', {})
     checksum_algorithm = checksum_context.get('request_algorithm')
@@ -3304,7 +3314,7 @@ def conditionally_calculate_md5(params, **kwargs):
         # Skip for requests that will have a flexible checksum applied
         return
 
-    if _has_checksum_header(params):
+    if has_checksum_header(params):
         # Don't add a new header if one is already available.
         return
 

{botocore-1.35.99.dist-info → botocore-1.36.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: botocore
-Version: 1.35.99
+Version: 1.36.1
 Summary: Low-level, data-driven core of boto 3.
 Home-page: https://github.com/boto/botocore
 Author: Amazon Web Services
@@ -27,7 +27,7 @@ Requires-Dist: python-dateutil (<3.0.0,>=2.1)
 Requires-Dist: urllib3 (<1.27,>=1.25.4) ; python_version < "3.10"
 Requires-Dist: urllib3 (!=2.2.0,<3,>=1.25.4) ; python_version >= "3.10"
 Provides-Extra: crt
-Requires-Dist: awscrt (==0.22.0) ; extra == 'crt'
+Requires-Dist: awscrt (==0.23.4) ; extra == 'crt'
 
 botocore
 ========