swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/middleware/s3api/s3request.py

@@ -17,16 +17,16 @@ import base64
 import binascii
 from collections import defaultdict, OrderedDict
 from email.header import Header
-from hashlib import sha1, sha256, md5
+from hashlib import sha1, sha256
 import hmac
 import re
-import six
 # pylint: disable-msg=import-error
-from six.moves.urllib.parse import quote, unquote, parse_qsl
+from urllib.parse import quote, unquote, parse_qsl
 import string
 
-from swift.common.utils import split_path, json, get_swift_info, \
-    close_if_possible
+from swift.common.utils import split_path, json, md5, streq_const_time, \
+    get_policy_index, InputProxy
+from swift.common.registry import get_swift_info
 from swift.common import swob
 from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
     HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
@@ -34,18 +34,19 @@ from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
     HTTP_PARTIAL_CONTENT, HTTP_NOT_MODIFIED, HTTP_PRECONDITION_FAILED, \
     HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, HTTP_LENGTH_REQUIRED, \
     HTTP_BAD_REQUEST, HTTP_REQUEST_TIMEOUT, HTTP_SERVICE_UNAVAILABLE, \
-    HTTP_TOO_MANY_REQUESTS, HTTP_RATE_LIMITED, is_success
+    HTTP_TOO_MANY_REQUESTS, HTTP_RATE_LIMITED, is_success, \
+    HTTP_CLIENT_CLOSED_REQUEST
 
 from swift.common.constraints import check_utf8
-from swift.proxy.controllers.base import get_container_info, \
-    headers_to_container_info
+from swift.proxy.controllers.base import get_container_info
 from swift.common.request_helpers import check_path_header
 
 from swift.common.middleware.s3api.controllers import ServiceController, \
     ObjectController, AclController, MultiObjectDeleteController, \
     LocationController, LoggingStatusController, PartController, \
     UploadController, UploadsController, VersioningController, \
-    UnsupportedController, S3AclController, BucketController
+    UnsupportedController, S3AclController, BucketController, \
+    TaggingController, ObjectLockController
 from swift.common.middleware.s3api.s3response import AccessDenied, \
     InvalidArgument, InvalidDigest, BucketAlreadyOwnedByYou, \
     RequestTimeTooSkewed, S3Response, SignatureDoesNotMatch, \
@@ -54,14 +55,14 @@ from swift.common.middleware.s3api.s3response import AccessDenied, \
     MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \
     MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \
     BadDigest, AuthorizationHeaderMalformed, SlowDown, \
-    AuthorizationQueryParametersError, ServiceUnavailable
-from swift.common.middleware.s3api.exception import NotS3Request, \
-    BadSwiftRequest
+    AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU, \
+    InvalidPartNumber, InvalidPartArgument, XAmzContentSHA256Mismatch
+from swift.common.middleware.s3api.exception import NotS3Request
 from swift.common.middleware.s3api.utils import utf8encode, \
     S3Timestamp, mktime, MULTIUPLOAD_SUFFIX
 from swift.common.middleware.s3api.subresource import decode_acl, encode_acl
 from swift.common.middleware.s3api.utils import sysmeta_header, \
-    validate_bucket_name
+    validate_bucket_name, Config
 from swift.common.middleware.s3api.acl_utils import handle_acl_header
 
 
@@ -73,7 +74,8 @@ ALLOWED_SUB_RESOURCES = sorted([
     'versionId', 'versioning', 'versions', 'website',
     'response-cache-control', 'response-content-disposition',
     'response-content-encoding', 'response-content-language',
-    'response-content-type', 'response-expires', 'cors', 'tagging', 'restore'
+    'response-content-type', 'response-expires', 'cors', 'tagging', 'restore',
+    'object-lock'
 ])
 
 
@@ -102,6 +104,7 @@ def _header_acl_property(resource):
     """
     Set and retrieve the acl in self.headers
     """
+
     def getter(self):
         return getattr(self, '_%s' % resource)
 
@@ -116,33 +119,56 @@ def _header_acl_property(resource):
                     doc='Get and set the %s acl property' % resource)
 
 
-class HashingInput(object):
+class S3InputSHA256Mismatch(BaseException):
+    """
+    Client provided a X-Amz-Content-SHA256, but it doesn't match the data.
+
+    Inherit from BaseException (rather than Exception) so it cuts from the
+    proxy-server app (which will presumably be the one reading the input)
+    through all the layers of the pipeline back to us. It should never escape
+    the s3api middleware.
+    """
+    def __init__(self, expected, computed):
+        self.expected = expected
+        self.computed = computed
+
+
+class HashingInput(InputProxy):
     """
-    wsgi.input wrapper to verify the hash of the input as it's read.
+    wsgi.input wrapper to verify the SHA256 of the input as it's read.
     """
-    def __init__(self, reader, content_length, hasher, expected_hex_hash):
-        self._input = reader
-        self._to_read = content_length
-        self._hasher = hasher()
-        self._expected = expected_hex_hash
-
-    def read(self, size=None):
-        chunk = self._input.read(size)
+
+    def __init__(self, wsgi_input, content_length, expected_hex_hash):
+        super().__init__(wsgi_input)
+        self._expected_length = content_length
+        self._hasher = sha256()
+        self._expected_hash = expected_hex_hash
+        if content_length == 0 and \
+                self._hasher.hexdigest() != self._expected_hash.lower():
+            self.close()
+            raise XAmzContentSHA256Mismatch(
+                client_computed_content_s_h_a256=self._expected_hash,
+                s3_computed_content_s_h_a256=self._hasher.hexdigest(),
+            )
+
+    def chunk_update(self, chunk, eof, *args, **kwargs):
         self._hasher.update(chunk)
-        self._to_read -= len(chunk)
-        short_read = bool(chunk) if size is None else (len(chunk) < size)
-        if self._to_read < 0 or (short_read and self._to_read) or (
-                self._to_read == 0 and
-                self._hasher.hexdigest() != self._expected):
+
+        if self.bytes_received < self._expected_length:
+            error = eof
+        elif self.bytes_received == self._expected_length:
+            error = self._hasher.hexdigest() != self._expected_hash.lower()
+        else:
+            error = True
+
+        if error:
             self.close()
             # Since we don't return the last chunk, the PUT never completes
-            raise swob.HTTPUnprocessableEntity(
-                'The X-Amz-Content-SHA56 you specified did not match '
-                'what we received.')
-        return chunk
+            raise S3InputSHA256Mismatch(
+                self._expected_hash,
+                self._hasher.hexdigest())
 
-    def close(self):
-        close_if_possible(self._input)
+        return chunk
 
 
 class SigV4Mixin(object):
@@ -159,7 +185,7 @@ class SigV4Mixin(object):
                 derived_secret, scope_piece.encode('utf8'), sha256).digest()
         valid_signature = hmac.new(
             derived_secret, self.string_to_sign, sha256).hexdigest()
-        return user_signature == valid_signature
+        return streq_const_time(user_signature, valid_signature)
 
     @property
     def _is_query_auth(self):
@@ -188,11 +214,13 @@ class SigV4Mixin(object):
                         timestamp = mktime(self.headers.get('Date'))
             except (ValueError, TypeError):
                 raise AccessDenied('AWS authentication requires a valid Date '
-                                   'or x-amz-date header')
+                                   'or x-amz-date header',
+                                   reason='invalid_date')
 
             if timestamp < 0:
                 raise AccessDenied('AWS authentication requires a valid Date '
-                                   'or x-amz-date header')
+                                   'or x-amz-date header',
+                                   reason='invalid_date')
 
             try:
                 self._timestamp = S3Timestamp(timestamp)
@@ -213,7 +241,7 @@ class SigV4Mixin(object):
         try:
             expires = int(self.params['X-Amz-Expires'])
         except KeyError:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_expires')
         except ValueError:
             err = 'X-Amz-Expires should be a number'
         else:
@@ -229,14 +257,36 @@ class SigV4Mixin(object):
             raise AuthorizationQueryParametersError(err)
 
         if int(self.timestamp) + expires < S3Timestamp.now():
-            raise AccessDenied('Request has expired')
+            raise AccessDenied('Request has expired', reason='expired')
+
+    def _validate_sha256(self):
+        aws_sha256 = self.headers.get('x-amz-content-sha256')
+        looks_like_sha256 = (
+            aws_sha256 and len(aws_sha256) == 64 and
+            all(c in '0123456789abcdef' for c in aws_sha256.lower()))
+        if not aws_sha256:
+            if 'X-Amz-Credential' in self.params:
+                pass  # pre-signed URL; not required
+            else:
+                msg = 'Missing required header for this request: ' \
+                      'x-amz-content-sha256'
+                raise InvalidRequest(msg)
+        elif aws_sha256 == 'UNSIGNED-PAYLOAD':
+            pass
+        elif not looks_like_sha256 and 'X-Amz-Credential' not in self.params:
+            raise InvalidArgument(
+                'x-amz-content-sha256',
+                aws_sha256,
+                'x-amz-content-sha256 must be UNSIGNED-PAYLOAD, or '
+                'a valid sha256 value.')
+        return aws_sha256
 
     def _parse_credential(self, credential_string):
         parts = credential_string.split("/")
         # credential must be in following format:
         # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request
         if not parts[0] or len(parts) != 5:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_credential')
         return dict(zip(['access', 'date', 'region', 'service', 'terminal'],
                         parts))
 
@@ -253,15 +303,16 @@ class SigV4Mixin(object):
                                   self.params.get('X-Amz-Algorithm'))
         try:
             cred_param = self._parse_credential(
-                self.params['X-Amz-Credential'])
-            sig = self.params['X-Amz-Signature']
+                swob.wsgi_to_str(self.params['X-Amz-Credential']))
+            sig = swob.wsgi_to_str(self.params['X-Amz-Signature'])
             if not sig:
-                raise AccessDenied()
+                raise AccessDenied(reason='invalid_query_auth')
         except KeyError:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_query_auth')
 
         try:
-            signed_headers = self.params['X-Amz-SignedHeaders']
+            signed_headers = swob.wsgi_to_str(
+                self.params['X-Amz-SignedHeaders'])
         except KeyError:
             # TODO: make sure if is it malformed request?
             raise AuthorizationHeaderMalformed()
@@ -304,12 +355,12 @@ class SigV4Mixin(object):
         :raises: AuthorizationHeaderMalformed
         """
 
-        auth_str = self.headers['Authorization']
+        auth_str = swob.wsgi_to_str(self.headers['Authorization'])
         cred_param = self._parse_credential(auth_str.partition(
             "Credential=")[2].split(',')[0])
         sig = auth_str.partition("Signature=")[2].split(',')[0]
         if not sig:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_header_auth')
         signed_headers = auth_str.partition(
             "SignedHeaders=")[2].split(',', 1)[0]
         if not signed_headers:
@@ -370,7 +421,7 @@ class SigV4Mixin(object):
         else:  # mostly-functional fallback
             headers_lower_dict = dict(
                 (k.lower().strip(), ' '.join(_header_strip(v or '').split()))
-                for (k, v) in six.iteritems(self.headers))
+                for (k, v) in self.headers.items())
 
         if 'host' in headers_lower_dict and re.match(
                 'Boto/2.[0-9].[0-2]',
@@ -383,7 +434,7 @@ class SigV4Mixin(object):
 
         headers_to_sign = [
             (key, value) for key, value in sorted(headers_lower_dict.items())
-            if key in self._signed_headers]
+            if swob.wsgi_to_str(key) in self._signed_headers]
 
         if len(headers_to_sign) != len(self._signed_headers):
             # NOTE: if we are missing the header suggested via
@@ -399,7 +450,8 @@ class SigV4Mixin(object):
         """
         It won't require bucket name in canonical_uri for v4.
         """
-        return swob.wsgi_to_bytes(self.environ.get('RAW_PATH_INFO', self.path))
+        return swob.wsgi_to_bytes(swob.wsgi_quote(
+            self.environ.get('PATH_INFO', self.path), safe='-_.~/'))
 
     def _canonical_request(self):
         # prepare 'canonical_request'
@@ -417,7 +469,7 @@ class SigV4Mixin(object):
         #
 
         # 1. Add verb like: GET
-        cr = [swob.wsgi_to_bytes(self.method.upper())]
+        cr = [swob.wsgi_to_bytes(self.method)]
 
         # 2. Add path like: /
         path = self._canonical_uri()
@@ -439,30 +491,9 @@ class SigV4Mixin(object):
         cr.append(b';'.join(swob.wsgi_to_bytes(k) for k, v in headers_to_sign))
 
         # 6. Add payload string at the tail
-        if 'X-Amz-Credential' in self.params:
-            # V4 with query parameters only
-            hashed_payload = 'UNSIGNED-PAYLOAD'
-        elif 'X-Amz-Content-SHA256' not in self.headers:
-            msg = 'Missing required header for this request: ' \
-                  'x-amz-content-sha256'
-            raise InvalidRequest(msg)
-        else:
-            hashed_payload = self.headers['X-Amz-Content-SHA256']
-            if hashed_payload != 'UNSIGNED-PAYLOAD':
-                if self.content_length == 0:
-                    if hashed_payload != sha256().hexdigest():
-                        raise BadDigest(
-                            'The X-Amz-Content-SHA56 you specified did not '
-                            'match what we received.')
-                elif self.content_length:
-                    self.environ['wsgi.input'] = HashingInput(
-                        self.environ['wsgi.input'],
-                        self.content_length,
-                        sha256,
-                        hashed_payload)
-                # else, length not provided -- Swift will kick out a
-                # 411 Length Required which will get translated back
-                # to a S3-style response in S3Request._swift_error_codes
+        hashed_payload = self.headers.get('X-Amz-Content-SHA256',
+                                          'UNSIGNED-PAYLOAD')
+
         cr.append(swob.wsgi_to_bytes(hashed_payload))
         return b'\n'.join(cr)
 
@@ -524,18 +555,11 @@ class S3Request(swob.Request):
     bucket_acl = _header_acl_property('container')
     object_acl = _header_acl_property('object')
 
-    def __init__(self, env, app=None, slo_enabled=True, storage_domain='',
-                 location='us-east-1', force_request_log=False,
-                 dns_compliant_bucket_names=True, allow_multipart_uploads=True,
-                 allow_no_owner=False):
-        # NOTE: app and allow_no_owner are not used by this class, need for
-        #       compatibility of S3acl
+    def __init__(self, env, app=None, conf=None):
+        # NOTE: app is not used by this class, need for compatibility of S3acl
         swob.Request.__init__(self, env)
-        self.storage_domain = storage_domain
-        self.location = location
-        self.force_request_log = force_request_log
-        self.dns_compliant_bucket_names = dns_compliant_bucket_names
-        self.allow_multipart_uploads = allow_multipart_uploads
+        self.conf = conf or Config()
+        self.location = self.conf.location
         self._timestamp = None
         self.access_key, self.signature = self._parse_auth_info()
         self.bucket_in_host = self._parse_host()
@@ -551,20 +575,70 @@ class S3Request(swob.Request):
         }
         self.account = None
         self.user_id = None
-        self.slo_enabled = slo_enabled
+        self.policy_index = None
 
         # Avoids that swift.swob.Response replaces Location header value
         # by full URL when absolute path given. See swift.swob for more detail.
         self.environ['swift.leave_relative_location'] = True
 
+    def validate_part_number(self, parts_count=None, check_max=True):
+        """
+        Get the partNumber param, if it exists, and check it is valid.
+
+        To be valid, a partNumber must satisfy two criteria. First, it must be
+        an integer between 1 and the maximum allowed parts, inclusive. The
+        maximum allowed parts is the maximum of the configured
+        ``max_upload_part_num`` and, if given, ``parts_count``. Second, the
+        partNumber must be less than or equal to the ``parts_count``, if it is
+        given.
+
+        :param parts_count: if given, this is the number of parts in an
+            existing object.
+        :raises InvalidPartArgument: if the partNumber param is invalid i.e.
+            less than 1 or greater than the maximum allowed parts.
+        :raises InvalidPartNumber: if the partNumber param is valid but greater
+            than ``num_parts``.
+        :return: an integer part number if the partNumber param exists,
+            otherwise ``None``.
+        """
+        part_number = self.params.get('partNumber')
+        if part_number is None:
+            return None
+
+        if self.range:
+            raise InvalidRequest('Cannot specify both Range header and '
+                                 'partNumber query parameter')
+
+        try:
+            parts_count = int(parts_count)
+        except (TypeError, ValueError):
+            # an invalid/empty param is treated like parts_count=max_parts
+            parts_count = self.conf.max_upload_part_num
+        # max_parts may be raised to the number of existing parts
+        max_parts = max(self.conf.max_upload_part_num, parts_count)
+
+        try:
+            part_number = int(part_number)
+            if part_number < 1:
+                raise ValueError
+        except ValueError:
+            raise InvalidPartArgument(max_parts, part_number)  # 400
+
+        if check_max:
+            if part_number > max_parts:
+                raise InvalidPartArgument(max_parts, part_number)  # 400
+            if part_number > parts_count:
+                raise InvalidPartNumber()  # 416
+
+        return part_number
+
     def check_signature(self, secret):
         secret = utf8encode(secret)
         user_signature = self.signature
         valid_signature = base64.b64encode(hmac.new(
-            secret, self.string_to_sign, sha1).digest()).strip()
-        if not six.PY2:
-            valid_signature = valid_signature.decode('ascii')
-        return user_signature == valid_signature
+            secret, self.string_to_sign, sha1
+        ).digest()).strip().decode('ascii')
+        return streq_const_time(user_signature, valid_signature)
 
     @property
     def timestamp(self):
@@ -587,11 +661,13 @@ class S3Request(swob.Request):
                                          self.headers.get('Date')))
             except ValueError:
                 raise AccessDenied('AWS authentication requires a valid Date '
-                                   'or x-amz-date header')
+                                   'or x-amz-date header',
+                                   reason='invalid_date')
 
             if timestamp < 0:
                 raise AccessDenied('AWS authentication requires a valid Date '
-                                   'or x-amz-date header')
+                                   'or x-amz-date header',
+                                   reason='invalid_date')
             try:
                 self._timestamp = S3Timestamp(timestamp)
             except ValueError:
@@ -609,29 +685,30 @@ class S3Request(swob.Request):
         return 'AWSAccessKeyId' in self.params
 
     def _parse_host(self):
-        storage_domain = self.storage_domain
-        if not storage_domain:
+        if not self.conf.storage_domains:
             return None
 
-        if not storage_domain.startswith('.'):
-            storage_domain = '.' + storage_domain
-
         if 'HTTP_HOST' in self.environ:
             given_domain = self.environ['HTTP_HOST']
         elif 'SERVER_NAME' in self.environ:
             given_domain = self.environ['SERVER_NAME']
         else:
             return None
-
         port = ''
         if ':' in given_domain:
             given_domain, port = given_domain.rsplit(':', 1)
-        if given_domain.endswith(storage_domain):
-            return given_domain[:-len(storage_domain)]
+
+        for storage_domain in self.conf.storage_domains:
+            if not storage_domain.startswith('.'):
+                storage_domain = '.' + storage_domain
+
+            if given_domain.endswith(storage_domain):
+                return given_domain[:-len(storage_domain)]
 
         return None
 
     def _parse_uri(self):
+        # NB: returns WSGI strings
         if not check_utf8(swob.wsgi_to_str(self.environ['PATH_INFO'])):
             raise InvalidURI(self.path)
 
@@ -642,10 +719,10 @@ class S3Request(swob.Request):
         bucket, obj = self.split_path(0, 2, True)
 
         if bucket and not validate_bucket_name(
-                bucket, self.dns_compliant_bucket_names):
+                bucket, self.conf.dns_compliant_bucket_names):
             # Ignore GET service case
             raise InvalidBucketName(bucket)
-        return (bucket, obj)
+        return bucket, obj
 
     def _parse_query_authentication(self):
         """
@@ -658,14 +735,14 @@ class S3Request(swob.Request):
         :raises: AccessDenied
         """
         try:
-            access = self.params['AWSAccessKeyId']
-            expires = self.params['Expires']
-            sig = self.params['Signature']
+            access = swob.wsgi_to_str(self.params['AWSAccessKeyId'])
+            expires = swob.wsgi_to_str(self.params['Expires'])
+            sig = swob.wsgi_to_str(self.params['Signature'])
         except KeyError:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_query_auth')
 
         if not all([access, sig, expires]):
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_query_auth')
 
         return access, sig
 
@@ -676,9 +753,9 @@ class S3Request(swob.Request):
         :returns: a tuple of access_key and signature
         :raises: AccessDenied
         """
-        auth_str = self.headers['Authorization']
+        auth_str = swob.wsgi_to_str(self.headers['Authorization'])
         if not auth_str.startswith('AWS ') or ':' not in auth_str:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_header_auth')
         # This means signature format V2
         access, sig = auth_str.split(' ', 1)[1].rsplit(':', 1)
         return access, sig
@@ -709,15 +786,15 @@ class S3Request(swob.Request):
         try:
             ex = S3Timestamp(float(self.params['Expires']))
         except (KeyError, ValueError):
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_expires')
 
         if S3Timestamp.now() > ex:
-            raise AccessDenied('Request has expired')
+            raise AccessDenied('Request has expired', reason='expired')
 
         if ex >= 2 ** 31:
             raise AccessDenied(
                 'Invalid date (should be seconds since epoch): %s' %
-                self.params['Expires'])
+                self.params['Expires'], reason='invalid_expires')
 
     def _validate_dates(self):
         """
@@ -729,19 +806,23 @@ class S3Request(swob.Request):
         amz_date_header = self.headers.get('X-Amz-Date')
         if not date_header and not amz_date_header:
             raise AccessDenied('AWS authentication requires a valid Date '
-                               'or x-amz-date header')
+                               'or x-amz-date header',
+                               reason='invalid_date')
 
         # Anyways, request timestamp should be validated
         epoch = S3Timestamp(0)
         if self.timestamp < epoch:
-            raise AccessDenied()
+            raise AccessDenied(reason='invalid_date')
 
         # If the standard date is too far ahead or behind, it is an
         # error
-        delta = 60 * 5
-        if abs(int(self.timestamp) - int(S3Timestamp.now())) > delta:
+        delta = abs(int(self.timestamp) - int(S3Timestamp.now()))
+        if delta > self.conf.allowable_clock_skew:
             raise RequestTimeTooSkewed()
 
+    def _validate_sha256(self):
+        return self.headers.get('x-amz-content-sha256')
+
     def _validate_headers(self):
         if 'CONTENT_LENGTH' in self.environ:
             try:
@@ -752,21 +833,6 @@ class S3Request(swob.Request):
                 raise InvalidArgument('Content-Length',
                                       self.environ['CONTENT_LENGTH'])
 
-        value = _header_strip(self.headers.get('Content-MD5'))
-        if value is not None:
-            if not re.match('^[A-Za-z0-9+/]+={0,2}$', value):
-                # Non-base64-alphabet characters in value.
-                raise InvalidDigest(content_md5=value)
-            try:
-                self.headers['ETag'] = binascii.b2a_hex(
-                    binascii.a2b_base64(value))
-            except binascii.error:
-                # incorrect padding, most likely
-                raise InvalidDigest(content_md5=value)
-
-            if len(self.headers['ETag']) != 32:
-                raise InvalidDigest(content_md5=value)
-
         if self.method == 'PUT' and any(h in self.headers for h in (
                 'If-Match', 'If-None-Match',
                 'If-Modified-Since', 'If-Unmodified-Since')):
@@ -782,7 +848,6 @@ class S3Request(swob.Request):
                 raise InvalidArgument('x-amz-copy-source',
                                       self.headers['X-Amz-Copy-Source'],
                                       msg)
-
         if 'x-amz-metadata-directive' in self.headers:
             value = self.headers['x-amz-metadata-directive']
             if value not in ('COPY', 'REPLACE'):
@@ -813,6 +878,37 @@ class S3Request(swob.Request):
         if 'x-amz-website-redirect-location' in self.headers:
             raise S3NotImplemented('Website redirection is not supported.')
 
+        aws_sha256 = self._validate_sha256()
+        if (aws_sha256
+                and aws_sha256 != 'UNSIGNED-PAYLOAD'
+                and self.content_length is not None):
+            # Even if client-provided SHA doesn't look like a SHA, wrap the
+            # input anyway so we'll send the SHA of what the client sent in
+            # the eventual error
+            self.environ['wsgi.input'] = HashingInput(
+                self.environ['wsgi.input'],
+                self.content_length,
+                aws_sha256)
+        # If no content-length, either client's trying to do a HTTP chunked
+        # transfer, or a HTTP/1.0-style transfer (in which case swift will
+        # reject with length-required and we'll translate back to
+        # MissingContentLength)
+
+        value = _header_strip(self.headers.get('Content-MD5'))
+        if value is not None:
+            if not re.match('^[A-Za-z0-9+/]+={0,2}$', value):
+                # Non-base64-alphabet characters in value.
+                raise InvalidDigest(content_md5=value)
+            try:
+                self.headers['ETag'] = binascii.b2a_hex(
+                    binascii.a2b_base64(value))
+            except binascii.Error:
+                # incorrect padding, most likely
+                raise InvalidDigest(content_md5=value)
+
+            if len(self.headers['ETag']) != 32:
+                raise InvalidDigest(content_md5=value)
+
         # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
         # describes some of what would be required to support this
         if any(['aws-chunked' in self.headers.get('content-encoding', ''),
@@ -852,7 +948,13 @@ class S3Request(swob.Request):
 
         if te or ml:
             # Limit the read similar to how SLO handles manifests
-            body = self.body_file.read(max_length)
+            try:
+                body = self.body_file.read(max_length)
+            except S3InputSHA256Mismatch as err:
+                raise XAmzContentSHA256Mismatch(
+                    client_computed_content_s_h_a256=err.expected,
+                    s3_computed_content_s_h_a256=err.computed,
+                )
         else:
             # No (or zero) Content-Length provided, and not chunked transfer;
             # no body. Assume zero-length, and enforce a required body below.
@@ -865,7 +967,8 @@ class S3Request(swob.Request):
             raise InvalidRequest('Missing required header for this request: '
                                  'Content-MD5')
 
-        digest = base64.b64encode(md5(body).digest()).strip().decode('ascii')
+        digest = base64.b64encode(md5(
+            body, usedforsecurity=False).digest()).strip().decode('ascii')
         if self.environ['HTTP_CONTENT_MD5'] != digest:
             raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5'])
 
@@ -889,20 +992,16 @@ class S3Request(swob.Request):
         except KeyError:
             return None
 
-        if '?' in src_path:
-            src_path, qs = src_path.split('?', 1)
-            query = parse_qsl(qs, True)
-            if not query:
-                pass  # ignore it
-            elif len(query) > 1 or query[0][0] != 'versionId':
-                raise InvalidArgument('X-Amz-Copy-Source',
-                                      self.headers['X-Amz-Copy-Source'],
-                                      'Unsupported copy source parameter.')
-            elif query[0][1] != 'null':
-                # TODO: once we support versioning, we'll need to translate
-                # src_path to the proper location in the versions container
-                raise S3NotImplemented('Versioning is not yet supported')
-            self.headers['X-Amz-Copy-Source'] = src_path
+        src_path, qs = src_path.partition('?')[::2]
+        parsed = parse_qsl(qs, True)
+        if not parsed:
+            query = {}
+        elif len(parsed) == 1 and parsed[0][0] == 'versionId':
+            query = {'version-id': parsed[0][1]}
+        else:
+            raise InvalidArgument('X-Amz-Copy-Source',
+                                  self.headers['X-Amz-Copy-Source'],
+                                  'Unsupported copy source parameter.')
 
         src_path = unquote(src_path)
         src_path = src_path if src_path.startswith('/') else ('/' + src_path)
@@ -911,20 +1010,17 @@ class S3Request(swob.Request):
         headers = swob.HeaderKeyDict()
         headers.update(self._copy_source_headers())
 
-        src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj,
-                                     headers=headers)
+        src_resp = self.get_response(app, 'HEAD', src_bucket,
+                                     swob.str_to_wsgi(src_obj),
+                                     headers=headers, query=query)
         if src_resp.status_int == 304:  # pylint: disable-msg=E1101
             raise PreconditionFailed()
 
-        self.headers['X-Amz-Copy-Source'] = \
-            '/' + self.headers['X-Amz-Copy-Source'].lstrip('/')
-        source_container, source_obj = \
-            split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True)
-
-        if (self.container_name == source_container and
-                self.object_name == source_obj and
+        if (self.container_name == src_bucket and
+                self.object_name == src_obj and
                 self.headers.get('x-amz-metadata-directive',
-                                 'COPY') == 'COPY'):
+                                 'COPY') == 'COPY' and
+                not query):
             raise InvalidRequest("This copy request is illegal "
                                  "because it is trying to copy an "
                                  "object to itself without "
@@ -932,6 +1028,12 @@ class S3Request(swob.Request):
                                  "storage class, website redirect "
                                  "location or encryption "
                                  "attributes.")
+        # We've done some normalizing; write back so it's ready for
+        # to_swift_req
+        self.headers['X-Amz-Copy-Source'] = quote(src_path)
+        if query:
+            self.headers['X-Amz-Copy-Source'] += \
+                '?versionId=' + query['version-id']
         return src_resp
 
     def _canonical_uri(self):
@@ -979,7 +1081,7 @@ class S3Request(swob.Request):
         else:
             # Should have already raised NotS3Request in _parse_auth_info,
             # but as a sanity check...
-            raise AccessDenied()
+            raise AccessDenied(reason='not_s3')
 
         for key, value in sorted(amz_headers.items()):
             buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value)))
@@ -1018,7 +1120,7 @@ class S3Request(swob.Request):
         if self.is_service_request:
             return ServiceController
 
-        if not self.slo_enabled:
+        if not self.conf.allow_multipart_uploads:
             multi_part = ['partNumber', 'uploadId', 'uploads']
             if len([p for p in multi_part if p in self.params]):
                 raise S3NotImplemented("Multi-part feature isn't support")
@@ -1032,16 +1134,23 @@ class S3Request(swob.Request):
         if 'logging' in self.params:
             return LoggingStatusController
         if 'partNumber' in self.params:
-            return PartController
+            if self.method == 'PUT':
+                return PartController
+            else:
+                return ObjectController
         if 'uploadId' in self.params:
             return UploadController
         if 'uploads' in self.params:
             return UploadsController
         if 'versioning' in self.params:
             return VersioningController
+        if 'tagging' in self.params:
+            return TaggingController
+        if 'object-lock' in self.params:
+            return ObjectLockController
 
         unsupported = ('notification', 'policy', 'requestPayment', 'torrent',
-                       'website', 'cors', 'tagging', 'restore')
+                       'website', 'cors', 'restore')
         if set(unsupported) & set(self.params):
             return UnsupportedController
 
@@ -1071,11 +1180,12 @@ class S3Request(swob.Request):
         Create a Swift request based on this request's environment.
         """
         if self.account is None:
-            account = self.access_key
+            account = swob.str_to_wsgi(self.access_key)
         else:
             account = self.account
 
         env = self.environ.copy()
+        env['swift.infocache'] = self.environ.setdefault('swift.infocache', {})
 
         def sanitize(value):
             if set(value).issubset(string.printable):
@@ -1121,8 +1231,10 @@ class S3Request(swob.Request):
                 env['HTTP_X_OBJECT_META_' + key[16:]] = sanitize(env[key])
                 del env[key]
 
-        if 'HTTP_X_AMZ_COPY_SOURCE' in env:
-            env['HTTP_X_COPY_FROM'] = env['HTTP_X_AMZ_COPY_SOURCE']
+        copy_from_version_id = ''
+        if 'HTTP_X_AMZ_COPY_SOURCE' in env and env['REQUEST_METHOD'] == 'PUT':
+            env['HTTP_X_COPY_FROM'], copy_from_version_id = env[
+                'HTTP_X_AMZ_COPY_SOURCE'].partition('?versionId=')[::2]
             del env['HTTP_X_AMZ_COPY_SOURCE']
             env['CONTENT_LENGTH'] = '0'
             if env.pop('HTTP_X_AMZ_METADATA_DIRECTIVE', None) == 'REPLACE':
@@ -1141,7 +1253,7 @@ class S3Request(swob.Request):
                     if key.startswith('HTTP_X_OBJECT_META_'):
                         del env[key]
 
-        if self.force_request_log:
+        if self.conf.force_swift_request_proxy_log:
             env['swift.proxy_access_log_made'] = False
         env['swift.source'] = 'S3'
         if method is not None:
@@ -1155,16 +1267,16 @@ class S3Request(swob.Request):
             path = '/v1/%s' % (account)
         env['PATH_INFO'] = path
 
-        query_string = ''
+        params = []
         if query is not None:
-            params = []
             for key, value in sorted(query.items()):
                 if value is not None:
                     params.append('%s=%s' % (key, quote(str(value))))
                 else:
                     params.append(key)
-            query_string = '&'.join(params)
-        env['QUERY_STRING'] = query_string
+        if copy_from_version_id and not (query and query.get('version-id')):
+            params.append('version-id=' + copy_from_version_id)
+        env['QUERY_STRING'] = '&'.join(params)
 
         return swob.Request.blank(quote(path), environ=env, body=body,
                                   headers=headers)
@@ -1230,7 +1342,7 @@ class S3Request(swob.Request):
 
     def _bucket_put_accepted_error(self, container, app):
         sw_req = self.to_swift_req('HEAD', container, None)
-        info = get_container_info(sw_req.environ, app)
+        info = get_container_info(sw_req.environ, app, swift_source='S3')
         sysmeta = info.get('sysmeta', {})
         try:
             acl = json.loads(sysmeta.get('s3api-acl',
@@ -1288,6 +1400,16 @@ class S3Request(swob.Request):
                     return NoSuchKey(obj)
                 return NoSuchBucket(container)
 
+            # Since BadDigest ought to plumb in some client-provided values,
+            # defer evaluation until we know they're provided
+            def bad_digest_handler():
+                etag = binascii.hexlify(base64.b64decode(
+                    env['HTTP_CONTENT_MD5']))
+                return BadDigest(
+                    expected_digest=etag,  # yes, really hex
+                    # TODO: plumb in calculated_digest, as b64
+                )
+
             code_map = {
                 'HEAD': {
                     HTTP_NOT_FOUND: not_found_handler,
@@ -1296,14 +1418,15 @@ class S3Request(swob.Request):
                 'GET': {
                     HTTP_NOT_FOUND: not_found_handler,
                     HTTP_PRECONDITION_FAILED: PreconditionFailed,
-                    HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: InvalidRange,
                 },
                 'PUT': {
                     HTTP_NOT_FOUND: (NoSuchBucket, container),
-                    HTTP_UNPROCESSABLE_ENTITY: BadDigest,
+                    HTTP_UNPROCESSABLE_ENTITY: bad_digest_handler,
                     HTTP_REQUEST_ENTITY_TOO_LARGE: EntityTooLarge,
                     HTTP_LENGTH_REQUIRED: MissingContentLength,
                     HTTP_REQUEST_TIMEOUT: RequestTimeout,
+                    HTTP_PRECONDITION_FAILED: PreconditionFailed,
+                    HTTP_CLIENT_CLOSED_REQUEST: RequestTimeout,
                 },
                 'POST': {
                     HTTP_NOT_FOUND: not_found_handler,
@@ -1335,13 +1458,25 @@ class S3Request(swob.Request):
 
         try:
             sw_resp = sw_req.get_response(app)
-        except swob.HTTPException as err:
-            sw_resp = err
+        except S3InputSHA256Mismatch as err:
+            # hopefully by now any modifications to the path (e.g. tenant to
+            # account translation) will have been made by auth middleware
+            self.environ['s3api.backend_path'] = sw_req.environ['PATH_INFO']
+            raise XAmzContentSHA256Mismatch(
+                client_computed_content_s_h_a256=err.expected,
+                s3_computed_content_s_h_a256=err.computed,
+            )
         else:
             # reuse account
             _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
                                             2, 3, True)
+            # Update s3.backend_path from the response environ
+            self.environ['s3api.backend_path'] = sw_resp.environ['PATH_INFO']
 
+        # keep a record of the backend policy index so that the s3api can add
+        # it to the headers of whatever response it returns, which may not
+        # necessarily be this resp.
+        self.policy_index = get_policy_index(sw_req.headers, sw_resp.headers)
         resp = S3Response.from_swift_resp(sw_resp)
         status = resp.status_int  # pylint: disable-msg=E1101
 
@@ -1351,8 +1486,6 @@ class S3Request(swob.Request):
                 self.user_id = "%s:%s" % (
                     sw_resp.environ['HTTP_X_TENANT_NAME'],
                     sw_resp.environ['HTTP_X_USER_NAME'])
-                if six.PY2 and not isinstance(self.user_id, bytes):
-                    self.user_id = self.user_id.encode('utf8')
             else:
                 # tempauth
                 self.user_id = self.access_key
@@ -1371,20 +1504,43 @@ class S3Request(swob.Request):
                 error_codes[sw_resp.status_int]  # pylint: disable-msg=E1101
             if isinstance(err_resp, tuple):
                 raise err_resp[0](*err_resp[1:])
+            elif b'quota' in err_msg:
+                raise err_resp(err_msg)
             else:
                 raise err_resp()
 
         if status == HTTP_BAD_REQUEST:
-            raise BadSwiftRequest(err_msg.decode('utf8'))
+            err_str = err_msg.decode('utf8')
+            if 'X-Delete-At' in err_str:
+                raise InvalidArgument('X-Delete-At',
+                                      self.headers['X-Delete-At'],
+                                      err_str)
+            if 'X-Delete-After' in err_str:
+                raise InvalidArgument('X-Delete-After',
+                                      self.headers['X-Delete-After'],
+                                      err_str)
+            else:
+                raise InvalidRequest(msg=err_str)
         if status == HTTP_UNAUTHORIZED:
             raise SignatureDoesNotMatch(
                 **self.signature_does_not_match_kwargs())
         if status == HTTP_FORBIDDEN:
-            raise AccessDenied()
+            raise AccessDenied(reason='forbidden')
+        if status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
+            self.validate_part_number(
+                parts_count=resp.headers.get('x-amz-mp-parts-count'))
+            raise InvalidRange()
         if status == HTTP_SERVICE_UNAVAILABLE:
             raise ServiceUnavailable()
         if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):
+            if self.conf.ratelimit_as_client_error:
+                raise SlowDown(status='429 Slow Down')
             raise SlowDown()
+        if resp.status_int == HTTP_CONFLICT:
+            if self.method == 'GET':
+                raise BrokenMPU()
+            else:
+                raise ServiceUnavailable()
 
         raise InternalError('unexpected status code %d' % status)
 
@@ -1439,34 +1595,49 @@ class S3Request(swob.Request):
         :raises: NoSuchBucket when the container doesn't exist
         :raises: InternalError when the request failed without 404
         """
-        if self.is_authenticated:
-            # if we have already authenticated, yes we can use the account
-            # name like as AUTH_xxx for performance efficiency
-            sw_req = self.to_swift_req(app, self.container_name, None)
-            info = get_container_info(sw_req.environ, app)
-            if is_success(info['status']):
-                return info
-            elif info['status'] == 404:
-                raise NoSuchBucket(self.container_name)
-            else:
-                raise InternalError(
-                    'unexpected status code %d' % info['status'])
+        if not self.is_authenticated:
+            sw_req = self.to_swift_req('TEST', None, None, body='')
+            # don't show log message of this request
+            sw_req.environ['swift.proxy_access_log_made'] = True
+
+            sw_resp = sw_req.get_response(app)
+
+            if not sw_req.remote_user:
+                raise SignatureDoesNotMatch(
+                    **self.signature_does_not_match_kwargs())
+
+            _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
+                                            2, 3, True)
+        sw_req = self.to_swift_req('TEST', self.container_name, None)
+        info = get_container_info(sw_req.environ, app, swift_source='S3')
+        if is_success(info['status']):
+            return info
+        elif info['status'] == HTTP_NOT_FOUND:
+            raise NoSuchBucket(self.container_name)
+        elif info['status'] == HTTP_SERVICE_UNAVAILABLE:
+            raise ServiceUnavailable()
         else:
-            # otherwise we do naive HEAD request with the authentication
-            resp = self.get_response(app, 'HEAD', self.container_name, '')
-            headers = resp.sw_headers.copy()
-            headers.update(resp.sysmeta_headers)
-            return headers_to_container_info(
-                headers, resp.status_int)  # pylint: disable-msg=E1101
-
-    def gen_multipart_manifest_delete_query(self, app, obj=None):
-        if not self.allow_multipart_uploads:
-            return None
-        query = {'multipart-manifest': 'delete'}
+            raise InternalError(
+                'unexpected status code %d' % info['status'])
+
+    def gen_multipart_manifest_delete_query(self, app, obj=None, version=None):
+        if not self.conf.allow_multipart_uploads:
+            return {}
         if not obj:
             obj = self.object_name
-        resp = self.get_response(app, 'HEAD', obj=obj)
-        return query if resp.is_slo else None
+        query = {'symlink': 'get'}
+        if version is not None:
+            query['version-id'] = version
+        resp = self.get_response(app, 'HEAD', obj=obj, query=query)
+        if not resp.is_slo:
+            return {}
+        elif resp.sysmeta_headers.get(sysmeta_header('object', 'etag')):
+            # Even if allow_async_delete is turned off, SLO will just handle
+            # the delete synchronously, so we don't need to check before
+            # setting async=on
+            return {'multipart-manifest': 'delete', 'async': 'on'}
+        else:
+            return {'multipart-manifest': 'delete'}
 
     def set_acl_handler(self, handler):
         pass
@@ -1476,14 +1647,9 @@ class S3AclRequest(S3Request):
     """
     S3Acl request object.
     """
-    def __init__(self, env, app, slo_enabled=True, storage_domain='',
-                 location='us-east-1', force_request_log=False,
-                 dns_compliant_bucket_names=True, allow_multipart_uploads=True,
-                 allow_no_owner=False):
-        super(S3AclRequest, self).__init__(
-            env, app, slo_enabled, storage_domain, location, force_request_log,
-            dns_compliant_bucket_names, allow_multipart_uploads)
-        self.allow_no_owner = allow_no_owner
+
+    def __init__(self, env, app=None, conf=None):
+        super(S3AclRequest, self).__init__(env, app, conf)
         self.authenticate(app)
         self.acl_handler = None
 
@@ -1517,14 +1683,14 @@ class S3AclRequest(S3Request):
             # keystone
             self.user_id = "%s:%s" % (sw_resp.environ['HTTP_X_TENANT_NAME'],
                                       sw_resp.environ['HTTP_X_USER_NAME'])
-            if six.PY2 and not isinstance(self.user_id, bytes):
-                self.user_id = self.user_id.encode('utf8')
         else:
             # tempauth
             self.user_id = self.access_key
 
         sw_req.environ.get('swift.authorize', lambda req: None)(sw_req)
         self.environ['swift_owner'] = sw_req.environ.get('swift_owner', False)
+        if 'REMOTE_USER' in sw_req.environ:
+            self.environ['REMOTE_USER'] = sw_req.environ['REMOTE_USER']
 
         # Need to skip S3 authorization on subsequent requests to prevent
         # overwriting the account in PATH_INFO
@@ -1550,9 +1716,9 @@ class S3AclRequest(S3Request):
         resp = self._get_response(
             app, method, container, obj, headers, body, query)
         resp.bucket_acl = decode_acl(
-            'container', resp.sysmeta_headers, self.allow_no_owner)
+            'container', resp.sysmeta_headers, self.conf.allow_no_owner)
         resp.object_acl = decode_acl(
-            'object', resp.sysmeta_headers, self.allow_no_owner)
+            'object', resp.sysmeta_headers, self.conf.allow_no_owner)
 
         return resp