swift 2.33.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/common/middleware/s3api/s3request.py

@@ -56,7 +56,8 @@ from swift.common.middleware.s3api.s3response import AccessDenied, \
     MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \
     MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \
     BadDigest, AuthorizationHeaderMalformed, SlowDown, \
-    AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU
+    AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU, \
+    InvalidPartNumber, InvalidPartArgument, XAmzContentSHA256Mismatch
 from swift.common.middleware.s3api.exception import NotS3Request
 from swift.common.middleware.s3api.utils import utf8encode, \
     S3Timestamp, mktime, MULTIUPLOAD_SUFFIX
@@ -119,6 +120,20 @@ def _header_acl_property(resource):
                     doc='Get and set the %s acl property' % resource)
 
 
+class S3InputSHA256Mismatch(BaseException):
+    """
+    Client provided a X-Amz-Content-SHA256, but it doesn't match the data.
+
+    Inherit from BaseException (rather than Exception) so it cuts from the
+    proxy-server app (which will presumably be the one reading the input)
+    through all the layers of the pipeline back to us. It should never escape
+    the s3api middleware.
+    """
+    def __init__(self, expected, computed):
+        self.expected = expected
+        self.computed = computed
+
+
 class HashingInput(object):
     """
     wsgi.input wrapper to verify the hash of the input as it's read.
@@ -129,6 +144,13 @@ class HashingInput(object):
         self._to_read = content_length
         self._hasher = hasher()
         self._expected = expected_hex_hash
+        if content_length == 0 and \
+                self._hasher.hexdigest() != self._expected.lower():
+            self.close()
+            raise XAmzContentSHA256Mismatch(
+                client_computed_content_s_h_a256=self._expected,
+                s3_computed_content_s_h_a256=self._hasher.hexdigest(),
+            )
 
     def read(self, size=None):
         chunk = self._input.read(size)
@@ -137,12 +159,12 @@ class HashingInput(object):
         short_read = bool(chunk) if size is None else (len(chunk) < size)
         if self._to_read < 0 or (short_read and self._to_read) or (
                 self._to_read == 0 and
-                self._hasher.hexdigest() != self._expected):
+                self._hasher.hexdigest() != self._expected.lower()):
             self.close()
             # Since we don't return the last chunk, the PUT never completes
-            raise swob.HTTPUnprocessableEntity(
-                'The X-Amz-Content-SHA56 you specified did not match '
-                'what we received.')
+            raise S3InputSHA256Mismatch(
+                self._expected,
+                self._hasher.hexdigest())
         return chunk
 
     def close(self):
@@ -237,6 +259,28 @@ class SigV4Mixin(object):
         if int(self.timestamp) + expires < S3Timestamp.now():
             raise AccessDenied('Request has expired', reason='expired')
 
+    def _validate_sha256(self):
+        aws_sha256 = self.headers.get('x-amz-content-sha256')
+        looks_like_sha256 = (
+            aws_sha256 and len(aws_sha256) == 64 and
+            all(c in '0123456789abcdef' for c in aws_sha256.lower()))
+        if not aws_sha256:
+            if 'X-Amz-Credential' in self.params:
+                pass  # pre-signed URL; not required
+            else:
+                msg = 'Missing required header for this request: ' \
+                      'x-amz-content-sha256'
+                raise InvalidRequest(msg)
+        elif aws_sha256 == 'UNSIGNED-PAYLOAD':
+            pass
+        elif not looks_like_sha256 and 'X-Amz-Credential' not in self.params:
+            raise InvalidArgument(
+                'x-amz-content-sha256',
+                aws_sha256,
+                'x-amz-content-sha256 must be UNSIGNED-PAYLOAD, or '
+                'a valid sha256 value.')
+        return aws_sha256
+
     def _parse_credential(self, credential_string):
         parts = credential_string.split("/")
         # credential must be in following format:
@@ -447,30 +491,9 @@ class SigV4Mixin(object):
         cr.append(b';'.join(swob.wsgi_to_bytes(k) for k, v in headers_to_sign))
 
         # 6. Add payload string at the tail
-        if 'X-Amz-Credential' in self.params:
-            # V4 with query parameters only
-            hashed_payload = 'UNSIGNED-PAYLOAD'
-        elif 'X-Amz-Content-SHA256' not in self.headers:
-            msg = 'Missing required header for this request: ' \
-                  'x-amz-content-sha256'
-            raise InvalidRequest(msg)
-        else:
-            hashed_payload = self.headers['X-Amz-Content-SHA256']
-            if hashed_payload != 'UNSIGNED-PAYLOAD':
-                if self.content_length == 0:
-                    if hashed_payload.lower() != sha256().hexdigest():
-                        raise BadDigest(
-                            'The X-Amz-Content-SHA56 you specified did not '
-                            'match what we received.')
-                elif self.content_length:
-                    self.environ['wsgi.input'] = HashingInput(
-                        self.environ['wsgi.input'],
-                        self.content_length,
-                        sha256,
-                        hashed_payload.lower())
-                # else, length not provided -- Swift will kick out a
-                # 411 Length Required which will get translated back
-                # to a S3-style response in S3Request._swift_error_codes
+        hashed_payload = self.headers.get('X-Amz-Content-SHA256',
+                                          'UNSIGNED-PAYLOAD')
+
         cr.append(swob.wsgi_to_bytes(hashed_payload))
         return b'\n'.join(cr)
 
@@ -558,6 +581,57 @@ class S3Request(swob.Request):
         # by full URL when absolute path given. See swift.swob for more detail.
         self.environ['swift.leave_relative_location'] = True
 
+    def validate_part_number(self, parts_count=None, check_max=True):
+        """
+        Get the partNumber param, if it exists, and check it is valid.
+
+        To be valid, a partNumber must satisfy two criteria. First, it must be
+        an integer between 1 and the maximum allowed parts, inclusive. The
+        maximum allowed parts is the maximum of the configured
+        ``max_upload_part_num`` and, if given, ``parts_count``. Second, the
+        partNumber must be less than or equal to the ``parts_count``, if it is
+        given.
+
+        :param parts_count: if given, this is the number of parts in an
+            existing object.
+        :raises InvalidPartArgument: if the partNumber param is invalid i.e.
+            less than 1 or greater than the maximum allowed parts.
+        :raises InvalidPartNumber: if the partNumber param is valid but greater
+            than ``num_parts``.
+        :return: an integer part number if the partNumber param exists,
+            otherwise ``None``.
+        """
+        part_number = self.params.get('partNumber')
+        if part_number is None:
+            return None
+
+        if self.range:
+            raise InvalidRequest('Cannot specify both Range header and '
+                                 'partNumber query parameter')
+
+        try:
+            parts_count = int(parts_count)
+        except (TypeError, ValueError):
+            # an invalid/empty param is treated like parts_count=max_parts
+            parts_count = self.conf.max_upload_part_num
+        # max_parts may be raised to the number of existing parts
+        max_parts = max(self.conf.max_upload_part_num, parts_count)
+
+        try:
+            part_number = int(part_number)
+            if part_number < 1:
+                raise ValueError
+        except ValueError:
+            raise InvalidPartArgument(max_parts, part_number)  # 400
+
+        if check_max:
+            if part_number > max_parts:
+                raise InvalidPartArgument(max_parts, part_number)  # 400
+            if part_number > parts_count:
+                raise InvalidPartNumber()  # 416
+
+        return part_number
+
     def check_signature(self, secret):
         secret = utf8encode(secret)
         user_signature = self.signature
@@ -747,6 +821,9 @@ class S3Request(swob.Request):
         if delta > self.conf.allowable_clock_skew:
             raise RequestTimeTooSkewed()
 
+    def _validate_sha256(self):
+        return self.headers.get('x-amz-content-sha256')
+
     def _validate_headers(self):
         if 'CONTENT_LENGTH' in self.environ:
             try:
@@ -757,21 +834,6 @@ class S3Request(swob.Request):
                 raise InvalidArgument('Content-Length',
                                       self.environ['CONTENT_LENGTH'])
 
-        value = _header_strip(self.headers.get('Content-MD5'))
-        if value is not None:
-            if not re.match('^[A-Za-z0-9+/]+={0,2}$', value):
-                # Non-base64-alphabet characters in value.
-                raise InvalidDigest(content_md5=value)
-            try:
-                self.headers['ETag'] = binascii.b2a_hex(
-                    binascii.a2b_base64(value))
-            except binascii.Error:
-                # incorrect padding, most likely
-                raise InvalidDigest(content_md5=value)
-
-            if len(self.headers['ETag']) != 32:
-                raise InvalidDigest(content_md5=value)
-
         if self.method == 'PUT' and any(h in self.headers for h in (
                 'If-Match', 'If-None-Match',
                 'If-Modified-Since', 'If-Unmodified-Since')):
@@ -817,6 +879,38 @@ class S3Request(swob.Request):
         if 'x-amz-website-redirect-location' in self.headers:
             raise S3NotImplemented('Website redirection is not supported.')
 
+        aws_sha256 = self._validate_sha256()
+        if (aws_sha256
+                and aws_sha256 != 'UNSIGNED-PAYLOAD'
+                and self.content_length is not None):
+            # Even if client-provided SHA doesn't look like a SHA, wrap the
+            # input anyway so we'll send the SHA of what the client sent in
+            # the eventual error
+            self.environ['wsgi.input'] = HashingInput(
+                self.environ['wsgi.input'],
+                self.content_length,
+                sha256,
+                aws_sha256)
+        # If no content-length, either client's trying to do a HTTP chunked
+        # transfer, or a HTTP/1.0-style transfer (in which case swift will
+        # reject with length-required and we'll translate back to
+        # MissingContentLength)
+
+        value = _header_strip(self.headers.get('Content-MD5'))
+        if value is not None:
+            if not re.match('^[A-Za-z0-9+/]+={0,2}$', value):
+                # Non-base64-alphabet characters in value.
+                raise InvalidDigest(content_md5=value)
+            try:
+                self.headers['ETag'] = binascii.b2a_hex(
+                    binascii.a2b_base64(value))
+            except binascii.Error:
+                # incorrect padding, most likely
+                raise InvalidDigest(content_md5=value)
+
+            if len(self.headers['ETag']) != 32:
+                raise InvalidDigest(content_md5=value)
+
         # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
         # describes some of what would be required to support this
         if any(['aws-chunked' in self.headers.get('content-encoding', ''),
@@ -858,13 +952,11 @@ class S3Request(swob.Request):
             # Limit the read similar to how SLO handles manifests
             try:
                 body = self.body_file.read(max_length)
-            except swob.HTTPException as err:
-                if err.status_int == HTTP_UNPROCESSABLE_ENTITY:
-                    # Special case for HashingInput check
-                    raise BadDigest(
-                        'The X-Amz-Content-SHA56 you specified did not '
-                        'match what we received.')
-                raise
+            except S3InputSHA256Mismatch as err:
+                raise XAmzContentSHA256Mismatch(
+                    client_computed_content_s_h_a256=err.expected,
+                    s3_computed_content_s_h_a256=err.computed,
+                )
         else:
             # No (or zero) Content-Length provided, and not chunked transfer;
             # no body. Assume zero-length, and enforce a required body below.
@@ -1044,7 +1136,10 @@ class S3Request(swob.Request):
         if 'logging' in self.params:
             return LoggingStatusController
         if 'partNumber' in self.params:
-            return PartController
+            if self.method == 'PUT':
+                return PartController
+            else:
+                return ObjectController
         if 'uploadId' in self.params:
             return UploadController
         if 'uploads' in self.params:
@@ -1087,7 +1182,7 @@ class S3Request(swob.Request):
         Create a Swift request based on this request's environment.
         """
         if self.account is None:
-            account = self.access_key
+            account = swob.str_to_wsgi(self.access_key)
         else:
             account = self.account
 
@@ -1307,6 +1402,16 @@ class S3Request(swob.Request):
                     return NoSuchKey(obj)
                 return NoSuchBucket(container)
 
+            # Since BadDigest ought to plumb in some client-provided values,
+            # defer evaluation until we know they're provided
+            def bad_digest_handler():
+                etag = binascii.hexlify(base64.b64decode(
+                    env['HTTP_CONTENT_MD5']))
+                return BadDigest(
+                    expected_digest=etag,  # yes, really hex
+                    # TODO: plumb in calculated_digest, as b64
+                )
+
             code_map = {
                 'HEAD': {
                     HTTP_NOT_FOUND: not_found_handler,
@@ -1315,11 +1420,10 @@ class S3Request(swob.Request):
                 'GET': {
                     HTTP_NOT_FOUND: not_found_handler,
                     HTTP_PRECONDITION_FAILED: PreconditionFailed,
-                    HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: InvalidRange,
                 },
                 'PUT': {
                     HTTP_NOT_FOUND: (NoSuchBucket, container),
-                    HTTP_UNPROCESSABLE_ENTITY: BadDigest,
+                    HTTP_UNPROCESSABLE_ENTITY: bad_digest_handler,
                     HTTP_REQUEST_ENTITY_TOO_LARGE: EntityTooLarge,
                     HTTP_LENGTH_REQUIRED: MissingContentLength,
                     HTTP_REQUEST_TIMEOUT: RequestTimeout,
@@ -1356,13 +1460,14 @@ class S3Request(swob.Request):
 
         try:
             sw_resp = sw_req.get_response(app)
-        except swob.HTTPException as err:
-            # Maybe a 422 from HashingInput? Put something in
-            # s3api.backend_path - hopefully by now any modifications to the
-            # path (e.g. tenant to account translation) will have been made by
-            # auth middleware
+        except S3InputSHA256Mismatch as err:
+            # hopefully by now any modifications to the path (e.g. tenant to
+            # account translation) will have been made by auth middleware
             self.environ['s3api.backend_path'] = sw_req.environ['PATH_INFO']
-            sw_resp = err
+            raise XAmzContentSHA256Mismatch(
+                client_computed_content_s_h_a256=err.expected,
+                s3_computed_content_s_h_a256=err.computed,
+            )
         else:
             # reuse account
             _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
@@ -1414,7 +1519,7 @@ class S3Request(swob.Request):
                 raise InvalidArgument('X-Delete-At',
                                       self.headers['X-Delete-At'],
                                       err_str)
-            if 'X-Delete-After' in err_msg.decode('utf8'):
+            if 'X-Delete-After' in err_str:
                 raise InvalidArgument('X-Delete-After',
                                       self.headers['X-Delete-After'],
                                       err_str)
@@ -1425,6 +1530,10 @@ class S3Request(swob.Request):
                 **self.signature_does_not_match_kwargs())
         if status == HTTP_FORBIDDEN:
             raise AccessDenied(reason='forbidden')
+        if status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
+            self.validate_part_number(
+                parts_count=resp.headers.get('x-amz-mp-parts-count'))
+            raise InvalidRange()
         if status == HTTP_SERVICE_UNAVAILABLE:
             raise ServiceUnavailable()
         if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS):

swift/common/middleware/s3api/s3response.py

@@ -72,6 +72,8 @@ def translate_swift_to_s3(key, val):
         return key, val
     elif _key == 'x-object-version-id':
         return 'x-amz-version-id', val
+    elif _key == 'x-parts-count':
+        return 'x-amz-mp-parts-count', val
     elif _key == 'x-copied-from-version-id':
         return 'x-amz-copy-source-version-id', val
     elif _key == 'x-backend-content-type' and \
@@ -250,12 +252,17 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
         self.headers = HeaderKeyDict(self.headers)
 
     @property
-    def metric_name(self):
-        parts = [str(self.status_int), self._code]
+    def summary(self):
+        """Provide a summary of the error code and reason."""
         if self.reason:
-            parts.append(self.reason)
-        metric = '.'.join(parts)
-        return metric.replace(' ', '_')
+            summary = '.'.join([self._code, self.reason])
+        else:
+            summary = self._code
+        return summary.replace(' ', '_')
+
+    @property
+    def metric_name(self):
+        return '.'.join([str(self.status_int), self.summary])
 
     def _body_iter(self):
         error_elem = Element('Error')
@@ -320,6 +327,12 @@ class BadDigest(ErrorResponse):
     _msg = 'The Content-MD5 you specified did not match what we received.'
 
 
+class XAmzContentSHA256Mismatch(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = "The provided 'x-amz-content-sha256' header does not match what " \
+           "was computed."
+
+
 class BucketAlreadyExists(ErrorResponse):
     _status = '409 Conflict'
     _msg = 'The requested bucket name is not available. The bucket ' \
@@ -436,7 +449,7 @@ class InvalidBucketState(ErrorResponse):
 
 class InvalidDigest(ErrorResponse):
     _status = '400 Bad Request'
-    _msg = 'The Content-MD5 you specified was an invalid.'
+    _msg = 'The Content-MD5 you specified was invalid.'
 
 
 class InvalidLocationConstraint(ErrorResponse):
@@ -449,6 +462,17 @@ class InvalidObjectState(ErrorResponse):
     _msg = 'The operation is not valid for the current state of the object.'
 
 
+class InvalidPartArgument(InvalidArgument):
+    _code = 'InvalidArgument'
+
+    def __init__(self, max_parts, value):
+        err_msg = ('Part number must be an integer between '
+                   '1 and %s, inclusive' % max_parts)
+        super(InvalidArgument, self).__init__(err_msg,
+                                              argument_name='partNumber',
+                                              argument_value=value)
+
+
 class InvalidPart(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'One or more of the specified parts could not be found. The part ' \
@@ -478,6 +502,11 @@ class InvalidRange(ErrorResponse):
     _msg = 'The requested range cannot be satisfied.'
 
 
+class InvalidPartNumber(ErrorResponse):
+    _status = '416 Requested Range Not Satisfiable'
+    _msg = 'The requested partnumber is not satisfiable'
+
+
 class InvalidRequest(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'Invalid Request.'

swift/common/middleware/s3api/s3token.py

@@ -65,7 +65,7 @@ import six
 from six.moves import urllib
 
 from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
-    HTTPException
+    HTTPException, str_to_wsgi
 from swift.common.utils import config_true_value, split_path, get_logger, \
     cache_from_env, append_underscore
 from swift.common.wsgi import ConfigFileError
@@ -404,7 +404,7 @@ class S3Token(object):
         self._logger.debug('Connecting with tenant: %s', tenant_to_connect)
         new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect)
         environ['PATH_INFO'] = environ['PATH_INFO'].replace(
-            account, new_tenant_name, 1)
+            str_to_wsgi(account), str_to_wsgi(new_tenant_name), 1)
         return self._app(environ, start_response)
 
 

swift/common/middleware/s3api/utils.py

@@ -172,6 +172,7 @@ class Config(dict):
         'allow_no_owner': False,
         'allowable_clock_skew': 900,
         'ratelimit_as_client_error': False,
+        'max_upload_part_num': 1000,
     }
 
     def __init__(self, base=None):