swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/middleware/s3api/s3response.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 import re
-from collections import MutableMapping
+from collections.abc import MutableMapping
 from functools import partial
 
 from swift.common import header_key_dict
@@ -25,6 +25,8 @@ from swift.common.request_helpers import is_sys_meta
 from swift.common.middleware.s3api.utils import snake_to_camel, \
     sysmeta_prefix, sysmeta_header
 from swift.common.middleware.s3api.etree import Element, SubElement, tostring
+from swift.common.middleware.versioned_writes.object_versioning import \
+    DELETE_MARKER_CONTENT_TYPE
 
 
 class HeaderKeyDict(header_key_dict.HeaderKeyDict):
@@ -44,6 +46,57 @@ class HeaderKeyDict(header_key_dict.HeaderKeyDict):
         return s
 
 
+def translate_swift_to_s3(key, val):
+    _key = swob.bytes_to_wsgi(swob.wsgi_to_bytes(key).lower())
+
+    def translate_meta_key(_key):
+        if not _key.startswith('x-object-meta-'):
+            return _key
+        # Note that AWS allows user-defined metadata with underscores in the
+        # header, while WSGI (and other protocols derived from CGI) does not
+        # differentiate between an underscore and a dash. Fortunately,
+        # eventlet exposes the raw headers from the client, so we could
+        # translate '_' to '=5F' on the way in. Now, we translate back.
+        return 'x-amz-meta-' + _key[14:].replace('=5f', '_')
+
+    if _key.startswith('x-object-meta-'):
+        return translate_meta_key(_key), val
+    elif _key in ('accept-ranges', 'content-length', 'content-type',
+                  'content-range', 'content-encoding',
+                  'content-disposition', 'content-language',
+                  'etag', 'last-modified', 'x-robots-tag',
+                  'cache-control', 'expires'):
+        return key, val
+    elif _key == 'x-object-version-id':
+        return 'x-amz-version-id', val
+    elif _key == 'x-parts-count':
+        return 'x-amz-mp-parts-count', val
+    elif _key == 'x-copied-from-version-id':
+        return 'x-amz-copy-source-version-id', val
+    elif _key == 'x-backend-content-type' and \
+            val == DELETE_MARKER_CONTENT_TYPE:
+        return 'x-amz-delete-marker', 'true'
+    elif _key == 'access-control-expose-headers':
+        exposed_headers = val.split(', ')
+        exposed_headers.extend([
+            'x-amz-request-id',
+            'x-amz-id-2',
+        ])
+        return 'access-control-expose-headers', ', '.join(
+            translate_meta_key(h) for h in exposed_headers)
+    elif _key == 'access-control-allow-methods':
+        methods = val.split(', ')
+        try:
+            methods.remove('COPY')  # that's not a thing in S3
+        except ValueError:
+            pass  # not there? don't worry about it
+        return key, ', '.join(methods)
+    elif _key.startswith('access-control-'):
+        return key, val
+    # else, drop the header
+    return None
+
+
 class S3ResponseBase(object):
     """
     Base class for swift3 responses.
@@ -57,6 +110,7 @@ class S3Response(S3ResponseBase, swob.Response):
     headers instead of Swift's HeaderKeyDict.  This also translates Swift
     specific headers to S3 headers.
     """
+
     def __init__(self, *args, **kwargs):
         swob.Response.__init__(self, *args, **kwargs)
 
@@ -96,22 +150,13 @@ class S3Response(S3ResponseBase, swob.Response):
 
         # Handle swift headers
         for key, val in sw_headers.items():
-            _key = swob.bytes_to_wsgi(swob.wsgi_to_bytes(key).lower())
-
-            if _key.startswith('x-object-meta-'):
-                # Note that AWS ignores user-defined headers with '=' in the
-                # header name. We translated underscores to '=5F' on the way
-                # in, though.
-                headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val
-            elif _key in ('content-length', 'content-type',
-                          'content-range', 'content-encoding',
-                          'content-disposition', 'content-language',
-                          'etag', 'last-modified', 'x-robots-tag',
-                          'cache-control', 'expires'):
-                headers[key] = val
-            elif _key == 'x-static-large-object':
-                # for delete slo
-                self.is_slo = config_true_value(val)
+            s3_pair = translate_swift_to_s3(key, val)
+            if s3_pair is None:
+                continue
+            headers[s3_pair[0]] = s3_pair[1]
+
+        self.is_slo = config_true_value(sw_headers.get(
+            'x-static-large-object'))
 
         # Check whether we stored the AWS-style etag on upload
         override_etag = s3_sysmeta_headers.get(
@@ -184,16 +229,17 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
     _code = ''
     xml_declaration = True
 
-    def __init__(self, msg=None, *args, **kwargs):
+    def __init__(self, msg=None, reason=None, *args, **kwargs):
         if msg:
             self._msg = msg
         if not self._code:
             self._code = self.__class__.__name__
+        self.reason = reason
 
         self.info = kwargs.copy()
         for reserved_key in ('headers', 'body'):
             if self.info.get(reserved_key):
-                del(self.info[reserved_key])
+                del (self.info[reserved_key])
 
         swob.HTTPException.__init__(
             self, status=kwargs.pop('status', self._status),
@@ -202,6 +248,19 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
             **kwargs)
         self.headers = HeaderKeyDict(self.headers)
 
+    @property
+    def summary(self):
+        """Provide a summary of the error code and reason."""
+        if self.reason:
+            summary = '.'.join([self._code, self.reason])
+        else:
+            summary = self._code
+        return summary.replace(' ', '_')
+
+    @property
+    def metric_name(self):
+        return '.'.join([str(self.status_int), self.summary])
+
     def _body_iter(self):
         error_elem = Element('Error')
         SubElement(error_elem, 'Code').text = self._code
@@ -217,7 +276,7 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
 
     def _dict_to_etree(self, parent, d):
         for key, value in d.items():
-            tag = re.sub('\W', '', snake_to_camel(key))
+            tag = re.sub(r'\W', '', snake_to_camel(key))
             elem = SubElement(parent, tag)
 
             if isinstance(value, (dict, MutableMapping)):
@@ -265,6 +324,12 @@ class BadDigest(ErrorResponse):
     _msg = 'The Content-MD5 you specified did not match what we received.'
 
 
+class XAmzContentSHA256Mismatch(ErrorResponse):
+    _status = '400 Bad Request'
+    _msg = "The provided 'x-amz-content-sha256' header does not match what " \
+           "was computed."
+
+
 class BucketAlreadyExists(ErrorResponse):
     _status = '409 Conflict'
     _msg = 'The requested bucket name is not available. The bucket ' \
@@ -289,6 +354,12 @@ class BucketNotEmpty(ErrorResponse):
     _msg = 'The bucket you tried to delete is not empty'
 
 
+class VersionedBucketNotEmpty(BucketNotEmpty):
+    _msg = 'The bucket you tried to delete is not empty. ' \
+           'You must delete all versions in the bucket.'
+    _code = 'BucketNotEmpty'
+
+
 class CredentialsNotSupported(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'This request does not support credentials.'
@@ -375,7 +446,7 @@ class InvalidBucketState(ErrorResponse):
 
 class InvalidDigest(ErrorResponse):
     _status = '400 Bad Request'
-    _msg = 'The Content-MD5 you specified was an invalid.'
+    _msg = 'The Content-MD5 you specified was invalid.'
 
 
 class InvalidLocationConstraint(ErrorResponse):
@@ -388,6 +459,17 @@ class InvalidObjectState(ErrorResponse):
     _msg = 'The operation is not valid for the current state of the object.'
 
 
+class InvalidPartArgument(InvalidArgument):
+    _code = 'InvalidArgument'
+
+    def __init__(self, max_parts, value):
+        err_msg = ('Part number must be an integer between '
+                   '1 and %s, inclusive' % max_parts)
+        super(InvalidArgument, self).__init__(err_msg,
+                                              argument_name='partNumber',
+                                              argument_value=value)
+
+
 class InvalidPart(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'One or more of the specified parts could not be found. The part ' \
@@ -417,6 +499,11 @@ class InvalidRange(ErrorResponse):
     _msg = 'The requested range cannot be satisfied.'
 
 
+class InvalidPartNumber(ErrorResponse):
+    _status = '416 Requested Range Not Satisfiable'
+    _msg = 'The requested partnumber is not satisfiable'
+
+
 class InvalidRequest(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'Invalid Request.'
@@ -461,7 +548,7 @@ class InvalidURI(ErrorResponse):
         ErrorResponse.__init__(self, msg, uri=uri, *args, **kwargs)
 
 
-class KeyTooLong(ErrorResponse):
+class KeyTooLongError(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'Your key is too long.'
 
@@ -481,7 +568,7 @@ class MalformedPOSTRequest(ErrorResponse):
 class MalformedXML(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'The XML you provided was not well-formed or did not validate ' \
-           'against our published schema.'
+           'against our published schema'
 
 
 class MaxMessageLengthExceeded(ErrorResponse):
@@ -553,6 +640,16 @@ class NoSuchKey(ErrorResponse):
         ErrorResponse.__init__(self, msg, key=key, *args, **kwargs)
 
 
+class ObjectLockConfigurationNotFoundError(ErrorResponse):
+    _status = '404 Not found'
+    _msg = 'Object Lock configuration does not exist for this bucket'
+
+    def __init__(self, bucket, msg=None, *args, **kwargs):
+        if not bucket:
+            raise InternalError()
+        ErrorResponse.__init__(self, msg, bucket_name=bucket, *args, **kwargs)
+
+
 class NoSuchLifecycleConfiguration(ErrorResponse):
     _status = '404 Not Found'
     _msg = 'The lifecycle configuration does not exist. .'
@@ -689,3 +786,9 @@ class UserKeyMustBeSpecified(ErrorResponse):
     _status = '400 Bad Request'
     _msg = 'The bucket POST must contain the specified field name. If it is ' \
            'specified, please check the order of the fields.'
+
+
+class BrokenMPU(ErrorResponse):
+    # This is very much a Swift-ism, and we wish we didn't need it
+    _status = '409 Conflict'
+    _msg = 'Multipart upload has broken segment data.'

swift/common/middleware/s3api/s3token.py

@@ -61,11 +61,10 @@ from keystoneclient.v3 import client as keystone_client
 from keystoneauth1 import session as keystone_session
 from keystoneauth1 import loading as keystone_loading
 import requests
-import six
-from six.moves import urllib
+import urllib
 
 from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
-    HTTPException
+    HTTPException, str_to_wsgi
 from swift.common.utils import config_true_value, split_path, get_logger, \
     cache_from_env, append_underscore
 from swift.common.wsgi import ConfigFileError
@@ -179,10 +178,12 @@ class S3Token(object):
             self._verify = None
 
         self._secret_cache_duration = int(conf.get('secret_cache_duration', 0))
-        if self._secret_cache_duration > 0:
+        if self._secret_cache_duration < 0:
+            raise ValueError('secret_cache_duration must be non-negative')
+        if self._secret_cache_duration:
             try:
                 auth_plugin = keystone_loading.get_plugin_loader(
-                    conf.get('auth_type'))
+                    conf.get('auth_type', 'password'))
                 available_auth_options = auth_plugin.get_options()
                 auth_options = {}
                 for option in available_auth_options:
@@ -193,7 +194,9 @@ class S3Token(object):
 
                 auth = auth_plugin.load_from_options(**auth_options)
                 session = keystone_session.Session(auth=auth)
-                self.keystoneclient = keystone_client.Client(session=session)
+                self.keystoneclient = keystone_client.Client(
+                    session=session,
+                    region_name=conf.get('region_name'))
                 self._logger.info("Caching s3tokens for %s seconds",
                                   self._secret_cache_duration)
             except Exception:
@@ -213,9 +216,7 @@ class S3Token(object):
         error_msg = ('<?xml version="1.0" encoding="UTF-8"?>\r\n'
                      '<Error>\r\n  <Code>%s</Code>\r\n  '
                      '<Message>%s</Message>\r\n</Error>\r\n' %
-                     (code, message))
-        if six.PY3:
-            error_msg = error_msg.encode()
+                     (code, message)).encode()
         resp.body = error_msg
         return resp
 
@@ -262,18 +263,18 @@ class S3Token(object):
             return self._app(environ, start_response)
 
         access = s3_auth_details['access_key']
-        if isinstance(access, six.binary_type):
+        if isinstance(access, bytes):
             access = access.decode('utf-8')
 
         signature = s3_auth_details['signature']
-        if isinstance(signature, six.binary_type):
+        if isinstance(signature, bytes):
             signature = signature.decode('utf-8')
 
         string_to_sign = s3_auth_details['string_to_sign']
-        if isinstance(string_to_sign, six.text_type):
+        if isinstance(string_to_sign, str):
             string_to_sign = string_to_sign.encode('utf-8')
         token = base64.urlsafe_b64encode(string_to_sign)
-        if isinstance(token, six.binary_type):
+        if isinstance(token, bytes):
             token = token.decode('ascii')
 
         # NOTE(chmou): This is to handle the special case with nova
@@ -395,12 +396,10 @@ class S3Token(object):
 
         req.headers.update(headers)
         tenant_to_connect = force_tenant or tenant['id']
-        if six.PY2 and isinstance(tenant_to_connect, six.text_type):
-            tenant_to_connect = tenant_to_connect.encode('utf-8')
         self._logger.debug('Connecting with tenant: %s', tenant_to_connect)
         new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect)
-        environ['PATH_INFO'] = environ['PATH_INFO'].replace(account,
-                                                            new_tenant_name)
+        environ['PATH_INFO'] = environ['PATH_INFO'].replace(
+            str_to_wsgi(account), str_to_wsgi(new_tenant_name), 1)
         return self._app(environ, start_response)
 
 

swift/common/middleware/s3api/schema/delete.rng

@@ -5,7 +5,7 @@
       <interleave>
         <optional>
           <element name="Quiet">
-            <data type="boolean"/>
+            <data type="string"/>
           </element>
         </optional>
         <oneOrMore>

swift/common/middleware/s3api/subresource.py

@@ -43,8 +43,6 @@ http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html
 """
 from functools import partial
 
-import six
-
 from swift.common.utils import json
 
 from swift.common.middleware.s3api.s3response import InvalidArgument, \
@@ -182,18 +180,19 @@ class Grantee(object):
         """
         Convert a grantee string in the HTTP header to an Grantee instance.
         """
-        type, value = grantee.split('=', 1)
+        grantee_type, value = grantee.split('=', 1)
+        grantee_type = grantee_type.lower()
         value = value.strip('"\'')
-        if type == 'id':
+        if grantee_type == 'id':
             return User(value)
-        elif type == 'emailAddress':
+        elif grantee_type == 'emailaddress':
             raise S3NotImplemented()
-        elif type == 'uri':
+        elif grantee_type == 'uri':
             # return a subclass instance of Group class
             subclass = get_group_subclass_from_uri(value)
             return subclass()
         else:
-            raise InvalidArgument(type, value,
+            raise InvalidArgument(grantee_type, value,
                                   'Argument format not recognized')
 
 
@@ -232,7 +231,7 @@ class Owner(object):
     """
     def __init__(self, id, name):
         self.id = id
-        if not (name is None or isinstance(name, six.string_types)):
+        if not (name is None or isinstance(name, str)):
             raise TypeError('name must be a string or None')
         self.name = name
 
@@ -428,8 +427,6 @@ class ACL(object):
         return tostring(self.elem())
 
     def __repr__(self):
-        if six.PY2:
-            return self.__bytes__()
         return self.__bytes__().decode('utf8')
 
     @classmethod

swift/common/middleware/s3api/utils.py

@@ -15,9 +15,9 @@
 
 import base64
 import calendar
+import datetime
 import email.utils
 import re
-import six
 import time
 import uuid
 
@@ -53,8 +53,6 @@ def snake_to_camel(snake):
 
 def unique_id():
     result = base64.urlsafe_b64encode(str(uuid.uuid4()).encode('ascii'))
-    if six.PY2:
-        return result
     return result.decode('ascii')
 
 
@@ -95,8 +93,8 @@ def validate_bucket_name(name, dns_compliant_bucket_names):
     elif name.endswith('.'):
         # Bucket names must not end with dot
         return False
-    elif re.match("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)"
-                  "{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$",
+    elif re.match(r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)"
+                  r"{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$",
                   name):
         # Bucket names cannot be formatted as an IP Address
         return False
@@ -108,9 +106,19 @@ def validate_bucket_name(name, dns_compliant_bucket_names):
 
 
 class S3Timestamp(utils.Timestamp):
+    S3_XML_FORMAT = "%Y-%m-%dT%H:%M:%S.000Z"
+
     @property
     def s3xmlformat(self):
-        return self.isoformat[:-7] + '.000Z'
+        dt = datetime.datetime.fromtimestamp(self.ceil(), utils.UTC)
+        return dt.strftime(self.S3_XML_FORMAT)
+
+    @classmethod
+    def from_s3xmlformat(cls, date_string):
+        dt = datetime.datetime.strptime(date_string, cls.S3_XML_FORMAT)
+        dt = dt.replace(tzinfo=utils.UTC)
+        seconds = calendar.timegm(dt.timetuple())
+        return cls(seconds)
 
     @property
     def amz_date_format(self):
@@ -120,10 +128,6 @@ class S3Timestamp(utils.Timestamp):
         return self.isoformat.replace(
             '-', '').replace(':', '')[:-7] + 'Z'
 
-    @classmethod
-    def now(cls):
-        return cls(time.time())
-
 
 def mktime(timestamp_str, time_format='%Y-%m-%dT%H:%M:%S'):
     """
@@ -156,7 +160,20 @@ def mktime(timestamp_str, time_format='%Y-%m-%dT%H:%M:%S'):
 
 
 class Config(dict):
+    DEFAULTS = {
+        'storage_domains': [],
+        'location': 'us-east-1',
+        'force_swift_request_proxy_log': False,
+        'dns_compliant_bucket_names': True,
+        'allow_multipart_uploads': True,
+        'allow_no_owner': False,
+        'allowable_clock_skew': 900,
+        'ratelimit_as_client_error': False,
+        'max_upload_part_num': 1000,
+    }
+
     def __init__(self, base=None):
+        self.update(self.DEFAULTS)
         if base is not None:
             self.update(base)