swift 2.33.0__py2.py3-none-any.whl → 2.34.1__py2.py3-none-any.whl

swift/common/middleware/cname_lookup.py

@@ -29,8 +29,6 @@ rewritten and the request is passed further down the WSGI chain.
 
 import six
 
-from swift import gettext_ as _
-
 try:
     import dns.resolver
     import dns.exception
@@ -167,7 +165,7 @@ class CNAMELookupMiddleware(object):
                 elif self._domain_endswith_in_storage_domain(found_domain):
                     # Found it!
                     self.logger.info(
-                        _('Mapped %(given_domain)s to %(found_domain)s') %
+                        'Mapped %(given_domain)s to %(found_domain)s',
                         {'given_domain': given_domain,
                          'found_domain': found_domain})
                     if port:
@@ -180,8 +178,8 @@ class CNAMELookupMiddleware(object):
                 else:
                     # try one more deep in the chain
                     self.logger.debug(
-                        _('Following CNAME chain for  '
-                          '%(given_domain)s to %(found_domain)s') %
+                        'Following CNAME chain for  '
+                        '%(given_domain)s to %(found_domain)s',
                         {'given_domain': given_domain,
                          'found_domain': found_domain})
                     a_domain = found_domain

swift/common/middleware/container_sync.py

@@ -17,6 +17,7 @@ import os
 
 from swift.common.constraints import valid_api_version
 from swift.common.container_sync_realms import ContainerSyncRealms
+from swift.common.request_helpers import append_log_info
 from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify
 from swift.common.utils import (
     config_true_value, get_logger, streq_const_time)
@@ -109,20 +110,17 @@ class ContainerSync(object):
             valid = False
             auth = auth.split()
             if len(auth) != 3:
-                req.environ.setdefault('swift.log_info', []).append(
-                    'cs:not-3-args')
+                append_log_info(req.environ, 'cs:not-3-args')
             else:
                 realm, nonce, sig = auth
                 realm_key = self.realms_conf.key(realm)
                 realm_key2 = self.realms_conf.key2(realm)
                 if not realm_key:
-                    req.environ.setdefault('swift.log_info', []).append(
-                        'cs:no-local-realm-key')
+                    append_log_info(req.environ, 'cs:no-local-realm-key')
                 else:
                     user_key = info.get('sync_key')
                     if not user_key:
-                        req.environ.setdefault('swift.log_info', []).append(
-                            'cs:no-local-user-key')
+                        append_log_info(req.environ, 'cs:no-local-user-key')
                     else:
                         # x-timestamp headers get shunted by gatekeeper
                         if 'x-backend-inbound-x-timestamp' in req.headers:
@@ -139,11 +137,9 @@ class ContainerSync(object):
                             realm_key2, user_key) if realm_key2 else expected
                         if not streq_const_time(sig, expected) and \
                                 not streq_const_time(sig, expected2):
-                            req.environ.setdefault(
-                                'swift.log_info', []).append('cs:invalid-sig')
+                            append_log_info(req.environ, 'cs:invalid-sig')
                         else:
-                            req.environ.setdefault(
-                                'swift.log_info', []).append('cs:valid')
+                            append_log_info(req.environ, 'cs:valid')
                             valid = True
             if not valid:
                 exc = HTTPUnauthorized(

swift/common/middleware/crypto/crypto_utils.py

@@ -22,7 +22,6 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 import six
 from six.moves.urllib import parse as urlparse
 
-from swift import gettext_ as _
 from swift.common.exceptions import EncryptionException, UnknownSecretIdError
 from swift.common.swob import HTTPInternalServerError
 from swift.common.utils import get_logger
@@ -160,7 +159,7 @@ class CryptoWSGIContext(WSGIContext):
         try:
             fetch_crypto_keys = env[CRYPTO_KEY_CALLBACK]
         except KeyError:
-            self.logger.exception(_('ERROR get_keys() missing callback'))
+            self.logger.exception('ERROR get_keys() missing callback')
             raise HTTPInternalServerError(
                 "Unable to retrieve encryption keys.")
 
@@ -181,12 +180,12 @@ class CryptoWSGIContext(WSGIContext):
                 self.crypto.check_key(key)
                 continue
             except KeyError:
-                self.logger.exception(_("Missing key for %r") % name)
+                self.logger.exception("Missing key for %r", name)
             except TypeError:
-                self.logger.exception(_("Did not get a keys dict"))
+                self.logger.exception("Did not get a keys dict")
             except ValueError as e:
                 # don't include the key in any messages!
-                self.logger.exception(_("Bad key for %(name)r: %(err)s") %
+                self.logger.exception("Bad key for %(name)r: %(err)s",
                                       {'name': name, 'err': e})
             raise HTTPInternalServerError(
                 "Unable to retrieve encryption keys.")

swift/common/middleware/crypto/decrypter.py

@@ -16,7 +16,6 @@
 import base64
 import json
 
-from swift import gettext_ as _
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.http import is_success
 from swift.common.middleware.crypto.crypto_utils import CryptoWSGIContext, \
@@ -77,10 +76,10 @@ class BaseDecrypterContext(CryptoWSGIContext):
                                           crypto_meta['body_key'])
         except KeyError as err:
             self.logger.error(
-                _('Error decrypting %(resp_type)s: Missing %(key)s'),
+                'Error decrypting %(resp_type)s: Missing %(key)s',
                 {'resp_type': self.server_type, 'key': err})
         except ValueError as err:
-            self.logger.error(_('Error decrypting %(resp_type)s: %(reason)s'),
+            self.logger.error('Error decrypting %(resp_type)s: %(reason)s',
                               {'resp_type': self.server_type, 'reason': err})
         raise HTTPInternalServerError(
             body='Error decrypting %s' % self.server_type,
@@ -178,7 +177,7 @@ class DecrypterObjContext(BaseDecrypterContext):
                 value, key, required, bytes_to_wsgi)
         except EncryptionException as err:
             self.logger.error(
-                _("Error decrypting header %(header)s: %(error)s"),
+                "Error decrypting header %(header)s: %(error)s",
                 {'header': header, 'error': err})
             raise HTTPInternalServerError(
                 body='Error decrypting header',
@@ -313,7 +312,7 @@ class DecrypterObjContext(BaseDecrypterContext):
             try:
                 crypto_meta = self.get_crypto_meta(header, check)
             except EncryptionException as err:
-                self.logger.error(_('Error decrypting object: %s'), err)
+                self.logger.error('Error decrypting object: %s', err)
                 raise HTTPInternalServerError(
                     body='Error decrypting object', content_type='text/plain')
         return crypto_meta

swift/common/middleware/crypto/kms_keymaster.py

@@ -34,7 +34,7 @@ class KmsKeyMaster(BaseKeyMaster):
                       'domain_id', 'domain_name', 'project_id',
                       'project_domain_id', 'reauthenticate',
                       'auth_endpoint', 'api_class', 'key_id*',
-                      'active_root_secret_id')
+                      'barbican_endpoint', 'active_root_secret_id')
     keymaster_conf_section = 'kms_keymaster'
 
     def _get_root_secret(self, conf):
@@ -67,6 +67,7 @@ class KmsKeyMaster(BaseKeyMaster):
         oslo_conf = cfg.ConfigOpts()
         options.set_defaults(
             oslo_conf, auth_endpoint=conf.get('auth_endpoint'),
+            barbican_endpoint=conf.get('barbican_endpoint'),
             api_class=conf.get('api_class')
         )
         options.enable_logging()

swift/common/middleware/listing_formats.py

@@ -18,6 +18,7 @@ import six
 from xml.etree.cElementTree import Element, SubElement, tostring
 
 from swift.common.constraints import valid_api_version
+from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.http import HTTP_NO_CONTENT
 from swift.common.request_helpers import get_param
 from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \
@@ -178,52 +179,39 @@ class ListingFilter(object):
             start_response(status, headers)
             return resp_iter
 
-        header_to_index = {}
-        resp_content_type = resp_length = None
-        for i, (header, value) in enumerate(headers):
-            header = header.lower()
-            if header == 'content-type':
-                header_to_index[header] = i
-                resp_content_type = value.partition(';')[0]
-            elif header == 'content-length':
-                header_to_index[header] = i
-                resp_length = int(value)
-            elif header == 'vary':
-                header_to_index[header] = i
-
         if not status.startswith(('200 ', '204 ')):
             start_response(status, headers)
             return resp_iter
 
+        headers_dict = HeaderKeyDict(headers)
+        resp_content_type = headers_dict.get(
+            'content-type', '').partition(';')[0]
+        resp_length = headers_dict.get('content-length')
+
         if can_vary:
-            if 'vary' in header_to_index:
-                value = headers[header_to_index['vary']][1]
+            if 'vary' in headers_dict:
+                value = headers_dict['vary']
                 if 'accept' not in list_from_csv(value.lower()):
-                    headers[header_to_index['vary']] = (
-                        'Vary', value + ', Accept')
+                    headers_dict['vary'] = value + ', Accept'
             else:
-                headers.append(('Vary', 'Accept'))
+                headers_dict['vary'] = 'Accept'
 
         if resp_content_type != 'application/json':
-            start_response(status, headers)
+            start_response(status, list(headers_dict.items()))
             return resp_iter
 
-        if resp_length is None or \
-                resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
-            start_response(status, headers)
+        if req.method == 'HEAD':
+            headers_dict['content-type'] = out_content_type + '; charset=utf-8'
+            # proxy logging (and maybe other mw?) seem to be good about
+            # sticking this on HEAD/204 but we do it here to be responsible
+            # and explicit
+            headers_dict['content-length'] = 0
+            start_response(status, list(headers_dict.items()))
             return resp_iter
 
-        def set_header(header, value):
-            if value is None:
-                del headers[header_to_index[header]]
-            else:
-                headers[header_to_index[header]] = (
-                    headers[header_to_index[header]][0], str(value))
-
-        if req.method == 'HEAD':
-            set_header('content-type', out_content_type + '; charset=utf-8')
-            set_header('content-length', None)  # don't know, can't determine
-            start_response(status, headers)
+        if resp_length is None or \
+                int(resp_length) > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
+            start_response(status, list(headers_dict.items()))
             return resp_iter
 
         body = b''.join(resp_iter)
@@ -237,7 +225,7 @@ class ListingFilter(object):
         except ValueError:
             # Static web listing that's returning invalid JSON?
             # Just pass it straight through; that's about all we *can* do.
-            start_response(status, headers)
+            start_response(status, list(headers_dict.items()))
             return [body]
 
         if not req.allow_reserved_names:
@@ -257,16 +245,16 @@ class ListingFilter(object):
                 body = json.dumps(listing).encode('ascii')
         except KeyError:
             # listing was in a bad format -- funky static web listing??
-            start_response(status, headers)
+            start_response(status, list(headers_dict.items()))
             return [body]
 
         if not body:
             status = '%s %s' % (HTTP_NO_CONTENT,
                                 RESPONSE_REASONS[HTTP_NO_CONTENT][0])
 
-        set_header('content-type', out_content_type + '; charset=utf-8')
-        set_header('content-length', len(body))
-        start_response(status, headers)
+        headers_dict['content-type'] = out_content_type + '; charset=utf-8'
+        headers_dict['content-length'] = len(body)
+        start_response(status, list(headers_dict.items()))
         return [body]
 
 

swift/common/middleware/proxy_logging.py

@@ -88,7 +88,9 @@ bandwidth usage will want to only sum up logs with no ``swift.source``.
 import os
 import time
 
+from swift.common.constraints import valid_api_version
 from swift.common.middleware.catch_errors import enforce_byte_count
+from swift.common.request_helpers import get_log_info
 from swift.common.swob import Request
 from swift.common.utils import (get_logger, get_remote_client,
                                 config_true_value, reiterate,
@@ -247,6 +249,7 @@ class ProxyLoggingMiddleware(object):
         :param start_time: timestamp request started
         :param end_time: timestamp request completed
         :param resp_headers: dict of the response headers
+        :param ttfb: time to first byte
         :param wire_status_int: the on the wire status int
         """
         self.obscure_req(req)
@@ -269,10 +272,8 @@ class ProxyLoggingMiddleware(object):
         duration_time_str = "%.4f" % (end_time - start_time)
         policy_index = get_policy_index(req.headers, resp_headers)
 
-        acc, cont, obj = None, None, None
         swift_path = req.environ.get('swift.backend_path', req.path)
-        if swift_path.startswith('/v1/'):
-            _, acc, cont, obj = split_path(swift_path, 1, 4, True)
+        acc, cont, obj = self.get_aco_from_path(swift_path)
 
         replacements = {
             # Time information
@@ -317,8 +318,7 @@ class ProxyLoggingMiddleware(object):
             'transaction_id': req.environ.get('swift.trans_id'),
             'request_time': duration_time_str,
             'source': req.environ.get('swift.source'),
-            'log_info':
-                ','.join(req.environ.get('swift.log_info', '')),
+            'log_info': get_log_info(req.environ),
             'policy_index': policy_index,
             'ttfb': ttfb,
             'pid': self.pid,
@@ -347,22 +347,28 @@ class ProxyLoggingMiddleware(object):
             self.access_logger.update_stats(metric_name_policy + '.xfer',
                                             bytes_received + bytes_sent)
 
+    def get_aco_from_path(self, swift_path):
+        try:
+            version, acc, cont, obj = split_path(swift_path, 1, 4, True)
+            if not valid_api_version(version):
+                raise ValueError
+        except ValueError:
+            acc, cont, obj = None, None, None
+        return acc, cont, obj
+
     def get_metric_name_type(self, req):
         swift_path = req.environ.get('swift.backend_path', req.path)
-        if swift_path.startswith('/v1/'):
-            try:
-                stat_type = [None, 'account', 'container',
-                             'object'][swift_path.strip('/').count('/')]
-            except IndexError:
-                stat_type = 'object'
-        else:
-            stat_type = req.environ.get('swift.source')
-        return stat_type
+        acc, cont, obj = self.get_aco_from_path(swift_path)
+        if obj:
+            return 'object'
+        if cont:
+            return 'container'
+        if acc:
+            return 'account'
+        return req.environ.get('swift.source') or 'UNKNOWN'
 
     def statsd_metric_name(self, req, status_int, method):
         stat_type = self.get_metric_name_type(req)
-        if stat_type is None:
-            return None
         stat_method = method if method in self.valid_methods \
             else 'BAD_METHOD'
         return '.'.join((stat_type, stat_method, str(status_int)))

swift/common/middleware/ratelimit.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import time
-from swift import gettext_ as _
 
 import eventlet
 
@@ -261,7 +260,7 @@ class RateLimitMiddleware(object):
 
         if account_name in self.ratelimit_blacklist or \
                 account_global_ratelimit == 'BLACKLIST':
-            self.logger.error(_('Returning 497 because of blacklisting: %s'),
+            self.logger.error('Returning 497 because of blacklisting: %s',
                               account_name)
             eventlet.sleep(self.BLACK_LIST_SLEEP)
             return Response(status='497 Blacklisted',
@@ -276,8 +275,8 @@ class RateLimitMiddleware(object):
                 if self.log_sleep_time_seconds and \
                         need_to_sleep > self.log_sleep_time_seconds:
                     self.logger.warning(
-                        _("Ratelimit sleep log: %(sleep)s for "
-                          "%(account)s/%(container)s/%(object)s"),
+                        "Ratelimit sleep log: %(sleep)s for "
+                        "%(account)s/%(container)s/%(object)s",
                         {'sleep': need_to_sleep, 'account': account_name,
                          'container': container_name, 'object': obj_name})
                 if need_to_sleep > 0:
@@ -288,8 +287,8 @@ class RateLimitMiddleware(object):
                 else:
                     path = '/'.join((account_name, container_name))
                 self.logger.error(
-                    _('Returning 498 for %(meth)s to %(path)s. '
-                      'Ratelimit (Max Sleep) %(e)s'),
+                    'Returning 498 for %(meth)s to %(path)s. '
+                    'Ratelimit (Max Sleep) %(e)s',
                     {'meth': req.method, 'path': path, 'e': str(e)})
                 error_resp = Response(status='498 Rate Limited',
                                       body='Slow down', request=req)
@@ -309,7 +308,7 @@ class RateLimitMiddleware(object):
             self.memcache_client = cache_from_env(env)
         if not self.memcache_client:
             self.logger.warning(
-                _('Warning: Cannot ratelimit without a memcached client'))
+                'Warning: Cannot ratelimit without a memcached client')
             return self.app(env, start_response)
         try:
             version, account, container, obj = req.split_path(1, 4, True)

swift/common/middleware/recon.py

@@ -20,7 +20,6 @@ import time
 from resource import getpagesize
 
 from swift import __version__ as swiftver
-from swift import gettext_ as _
 from swift.common.constraints import check_mount
 from swift.common.storage_policy import POLICIES
 from swift.common.swob import Request, Response
@@ -89,11 +88,11 @@ class ReconMiddleware(object):
             if err.errno == errno.ENOENT and ignore_missing:
                 pass
             else:
-                self.logger.exception(_('Error reading recon cache file'))
+                self.logger.exception('Error reading recon cache file')
         except ValueError:
-            self.logger.exception(_('Error parsing recon cache file'))
+            self.logger.exception('Error parsing recon cache file')
         except Exception:
-            self.logger.exception(_('Error retrieving recon data'))
+            self.logger.exception('Error retrieving recon data')
         return dict((key, None) for key in cache_keys)
 
     def get_version(self):
@@ -181,7 +180,7 @@ class ReconMiddleware(object):
         try:
             return {self.devices: os.listdir(self.devices)}
         except Exception:
-            self.logger.exception(_('Error listing devices'))
+            self.logger.exception('Error listing devices')
             return {self.devices: None}
 
     def get_updater_info(self, recon_type):
@@ -277,7 +276,7 @@ class ReconMiddleware(object):
                 except IOError as err:
                     sums[ringfile] = None
                     if err.errno != errno.ENOENT:
-                        self.logger.exception(_('Error reading ringfile'))
+                        self.logger.exception('Error reading ringfile')
         return sums
 
     def get_swift_conf_md5(self):
@@ -287,7 +286,7 @@ class ReconMiddleware(object):
             hexsum = md5_hash_for_file(SWIFT_CONF_FILE)
         except IOError as err:
             if err.errno != errno.ENOENT:
-                self.logger.exception(_('Error reading swift.conf'))
+                self.logger.exception('Error reading swift.conf')
         return {SWIFT_CONF_FILE: hexsum}
 
     def get_quarantine_count(self):

swift/common/middleware/s3api/acl_handlers.py

@@ -133,6 +133,15 @@ class BaseAclHandler(object):
                 query = {}
             else:
                 query = {'version-id': version_id}
+            if self.req.method == 'HEAD':
+                # This HEAD for ACL is going to also be the definitive response
+                # to the client so we need to include client params. We don't
+                # do this for other client request methods because they may
+                # have invalid combinations of params and headers for a swift
+                # HEAD request.
+                part_number = self.req.params.get('partNumber')
+                if part_number is not None:
+                    query['part-number'] = part_number
             resp = self.req.get_acl_response(app, 'HEAD',
                                              container, obj,
                                              headers, query=query)

swift/common/middleware/s3api/controllers/multi_upload.py

@@ -196,15 +196,7 @@ class PartController(Controller):
             raise InvalidArgument('ResourceType', 'partNumber',
                                   'Unexpected query string parameter')
 
-        try:
-            part_number = int(get_param(req, 'partNumber'))
-            if part_number < 1 or self.conf.max_upload_part_num < part_number:
-                raise Exception()
-        except Exception:
-            err_msg = 'Part number must be an integer between 1 and %d,' \
-                      ' inclusive' % self.conf.max_upload_part_num
-            raise InvalidArgument('partNumber', get_param(req, 'partNumber'),
-                                  err_msg)
+        part_number = req.validate_part_number()
 
         upload_id = get_param(req, 'uploadId')
         _get_upload_info(req, self.app, upload_id)

swift/common/middleware/s3api/controllers/obj.py

@@ -90,8 +90,14 @@ class ObjectController(Controller):
         if version_id not in ('null', None) and \
                 'object_versioning' not in get_swift_info():
             raise S3NotImplemented()
+        part_number = req.validate_part_number(check_max=False)
+
+        query = {}
+        if version_id is not None:
+            query['version-id'] = version_id
+        if part_number is not None:
+            query['part-number'] = part_number
 
-        query = {} if version_id is None else {'version-id': version_id}
         if version_id not in ('null', None):
             container_info = req.get_container_info(self.app)
             if not container_info.get(
@@ -101,6 +107,19 @@ class ObjectController(Controller):
 
         resp = req.get_response(self.app, query=query)
 
+        if not resp.is_slo:
+            # SLO ignores part_number for non-slo objects, but s3api only
+            # allows the query param for non-MPU if it's exactly 1.
+            part_number = req.validate_part_number(parts_count=1)
+            if part_number == 1:
+                # When the query param *is* exactly 1 the response status code
+                # and headers are updated.
+                resp.status = HTTP_PARTIAL_CONTENT
+                resp.headers['Content-Range'] = \
+                    'bytes 0-%d/%s' % (int(resp.headers['Content-Length']) - 1,
+                                       resp.headers['Content-Length'])
+            # else: part_number is None
+
         if req.method == 'HEAD':
             resp.app_iter = None
 

swift/common/middleware/s3api/s3api.py

@@ -149,6 +149,7 @@ from six.moves.urllib.parse import parse_qs
 from swift.common.constraints import valid_api_version
 from swift.common.middleware.listing_formats import \
     MAX_CONTAINER_LISTING_CONTENT_LENGTH
+from swift.common.request_helpers import append_log_info
 from swift.common.wsgi import PipelineWrapper, loadcontext, WSGIContext
 
 from swift.common.middleware import app_property
@@ -353,6 +354,7 @@ class S3ApiMiddleware(object):
             self.logger.debug(e.cause)
         except ErrorResponse as err_resp:
             self.logger.increment(err_resp.metric_name)
+            append_log_info(env, 's3:err:%s' % err_resp.summary)
             if isinstance(err_resp, InternalError):
                 self.logger.exception(err_resp)
             resp = err_resp