swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/middleware/gatekeeper.py

@@ -36,7 +36,7 @@ from swift.common.utils import get_logger, config_true_value
 from swift.common.request_helpers import (
     remove_items, get_sys_meta_prefix, OBJECT_TRANSIENT_SYSMETA_PREFIX
 )
-from six.moves.urllib.parse import urlsplit
+from urllib.parse import urlsplit
 import re
 
 #: A list of python regular expressions that will be used to
@@ -75,6 +75,8 @@ class GatekeeperMiddleware(object):
         self.outbound_condition = make_exclusion_test(outbound_exclusions)
         self.shunt_x_timestamp = config_true_value(
             conf.get('shunt_inbound_x_timestamp', 'true'))
+        self.allow_reserved_names_header = config_true_value(
+            conf.get('allow_reserved_names_header', 'false'))
 
     def __call__(self, env, start_response):
         req = Request(env)
@@ -89,6 +91,11 @@ class GatekeeperMiddleware(object):
             self.logger.debug('shunted request headers: %s' %
                               [('X-Timestamp', ts)])
 
+        if 'X-Allow-Reserved-Names' in req.headers \
+                and self.allow_reserved_names_header:
+            req.headers['X-Backend-Allow-Reserved-Names'] = \
+                req.headers.pop('X-Allow-Reserved-Names')
+
         def gatekeeper_response(status, response_headers, exc_info=None):
             def fixed_response_headers():
                 def relative_path(value):

swift/common/middleware/keystoneauth.py

@@ -178,9 +178,13 @@ class KeystoneAuth(object):
             config_read_reseller_options(conf,
                                          dict(operator_roles=['admin',
                                                               'swiftoperator'],
-                                              service_roles=[]))
+                                              service_roles=[],
+                                              project_reader_roles=[]))
         self.reseller_admin_role = conf.get('reseller_admin_role',
                                             'ResellerAdmin').lower()
+        self.system_reader_roles = {role.lower() for role in list_from_csv(
+            conf.get('system_reader_roles', ''))}
+
         config_is_admin = conf.get('is_admin', "false").lower()
         if swift_utils.config_true_value(config_is_admin):
             self.logger.warning("The 'is_admin' option for keystoneauth is no "
@@ -415,14 +419,26 @@ class KeystoneAuth(object):
         user_service_roles = [r.lower() for r in env_identity.get(
                               'service_roles', [])]
 
-        # Give unconditional access to a user with the reseller_admin
-        # role.
+        # Give unconditional access to a user with the reseller_admin role.
         if self.reseller_admin_role in user_roles:
             msg = 'User %s has reseller admin authorizing'
             self.logger.debug(msg, tenant_id)
             req.environ['swift_owner'] = True
             return
 
+        # Being in system_reader_roles is almost as good as reseller_admin.
+        if self.system_reader_roles.intersection(user_roles):
+            # Note that if a system reader is trying to write, we're letting
+            # the request fall on other access checks below. This way,
+            # a compliance auditor can write a log file as a normal member.
+            if req.method in ('GET', 'HEAD'):
+                msg = 'User %s has system reader authorizing'
+                self.logger.debug(msg, tenant_id)
+                # We aren't setting 'swift_owner' nor 'reseller_request'
+                # because they are only ever used for something that modifies
+                # the contents of the cluster (setting ACL, deleting accounts).
+                return
+
         # If we are not reseller admin and user is trying to delete its own
         # account then deny it.
         if not container and not obj and req.method == 'DELETE':
@@ -485,6 +501,20 @@ class KeystoneAuth(object):
             req.environ['swift_owner'] = True
             return
 
+        # The project_reader_roles is almost as good as operator_roles. But
+        # it does not work with service tokens and does not get 'swift_owner'.
+        # And, it only serves GET requests, obviously.
+        project_reader_roles = self.account_rules[account_prefix][
+            'project_reader_roles']
+        have_reader_role = set(project_reader_roles).intersection(
+            set(user_roles))
+        if have_reader_role:
+            if req.method in ('GET', 'HEAD'):
+                msg = 'User %s with role(s) %s has project reader authorizing'
+                self.logger.debug(msg, tenant_id,
+                                  ','.join(project_reader_roles))
+                return
+
         if acl_authorized is not None:
             return self.denied_response(req)
 

swift/common/middleware/list_endpoints.py

@@ -52,9 +52,9 @@ of endpoints having the same form as described above, and a key 'headers' that
 maps to a dictionary of headers that should be sent with a request made to
 the endpoints, e.g.::
 
-    { "endpoints": {"http://10.1.1.1:6010/sda1/2/a/c3/o1",
-                    "http://10.1.1.1:6030/sda3/2/a/c3/o1",
-                    "http://10.1.1.1:6040/sda4/2/a/c3/o1"},
+    { "endpoints": {"http://10.1.1.1:6210/sda1/2/a/c3/o1",
+                    "http://10.1.1.1:6230/sda3/2/a/c3/o1",
+                    "http://10.1.1.1:6240/sda4/2/a/c3/o1"},
       "headers": {"X-Backend-Storage-Policy-Index": "1"}}
 
 In this example, the 'headers' dictionary indicates that requests to the
@@ -80,7 +80,7 @@ environment (everyone can query the locality data using this middleware).
 
 import json
 
-from six.moves.urllib.parse import quote, unquote
+from urllib.parse import quote, unquote
 
 from swift.common.ring import Ring
 from swift.common.utils import get_logger, split_path

swift/common/middleware/listing_formats.py

@@ -14,14 +14,15 @@
 # limitations under the License.
 
 import json
-import six
 from xml.etree.cElementTree import Element, SubElement, tostring
 
 from swift.common.constraints import valid_api_version
+from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.http import HTTP_NO_CONTENT
 from swift.common.request_helpers import get_param
 from swift.common.swob import HTTPException, HTTPNotAcceptable, Request, \
-    RESPONSE_REASONS, HTTPBadRequest
+    RESPONSE_REASONS, HTTPBadRequest, wsgi_quote, wsgi_to_bytes
+from swift.common.utils import RESERVED, get_logger, list_from_csv
 
 
 #: Mapping of query string ``format=`` values to their corresponding
@@ -73,8 +74,6 @@ def to_xml(document_element):
 
 
 def account_to_xml(listing, account_name):
-    if isinstance(account_name, bytes):
-        account_name = account_name.decode('utf-8')
     doc = Element('account', name=account_name)
     doc.text = '\n'
     for record in listing:
@@ -84,15 +83,15 @@ def account_to_xml(listing, account_name):
         else:
             sub = SubElement(doc, 'container')
             for field in ('name', 'count', 'bytes', 'last_modified'):
-                SubElement(sub, field).text = six.text_type(
-                    record.pop(field))
+                SubElement(sub, field).text = str(record.pop(field))
+            for field in ('storage_policy',):
+                if field in record:
+                    SubElement(sub, field).text = str(record.pop(field))
         sub.tail = '\n'
     return to_xml(doc)
 
 
 def container_to_xml(listing, base_name):
-    if isinstance(base_name, bytes):
-        base_name = base_name.decode('utf-8')
     doc = Element('container', name=base_name)
     for record in listing:
         if 'subdir' in record:
@@ -103,8 +102,7 @@ def container_to_xml(listing, base_name):
             sub = SubElement(doc, 'object')
             for field in ('name', 'hash', 'bytes', 'content_type',
                           'last_modified'):
-                SubElement(sub, field).text = six.text_type(
-                    record.pop(field))
+                SubElement(sub, field).text = str(record.pop(field))
     return to_xml(doc)
 
 
@@ -119,8 +117,31 @@ def listing_to_text(listing):
 
 
 class ListingFilter(object):
-    def __init__(self, app):
+    def __init__(self, app, conf, logger=None):
         self.app = app
+        self.logger = logger or get_logger(conf, log_route='listing-filter')
+
+    def filter_reserved(self, listing, account, container):
+        new_listing = []
+        for entry in list(listing):
+            for key in ('name', 'subdir'):
+                value = entry.get(key, '')
+                if RESERVED in value:
+                    if container:
+                        self.logger.warning(
+                            'Container listing for %s/%s had '
+                            'reserved byte in %s: %r',
+                            wsgi_quote(account), wsgi_quote(container),
+                            key, value)
+                    else:
+                        self.logger.warning(
+                            'Account listing for %s had '
+                            'reserved byte in %s: %r',
+                            wsgi_quote(account), key, value)
+                    break  # out of the *key* loop; check next entry
+            else:
+                new_listing.append(entry)
+        return new_listing
 
     def __call__(self, env, start_response):
         req = Request(env)
@@ -128,10 +149,10 @@ class ListingFilter(object):
             # account and container only
             version, acct, cont = req.split_path(2, 3)
         except ValueError:
-            is_container_req = False
+            is_account_or_container_req = False
         else:
-            is_container_req = True
-        if not is_container_req:
+            is_account_or_container_req = True
+        if not is_account_or_container_req:
             return self.app(env, start_response)
 
         if not valid_api_version(version) or req.method not in ('GET', 'HEAD'):
@@ -145,46 +166,50 @@ class ListingFilter(object):
             return err(env, start_response)
 
         params = req.params
+        can_vary = 'format' not in params
         params['format'] = 'json'
         req.params = params
 
+        # Give other middlewares a chance to be in charge
+        env.setdefault('swift.format_listing', True)
         status, headers, resp_iter = req.call_application(self.app)
-
-        header_to_index = {}
-        resp_content_type = resp_length = None
-        for i, (header, value) in enumerate(headers):
-            header = header.lower()
-            if header == 'content-type':
-                header_to_index[header] = i
-                resp_content_type = value.partition(';')[0]
-            elif header == 'content-length':
-                header_to_index[header] = i
-                resp_length = int(value)
-
-        if not status.startswith('200 '):
+        if not env.get('swift.format_listing'):
             start_response(status, headers)
             return resp_iter
 
-        if resp_content_type != 'application/json':
+        if not status.startswith(('200 ', '204 ')):
             start_response(status, headers)
             return resp_iter
 
-        if resp_length is None or \
-                resp_length > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
-            start_response(status, headers)
-            return resp_iter
+        headers_dict = HeaderKeyDict(headers)
+        resp_content_type = headers_dict.get(
+            'content-type', '').partition(';')[0]
+        resp_length = headers_dict.get('content-length')
 
-        def set_header(header, value):
-            if value is None:
-                del headers[header_to_index[header]]
+        if can_vary:
+            if 'vary' in headers_dict:
+                value = headers_dict['vary']
+                if 'accept' not in list_from_csv(value.lower()):
+                    headers_dict['vary'] = value + ', Accept'
             else:
-                headers[header_to_index[header]] = (
-                    headers[header_to_index[header]][0], str(value))
+                headers_dict['vary'] = 'Accept'
+
+        if resp_content_type != 'application/json':
+            start_response(status, list(headers_dict.items()))
+            return resp_iter
 
         if req.method == 'HEAD':
-            set_header('content-type', out_content_type + '; charset=utf-8')
-            set_header('content-length', None)  # don't know, can't determine
-            start_response(status, headers)
+            headers_dict['content-type'] = out_content_type + '; charset=utf-8'
+            # proxy logging (and maybe other mw?) seem to be good about
+            # sticking this on HEAD/204 but we do it here to be responsible
+            # and explicit
+            headers_dict['content-length'] = 0
+            start_response(status, list(headers_dict.items()))
+            return resp_iter
+
+        if resp_length is None or \
+                int(resp_length) > MAX_CONTAINER_LISTING_CONTENT_LENGTH:
+            start_response(status, list(headers_dict.items()))
             return resp_iter
 
         body = b''.join(resp_iter)
@@ -198,32 +223,43 @@ class ListingFilter(object):
         except ValueError:
             # Static web listing that's returning invalid JSON?
             # Just pass it straight through; that's about all we *can* do.
-            start_response(status, headers)
+            start_response(status, list(headers_dict.items()))
             return [body]
 
+        if not req.allow_reserved_names:
+            listing = self.filter_reserved(listing, acct, cont)
+
         try:
             if out_content_type.endswith('/xml'):
                 if cont:
-                    body = container_to_xml(listing, cont)
+                    body = container_to_xml(
+                        listing, wsgi_to_bytes(cont).decode('utf-8'))
                 else:
-                    body = account_to_xml(listing, acct)
+                    body = account_to_xml(
+                        listing, wsgi_to_bytes(acct).decode('utf-8'))
             elif out_content_type == 'text/plain':
                 body = listing_to_text(listing)
-            # else, json -- we continue down here to be sure we set charset
+            else:
+                body = json.dumps(listing).encode('ascii')
         except KeyError:
             # listing was in a bad format -- funky static web listing??
-            start_response(status, headers)
+            start_response(status, list(headers_dict.items()))
             return [body]
 
         if not body:
             status = '%s %s' % (HTTP_NO_CONTENT,
                                 RESPONSE_REASONS[HTTP_NO_CONTENT][0])
 
-        set_header('content-type', out_content_type + '; charset=utf-8')
-        set_header('content-length', len(body))
-        start_response(status, headers)
+        headers_dict['content-type'] = out_content_type + '; charset=utf-8'
+        headers_dict['content-length'] = len(body)
+        start_response(status, list(headers_dict.items()))
         return [body]
 
 
 def filter_factory(global_conf, **local_conf):
-    return ListingFilter
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def listing_filter(app):
+        return ListingFilter(app, conf)
+    return listing_filter

swift/common/middleware/memcache.py

@@ -13,14 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
-from eventlet.green import ssl
-from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
-
-from swift.common.memcached import (MemcacheRing, CONN_TIMEOUT, POOL_TIMEOUT,
-                                    IO_TIMEOUT, TRY_COUNT)
-from swift.common.utils import config_true_value
+from swift.common.memcached import load_memcache
+from swift.common.utils import get_logger
 
 
 class MemcacheMiddleware(object):
@@ -30,93 +24,8 @@ class MemcacheMiddleware(object):
 
     def __init__(self, app, conf):
         self.app = app
-        self.memcache_servers = conf.get('memcache_servers')
-        serialization_format = conf.get('memcache_serialization_support')
-        try:
-            # Originally, while we documented using memcache_max_connections
-            # we only accepted max_connections
-            max_conns = int(conf.get('memcache_max_connections',
-                                     conf.get('max_connections', 0)))
-        except ValueError:
-            max_conns = 0
-
-        memcache_options = {}
-        if (not self.memcache_servers
-                or serialization_format is None
-                or max_conns <= 0):
-            path = os.path.join(conf.get('swift_dir', '/etc/swift'),
-                                'memcache.conf')
-            memcache_conf = ConfigParser()
-            if memcache_conf.read(path):
-                # if memcache.conf exists we'll start with those base options
-                try:
-                    memcache_options = dict(memcache_conf.items('memcache'))
-                except NoSectionError:
-                    pass
-
-                if not self.memcache_servers:
-                    try:
-                        self.memcache_servers = \
-                            memcache_conf.get('memcache', 'memcache_servers')
-                    except (NoSectionError, NoOptionError):
-                        pass
-                if serialization_format is None:
-                    try:
-                        serialization_format = \
-                            memcache_conf.get('memcache',
-                                              'memcache_serialization_support')
-                    except (NoSectionError, NoOptionError):
-                        pass
-                if max_conns <= 0:
-                    try:
-                        new_max_conns = \
-                            memcache_conf.get('memcache',
-                                              'memcache_max_connections')
-                        max_conns = int(new_max_conns)
-                    except (NoSectionError, NoOptionError, ValueError):
-                        pass
-
-        # while memcache.conf options are the base for the memcache
-        # middleware, if you set the same option also in the filter
-        # section of the proxy config it is more specific.
-        memcache_options.update(conf)
-        connect_timeout = float(memcache_options.get(
-            'connect_timeout', CONN_TIMEOUT))
-        pool_timeout = float(memcache_options.get(
-            'pool_timeout', POOL_TIMEOUT))
-        tries = int(memcache_options.get('tries', TRY_COUNT))
-        io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT))
-        if config_true_value(memcache_options.get('tls_enabled', 'false')):
-            tls_cafile = memcache_options.get('tls_cafile')
-            tls_certfile = memcache_options.get('tls_certfile')
-            tls_keyfile = memcache_options.get('tls_keyfile')
-            self.tls_context = ssl.create_default_context(
-                cafile=tls_cafile)
-            if tls_certfile:
-                self.tls_context.load_cert_chain(tls_certfile,
-                                                 tls_keyfile)
-        else:
-            self.tls_context = None
-
-        if not self.memcache_servers:
-            self.memcache_servers = '127.0.0.1:11211'
-        if max_conns <= 0:
-            max_conns = 2
-        if serialization_format is None:
-            serialization_format = 2
-        else:
-            serialization_format = int(serialization_format)
-
-        self.memcache = MemcacheRing(
-            [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
-            connect_timeout=connect_timeout,
-            pool_timeout=pool_timeout,
-            tries=tries,
-            io_timeout=io_timeout,
-            allow_pickle=(serialization_format == 0),
-            allow_unpickle=(serialization_format <= 1),
-            max_conns=max_conns,
-            tls_context=self.tls_context)
+        self.logger = get_logger(conf, log_route='memcache')
+        self.memcache = load_memcache(conf, self.logger)
 
     def __call__(self, env, start_response):
         env['swift.cache'] = self.memcache

swift/common/middleware/name_check.py

@@ -41,14 +41,15 @@ The filter returns HTTPBadRequest if path is invalid.
 '''
 
 import re
-from swift.common.utils import get_logger, register_swift_info
+from swift.common.utils import get_logger
+from swift.common.registry import register_swift_info
 
 from swift.common.swob import Request, HTTPBadRequest
 
 
 FORBIDDEN_CHARS = "\'\"`<>"
 MAX_LENGTH = 255
-FORBIDDEN_REGEXP = "/\./|/\.\./|/\.$|/\.\.$"
+FORBIDDEN_REGEXP = r"/\./|/\.\./|/\.$|/\.\.$"
 
 
 class NameCheckMiddleware(object):