swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/storage_policy.py

@@ -17,8 +17,7 @@ import os
 import string
 import sys
 import textwrap
-import six
-from six.moves.configparser import ConfigParser
+from configparser import ConfigParser
 from swift.common.utils import (
     config_true_value, quorum_size, whataremyips, list_from_csv,
     config_positive_int_value, get_zero_indexed_base_string, load_pkg_resource)
@@ -37,11 +36,11 @@ DEFAULT_EC_OBJECT_SEGMENT_SIZE = 1048576
 
 
 class BindPortsCache(object):
-    def __init__(self, swift_dir, bind_ip):
+    def __init__(self, swift_dir, ring_ip):
         self.swift_dir = swift_dir
         self.mtimes_by_ring_path = {}
         self.portsets_by_ring_path = {}
-        self.my_ips = set(whataremyips(bind_ip))
+        self.my_ips = set(whataremyips(ring_ip))
 
     def all_bind_ports_for_node(self):
         """
@@ -80,8 +79,10 @@ class BindPortsCache(object):
                 # the first one we notice.
 
         # Return the requested set of ports from our (now-freshened) cache
-        return six.moves.reduce(set.union,
-                                self.portsets_by_ring_path.values(), set())
+        res = set()
+        for ports in self.portsets_by_ring_path.values():
+            res.update(ports)
+        return res
 
 
 class PolicyError(ValueError):
@@ -160,7 +161,7 @@ class BaseStoragePolicy(object):
                  object_ring=None, aliases='',
                  diskfile_module='egg:swift#replication.fs'):
         # do not allow BaseStoragePolicy class to be instantiated directly
-        if type(self) == BaseStoragePolicy:
+        if type(self) is BaseStoragePolicy:
             raise TypeError("Can't instantiate BaseStoragePolicy directly")
         # policy parameter validation
         try:
@@ -366,15 +367,26 @@ class BaseStoragePolicy(object):
             self._validate_policy_name(name)
         self.alias_list.insert(0, name)
 
-    def load_ring(self, swift_dir):
+    def validate_ring_data(self, ring_data):
+        """
+        Validation hook used when loading the ring; currently only used for EC
+        """
+
+    def load_ring(self, swift_dir, reload_time=None):
         """
         Load the ring for this policy immediately.
 
         :param swift_dir: path to rings
+        :param reload_time: time interval in seconds to check for a ring change
         """
         if self.object_ring:
+            if reload_time is not None:
+                self.object_ring.reload_time = reload_time
             return
-        self.object_ring = Ring(swift_dir, ring_name=self.ring_name)
+
+        self.object_ring = Ring(
+            swift_dir, ring_name=self.ring_name,
+            validation_hook=self.validate_ring_data, reload_time=reload_time)
 
     @property
     def quorum(self):
@@ -403,7 +415,7 @@ class BaseStoragePolicy(object):
                 (self.diskfile_module, self.name, err))
         try:
             dfm_cls.check_policy(self)
-        except ValueError as err:
+        except ValueError:
             raise PolicyError(
                 'Invalid diskfile_module %s for policy %s:%s (%s)' %
                 (self.diskfile_module, int(self), self.name, self.policy_type))
@@ -643,38 +655,25 @@ class ECStoragePolicy(BaseStoragePolicy):
         """
         return self._ec_quorum_size * self.ec_duplication_factor
 
-    def load_ring(self, swift_dir):
+    def validate_ring_data(self, ring_data):
         """
-        Load the ring for this policy immediately.
+        EC specific validation
 
-        :param swift_dir: path to rings
+        Replica count check - we need _at_least_ (#data + #parity) replicas
+        configured.  Also if the replica count is larger than exactly that
+        number there's a non-zero risk of error for code that is
+        considering the number of nodes in the primary list from the ring.
         """
-        if self.object_ring:
-            return
 
-        def validate_ring_data(ring_data):
-            """
-            EC specific validation
-
-            Replica count check - we need _at_least_ (#data + #parity) replicas
-            configured.  Also if the replica count is larger than exactly that
-            number there's a non-zero risk of error for code that is
-            considering the number of nodes in the primary list from the ring.
-            """
-
-            configured_fragment_count = ring_data.replica_count
-            required_fragment_count = \
-                (self.ec_n_unique_fragments) * self.ec_duplication_factor
-            if configured_fragment_count != required_fragment_count:
-                raise RingLoadError(
-                    'EC ring for policy %s needs to be configured with '
-                    'exactly %d replicas. Got %s.' % (
-                        self.name, required_fragment_count,
-                        configured_fragment_count))
-
-        self.object_ring = Ring(
-            swift_dir, ring_name=self.ring_name,
-            validation_hook=validate_ring_data)
+        configured_fragment_count = ring_data.replica_count
+        required_fragment_count = \
+            (self.ec_n_unique_fragments) * self.ec_duplication_factor
+        if configured_fragment_count != required_fragment_count:
+            raise RingLoadError(
+                'EC ring for policy %s needs to be configured with '
+                'exactly %d replicas. Got %s.' % (
+                    self.name, required_fragment_count,
+                    configured_fragment_count))
 
     def get_backend_index(self, node_index):
         """
@@ -819,6 +818,15 @@ class StoragePolicyCollection(object):
                 return None
         return self.by_index.get(index)
 
+    def get_by_name_or_index(self, name_or_index):
+        by_name = self.get_by_name(name_or_index)
+        by_index = self.get_by_index(name_or_index)
+        if by_name and by_index and by_name != by_index:
+            raise PolicyError(
+                "Found different polices when searching by "
+                "name (%s) and by index (%s)" % (by_name, by_index))
+        return by_name or by_index
+
     @property
     def legacy(self):
         return self.get_by_index(None)
@@ -968,12 +976,9 @@ def reload_storage_policies():
     Reload POLICIES from ``swift.conf``.
     """
     global _POLICIES
-    if six.PY2:
-        policy_conf = ConfigParser()
-    else:
-        # Python 3.2 disallows section or option duplicates by default
-        # strict=False allows us to preserve the older behavior
-        policy_conf = ConfigParser(strict=False)
+    # Python disallows section or option duplicates by default
+    # strict=False allows them, which Swift has always done
+    policy_conf = ConfigParser(strict=False)
     policy_conf.read(utils.SWIFT_CONF_FILE)
     try:
         _POLICIES = parse_storage_policies(policy_conf)

swift/common/swob.py

@@ -35,7 +35,8 @@ place to keep Swift working every time webob decides some interface
 needs to change.
 """
 
-from collections import defaultdict, MutableMapping
+from collections import defaultdict
+from collections.abc import MutableMapping
 import time
 from functools import partial
 from datetime import datetime
@@ -43,16 +44,14 @@ from email.utils import parsedate
 import re
 import random
 import functools
-import inspect
+from io import BytesIO
 
-import six
-from six import BytesIO
-from six import StringIO
-from six.moves import urllib
+from io import StringIO
+import urllib
 
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.utils import UTC, reiterate, split_path, Timestamp, pairs, \
-    close_if_possible, closing_if_possible
+    close_if_possible, closing_if_possible, config_true_value, friendly_close
 from swift.common.exceptions import InvalidTimestamp
 
 
@@ -111,6 +110,8 @@ RESPONSE_REASONS = {
           'backend server.'),
     507: ('Insufficient Storage', 'There was not enough space to save the '
           'resource. Drive: %(drive)s'),
+    529: ('Too Many Backend Requests', 'The server is incapable of performing '
+          'the requested operation due to too many requests. Slow down.')
 }
 
 MAX_RANGE_OVERLAPS = 2
@@ -152,7 +153,7 @@ def _datetime_property(header):
                 return None
 
     def setter(self, value):
-        if isinstance(value, (float,) + six.integer_types):
+        if isinstance(value, (float, int)):
             self.headers[header] = time.strftime(
                 "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(value))
         elif isinstance(value, datetime):
@@ -246,9 +247,7 @@ class HeaderEnvironProxy(MutableMapping):
     def __setitem__(self, key, value):
         if value is None:
             self.environ.pop(header_to_environ_key(key), None)
-        elif six.PY2 and isinstance(value, six.text_type):
-            self.environ[header_to_environ_key(key)] = value.encode('utf-8')
-        elif not six.PY2 and isinstance(value, six.binary_type):
+        elif isinstance(value, bytes):
             self.environ[header_to_environ_key(key)] = value.decode('latin1')
         else:
             self.environ[header_to_environ_key(key)] = str(value)
@@ -274,70 +273,42 @@ class HeaderEnvironProxy(MutableMapping):
 def wsgi_to_bytes(wsgi_str):
     if wsgi_str is None:
         return None
-    if six.PY2:
-        return wsgi_str
     return wsgi_str.encode('latin1')
 
 
 def wsgi_to_str(wsgi_str):
     if wsgi_str is None:
         return None
-    if six.PY2:
-        return wsgi_str
     return wsgi_to_bytes(wsgi_str).decode('utf8', errors='surrogateescape')
 
 
 def bytes_to_wsgi(byte_str):
-    if six.PY2:
-        return byte_str
     return byte_str.decode('latin1')
 
 
 def str_to_wsgi(native_str):
-    if six.PY2:
-        return native_str
     return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
 
 
 def wsgi_quote(wsgi_str, safe='/'):
-    if six.PY2:
-        if not isinstance(wsgi_str, bytes):
-            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
-        return urllib.parse.quote(wsgi_str, safe=safe)
-
     if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
         raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
     return urllib.parse.quote(wsgi_str, safe=safe, encoding='latin-1')
 
 
 def wsgi_unquote(wsgi_str):
-    if six.PY2:
-        if not isinstance(wsgi_str, bytes):
-            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
-        return urllib.parse.unquote(wsgi_str)
-
     if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
         raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
     return urllib.parse.unquote(wsgi_str, encoding='latin-1')
 
 
 def wsgi_quote_plus(wsgi_str):
-    if six.PY2:
-        if not isinstance(wsgi_str, bytes):
-            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
-        return urllib.parse.quote_plus(wsgi_str)
-
     if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
         raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
     return urllib.parse.quote_plus(wsgi_str, encoding='latin-1')
 
 
 def wsgi_unquote_plus(wsgi_str):
-    if six.PY2:
-        if not isinstance(wsgi_str, bytes):
-            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
-        return urllib.parse.unquote_plus(wsgi_str)
-
     if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
         raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
     return urllib.parse.unquote_plus(wsgi_str, encoding='latin-1')
@@ -355,7 +326,7 @@ def _resp_status_property():
         return '%s %s' % (self.status_int, self.title)
 
     def setter(self, value):
-        if isinstance(value, six.integer_types):
+        if isinstance(value, int):
             self.status_int = value
             self.explanation = self.title = RESPONSE_REASONS[value][0]
         else:
@@ -383,9 +354,9 @@ def _resp_body_property():
         return self._body
 
     def setter(self, value):
-        if isinstance(value, six.text_type):
+        if isinstance(value, str):
             raise TypeError('WSGI responses must be bytes')
-        if isinstance(value, six.binary_type):
+        if isinstance(value, bytes):
             self.content_length = len(value)
             close_if_possible(self._app_iter)
             self._app_iter = None
@@ -689,6 +660,12 @@ class Range(object):
         return all_ranges
 
 
+def normalize_etag(tag):
+    if tag and tag.startswith('"') and tag.endswith('"') and tag != '"':
+        return tag[1:-1]
+    return tag
+
+
 class Match(object):
     """
     Wraps a Request's If-[None-]Match header as a friendly object.
@@ -701,15 +678,10 @@ class Match(object):
             tag = tag.strip()
             if not tag:
                 continue
-            if tag.startswith('"') and tag.endswith('"'):
-                self.tags.add(tag[1:-1])
-            else:
-                self.tags.add(tag)
+            self.tags.add(normalize_etag(tag))
 
     def __contains__(self, val):
-        if val and val.startswith('"') and val.endswith('"'):
-            val = val[1:-1]
-        return '*' in self.tags or val in self.tags
+        return '*' in self.tags or normalize_etag(val) in self.tags
 
     def __repr__(self):
         return '%s(%r)' % (
@@ -724,7 +696,7 @@ class Accept(object):
     """
 
     # RFC 2616 section 2.2
-    token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+'
+    token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+'  # nosec B105
     qdtext = r'[^"]'
     quoted_pair = r'(?:\\.)'
     quoted_string = r'"(?:' + qdtext + r'|' + quoted_pair + r')*"'
@@ -799,19 +771,13 @@ def _req_environ_property(environ_field, is_wsgi_string_field=True):
         return self.environ.get(environ_field, None)
 
     def setter(self, value):
-        if six.PY2:
-            if isinstance(value, six.text_type):
-                self.environ[environ_field] = value.encode('utf-8')
-            else:
-                self.environ[environ_field] = value
-        else:
-            if is_wsgi_string_field:
-                # Check that input is valid before setting
-                if isinstance(value, str):
-                    value.encode('latin1').decode('utf-8')
-                if isinstance(value, bytes):
-                    value = value.decode('latin1')
-            self.environ[environ_field] = value
+        if is_wsgi_string_field:
+            # Check that input is valid before setting
+            if isinstance(value, str):
+                value.encode('latin1').decode('utf-8')
+            if isinstance(value, bytes):
+                value = value.decode('latin1')
+        self.environ[environ_field] = value
 
     return property(getter, setter, doc=("Get and set the %s property "
                     "in the WSGI environment") % environ_field)
@@ -829,7 +795,7 @@ def _req_body_property():
         return body
 
     def setter(self, value):
-        if not isinstance(value, six.binary_type):
+        if not isinstance(value, bytes):
             value = value.encode('utf8')
         self.environ['wsgi.input'] = WsgiBytesIO(value)
         self.environ['CONTENT_LENGTH'] = str(len(value))
@@ -926,15 +892,11 @@ class Request(object):
         """
         headers = headers or {}
         environ = environ or {}
-        if six.PY2:
-            if isinstance(path, six.text_type):
-                path = path.encode('utf-8')
+        if isinstance(path, bytes):
+            path = path.decode('latin1')
         else:
-            if isinstance(path, six.binary_type):
-                path = path.decode('latin1')
-            else:
-                # Check that the input is valid
-                path.encode('latin1')
+            # Check that the input is valid
+            path.encode('latin1')
 
         parsed_path = urllib.parse.urlparse(path)
         server_name = 'localhost'
@@ -964,7 +926,7 @@ class Request(object):
         }
         env.update(environ)
         if body is not None:
-            if not isinstance(body, six.binary_type):
+            if not isinstance(body, bytes):
                 body = body.encode('utf8')
             env['wsgi.input'] = WsgiBytesIO(body)
             env['CONTENT_LENGTH'] = str(len(body))
@@ -990,13 +952,9 @@ class Request(object):
         "Provides QUERY_STRING parameters as a dictionary"
         if self._params_cache is None:
             if 'QUERY_STRING' in self.environ:
-                if six.PY2:
-                    self._params_cache = dict(urllib.parse.parse_qsl(
-                        self.environ['QUERY_STRING'], True))
-                else:
-                    self._params_cache = dict(urllib.parse.parse_qsl(
-                        self.environ['QUERY_STRING'],
-                        keep_blank_values=True, encoding='latin-1'))
+                self._params_cache = dict(urllib.parse.parse_qsl(
+                    self.environ['QUERY_STRING'],
+                    keep_blank_values=True, encoding='latin-1'))
             else:
                 self._params_cache = {}
         return self._params_cache
@@ -1005,11 +963,35 @@ class Request(object):
     @params.setter
     def params(self, param_pairs):
         self._params_cache = None
-        if six.PY2:
-            self.query_string = urllib.parse.urlencode(param_pairs)
+        self.query_string = urllib.parse.urlencode(param_pairs,
+                                                   encoding='latin-1')
+
+    def ensure_x_timestamp(self):
+        """
+        Similar to :attr:`timestamp`, but the ``X-Timestamp`` header will be
+        set if not present.
+
+        :raises HTTPBadRequest: if X-Timestamp is already set but not a valid
+                                :class:`~swift.common.utils.Timestamp`
+        :returns: the request's X-Timestamp header,
+                  as a :class:`~swift.common.utils.Timestamp`
+        """
+        # The container sync feature includes an x-timestamp header with
+        # requests. If present this is checked and preserved, otherwise a fresh
+        # timestamp is added.
+        if 'HTTP_X_TIMESTAMP' in self.environ:
+            try:
+                self._timestamp = Timestamp(self.environ['HTTP_X_TIMESTAMP'])
+            except ValueError:
+                raise HTTPBadRequest(
+                    request=self, content_type='text/plain',
+                    body='X-Timestamp should be a UNIX timestamp float value; '
+                         'was %r' % self.environ['HTTP_X_TIMESTAMP'])
         else:
-            self.query_string = urllib.parse.urlencode(param_pairs,
-                                                       encoding='latin-1')
+            self._timestamp = Timestamp.now()
+        # Always normalize it to the internal form
+        self.environ['HTTP_X_TIMESTAMP'] = self._timestamp.internal
+        return self._timestamp
 
     @property
     def timestamp(self):
@@ -1063,6 +1045,11 @@ class Request(object):
         "Provides the full url of the request"
         return self.host_url + self.path_qs
 
+    @property
+    def allow_reserved_names(self):
+        return config_true_value(self.environ.get(
+            'HTTP_X_BACKEND_ALLOW_RESERVED_NAMES'))
+
     def as_referer(self):
         return self.method + ' ' + self.url
 
@@ -1227,7 +1214,7 @@ class Response(object):
         self.request = request
         self._app_iter = None
         # Allow error messages to come as natural strings on py3.
-        if isinstance(body, six.text_type):
+        if isinstance(body, str):
             body = body.encode('utf8')
         self.body = body
         self.app_iter = app_iter
@@ -1357,11 +1344,15 @@ class Response(object):
             if empty_resp is not None:
                 self.status = empty_resp
                 self.content_length = 0
+                # the existing successful response and it's app_iter have been
+                # determined to not meet the conditions of the reqeust, the
+                # response app_iter should be closed but not drained.
                 close_if_possible(app_iter)
                 return [b'']
 
         if self.request and self.request.method == 'HEAD':
             # We explicitly do NOT want to set self.content_length to 0 here
+            friendly_close(app_iter)  # be friendly to our app_iter
             return [b'']
 
         if self.conditional_response and self.request and \
@@ -1530,23 +1521,15 @@ def wsgify(func):
     return a Response object into WSGI callables.  Also catches any raised
     HTTPExceptions and treats them as a returned Response.
     """
-    argspec = inspect.getargspec(func)
-    if argspec.args and argspec.args[0] == 'self':
-        @functools.wraps(func)
-        def _wsgify_self(self, env, start_response):
-            try:
-                return func(self, Request(env))(env, start_response)
-            except HTTPException as err_resp:
-                return err_resp(env, start_response)
-        return _wsgify_self
-    else:
-        @functools.wraps(func)
-        def _wsgify_bare(env, start_response):
-            try:
-                return func(Request(env))(env, start_response)
-            except HTTPException as err_resp:
-                return err_resp(env, start_response)
-        return _wsgify_bare
+    @functools.wraps(func)
+    def _wsgify(*args):
+        env, start_response = args[-2:]
+        new_args = args[:-2] + (Request(env), )
+        try:
+            return func(*new_args)(env, start_response)
+        except HTTPException as err_resp:
+            return err_resp(env, start_response)
+    return _wsgify
 
 
 class StatusMap(object):
@@ -1591,3 +1574,4 @@ HTTPNotImplemented = status_map[501]
 HTTPBadGateway = status_map[502]
 HTTPServiceUnavailable = status_map[503]
 HTTPInsufficientStorage = status_map[507]
+HTTPTooManyBackendRequests = status_map[529]