swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/direct_client.py

@@ -23,17 +23,18 @@ import os
 import socket
 
 from eventlet import sleep, Timeout
-import six
-import six.moves.cPickle as pickle
-from six.moves.http_client import HTTPException
+import pickle  # nosec: B403
+from http.client import HTTPException
 
-from swift.common.bufferedhttp import http_connect
+from swift.common.bufferedhttp import http_connect, http_connect_raw
 from swift.common.exceptions import ClientException
-from swift.common.utils import Timestamp, FileLikeIter
+from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER, \
+    get_ip_port
+from swift.common.swob import normalize_etag
+from swift.common.utils import Timestamp, FileLikeIter, quote
 from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
     is_success, is_server_error
 from swift.common.header_key_dict import HeaderKeyDict
-from swift.common.utils import quote
 
 
 class DirectClientException(ClientException):
@@ -42,7 +43,7 @@ class DirectClientException(ClientException):
         # host can be used to override the node ip and port reported in
         # the exception
         host = host if host is not None else node
-        if not isinstance(path, six.text_type):
+        if isinstance(path, bytes):
             path = path.decode("utf-8")
         full_path = quote('/%s/%s%s' % (node['device'], part, path))
         msg = '%s server %s:%s direct %s %r gave status %s' % (
@@ -54,9 +55,23 @@ class DirectClientException(ClientException):
             http_reason=resp.reason, http_headers=headers)
 
 
+class DirectClientReconException(ClientException):
+
+    def __init__(self, method, node, path, resp):
+        if isinstance(path, bytes):
+            path = path.decode("utf-8")
+        msg = 'server %s:%s direct %s %r gave status %s' % (
+            node['ip'], node['port'], method, path, resp.status)
+        headers = HeaderKeyDict(resp.getheaders())
+        super(DirectClientReconException, self).__init__(
+            msg, http_host=node['ip'], http_port=node['port'],
+            http_status=resp.status, http_reason=resp.reason,
+            http_headers=headers)
+
+
 def _make_path(*components):
     return u'/' + u'/'.join(
-        x.decode('utf-8') if isinstance(x, six.binary_type) else x
+        x.decode('utf-8') if isinstance(x, bytes) else x
         for x in components)
 
 
@@ -95,13 +110,15 @@ def _make_req(node, part, method, path, headers, stype,
                     content_length = int(v)
         if not contents:
             headers['Content-Length'] = '0'
-        if isinstance(contents, six.string_types):
+        if isinstance(contents, str):
             contents = [contents]
         if content_length is None:
             headers['Transfer-Encoding'] = 'chunked'
 
+    ip, port = get_ip_port(node, headers)
+    headers.setdefault('X-Backend-Allow-Reserved-Names', 'true')
     with Timeout(conn_timeout):
-        conn = http_connect(node['ip'], node['port'], node['device'], part,
+        conn = http_connect(ip, port, node['device'], part,
                             method, path, headers=headers)
 
     if contents is not None:
@@ -138,28 +155,50 @@ def _get_direct_account_container(path, stype, node, part,
                                   marker=None, limit=None,
                                   prefix=None, delimiter=None,
                                   conn_timeout=5, response_timeout=15,
-                                  end_marker=None, reverse=None, headers=None):
-    """Base class for get direct account and container.
+                                  end_marker=None, reverse=None, headers=None,
+                                  extra_params=None):
+    """Base function for get direct account and container.
 
-    Do not use directly use the get_direct_account or
-    get_direct_container instead.
+    Do not use directly use the direct_get_account or
+    direct_get_container instead.
     """
-    params = ['format=json']
+    if headers is None:
+        headers = {}
+
+    params = {'format': 'json'}
+    if extra_params:
+        for key, value in extra_params.items():
+            if value is not None:
+                params[key] = value
     if marker:
-        params.append('marker=%s' % quote(marker))
+        if 'marker' in params:
+            raise TypeError('duplicate values for keyword arg: marker')
+        params['marker'] = quote(marker)
     if limit:
-        params.append('limit=%d' % limit)
+        if 'limit' in params:
+            raise TypeError('duplicate values for keyword arg: limit')
+        params['limit'] = '%d' % limit
     if prefix:
-        params.append('prefix=%s' % quote(prefix))
+        if 'prefix' in params:
+            raise TypeError('duplicate values for keyword arg: prefix')
+        params['prefix'] = quote(prefix)
     if delimiter:
-        params.append('delimiter=%s' % quote(delimiter))
+        if 'delimiter' in params:
+            raise TypeError('duplicate values for keyword arg: delimiter')
+        params['delimiter'] = quote(delimiter)
     if end_marker:
-        params.append('end_marker=%s' % quote(end_marker))
+        if 'end_marker' in params:
+            raise TypeError('duplicate values for keyword arg: end_marker')
+        params['end_marker'] = quote(end_marker)
     if reverse:
-        params.append('reverse=%s' % quote(reverse))
-    qs = '&'.join(params)
+        if 'reverse' in params:
+            raise TypeError('duplicate values for keyword arg: reverse')
+        params['reverse'] = quote(reverse)
+    qs = '&'.join('%s=%s' % (k, v) for k, v in params.items())
+
+    ip, port = get_ip_port(node, headers)
     with Timeout(conn_timeout):
-        conn = http_connect(node['ip'], node['port'], node['device'], part,
+        conn = http_connect(ip, port, node['device'], part,
                             'GET', path, query_string=qs,
                             headers=gen_headers(hdrs_in=headers))
     with Timeout(response_timeout):
@@ -193,12 +232,14 @@ def gen_headers(hdrs_in=None, add_ts=True):
         hdrs_out['X-Timestamp'] = Timestamp.now().internal
     if 'user-agent' not in hdrs_out:
         hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
+    hdrs_out.setdefault('X-Backend-Allow-Reserved-Names', 'true')
     return hdrs_out
 
 
 def direct_get_account(node, part, account, marker=None, limit=None,
                        prefix=None, delimiter=None, conn_timeout=5,
-                       response_timeout=15, end_marker=None, reverse=None):
+                       response_timeout=15, end_marker=None, reverse=None,
+                       headers=None):
     """
     Get listings directly from the account server.
 
@@ -218,6 +259,7 @@ def direct_get_account(node, part, account, marker=None, limit=None,
     """
     path = _make_path(account)
     return _get_direct_account_container(path, "Account", node, part,
+                                         headers=headers,
                                          marker=marker,
                                          limit=limit, prefix=prefix,
                                          delimiter=delimiter,
@@ -238,7 +280,7 @@ def direct_delete_account(node, part, account, conn_timeout=5,
 
 
 def direct_head_container(node, part, account, container, conn_timeout=5,
-                          response_timeout=15):
+                          response_timeout=15, headers=None):
     """
     Request container information directly from the container server.
 
@@ -251,8 +293,11 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
     :returns: a dict containing the response's headers in a HeaderKeyDict
     :raises ClientException: HTTP HEAD request failed
     """
+    if headers is None:
+        headers = {}
+
     path = _make_path(account, container)
-    resp = _make_req(node, part, 'HEAD', path, gen_headers(),
+    resp = _make_req(node, part, 'HEAD', path, gen_headers(headers),
                      'Container', conn_timeout, response_timeout)
 
     resp_headers = HeaderKeyDict()
@@ -264,7 +309,7 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
 def direct_get_container(node, part, account, container, marker=None,
                          limit=None, prefix=None, delimiter=None,
                          conn_timeout=5, response_timeout=15, end_marker=None,
-                         reverse=None, headers=None):
+                         reverse=None, headers=None, extra_params=None):
     """
     Get container listings directly from the container server.
 
@@ -281,6 +326,12 @@ def direct_get_container(node, part, account, container, marker=None,
     :param end_marker: end_marker query
     :param reverse: reverse the returned listing
     :param headers: headers to be included in the request
+    :param extra_params: a dict of extra parameters to be included in the
+        request. It can be used to pass additional parameters, e.g,
+        {'states':'updating'} can be used with shard_range/namespace listing.
+        It can also be used to pass the existing keyword args, like 'marker' or
+        'limit', but if the same parameter appears twice in both keyword arg
+        (not None) and extra_params, this function will raise TypeError.
     :returns: a tuple of (response headers, a list of objects) The response
               headers will be a HeaderKeyDict.
     """
@@ -293,7 +344,8 @@ def direct_get_container(node, part, account, container, marker=None,
                                          reverse=reverse,
                                          conn_timeout=conn_timeout,
                                          response_timeout=response_timeout,
-                                         headers=headers)
+                                         headers=headers,
+                                         extra_params=extra_params)
 
 
 def direct_delete_container(node, part, account, container, conn_timeout=5,
@@ -349,6 +401,31 @@ def direct_put_container(node, part, account, container, conn_timeout=5,
               content_length=content_length, chunk_size=chunk_size)
 
 
+def direct_post_container(node, part, account, container, conn_timeout=5,
+                          response_timeout=15, headers=None):
+    """
+    Make a POST request to a container server.
+
+    :param node: node dictionary from the ring
+    :param part: partition the container is on
+    :param account: account name
+    :param container: container name
+    :param conn_timeout: timeout in seconds for establishing the connection
+    :param response_timeout: timeout in seconds for getting the response
+    :param headers: additional headers to include in the request
+    :raises ClientException: HTTP PUT request failed
+    """
+    if headers is None:
+        headers = {}
+
+    lower_headers = set(k.lower() for k in headers)
+    headers_out = gen_headers(headers,
+                              add_ts='x-timestamp' not in lower_headers)
+    path = _make_path(account, container)
+    return _make_req(node, part, 'POST', path, headers_out, 'Container',
+                     conn_timeout, response_timeout)
+
+
 def direct_put_container_object(node, part, account, container, obj,
                                 conn_timeout=5, response_timeout=15,
                                 headers=None):
@@ -429,9 +506,10 @@ def direct_get_object(node, part, account, container, obj, conn_timeout=5,
     if headers is None:
         headers = {}
 
+    ip, port = get_ip_port(node, headers)
     path = _make_path(account, container, obj)
     with Timeout(conn_timeout):
-        conn = http_connect(node['ip'], node['port'], node['device'], part,
+        conn = http_connect(ip, port, node['device'], part,
                             'GET', path, headers=gen_headers(headers))
     with Timeout(response_timeout):
         resp = conn.getresponse()
@@ -483,7 +561,7 @@ def direct_put_object(node, part, account, container, name, contents,
     if headers is None:
         headers = {}
     if etag:
-        headers['ETag'] = etag.strip('"')
+        headers['ETag'] = normalize_etag(etag)
     if content_type is not None:
         headers['Content-Type'] = content_type
     else:
@@ -496,7 +574,7 @@ def direct_put_object(node, part, account, container, name, contents,
         'Object', conn_timeout, response_timeout, contents=contents,
         content_length=content_length, chunk_size=chunk_size)
 
-    return resp.getheader('etag').strip('"')
+    return normalize_etag(resp.getheader('etag'))
 
 
 def direct_post_object(node, part, account, container, name, headers,
@@ -549,6 +627,9 @@ def direct_get_suffix_hashes(node, part, suffixes, conn_timeout=5,
     """
     Get suffix hashes directly from the object server.
 
+    Note that unlike other ``direct_client`` functions, this one defaults
+    to using the replication network to make requests.
+
     :param node: node dictionary from the ring
     :param part: partition the container is on
     :param conn_timeout: timeout in seconds for establishing the connection
@@ -560,9 +641,11 @@ def direct_get_suffix_hashes(node, part, suffixes, conn_timeout=5,
     if headers is None:
         headers = {}
 
+    headers.setdefault(USE_REPLICATION_NETWORK_HEADER, 'true')
+    ip, port = get_ip_port(node, headers)
     path = '/%s' % '-'.join(suffixes)
     with Timeout(conn_timeout):
-        conn = http_connect(node['replication_ip'], node['replication_port'],
+        conn = http_connect(ip, port,
                             node['device'], part, 'REPLICATE', path,
                             headers=gen_headers(headers))
     with Timeout(response_timeout):
@@ -573,7 +656,7 @@ def direct_get_suffix_hashes(node, part, suffixes, conn_timeout=5,
                                     host={'ip': node['replication_ip'],
                                           'port': node['replication_port']}
                                     )
-    return pickle.loads(resp.read())
+    return pickle.loads(resp.read())  # nosec: B301
 
 
 def retry(func, *args, **kwargs):
@@ -619,3 +702,31 @@ def retry(func, *args, **kwargs):
                               http_device=args[0]['device'])
     else:
         raise ClientException('Raise too many retries')
+
+
+def direct_get_recon(node, recon_command, conn_timeout=5, response_timeout=15,
+                     headers=None):
+    """
+    Get recon json directly from the storage server.
+
+    :param node: node dictionary from the ring
+    :param recon_command: recon string (post /recon/)
+    :param conn_timeout: timeout in seconds for establishing the connection
+    :param response_timeout: timeout in seconds for getting the response
+    :param headers: dict to be passed into HTTPConnection headers
+    :returns: deserialized json response
+    :raises DirectClientReconException: HTTP GET request failed
+    """
+    if headers is None:
+        headers = {}
+
+    ip, port = get_ip_port(node, headers)
+    path = '/recon/%s' % recon_command
+    with Timeout(conn_timeout):
+        conn = http_connect_raw(ip, port, 'GET', path,
+                                headers=gen_headers(headers))
+    with Timeout(response_timeout):
+        resp = conn.getresponse()
+    if not is_success(resp.status):
+        raise DirectClientReconException('GET', node, path, resp)
+    return json.loads(resp.read())

swift/common/error_limiter.py

@@ -0,0 +1,93 @@
+# Copyright (c) 2021 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import collections
+from time import time
+
+from swift.common.utils import node_to_string
+
+
+class ErrorLimiter(object):
+    """
+    Tracks the number of errors that have occurred for nodes. A node will be
+    considered to be error-limited for a given interval of time after it has
+    accumulated more errors than a given limit.
+
+    :param suppression_interval: The number of seconds for which a node is
+        error-limited once it has accumulated more than ``suppression_limit``
+        errors. Should be a float value.
+    :param suppression_limit: The number of errors that a node must accumulate
+        before it is considered to be error-limited. Should be an int value.
+    """
+    def __init__(self, suppression_interval, suppression_limit):
+        self.suppression_interval = float(suppression_interval)
+        self.suppression_limit = int(suppression_limit)
+        self.stats = collections.defaultdict(dict)
+
+    def node_key(self, node):
+        """
+        Get the key under which a node's error stats will be stored.
+
+        :param node: dictionary describing a node.
+        :return: string key.
+        """
+        return node_to_string(node)
+
+    def is_limited(self, node):
+        """
+        Check if the node is currently error limited.
+
+        :param node: dictionary of node to check
+        :returns: True if error limited, False otherwise
+        """
+        now = time()
+        node_key = self.node_key(node)
+        error_stats = self.stats.get(node_key)
+
+        if error_stats is None or 'errors' not in error_stats:
+            return False
+
+        if 'last_error' in error_stats and error_stats['last_error'] < \
+                now - self.suppression_interval:
+            self.stats.pop(node_key)
+            return False
+        return error_stats['errors'] > self.suppression_limit
+
+    def limit(self, node):
+        """
+        Mark a node as error limited. This immediately pretends the
+        node received enough errors to trigger error suppression. Use
+        this for errors like Insufficient Storage. For other errors
+        use :func:`increment`.
+
+        :param node: dictionary of node to error limit
+        """
+        node_key = self.node_key(node)
+        error_stats = self.stats[node_key]
+        error_stats['errors'] = self.suppression_limit + 1
+        error_stats['last_error'] = time()
+
+    def increment(self, node):
+        """
+        Increment the error count and update the time of the last error for
+        the given ``node``.
+
+        :param node: dictionary describing a node.
+        :returns: True if suppression_limit is exceeded, False otherwise
+        """
+        node_key = self.node_key(node)
+        error_stats = self.stats[node_key]
+        error_stats['errors'] = error_stats.get('errors', 0) + 1
+        error_stats['last_error'] = time()
+        return error_stats['errors'] > self.suppression_limit

swift/common/exceptions.py

@@ -113,7 +113,11 @@ class DeviceUnavailable(SwiftException):
     pass
 
 
-class InvalidAccountInfo(SwiftException):
+class DatabaseAuditorException(SwiftException):
+    pass
+
+
+class InvalidAccountInfo(DatabaseAuditorException):
     pass
 
 
@@ -215,6 +219,10 @@ class ReplicationLockTimeout(LockTimeout):
     pass
 
 
+class PartitionLockTimeout(LockTimeout):
+    pass
+
+
 class MimeInvalid(SwiftException):
     pass
 
@@ -231,6 +239,22 @@ class UnknownSecretIdError(EncryptionException):
     pass
 
 
+class QuarantineRequest(SwiftException):
+    pass
+
+
+class MemcacheConnectionError(Exception):
+    pass
+
+
+class MemcacheIncrNotFoundError(MemcacheConnectionError):
+    pass
+
+
+class MemcachePoolTimeout(Timeout):
+    pass
+
+
 class ClientException(Exception):
 
     def __init__(self, msg, http_scheme='', http_host='', http_port='',

swift/common/header_key_dict.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import six
-
 
 class HeaderKeyDict(dict):
     """
@@ -31,10 +29,7 @@ class HeaderKeyDict(dict):
 
     @staticmethod
     def _title(s):
-        if six.PY2:
-            return s.title()
-        else:
-            return s.encode('latin1').title().decode('latin1')
+        return s.encode('latin1').title().decode('latin1')
 
     def update(self, other):
         if hasattr(other, 'keys'):
@@ -51,9 +46,7 @@ class HeaderKeyDict(dict):
         key = self._title(key)
         if value is None:
             self.pop(key, None)
-        elif six.PY2 and isinstance(value, six.text_type):
-            return dict.__setitem__(self, key, value.encode('utf-8'))
-        elif six.PY3 and isinstance(value, six.binary_type):
+        elif isinstance(value, bytes):
             return dict.__setitem__(self, key, value.decode('latin-1'))
         else:
             return dict.__setitem__(self, key, str(value))