swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/internal_client.py

@@ -13,29 +13,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from eventlet import sleep, Timeout
-from eventlet.green import httplib, socket
+from eventlet import sleep, Timeout, spawn
+from eventlet.green import socket
+from eventlet.green.http import client as http_client
+from eventlet.green.urllib import request as urllib_request
 import json
-import six
-from six.moves import range
-from six.moves import urllib
+import urllib
 import struct
-from sys import exc_info, exit
+from sys import exit
 import zlib
 from time import gmtime, strftime, time
 from zlib import compressobj
 
 from swift.common.exceptions import ClientException
 from swift.common.http import (HTTP_NOT_FOUND, HTTP_MULTIPLE_CHOICES,
-                               is_server_error)
+                               is_client_error, is_server_error)
+from swift.common.middleware.gatekeeper import GatekeeperMiddleware
+from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.swob import Request, bytes_to_wsgi
-from swift.common.utils import quote, closing_if_possible
-from swift.common.wsgi import loadapp, pipeline_property
-
-if six.PY3:
-    from eventlet.green.urllib import request as urllib2
-else:
-    from eventlet.green import urllib2
+from swift.common.utils import quote, close_if_possible, drain_and_close
+from swift.common.wsgi import loadapp
 
 
 class UnexpectedResponse(Exception):
@@ -125,8 +122,6 @@ class CompressingFileReader(object):
             return chunk
         raise StopIteration
 
-    next = __next__
-
     def seek(self, offset, whence=0):
         if not (offset == 0 and whence == 0):
             raise NotImplementedError('Seek implemented on offset 0 only')
@@ -143,22 +138,48 @@ class InternalClient(object):
     :param user_agent: User agent to be sent to requests to Swift.
     :param request_tries: Number of tries before InternalClient.make_request()
                           gives up.
+    :param use_replication_network: Force the client to use the replication
+        network over the cluster.
+    :param global_conf: a dict of options to update the loaded proxy config.
+        Options in ``global_conf`` will override those in ``conf_path`` except
+        where the ``conf_path`` option is preceded by ``set``.
+    :param app: Optionally provide a WSGI app for the internal client to use.
     """
 
     def __init__(self, conf_path, user_agent, request_tries,
-                 allow_modify_pipeline=False):
+                 use_replication_network=False, global_conf=None, app=None,
+                 **kwargs):
         if request_tries < 1:
             raise ValueError('request_tries must be positive')
-        self.app = loadapp(conf_path,
-                           allow_modify_pipeline=allow_modify_pipeline)
-        self.user_agent = user_agent
+        # Internal clients don't use the gatekeeper and the pipeline remains
+        # static so we never allow anything to modify the proxy pipeline.
+        if kwargs.get('allow_modify_pipeline'):
+            raise ValueError("'allow_modify_pipeline' is no longer supported")
+        self.app = app or loadapp(conf_path, global_conf=global_conf,
+                                  allow_modify_pipeline=False,)
+        self.check_gatekeeper_not_loaded(self.app)
+        self.user_agent = \
+            self.app._pipeline_final_app.backend_user_agent = user_agent
         self.request_tries = request_tries
-
-    get_object_ring = pipeline_property('get_object_ring')
-    container_ring = pipeline_property('container_ring')
-    account_ring = pipeline_property('account_ring')
-    auto_create_account_prefix = pipeline_property(
-        'auto_create_account_prefix', default='.')
+        self.use_replication_network = use_replication_network
+        self.get_object_ring = self.app._pipeline_final_app.get_object_ring
+        self.container_ring = self.app._pipeline_final_app.container_ring
+        self.account_ring = self.app._pipeline_final_app.account_ring
+        self.auto_create_account_prefix = \
+            self.app._pipeline_final_app.auto_create_account_prefix
+
+    @staticmethod
+    def check_gatekeeper_not_loaded(app):
+        # the Gatekeeper middleware would prevent an InternalClient passing
+        # X-Backend-* headers to the proxy app, so ensure it's not present
+        try:
+            for app in app._pipeline:
+                if isinstance(app, GatekeeperMiddleware):
+                    raise ValueError(
+                        "Gatekeeper middleware is not allowed in the "
+                        "InternalClient proxy pipeline")
+        except AttributeError:
+            pass
 
     def make_request(
             self, method, path, headers, acceptable_statuses, body_file=None,
@@ -184,8 +205,12 @@ class InternalClient(object):
 
         headers = dict(headers)
         headers['user-agent'] = self.user_agent
+        headers.setdefault('x-backend-allow-reserved-names', 'true')
+        if self.use_replication_network:
+            headers.setdefault(USE_REPLICATION_NETWORK_HEADER, 'true')
+
         for attempt in range(self.request_tries):
-            resp = exc_type = exc_value = exc_traceback = None
+            resp = err = None
             req = Request.blank(
                 path, environ={'REQUEST_METHOD': method}, headers=headers)
             if body_file is not None:
@@ -195,9 +220,10 @@ class InternalClient(object):
             if params:
                 req.params = params
             try:
-                resp = req.get_response(self.app)
-            except (Exception, Timeout):
-                exc_type, exc_value, exc_traceback = exc_info()
+                # execute in a separate greenthread to not polute corolocals
+                resp = spawn(req.get_response, self.app).wait()
+            except (Exception, Timeout) as e:
+                err = e
             else:
                 if resp.status_int in acceptable_statuses or \
                         resp.status_int // 100 in acceptable_statuses:
@@ -208,13 +234,13 @@ class InternalClient(object):
             # sleep only between tries, not after each one
             if attempt < self.request_tries - 1:
                 if resp:
-                    # always close any resp.app_iter before we discard it
-                    with closing_if_possible(resp.app_iter):
-                        # for non 2XX requests it's safe and useful to drain
-                        # the response body so we log the correct status code
-                        if resp.status_int // 100 != 2:
-                            for iter_body in resp.app_iter:
-                                pass
+                    # for non 2XX requests it's safe and useful to drain
+                    # the response body so we log the correct status code
+                    if resp.status_int // 100 != 2:
+                        drain_and_close(resp)
+                    else:
+                        # Just close; the 499 is appropriate
+                        close_if_possible(resp.app_iter)
                 sleep(2 ** (attempt + 1))
         if resp:
             msg = 'Unexpected response: %s' % resp.status
@@ -223,9 +249,14 @@ class InternalClient(object):
                 # non 2XX responses
                 msg += ' (%s)' % resp.body
             raise UnexpectedResponse(msg, resp)
-        if exc_type:
-            # To make pep8 tool happy, in place of raise t, v, tb:
-            six.reraise(exc_type, exc_value, exc_traceback)
+        if err:
+            raise err
+
+    def handle_request(self, *args, **kwargs):
+        resp = self.make_request(*args, **kwargs)
+        # Drain the response body to prevent unexpected disconnect
+        # in proxy-server
+        drain_and_close(resp)
 
     def _get_metadata(
             self, path, metadata_prefix='', acceptable_statuses=(2,),
@@ -356,7 +387,7 @@ class InternalClient(object):
                 headers[k] = v
             else:
                 headers['%s%s' % (metadata_prefix, k)] = v
-        self.make_request('POST', path, headers, acceptable_statuses)
+        self.handle_request('POST', path, headers, acceptable_statuses)
 
     # account methods
 
@@ -384,6 +415,34 @@ class InternalClient(object):
         return self._iter_items(path, marker, end_marker, prefix,
                                 acceptable_statuses)
 
+    def create_account(self, account):
+        """
+        Creates an account.
+
+        :param account: Account to create.
+        :raises UnexpectedResponse: Exception raised when requests fail
+                                    to get a response with an acceptable status
+        :raises Exception: Exception is raised when code fails in an
+                           unexpected way.
+        """
+        path = self.make_path(account)
+        self.handle_request('PUT', path, {}, (201, 202))
+
+    def delete_account(self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)):
+        """
+        Deletes an account.
+
+        :param account: Account to delete.
+        :param acceptable_statuses: List of status for valid responses,
+                                    defaults to (2, HTTP_NOT_FOUND).
+        :raises UnexpectedResponse: Exception raised when requests fail
+                                    to get a response with an acceptable status
+        :raises Exception: Exception is raised when code fails in an
+                           unexpected way.
+        """
+        path = self.make_path(account)
+        self.handle_request('DELETE', path, {}, acceptable_statuses)
+
     def get_account_info(
             self, account, acceptable_statuses=(2, HTTP_NOT_FOUND)):
         """
@@ -496,10 +555,11 @@ class InternalClient(object):
 
         headers = headers or {}
         path = self.make_path(account, container)
-        self.make_request('PUT', path, headers, acceptable_statuses)
+        self.handle_request('PUT', path, headers, acceptable_statuses)
 
     def delete_container(
-            self, account, container, acceptable_statuses=(2, HTTP_NOT_FOUND)):
+            self, account, container, headers=None,
+            acceptable_statuses=(2, HTTP_NOT_FOUND)):
         """
         Deletes a container.
 
@@ -514,8 +574,9 @@ class InternalClient(object):
                            unexpected way.
         """
 
+        headers = headers or {}
         path = self.make_path(account, container)
-        self.make_request('DELETE', path, {}, acceptable_statuses)
+        self.handle_request('DELETE', path, headers, acceptable_statuses)
 
     def get_container_metadata(
             self, account, container, metadata_prefix='',
@@ -618,7 +679,8 @@ class InternalClient(object):
         """
 
         path = self.make_path(account, container, obj)
-        self.make_request('DELETE', path, (headers or {}), acceptable_statuses)
+        self.handle_request('DELETE', path, (headers or {}),
+                            acceptable_statuses)
 
     def get_object_metadata(
             self, account, container, obj, metadata_prefix='',
@@ -647,7 +709,7 @@ class InternalClient(object):
         return self._get_metadata(path, metadata_prefix, acceptable_statuses,
                                   headers=headers, params=params)
 
-    def get_object(self, account, container, obj, headers,
+    def get_object(self, account, container, obj, headers=None,
                    acceptable_statuses=(2,), params=None):
         """
         Gets an object.
@@ -749,13 +811,17 @@ class InternalClient(object):
             path, metadata, metadata_prefix, acceptable_statuses)
 
     def upload_object(
-            self, fobj, account, container, obj, headers=None):
+            self, fobj, account, container, obj, headers=None,
+            acceptable_statuses=(2,), params=None):
         """
         :param fobj: File object to read object's content from.
         :param account: The object's account.
         :param container: The object's container.
         :param obj: The object.
         :param headers: Headers to send with request, defaults to empty dict.
+        :param acceptable_statuses: List of acceptable statuses for request.
+        :param params: A dict of params to be set in request query string,
+                       defaults to None.
 
         :raises UnexpectedResponse: Exception raised when requests fail
                                     to get a response with an acceptable status
@@ -767,16 +833,17 @@ class InternalClient(object):
         if 'Content-Length' not in headers:
             headers['Transfer-Encoding'] = 'chunked'
         path = self.make_path(account, container, obj)
-        self.make_request('PUT', path, headers, (2,), fobj)
+        self.handle_request('PUT', path, headers, acceptable_statuses, fobj,
+                            params=params)
 
 
 def get_auth(url, user, key, auth_version='1.0', **kwargs):
     if auth_version != '1.0':
         exit('ERROR: swiftclient missing, only auth v1.0 supported')
-    req = urllib2.Request(url)
+    req = urllib_request.Request(url)
     req.add_header('X-Auth-User', user)
     req.add_header('X-Auth-Key', key)
-    conn = urllib2.urlopen(req)
+    conn = urllib_request.urlopen(req)
     headers = conn.info()
     return (
         headers.getheader('X-Storage-Url'),
@@ -839,12 +906,12 @@ class SimpleClient(object):
 
             url += '?' + '&'.join(params)
 
-        req = urllib2.Request(url, headers=headers, data=contents)
+        req = urllib_request.Request(url, headers=headers, data=contents)
         if proxy:
             proxy = urllib.parse.urlparse(proxy)
             req.set_proxy(proxy.netloc, proxy.scheme)
         req.get_method = lambda: method
-        conn = urllib2.urlopen(req, timeout=timeout)
+        conn = urllib_request.urlopen(req, timeout=timeout)
         body = conn.read()
         info = conn.info()
         try:
@@ -886,14 +953,17 @@ class SimpleClient(object):
             self.attempts += 1
             try:
                 return self.base_request(method, **kwargs)
-            except (socket.error, httplib.HTTPException, urllib2.URLError) \
-                    as err:
+            except urllib_request.HTTPError as err:
+                if is_client_error(err.getcode() or 500):
+                    raise ClientException('Client error',
+                                          http_status=err.getcode())
+                elif self.attempts > retries:
+                    raise ClientException('Raise too many retries',
+                                          http_status=err.getcode())
+            except (socket.error, http_client.HTTPException,
+                    urllib_request.URLError):
                 if self.attempts > retries:
-                    if isinstance(err, urllib2.HTTPError):
-                        raise ClientException('Raise too many retries',
-                                              http_status=err.getcode())
-                    else:
-                        raise
+                    raise
             sleep(backoff)
             backoff = min(backoff * 2, self.max_backoff)
 

swift/common/linkat.py

@@ -17,8 +17,6 @@ import os
 import ctypes
 from ctypes.util import find_library
 
-import six
-
 __all__ = ['linkat']
 
 
@@ -72,12 +70,13 @@ class Linkat(object):
         if not isinstance(olddirfd, int) or not isinstance(newdirfd, int):
             raise TypeError("fd must be an integer.")
 
-        if isinstance(oldpath, six.text_type):
+        if isinstance(oldpath, str):
             oldpath = oldpath.encode('utf8')
-        if isinstance(newpath, six.text_type):
+        if isinstance(newpath, str):
             newpath = newpath.encode('utf8')
 
         return self._c_linkat(olddirfd, oldpath, newdirfd, newpath, flags)
 
+
 linkat = Linkat()
 del Linkat