swift 2.32.1__py2.py3-none-any.whl → 2.33.1__py2.py3-none-any.whl

swift/proxy/controllers/base.py

@@ -39,12 +39,13 @@ from sys import exc_info
 from eventlet.timeout import Timeout
 import six
 
+from swift.common.memcached import MemcacheConnectionError
 from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request
 from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \
     public, split_path, list_from_csv, GreenthreadSafeIterator, \
     GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \
-    document_iters_to_http_response_body, ShardRange, cache_from_env, \
-    MetricsPrefixLoggerAdapter, CooperativeIterator
+    document_iters_to_http_response_body, cache_from_env, \
+    CooperativeIterator, NamespaceBoundList, Namespace
 from swift.common.bufferedhttp import http_connect
 from swift.common import constraints
 from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \
@@ -423,9 +424,9 @@ def _record_ac_info_cache_metrics(
         logger = None
     else:
         logger = proxy_app.logger
-    op_type = 'container.info' if container else 'account.info'
+    server_type = 'container' if container else 'account'
     if logger:
-        record_cache_op_metrics(logger, op_type, cache_state, resp)
+        record_cache_op_metrics(logger, server_type, 'info', cache_state, resp)
 
 
 def get_container_info(env, app, swift_source=None, cache_only=False):
@@ -479,8 +480,7 @@ def get_container_info(env, app, swift_source=None, cache_only=False):
         # account is successful whether the account actually has .db files
         # on disk or not.
         is_autocreate_account = account.startswith(
-            getattr(proxy_app, 'auto_create_account_prefix',
-                    constraints.AUTO_CREATE_ACCOUNT_PREFIX))
+            constraints.AUTO_CREATE_ACCOUNT_PREFIX)
         if not is_autocreate_account:
             account_info = get_account_info(env, logged_app, swift_source)
             if not account_info or not is_success(account_info['status']):
@@ -774,11 +774,12 @@ def _get_info_from_infocache(env, account, container=None):
 
 
 def record_cache_op_metrics(
-        logger, op_type, cache_state, resp=None):
+        logger, server_type, op_type, cache_state, resp=None):
     """
     Record a single cache operation into its corresponding metrics.
 
     :param  logger: the metrics logger
+    :param  server_type: 'account' or 'container'
     :param  op_type: the name of the operation type, includes 'shard_listing',
               'shard_updating', and etc.
     :param  cache_state: the state of this cache operation. When it's
@@ -787,21 +788,23 @@ def record_cache_op_metrics(
               which will make to backend, expect a valid 'resp'.
     :param  resp: the response from backend for all cases except cache hits.
     """
+    server_type = server_type.lower()
     if cache_state == 'infocache_hit':
-        logger.increment('%s.infocache.hit' % op_type)
+        logger.increment('%s.%s.infocache.hit' % (server_type, op_type))
     elif cache_state == 'hit':
         # memcache hits.
-        logger.increment('%s.cache.hit' % op_type)
+        logger.increment('%s.%s.cache.hit' % (server_type, op_type))
     else:
         # the cases of cache_state is memcache miss, error, skip, force_skip
         # or disabled.
         if resp:
-            logger.increment(
-                '%s.cache.%s.%d' % (op_type, cache_state, resp.status_int))
+            logger.increment('%s.%s.cache.%s.%d' % (
+                server_type, op_type, cache_state, resp.status_int))
         else:
             # In some situation, we choose not to lookup backend after cache
             # miss.
-            logger.increment('%s.cache.%s' % (op_type, cache_state))
+            logger.increment('%s.%s.cache.%s' % (
+                server_type, op_type, cache_state))
 
 
 def _get_info_from_memcache(app, env, account, container=None):
@@ -886,6 +889,77 @@ def _get_info_from_caches(app, env, account, container=None):
     return info, cache_state
 
 
+def get_namespaces_from_cache(req, cache_key, skip_chance):
+    """
+    Get cached namespaces from infocache or memcache.
+
+    :param req: a :class:`swift.common.swob.Request` object.
+    :param cache_key: the cache key for both infocache and memcache.
+    :param skip_chance: the probability of skipping the memcache look-up.
+    :return: a tuple of (value, cache state). Value is an instance of
+        :class:`swift.common.utils.NamespaceBoundList` if a non-empty list is
+        found in memcache. Otherwise value is ``None``, for example if memcache
+        look-up was skipped, or no value was found, or an empty list was found.
+    """
+    # try get namespaces from infocache first
+    infocache = req.environ.setdefault('swift.infocache', {})
+    ns_bound_list = infocache.get(cache_key)
+    if ns_bound_list:
+        return ns_bound_list, 'infocache_hit'
+
+    # then try get them from memcache
+    memcache = cache_from_env(req.environ, True)
+    if not memcache:
+        return None, 'disabled'
+    if skip_chance and random.random() < skip_chance:
+        return None, 'skip'
+    try:
+        bounds = memcache.get(cache_key, raise_on_error=True)
+        cache_state = 'hit' if bounds else 'miss'
+    except MemcacheConnectionError:
+        bounds = None
+        cache_state = 'error'
+
+    if bounds:
+        if six.PY2:
+            # json.loads() in memcache.get will convert json 'string' to
+            # 'unicode' with python2, here we cast 'unicode' back to 'str'
+            bounds = [
+                [lower.encode('utf-8'), name.encode('utf-8')]
+                for lower, name in bounds]
+        ns_bound_list = NamespaceBoundList(bounds)
+        infocache[cache_key] = ns_bound_list
+    else:
+        ns_bound_list = None
+    return ns_bound_list, cache_state
+
+
+def set_namespaces_in_cache(req, cache_key, ns_bound_list, time):
+    """
+    Set a list of namespace bounds in infocache and memcache.
+
+    :param req: a :class:`swift.common.swob.Request` object.
+    :param cache_key: the cache key for both infocache and memcache.
+    :param ns_bound_list: a :class:`swift.common.utils.NamespaceBoundList`.
+    :param time: how long the namespaces should remain in memcache.
+    :return: the cache_state.
+    """
+    infocache = req.environ.setdefault('swift.infocache', {})
+    infocache[cache_key] = ns_bound_list
+    memcache = cache_from_env(req.environ, True)
+    if memcache and ns_bound_list:
+        try:
+            memcache.set(cache_key, ns_bound_list.bounds, time=time,
+                         raise_on_error=True)
+        except MemcacheConnectionError:
+            cache_state = 'set_error'
+        else:
+            cache_state = 'set'
+    else:
+        cache_state = 'disabled'
+    return cache_state
+
+
 def _prepare_pre_auth_info_request(env, path, swift_source):
     """
     Prepares a pre authed request to obtain info using a HEAD.
@@ -1065,6 +1139,14 @@ class ByteCountEnforcer(object):
 
 
 class GetterSource(object):
+    """
+    Encapsulates properties of a source from which a GET response is read.
+
+    :param app: a proxy app.
+    :param resp: an instance of ``HTTPResponse``.
+    :param node: a dict describing the node from which the response was
+        returned.
+    """
     __slots__ = ('app', 'resp', 'node', '_parts_iter')
 
     def __init__(self, app, resp, node):
@@ -1101,8 +1183,26 @@ class GetterSource(object):
 
 
 class GetterBase(object):
+    """
+    This base class provides helper methods for handling GET requests to
+    backend servers.
+
+    :param app: a proxy app.
+    :param req: an instance of ``swob.Request``.
+    :param node_iter: an iterator yielding nodes.
+    :param partition: partition.
+    :param policy: the policy instance, or None if Account or Container.
+    :param path: path for the request.
+    :param backend_headers: a dict of headers to be sent with backend requests.
+    :param node_timeout: the timeout value for backend requests.
+    :param resource_type: a string description of the type of resource being
+        accessed; ``resource type`` is used in logs and isn't necessarily the
+        server type.
+    :param logger: a logger instance.
+    """
     def __init__(self, app, req, node_iter, partition, policy,
-                 path, backend_headers, logger=None):
+                 path, backend_headers, node_timeout, resource_type,
+                 logger=None):
         self.app = app
         self.req = req
         self.node_iter = node_iter
@@ -1110,6 +1210,9 @@ class GetterBase(object):
         self.policy = policy
         self.path = path
         self.backend_headers = backend_headers
+        # resource type is used in logs and isn't necessarily the server type
+        self.resource_type = resource_type
+        self.node_timeout = node_timeout
         self.logger = logger or app.logger
         self.bytes_used_from_backend = 0
         self.source = None
@@ -1122,18 +1225,44 @@ class GetterBase(object):
         :return: ``True`` if ``self.source`` has been updated, ``False``
             otherwise.
         """
-        # Subclasses must implement this method
+        # Subclasses must implement this method, but _replace_source should be
+        # called to get a source installed
         raise NotImplementedError()
 
-    def _replace_source(self, err_msg):
-        # _find_source can modify self.source so stash current source
-        old_source = self.source
-        if not self._find_source():
-            return False
+    def _replace_source(self, err_msg=''):
+        if self.source:
+            self.app.error_occurred(self.source.node, err_msg)
+            self.source.close()
+        return self._find_source()
 
-        self.app.error_occurred(old_source.node, err_msg)
-        old_source.close()
-        return True
+    def _get_next_response_part(self):
+        # return the next part of the response body; there may only be one part
+        # unless it's a multipart/byteranges response
+        while True:
+            # the loop here is to resume if trying to parse
+            # multipart/byteranges response raises a ChunkReadTimeout
+            # and resets the source_parts_iter
+            try:
+                with WatchdogTimeout(self.app.watchdog, self.node_timeout,
+                                     ChunkReadTimeout):
+                    # If we don't have a multipart/byteranges response,
+                    # but just a 200 or a single-range 206, then this
+                    # performs no IO, and either just returns source or
+                    # raises StopIteration.
+                    # Otherwise, this call to next() performs IO when
+                    # we have a multipart/byteranges response, as it
+                    # will read the MIME boundary and part headers. In this
+                    # case, ChunkReadTimeout may also be raised.
+                    # If StopIteration is raised, it escapes and is
+                    # handled elsewhere.
+                    start_byte, end_byte, length, headers, part = next(
+                        self.source.parts_iter)
+                return (start_byte, end_byte, length, headers, part)
+            except ChunkReadTimeout:
+                if not self._replace_source(
+                        'Trying to read next part of %s multi-part GET '
+                        '(retrying)' % self.resource_type):
+                    raise
 
     def fast_forward(self, num_bytes):
         """
@@ -1240,32 +1369,43 @@ class GetterBase(object):
 
 
 class GetOrHeadHandler(GetterBase):
+    """
+    Handles GET requests to backend servers.
+
+    :param app: a proxy app.
+    :param req: an instance of ``swob.Request``.
+    :param server_type: server type used in logging
+    :param node_iter: an iterator yielding nodes.
+    :param partition: partition.
+    :param path: path for the request.
+    :param backend_headers: a dict of headers to be sent with backend requests.
+    :param concurrency: number of requests to run concurrently.
+    :param policy: the policy instance, or None if Account or Container.
+    :param logger: a logger instance.
+    """
     def __init__(self, app, req, server_type, node_iter, partition, path,
-                 backend_headers, concurrency=1, policy=None,
-                 newest=None, logger=None):
+                 backend_headers, concurrency=1, policy=None, logger=None):
+        newest = config_true_value(req.headers.get('x-newest', 'f'))
+        if server_type == 'Object' and not newest:
+            node_timeout = app.recoverable_node_timeout
+        else:
+            node_timeout = app.node_timeout
         super(GetOrHeadHandler, self).__init__(
-            app=app, req=req, node_iter=node_iter,
-            partition=partition, policy=policy, path=path,
-            backend_headers=backend_headers, logger=logger)
+            app=app, req=req, node_iter=node_iter, partition=partition,
+            policy=policy, path=path, backend_headers=backend_headers,
+            node_timeout=node_timeout, resource_type=server_type.lower(),
+            logger=logger)
+        self.newest = newest
         self.server_type = server_type
         self.used_nodes = []
         self.used_source_etag = None
         self.concurrency = concurrency
         self.latest_404_timestamp = Timestamp(0)
-        if self.server_type == 'Object':
-            self.node_timeout = self.app.recoverable_node_timeout
-        else:
-            self.node_timeout = self.app.node_timeout
         policy_options = self.app.get_policy_options(self.policy)
         self.rebalance_missing_suppression_count = min(
             policy_options.rebalance_missing_suppression_count,
             node_iter.num_primary_nodes - 1)
 
-        if newest is None:
-            self.newest = config_true_value(req.headers.get('x-newest', 'f'))
-        else:
-            self.newest = newest
-
         # populated when finding source
         self.statuses = []
         self.reasons = []
@@ -1276,31 +1416,6 @@ class GetOrHeadHandler(GetterBase):
         # populated from response headers
         self.start_byte = self.end_byte = self.length = None
 
-    def _get_next_response_part(self):
-        # return the next part of the response body; there may only be one part
-        # unless it's a multipart/byteranges response
-        while True:
-            try:
-                # This call to next() performs IO when we have a
-                # multipart/byteranges response; it reads the MIME
-                # boundary and part headers.
-                #
-                # If we don't have a multipart/byteranges response,
-                # but just a 200 or a single-range 206, then this
-                # performs no IO, and either just returns source or
-                # raises StopIteration.
-                with WatchdogTimeout(self.app.watchdog, self.node_timeout,
-                                     ChunkReadTimeout):
-                    # if StopIteration is raised, it escapes and is
-                    # handled elsewhere
-                    start_byte, end_byte, length, headers, part = next(
-                        self.source.parts_iter)
-                return (start_byte, end_byte, length, headers, part)
-            except ChunkReadTimeout:
-                if not self._replace_source(
-                        'Trying to read object during GET (retrying)'):
-                    raise StopIteration()
-
     def _iter_bytes_from_response_part(self, part_file, nbytes):
         # yield chunks of bytes from a single response part; if an error
         # occurs, try to resume yielding bytes from a different source
@@ -1345,7 +1460,7 @@ class GetOrHeadHandler(GetterBase):
                     self.bytes_used_from_backend += len(chunk)
                     yield chunk
 
-    def _iter_parts_from_response(self, req):
+    def _iter_parts_from_response(self):
         # iterate over potentially multiple response body parts; for each
         # part, yield an iterator over the part's bytes
         try:
@@ -1370,20 +1485,17 @@ class GetOrHeadHandler(GetterBase):
                            'part_iter': part_iter}
                     self.pop_range()
             except StopIteration:
-                req.environ['swift.non_client_disconnect'] = True
+                self.req.environ['swift.non_client_disconnect'] = True
             finally:
                 if part_iter:
                     part_iter.close()
 
-        except ChunkReadTimeout:
-            self.app.exception_occurred(self.source.node, 'Object',
-                                        'Trying to read during GET')
-            raise
         except ChunkWriteTimeout:
             self.logger.info(
                 'Client did not read from proxy within %ss',
                 self.app.client_timeout)
-            self.logger.increment('client_timeouts')
+            self.logger.increment('%s.client_timeouts' %
+                                  self.server_type.lower())
         except GeneratorExit:
             warn = True
             req_range = self.backend_headers['Range']
@@ -1394,7 +1506,8 @@ class GetOrHeadHandler(GetterBase):
                     if end is not None and begin is not None:
                         if end - begin + 1 == self.bytes_used_from_backend:
                             warn = False
-            if not req.environ.get('swift.non_client_disconnect') and warn:
+            if (warn and
+                    not self.req.environ.get('swift.non_client_disconnect')):
                 self.logger.info('Client disconnected on read of %r',
                                  self.path)
             raise
@@ -1418,7 +1531,7 @@ class GetOrHeadHandler(GetterBase):
         else:
             return None
 
-    def _make_node_request(self, node, node_timeout, logger_thread_locals):
+    def _make_node_request(self, node, logger_thread_locals):
         # make a backend request; return True if the response is deemed good
         # (has an acceptable status code), useful (matches any previously
         # discovered etag) and sufficient (a single good response is
@@ -1426,6 +1539,7 @@ class GetOrHeadHandler(GetterBase):
         self.logger.thread_locals = logger_thread_locals
         if node in self.used_nodes:
             return False
+
         req_headers = dict(self.backend_headers)
         ip, port = get_ip_port(node, req_headers)
         start_node_timing = time.time()
@@ -1438,7 +1552,7 @@ class GetOrHeadHandler(GetterBase):
                     query_string=self.req.query_string)
             self.app.set_node_timing(node, time.time() - start_node_timing)
 
-            with Timeout(node_timeout):
+            with Timeout(self.node_timeout):
                 possible_source = conn.getresponse()
                 # See NOTE: swift_conn at top of file about this.
                 possible_source.swift_conn = conn
@@ -1530,14 +1644,10 @@ class GetOrHeadHandler(GetterBase):
 
         nodes = GreenthreadSafeIterator(self.node_iter)
 
-        node_timeout = self.app.node_timeout
-        if self.server_type == 'Object' and not self.newest:
-            node_timeout = self.app.recoverable_node_timeout
-
         pile = GreenAsyncPile(self.concurrency)
 
         for node in nodes:
-            pile.spawn(self._make_node_request, node, node_timeout,
+            pile.spawn(self._make_node_request, node,
                        self.logger.thread_locals)
             _timeout = self.app.get_policy_options(
                 self.policy).concurrency_timeout \
@@ -1573,13 +1683,12 @@ class GetOrHeadHandler(GetterBase):
             return True
         return False
 
-    def _make_app_iter(self, req):
+    def _make_app_iter(self):
         """
         Returns an iterator over the contents of the source (via its read
         func).  There is also quite a bit of cleanup to ensure garbage
         collection works and the underlying socket of the source is closed.
 
-        :param req: incoming request object
         :return: an iterator that yields chunks of response body bytes
         """
 
@@ -1596,7 +1705,7 @@ class GetOrHeadHandler(GetterBase):
             # furnished one for us, so we'll just re-use it
             boundary = dict(content_type_attrs)["boundary"]
 
-        parts_iter = self._iter_parts_from_response(req)
+        parts_iter = self._iter_parts_from_response()
 
         def add_content_type(response_part):
             response_part["content_type"] = \
@@ -1607,15 +1716,15 @@ class GetOrHeadHandler(GetterBase):
             (add_content_type(pi) for pi in parts_iter),
             boundary, is_multipart, self.logger)
 
-    def get_working_response(self, req):
+    def get_working_response(self):
         res = None
-        if self._find_source():
-            res = Response(request=req)
+        if self._replace_source():
+            res = Response(request=self.req)
             res.status = self.source.resp.status
             update_headers(res, self.source.resp.getheaders())
-            if req.method == 'GET' and \
+            if self.req.method == 'GET' and \
                     self.source.resp.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
-                res.app_iter = self._make_app_iter(req)
+                res.app_iter = self._make_app_iter()
                 # See NOTE: swift_conn at top of file about this.
                 res.swift_conn = self.source.resp.swift_conn
             if not res.environ:
@@ -1644,6 +1753,7 @@ class NodeIter(object):
     may not, depending on how logging is configured, the vagaries of
     socket IO and eventlet, and the phase of the moon.)
 
+    :param server_type: one of 'account', 'container', or 'object'
     :param app: a proxy app
     :param ring: ring to get yield nodes from
     :param partition: ring partition to yield nodes for
@@ -1656,8 +1766,9 @@ class NodeIter(object):
         None for an account or container ring.
     """
 
-    def __init__(self, app, ring, partition, logger, request, node_iter=None,
-                 policy=None):
+    def __init__(self, server_type, app, ring, partition, logger, request,
+                 node_iter=None, policy=None):
+        self.server_type = server_type
         self.app = app
         self.ring = ring
         self.partition = partition
@@ -1704,12 +1815,14 @@ class NodeIter(object):
             return
         extra_handoffs = handoffs - self.expected_handoffs
         if extra_handoffs > 0:
-            self.logger.increment('handoff_count')
+            self.logger.increment('%s.handoff_count' %
+                                  self.server_type.lower())
             self.logger.warning(
                 'Handoff requested (%d)' % handoffs)
             if (extra_handoffs == self.num_primary_nodes):
                 # all the primaries were skipped, and handoffs didn't help
-                self.logger.increment('handoff_all_count')
+                self.logger.increment('%s.handoff_all_count' %
+                                      self.server_type.lower())
 
     def set_node_provider(self, callback):
         """
@@ -1786,9 +1899,10 @@ class Controller(object):
         self.trans_id = '-'
         self._allowed_methods = None
         self._private_methods = None
-        # adapt the app logger to prefix statsd metrics with the server type
-        self.logger = MetricsPrefixLoggerAdapter(
-            self.app.logger, {}, self.server_type.lower())
+
+    @property
+    def logger(self):
+        return self.app.logger
 
     @property
     def allowed_methods(self):
@@ -2006,9 +2120,8 @@ class Controller(object):
         :param node_iterator: optional node iterator.
         :returns: a swob.Response object
         """
-        nodes = GreenthreadSafeIterator(
-            node_iterator or NodeIter(self.app, ring, part, self.logger, req)
-        )
+        nodes = GreenthreadSafeIterator(node_iterator or NodeIter(
+            self.server_type.lower(), self.app, ring, part, self.logger, req))
         node_number = node_count or len(ring.get_part_nodes(part))
         pile = GreenAsyncPile(node_number)
 
@@ -2209,7 +2322,7 @@ class Controller(object):
                                    partition, path, backend_headers,
                                    concurrency, policy=policy,
                                    logger=self.logger)
-        res = handler.get_working_response(req)
+        res = handler.get_working_response()
 
         if not res:
             res = self.best_response(
@@ -2385,7 +2498,7 @@ class Controller(object):
         data = self._parse_listing_response(req, response)
         return data, response
 
-    def _parse_shard_ranges(self, req, listing, response):
+    def _parse_namespaces(self, req, listing, response):
         if listing is None:
             return None
 
@@ -2397,38 +2510,15 @@ class Controller(object):
             return None
 
         try:
-            return [ShardRange.from_dict(shard_range)
-                    for shard_range in listing]
+            # Note: a legacy container-server could return a list of
+            # ShardRanges, but that's ok: namespaces just need 'name', 'lower'
+            # and 'upper' keys. If we ever need to know we can look for a
+            # 'x-backend-record-shard-format' header from newer container
+            # servers.
+            return [Namespace(data['name'], data['lower'], data['upper'])
+                    for data in listing]
         except (ValueError, TypeError, KeyError) as err:
             self.logger.error(
-                "Failed to get shard ranges from %s: invalid data: %r",
+                "Failed to get namespaces from %s: invalid data: %r",
                 req.path_qs, err)
             return None
-
-    def _get_shard_ranges(
-            self, req, account, container, includes=None, states=None):
-        """
-        Fetch shard ranges from given `account/container`. If `includes` is
-        given then the shard range for that object name is requested, otherwise
-        all shard ranges are requested.
-
-        :param req: original Request instance.
-        :param account: account from which shard ranges should be fetched.
-        :param container: container from which shard ranges should be fetched.
-        :param includes: (optional) restricts the list of fetched shard ranges
-            to those which include the given name.
-        :param states: (optional) the states of shard ranges to be fetched.
-        :return: a list of instances of :class:`swift.common.utils.ShardRange`,
-            or None if there was a problem fetching the shard ranges
-        """
-        params = req.params.copy()
-        params.pop('limit', None)
-        params['format'] = 'json'
-        if includes:
-            params['includes'] = str_to_wsgi(includes)
-        if states:
-            params['states'] = states
-        headers = {'X-Backend-Record-Type': 'shard'}
-        listing, response = self._get_container_listing(
-            req, account, container, headers=headers, params=params)
-        return self._parse_shard_ranges(req, listing, response), response