swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/proxy/controllers/base.py

@@ -24,28 +24,26 @@
 #   These shenanigans are to ensure all related objects can be garbage
 # collected. We've seen objects hang around forever otherwise.
 
-from six.moves.urllib.parse import quote
+from urllib.parse import quote
 
-import os
 import time
 import json
 import functools
 import inspect
 import itertools
 import operator
+import random
 from copy import deepcopy
-from sys import exc_info
-from swift import gettext_ as _
 
-from eventlet import sleep
 from eventlet.timeout import Timeout
-import six
 
+from swift.common.memcached import MemcacheConnectionError
 from swift.common.wsgi import make_pre_authed_env, make_pre_authed_request
-from swift.common.utils import Timestamp, config_true_value, \
+from swift.common.utils import Timestamp, WatchdogTimeout, config_true_value, \
     public, split_path, list_from_csv, GreenthreadSafeIterator, \
-    GreenAsyncPile, quorum_size, parse_content_type, close_if_possible, \
-    document_iters_to_http_response_body, ShardRange, find_shard_range
+    GreenAsyncPile, quorum_size, parse_content_type, drain_and_close, \
+    document_iters_to_http_response_body, cache_from_env, \
+    CooperativeIterator, NamespaceBoundList, Namespace, ClosingMapper
 from swift.common.bufferedhttp import http_connect
 from swift.common import constraints
 from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout, \
@@ -54,20 +52,23 @@ from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.http import is_informational, is_success, is_redirection, \
     is_server_error, HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_MULTIPLE_CHOICES, \
     HTTP_BAD_REQUEST, HTTP_NOT_FOUND, HTTP_SERVICE_UNAVAILABLE, \
-    HTTP_INSUFFICIENT_STORAGE, HTTP_UNAUTHORIZED, HTTP_CONTINUE, HTTP_GONE
+    HTTP_UNAUTHORIZED, HTTP_CONTINUE, HTTP_GONE, \
+    HTTP_REQUESTED_RANGE_NOT_SATISFIABLE
 from swift.common.swob import Request, Response, Range, \
     HTTPException, HTTPRequestedRangeNotSatisfiable, HTTPServiceUnavailable, \
-    status_map, wsgi_to_str, str_to_wsgi, wsgi_quote
+    status_map, wsgi_to_str, str_to_wsgi, wsgi_quote, wsgi_unquote, \
+    normalize_etag
 from swift.common.request_helpers import strip_sys_meta_prefix, \
     strip_user_meta_prefix, is_user_meta, is_sys_meta, is_sys_or_user_meta, \
     http_response_to_document_iters, is_object_transient_sysmeta, \
-    strip_object_transient_sysmeta_prefix
+    strip_object_transient_sysmeta_prefix, get_ip_port, get_user_meta_prefix, \
+    get_sys_meta_prefix, is_use_replication_network
 from swift.common.storage_policy import POLICIES
 
-
 DEFAULT_RECHECK_ACCOUNT_EXISTENCE = 60  # seconds
 DEFAULT_RECHECK_CONTAINER_EXISTENCE = 60  # seconds
 DEFAULT_RECHECK_UPDATING_SHARD_RANGES = 3600  # seconds
+DEFAULT_RECHECK_LISTING_SHARD_RANGES = 600  # seconds
 
 
 def update_headers(response, headers):
@@ -88,19 +89,6 @@ def update_headers(response, headers):
             response.headers[name] = value
 
 
-def source_key(resp):
-    """
-    Provide the timestamp of the swift http response as a floating
-    point value.  Used as a sort key.
-
-    :param resp: bufferedhttp response object
-    """
-    return Timestamp(resp.getheader('x-backend-data-timestamp') or
-                     resp.getheader('x-backend-timestamp') or
-                     resp.getheader('x-put-timestamp') or
-                     resp.getheader('x-timestamp') or 0)
-
-
 def delay_denial(func):
     """
     Decorator to declare which methods should have any swift.authorize call
@@ -179,6 +167,7 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
         'status': status_int,
         'read_acl': headers.get('x-container-read'),
         'write_acl': headers.get('x-container-write'),
+        'sync_to': headers.get('x-container-sync-to'),
         'sync_key': headers.get('x-container-sync-key'),
         'object_count': headers.get('x-container-object-count'),
         'bytes': headers.get('x-container-bytes-used'),
@@ -192,9 +181,100 @@ def headers_to_container_info(headers, status_int=HTTP_OK):
         'meta': meta,
         'sysmeta': sysmeta,
         'sharding_state': headers.get('x-backend-sharding-state', 'unsharded'),
+        # the 'internal' format version of timestamps is cached since the
+        # normal format can be derived from this when required
+        'created_at': headers.get('x-backend-timestamp'),
+        'put_timestamp': headers.get('x-backend-put-timestamp'),
+        'delete_timestamp': headers.get('x-backend-delete-timestamp'),
+        'status_changed_at': headers.get('x-backend-status-changed-at'),
     }
 
 
+def headers_from_container_info(info):
+    """
+    Construct a HeaderKeyDict from a container info dict.
+
+    :param info: a dict of container metadata
+    :returns: a HeaderKeyDict or None if info is None or any required headers
+        could not be constructed
+    """
+    if not info:
+        return None
+
+    required = (
+        ('x-backend-timestamp', 'created_at'),
+        ('x-backend-put-timestamp', 'put_timestamp'),
+        ('x-backend-delete-timestamp', 'delete_timestamp'),
+        ('x-backend-status-changed-at', 'status_changed_at'),
+        ('x-backend-storage-policy-index', 'storage_policy'),
+        ('x-container-object-count', 'object_count'),
+        ('x-container-bytes-used', 'bytes'),
+        ('x-backend-sharding-state', 'sharding_state'),
+    )
+    required_normal_format_timestamps = (
+        ('x-timestamp', 'created_at'),
+        ('x-put-timestamp', 'put_timestamp'),
+    )
+    optional = (
+        ('x-container-read', 'read_acl'),
+        ('x-container-write', 'write_acl'),
+        ('x-container-sync-key', 'sync_key'),
+        ('x-container-sync-to', 'sync_to'),
+        ('x-versions-location', 'versions'),
+    )
+    cors_optional = (
+        ('access-control-allow-origin', 'allow_origin'),
+        ('access-control-expose-headers', 'expose_headers'),
+        ('access-control-max-age', 'max_age')
+    )
+
+    def lookup(info, key):
+        # raises KeyError or ValueError
+        val = info[key]
+        if val is None:
+            raise ValueError
+        return val
+
+    # note: required headers may be missing from info for example during
+    # upgrade when stale info is still in cache
+    headers = HeaderKeyDict()
+    for hdr, key in required:
+        try:
+            headers[hdr] = lookup(info, key)
+        except (KeyError, ValueError):
+            return None
+
+    for hdr, key in required_normal_format_timestamps:
+        try:
+            headers[hdr] = Timestamp(lookup(info, key)).normal
+        except (KeyError, ValueError):
+            return None
+
+    for hdr, key in optional:
+        try:
+            headers[hdr] = lookup(info, key)
+        except (KeyError, ValueError):
+            pass
+
+    policy_index = info.get('storage_policy')
+    headers['x-storage-policy'] = POLICIES[int(policy_index)].name
+    prefix = get_user_meta_prefix('container')
+    headers.update(
+        (prefix + k, v)
+        for k, v in info.get('meta', {}).items())
+    for hdr, key in cors_optional:
+        try:
+            headers[prefix + hdr] = lookup(info.get('cors'), key)
+        except (KeyError, ValueError):
+            pass
+    prefix = get_sys_meta_prefix('container')
+    headers.update(
+        (prefix + k, v)
+        for k, v in info.get('sysmeta', {}).items())
+
+    return headers
+
+
 def headers_to_object_info(headers, status_int=HTTP_OK):
     """
     Construct a cacheable dict of object info based on response headers.
@@ -282,6 +362,10 @@ def cors_validation(func):
                     resp.headers['Access-Control-Allow-Origin'] = '*'
                 else:
                     resp.headers['Access-Control-Allow-Origin'] = req_origin
+                    if 'Vary' in resp.headers:
+                        resp.headers['Vary'] += ', Origin'
+                    else:
+                        resp.headers['Vary'] = 'Origin'
 
             return resp
         else:
@@ -319,11 +403,43 @@ def get_object_info(env, app, path=None, swift_source=None):
     return info
 
 
-def get_container_info(env, app, swift_source=None):
+def _record_ac_info_cache_metrics(
+        app, cache_state, container=None, resp=None):
+    """
+    Record a single cache operation by account or container lookup into its
+    corresponding metrics.
+
+    :param  app: the application object
+    :param  cache_state: the state of this cache operation, includes
+                infocache_hit, memcache hit, miss, error, skip, force_skip
+                and disabled.
+    :param  container: the container name
+    :param  resp: the response from either backend or cache hit.
+    """
+    try:
+        proxy_app = app._pipeline_final_app
+    except AttributeError:
+        logger = None
+    else:
+        logger = proxy_app.logger
+    server_type = 'container' if container else 'account'
+    if logger:
+        record_cache_op_metrics(logger, server_type, 'info', cache_state, resp)
+
+
+def get_container_info(env, app, swift_source=None, cache_only=False):
     """
     Get the info structure for a container, based on env and app.
     This is useful to middlewares.
 
+    :param env: the environment used by the current request
+    :param app: the application object
+    :param swift_source: Used to mark the request as originating out of
+                         middleware. Will be logged in proxy logs.
+    :param cache_only: If true, indicates that caller doesn't want to HEAD the
+                       backend container when cache miss.
+    :returns: the object info
+
     .. note::
 
         This call bypasses auth. Success does not imply that the request has
@@ -331,13 +447,28 @@ def get_container_info(env, app, swift_source=None):
     """
     (version, wsgi_account, wsgi_container, unused) = \
         split_path(env['PATH_INFO'], 3, 4, True)
+
+    if not constraints.valid_api_version(version):
+        # Not a valid Swift request; return 0 like we do
+        # if there's an account failure
+        return headers_to_container_info({}, 0)
+
     account = wsgi_to_str(wsgi_account)
     container = wsgi_to_str(wsgi_container)
 
+    # Try to cut through all the layers to the proxy app
+    # (while also preserving logging)
+    try:
+        logged_app = app._pipeline_request_logging_app
+        proxy_app = app._pipeline_final_app
+    except AttributeError:
+        logged_app = proxy_app = app
     # Check in environment cache and in memcache (in that order)
-    info = _get_info_from_caches(app, env, account, container)
+    info, cache_state = _get_info_from_caches(
+        proxy_app, env, account, container)
 
-    if not info:
+    resp = None
+    if not info and not cache_only:
         # Cache miss; go HEAD the container and populate the caches
         env.setdefault('swift.infocache', {})
         # Before checking the container, make sure the account exists.
@@ -347,29 +478,35 @@ def get_container_info(env, app, swift_source=None):
         # account is successful whether the account actually has .db files
         # on disk or not.
         is_autocreate_account = account.startswith(
-            getattr(app, 'auto_create_account_prefix', '.'))
+            constraints.AUTO_CREATE_ACCOUNT_PREFIX)
         if not is_autocreate_account:
-            account_info = get_account_info(env, app, swift_source)
+            account_info = get_account_info(env, logged_app, swift_source)
             if not account_info or not is_success(account_info['status']):
+                _record_ac_info_cache_metrics(
+                    logged_app, cache_state, container)
                 return headers_to_container_info({}, 0)
 
         req = _prepare_pre_auth_info_request(
             env, ("/%s/%s/%s" % (version, wsgi_account, wsgi_container)),
             (swift_source or 'GET_CONTAINER_INFO'))
-        resp = req.get_response(app)
-        close_if_possible(resp.app_iter)
+        # *Always* allow reserved names for get-info requests -- it's on the
+        # caller to keep the result private-ish
+        req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
+        resp = req.get_response(logged_app)
+        drain_and_close(resp)
         # Check in infocache to see if the proxy (or anyone else) already
         # populated the cache for us. If they did, just use what's there.
         #
         # See similar comment in get_account_info() for justification.
         info = _get_info_from_infocache(env, account, container)
         if info is None:
-            info = set_info_cache(app, env, account, container, resp)
+            info = set_info_cache(env, account, container, resp)
 
     if info:
         info = deepcopy(info)  # avoid mutating what's in swift.infocache
     else:
-        info = headers_to_container_info({}, 503)
+        status_int = 0 if cache_only else 503
+        info = headers_to_container_info({}, status_int)
 
     # Old data format in memcache immediately after a Swift upgrade; clean
     # it up so consumers of get_container_info() aren't exposed to it.
@@ -385,6 +522,18 @@ def get_container_info(env, app, swift_source=None):
     if info.get('sharding_state') is None:
         info['sharding_state'] = 'unsharded'
 
+    versions_cont = info.get('sysmeta', {}).get('versions-container', '')
+    if versions_cont:
+        versions_cont = wsgi_unquote(str_to_wsgi(
+            versions_cont)).split('/')[0]
+        versions_req = _prepare_pre_auth_info_request(
+            env, ("/%s/%s/%s" % (version, wsgi_account, versions_cont)),
+            (swift_source or 'GET_CONTAINER_INFO'))
+        versions_req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
+        versions_info = get_container_info(versions_req.environ, app)
+        info['bytes'] = info['bytes'] + versions_info['bytes']
+
+    _record_ac_info_cache_metrics(logged_app, cache_state, container, resp)
     return info
 
 
@@ -401,19 +550,34 @@ def get_account_info(env, app, swift_source=None):
     :raises ValueError: when path doesn't contain an account
     """
     (version, wsgi_account, _junk) = split_path(env['PATH_INFO'], 2, 3, True)
+
+    if not constraints.valid_api_version(version):
+        return headers_to_account_info({}, 0)
+
     account = wsgi_to_str(wsgi_account)
 
+    # Try to cut through all the layers to the proxy app
+    # (while also preserving logging)
+    try:
+        app = app._pipeline_request_logging_app
+    except AttributeError:
+        pass
     # Check in environment cache and in memcache (in that order)
-    info = _get_info_from_caches(app, env, account)
+    info, cache_state = _get_info_from_caches(app, env, account)
 
     # Cache miss; go HEAD the account and populate the caches
-    if not info:
+    if info:
+        resp = None
+    else:
         env.setdefault('swift.infocache', {})
         req = _prepare_pre_auth_info_request(
             env, "/%s/%s" % (version, wsgi_account),
             (swift_source or 'GET_ACCOUNT_INFO'))
+        # *Always* allow reserved names for get-info requests -- it's on the
+        # caller to keep the result private-ish
+        req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
         resp = req.get_response(app)
-        close_if_possible(resp.app_iter)
+        drain_and_close(resp)
         # Check in infocache to see if the proxy (or anyone else) already
         # populated the cache for us. If they did, just use what's there.
         #
@@ -430,7 +594,7 @@ def get_account_info(env, app, swift_source=None):
         # memcache would defeat the purpose.
         info = _get_info_from_infocache(env, account)
         if info is None:
-            info = set_info_cache(app, env, account, None, resp)
+            info = set_info_cache(env, account, None, resp)
 
     if info:
         info = info.copy()  # avoid mutating what's in swift.infocache
@@ -443,6 +607,7 @@ def get_account_info(env, app, swift_source=None):
         else:
             info[field] = int(info[field])
 
+    _record_ac_info_cache_metrics(app, cache_state, container=None, resp=resp)
     return info
 
 
@@ -459,16 +624,10 @@ def get_cache_key(account, container=None, obj=None, shard=None):
                   with obj)
     :returns: a (native) string cache_key
     """
-    if six.PY2:
-        def to_native(s):
-            if s is None or isinstance(s, str):
-                return s
-            return s.encode('utf8')
-    else:
-        def to_native(s):
-            if s is None or isinstance(s, str):
-                return s
-            return s.decode('utf8', 'surrogateescape')
+    def to_native(s):
+        if s is None or isinstance(s, str):
+            return s
+        return s.decode('utf8', 'surrogateescape')
 
     account = to_native(account)
     container = to_native(container)
@@ -479,7 +638,7 @@ def get_cache_key(account, container=None, obj=None, shard=None):
             raise ValueError('Shard cache key requires account and container')
         if obj:
             raise ValueError('Shard cache key cannot have obj')
-        cache_key = 'shard-%s/%s/%s' % (shard, account, container)
+        cache_key = 'shard-%s-v2/%s/%s' % (shard, account, container)
     elif obj:
         if not (account and container):
             raise ValueError('Object cache key requires account and container')
@@ -497,11 +656,11 @@ def get_cache_key(account, container=None, obj=None, shard=None):
     return cache_key
 
 
-def set_info_cache(app, env, account, container, resp):
+def set_info_cache(env, account, container, resp):
     """
     Cache info in both memcache and env.
 
-    :param  app: the application object
+    :param  env: the WSGI request environment
     :param  account: the unquoted account name
     :param  container: the unquoted container name or None
     :param  resp: the response received or None if info cache should be cleared
@@ -509,31 +668,27 @@ def set_info_cache(app, env, account, container, resp):
     :returns: the info that was placed into the cache, or None if the
               request status was not in (404, 410, 2xx).
     """
+    cache_key = get_cache_key(account, container)
     infocache = env.setdefault('swift.infocache', {})
+    memcache = cache_from_env(env, True)
+    if resp is None:
+        clear_info_cache(env, account, container)
+        return
 
-    cache_time = None
-    if container and resp:
+    if container:
         cache_time = int(resp.headers.get(
             'X-Backend-Recheck-Container-Existence',
             DEFAULT_RECHECK_CONTAINER_EXISTENCE))
-    elif resp:
+    else:
         cache_time = int(resp.headers.get(
             'X-Backend-Recheck-Account-Existence',
             DEFAULT_RECHECK_ACCOUNT_EXISTENCE))
-    cache_key = get_cache_key(account, container)
 
-    if resp:
-        if resp.status_int in (HTTP_NOT_FOUND, HTTP_GONE):
-            cache_time *= 0.1
-        elif not is_success(resp.status_int):
-            cache_time = None
-
-    # Next actually set both memcache and the env cache
-    memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
-    if cache_time is None:
-        infocache.pop(cache_key, None)
-        if memcache:
-            memcache.delete(cache_key)
+    if resp.status_int in (HTTP_NOT_FOUND, HTTP_GONE):
+        cache_time *= 0.1
+    elif not is_success(resp.status_int):
+        # If we got a response, it was unsuccessful, and it wasn't an
+        # "authoritative" failure, bail without touching caches.
         return
 
     if container:
@@ -574,16 +729,23 @@ def set_object_info_cache(app, env, account, container, obj, resp):
     return info
 
 
-def clear_info_cache(app, env, account, container=None):
+def clear_info_cache(env, account, container=None, shard=None):
     """
     Clear the cached info in both memcache and env
 
-    :param  app: the application object
-    :param  env: the WSGI environment
+    :param  env: the WSGI request environment
     :param  account: the account name
-    :param  container: the containr name or None if setting info for containers
+    :param  container: the container name if clearing info for containers, or
+              None
+    :param  shard: the sharding state if clearing info for container shard
+              ranges, or None
     """
-    set_info_cache(app, env, account, container, None)
+    cache_key = get_cache_key(account, container, shard=shard)
+    infocache = env.setdefault('swift.infocache', {})
+    memcache = cache_from_env(env, True)
+    infocache.pop(cache_key, None)
+    if memcache:
+        memcache.delete(cache_key)
 
 
 def _get_info_from_infocache(env, account, container=None):
@@ -603,6 +765,40 @@ def _get_info_from_infocache(env, account, container=None):
     return None
 
 
+def record_cache_op_metrics(
+        logger, server_type, op_type, cache_state, resp=None):
+    """
+    Record a single cache operation into its corresponding metrics.
+
+    :param  logger: the metrics logger
+    :param  server_type: 'account' or 'container'
+    :param  op_type: the name of the operation type, includes 'shard_listing',
+              'shard_updating', and etc.
+    :param  cache_state: the state of this cache operation. When it's
+              'infocache_hit' or memcache 'hit', expect it succeeded and 'resp'
+              will be None; for all other cases like memcache 'miss' or 'skip'
+              which will make to backend, expect a valid 'resp'.
+    :param  resp: the response from backend for all cases except cache hits.
+    """
+    server_type = server_type.lower()
+    if cache_state == 'infocache_hit':
+        logger.increment('%s.%s.infocache.hit' % (server_type, op_type))
+    elif cache_state == 'hit':
+        # memcache hits.
+        logger.increment('%s.%s.cache.hit' % (server_type, op_type))
+    else:
+        # the cases of cache_state is memcache miss, error, skip, force_skip
+        # or disabled.
+        if resp:
+            logger.increment('%s.%s.cache.%s.%d' % (
+                server_type, op_type, cache_state, resp.status_int))
+        else:
+            # In some situation, we choose not to lookup backend after cache
+            # miss.
+            logger.increment('%s.%s.cache.%s' % (
+                server_type, op_type, cache_state))
+
+
 def _get_info_from_memcache(app, env, account, container=None):
     """
     Get cached account or container information from memcache
@@ -612,38 +808,37 @@ def _get_info_from_memcache(app, env, account, container=None):
     :param  account: the account name
     :param  container: the container name
 
-    :returns: a dictionary of cached info on cache hit, None on miss. Also
-      returns None if memcache is not in use.
+    :returns: a tuple of two values, the first is a dictionary of cached info
+      on cache hit, None on miss or if memcache is not in use; the second is
+      cache state.
     """
+    memcache = cache_from_env(env, True)
+    if not memcache:
+        return None, 'disabled'
+
+    try:
+        proxy_app = app._pipeline_final_app
+    except AttributeError:
+        # Only the middleware entry-points get a reference to the
+        # proxy-server app; if a middleware composes itself as multiple
+        # filters, we'll just have to choose a reasonable default
+        skip_chance = 0.0
+    else:
+        if container:
+            skip_chance = proxy_app.container_existence_skip_cache
+        else:
+            skip_chance = proxy_app.account_existence_skip_cache
+
     cache_key = get_cache_key(account, container)
-    memcache = getattr(app, 'memcache', None) or env.get('swift.cache')
-    if memcache:
+    if skip_chance and random.random() < skip_chance:
+        info = None
+        cache_state = 'skip'
+    else:
         info = memcache.get(cache_key)
-        if info and six.PY2:
-            # Get back to native strings
-            new_info = {}
-            for key in info:
-                new_key = key.encode("utf-8") if isinstance(
-                    key, six.text_type) else key
-                if isinstance(info[key], six.text_type):
-                    new_info[new_key] = info[key].encode("utf-8")
-                elif isinstance(info[key], dict):
-                    new_info[new_key] = {}
-                    for subkey, value in info[key].items():
-                        new_subkey = subkey.encode("utf-8") if isinstance(
-                            subkey, six.text_type) else subkey
-                        if isinstance(value, six.text_type):
-                            new_info[new_key][new_subkey] = \
-                                value.encode("utf-8")
-                        else:
-                            new_info[new_key][new_subkey] = value
-                else:
-                    new_info[new_key] = info[key]
-            info = new_info
-        if info:
-            env.setdefault('swift.infocache', {})[cache_key] = info
-        return info
-    return None
+        cache_state = 'hit' if info else 'miss'
+    if info:
+        env.setdefault('swift.infocache', {})[cache_key] = info
+    return info, cache_state
 
 
 def _get_info_from_caches(app, env, account, container=None):
@@ -653,13 +848,81 @@ def _get_info_from_caches(app, env, account, container=None):
 
     :param  app: the application object
     :param  env: the environment used by the current request
-    :returns: the cached info or None if not cached
+    :returns: a tuple of (the cached info or None if not cached, cache state)
     """
 
     info = _get_info_from_infocache(env, account, container)
-    if info is None:
-        info = _get_info_from_memcache(app, env, account, container)
-    return info
+    if info:
+        cache_state = 'infocache_hit'
+    else:
+        info, cache_state = _get_info_from_memcache(
+            app, env, account, container)
+    return info, cache_state
+
+
+def get_namespaces_from_cache(req, cache_key, skip_chance):
+    """
+    Get cached namespaces from infocache or memcache.
+
+    :param req: a :class:`swift.common.swob.Request` object.
+    :param cache_key: the cache key for both infocache and memcache.
+    :param skip_chance: the probability of skipping the memcache look-up.
+    :return: a tuple of (value, cache state). Value is an instance of
+        :class:`swift.common.utils.NamespaceBoundList` if a non-empty list is
+        found in memcache. Otherwise value is ``None``, for example if memcache
+        look-up was skipped, or no value was found, or an empty list was found.
+    """
+    # try get namespaces from infocache first
+    infocache = req.environ.setdefault('swift.infocache', {})
+    ns_bound_list = infocache.get(cache_key)
+    if ns_bound_list:
+        return ns_bound_list, 'infocache_hit'
+
+    # then try get them from memcache
+    memcache = cache_from_env(req.environ, True)
+    if not memcache:
+        return None, 'disabled'
+    if skip_chance and random.random() < skip_chance:
+        return None, 'skip'
+    try:
+        bounds = memcache.get(cache_key, raise_on_error=True)
+        cache_state = 'hit' if bounds else 'miss'
+    except MemcacheConnectionError:
+        bounds = None
+        cache_state = 'error'
+
+    if bounds:
+        ns_bound_list = NamespaceBoundList(bounds)
+        infocache[cache_key] = ns_bound_list
+    else:
+        ns_bound_list = None
+    return ns_bound_list, cache_state
+
+
+def set_namespaces_in_cache(req, cache_key, ns_bound_list, time):
+    """
+    Set a list of namespace bounds in infocache and memcache.
+
+    :param req: a :class:`swift.common.swob.Request` object.
+    :param cache_key: the cache key for both infocache and memcache.
+    :param ns_bound_list: a :class:`swift.common.utils.NamespaceBoundList`.
+    :param time: how long the namespaces should remain in memcache.
+    :return: the cache_state.
+    """
+    infocache = req.environ.setdefault('swift.infocache', {})
+    infocache[cache_key] = ns_bound_list
+    memcache = cache_from_env(req.environ, True)
+    if memcache and ns_bound_list:
+        try:
+            memcache.set(cache_key, ns_bound_list.bounds, time=time,
+                         raise_on_error=True)
+        except MemcacheConnectionError:
+            cache_state = 'set_error'
+        else:
+            cache_state = 'set'
+    else:
+        cache_state = 'disabled'
+    return cache_state
 
 
 def _prepare_pre_auth_info_request(env, path, swift_source):
@@ -739,6 +1002,9 @@ def _get_object_info(app, env, account, container, obj, swift_source=None):
     # Not in cache, let's try the object servers
     path = '/v1/%s/%s/%s' % (account, container, obj)
     req = _prepare_pre_auth_info_request(env, path, swift_source)
+    # *Always* allow reserved names for get-info requests -- it's on the
+    # caller to keep the result private-ish
+    req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
     resp = req.get_response(app)
     # Unlike get_account_info() and get_container_info(), we don't save
     # things in memcache, so we can store the info without network traffic,
@@ -791,6 +1057,33 @@ def bytes_to_skip(record_size, range_start):
     return (record_size - (range_start % record_size)) % record_size
 
 
+def is_good_source(status, server_type):
+    """
+    Indicates whether or not the request made to the backend found
+    what it was looking for.
+
+    :param resp: the response from the backend.
+    :param server_type: the type of server: 'Account', 'Container' or 'Object'.
+    :returns: True if the response status code is acceptable, False if not.
+    """
+    if (server_type == 'Object' and
+            status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE):
+        return True
+    return is_success(status) or is_redirection(status)
+
+
+def is_useful_response(resp, node):
+    if not resp:
+        return False
+    if ('handoff_index' in node
+            and resp.status == 404
+            and resp.getheader('x-backend-timestamp') is None):
+        # a 404 from a handoff are not considered authoritative unless they
+        # have an x-backend-timestamp that indicates that there is a tombstone
+        return False
+    return True
+
+
 class ByteCountEnforcer(object):
     """
     Enforces that successive calls to file_like.read() give at least
@@ -822,44 +1115,131 @@ class ByteCountEnforcer(object):
             return chunk
 
 
-class ResumingGetter(object):
-    def __init__(self, app, req, server_type, node_iter, partition, path,
-                 backend_headers, concurrency=1, client_chunk_size=None,
-                 newest=None, header_provider=None):
+class GetterSource(object):
+    """
+    Encapsulates properties of a source from which a GET response is read.
+
+    :param app: a proxy app.
+    :param resp: an instance of ``HTTPResponse``.
+    :param node: a dict describing the node from which the response was
+        returned.
+    """
+    __slots__ = ('app', 'resp', 'node', '_parts_iter')
+
+    def __init__(self, app, resp, node):
+        self.app = app
+        self.resp = resp
+        self.node = node
+        self._parts_iter = None
+
+    @property
+    def timestamp(self):
+        """
+        Provide the timestamp of the swift http response as a floating
+        point value.  Used as a sort key.
+
+        :return: an instance of ``utils.Timestamp``
+        """
+        return Timestamp(self.resp.getheader('x-backend-data-timestamp') or
+                         self.resp.getheader('x-backend-timestamp') or
+                         self.resp.getheader('x-put-timestamp') or
+                         self.resp.getheader('x-timestamp') or 0)
+
+    @property
+    def parts_iter(self):
+        # lazy load a source response body parts iter if and when the source is
+        # actually read
+        if self.resp and not self._parts_iter:
+            self._parts_iter = http_response_to_document_iters(
+                self.resp, read_chunk_size=self.app.object_chunk_size)
+        return self._parts_iter
+
+    def close(self):
+        # Close-out the connection as best as possible.
+        close_swift_conn(self.resp)
+
+
+class GetterBase(object):
+    """
+    This base class provides helper methods for handling GET requests to
+    backend servers.
+
+    :param app: a proxy app.
+    :param req: an instance of ``swob.Request``.
+    :param node_iter: an iterator yielding nodes.
+    :param partition: partition.
+    :param policy: the policy instance, or None if Account or Container.
+    :param path: path for the request.
+    :param backend_headers: a dict of headers to be sent with backend requests.
+    :param node_timeout: the timeout value for backend requests.
+    :param resource_type: a string description of the type of resource being
+        accessed; ``resource type`` is used in logs and isn't necessarily the
+        server type.
+    :param logger: a logger instance.
+    """
+    def __init__(self, app, req, node_iter, partition, policy,
+                 path, backend_headers, node_timeout, resource_type,
+                 logger=None):
         self.app = app
+        self.req = req
         self.node_iter = node_iter
-        self.server_type = server_type
         self.partition = partition
+        self.policy = policy
         self.path = path
         self.backend_headers = backend_headers
-        self.client_chunk_size = client_chunk_size
-        self.skip_bytes = 0
+        # resource type is used in logs and isn't necessarily the server type
+        self.resource_type = resource_type
+        self.node_timeout = node_timeout
+        self.logger = logger or app.logger
         self.bytes_used_from_backend = 0
-        self.used_nodes = []
-        self.used_source_etag = ''
-        self.concurrency = concurrency
-        self.node = None
-        self.header_provider = header_provider
-        self.latest_404_timestamp = Timestamp(0)
-
-        # stuff from request
-        self.req_method = req.method
-        self.req_path = req.path
-        self.req_query_string = req.query_string
-        if newest is None:
-            self.newest = config_true_value(req.headers.get('x-newest', 'f'))
-        else:
-            self.newest = newest
+        self.source = None
 
-        # populated when finding source
-        self.statuses = []
-        self.reasons = []
-        self.bodies = []
-        self.source_headers = []
-        self.sources = []
+    def _find_source(self):
+        """
+        Look for a suitable new source and if one is found then set
+        ``self.source``.
 
-        # populated from response headers
-        self.start_byte = self.end_byte = self.length = None
+        :return: ``True`` if ``self.source`` has been updated, ``False``
+            otherwise.
+        """
+        # Subclasses must implement this method, but _replace_source should be
+        # called to get a source installed
+        raise NotImplementedError()
+
+    def _replace_source(self, err_msg=''):
+        if self.source:
+            self.app.error_occurred(self.source.node, err_msg)
+            self.source.close()
+        return self._find_source()
+
+    def _get_next_response_part(self):
+        # return the next part of the response body; there may only be one part
+        # unless it's a multipart/byteranges response
+        while True:
+            # the loop here is to resume if trying to parse
+            # multipart/byteranges response raises a ChunkReadTimeout
+            # and resets the source_parts_iter
+            try:
+                with WatchdogTimeout(self.app.watchdog, self.node_timeout,
+                                     ChunkReadTimeout):
+                    # If we don't have a multipart/byteranges response,
+                    # but just a 200 or a single-range 206, then this
+                    # performs no IO, and either just returns source or
+                    # raises StopIteration.
+                    # Otherwise, this call to next() performs IO when
+                    # we have a multipart/byteranges response, as it
+                    # will read the MIME boundary and part headers. In this
+                    # case, ChunkReadTimeout may also be raised.
+                    # If StopIteration is raised, it escapes and is
+                    # handled elsewhere.
+                    start_byte, end_byte, length, headers, part = next(
+                        self.source.parts_iter)
+                return (start_byte, end_byte, length, headers, part)
+            except ChunkReadTimeout:
+                if not self._replace_source(
+                        'Trying to read next part of %s multi-part GET '
+                        '(retrying)' % self.resource_type):
+                    raise
 
     def fast_forward(self, num_bytes):
         """
@@ -873,6 +1253,9 @@ class ResumingGetter(object):
                                                   > end of range + 1
         :raises RangeAlreadyComplete: if begin + num_bytes == end of range + 1
         """
+        self.backend_headers.pop(
+            'X-Backend-Ignore-Range-If-Metadata-Present', None)
+
         try:
             req_range = Range(self.backend_headers.get('Range'))
         except ValueError:
@@ -935,9 +1318,6 @@ class ResumingGetter(object):
 
     def learn_size_from_content_range(self, start, end, length):
         """
-        If client_chunk_size is set, makes sure we yield things starting on
-        chunk boundaries based on the Content-Range header in the response.
-
         Sets our Range header's first byterange to the value learned from
         the Content-Range header in the response; if we were given a
         fully-specified range (e.g. "bytes=123-456"), this is a no-op.
@@ -950,9 +1330,6 @@ class ResumingGetter(object):
         if length == 0:
             return
 
-        if self.client_chunk_size:
-            self.skip_bytes = bytes_to_skip(self.client_chunk_size, start)
-
         if 'Range' in self.backend_headers:
             try:
                 req_range = Range(self.backend_headers['Range'])
@@ -967,221 +1344,134 @@ class ResumingGetter(object):
                                             e if e is not None else '')
                                  for s, e in new_ranges)))
 
-    def is_good_source(self, src):
-        """
-        Indicates whether or not the request made to the backend found
-        what it was looking for.
 
-        :param src: the response from the backend
-        :returns: True if found, False if not
-        """
-        if self.server_type == 'Object' and src.status == 416:
-            return True
-        return is_success(src.status) or is_redirection(src.status)
+class GetOrHeadHandler(GetterBase):
+    """
+    Handles GET requests to backend servers.
+
+    :param app: a proxy app.
+    :param req: an instance of ``swob.Request``.
+    :param server_type: server type used in logging
+    :param node_iter: an iterator yielding nodes.
+    :param partition: partition.
+    :param path: path for the request.
+    :param backend_headers: a dict of headers to be sent with backend requests.
+    :param concurrency: number of requests to run concurrently.
+    :param policy: the policy instance, or None if Account or Container.
+    :param logger: a logger instance.
+    """
+    def __init__(self, app, req, server_type, node_iter, partition, path,
+                 backend_headers, concurrency=1, policy=None, logger=None):
+        newest = config_true_value(req.headers.get('x-newest', 'f'))
+        if server_type == 'Object' and not newest:
+            node_timeout = app.recoverable_node_timeout
+        else:
+            node_timeout = app.node_timeout
+        super(GetOrHeadHandler, self).__init__(
+            app=app, req=req, node_iter=node_iter, partition=partition,
+            policy=policy, path=path, backend_headers=backend_headers,
+            node_timeout=node_timeout, resource_type=server_type.lower(),
+            logger=logger)
+        self.newest = newest
+        self.server_type = server_type
+        self.used_nodes = []
+        self.used_source_etag = None
+        self.concurrency = concurrency
+        self.latest_404_timestamp = Timestamp(0)
+        policy_options = self.app.get_policy_options(self.policy)
+        self.rebalance_missing_suppression_count = min(
+            policy_options.rebalance_missing_suppression_count,
+            node_iter.num_primary_nodes - 1)
 
-    def response_parts_iter(self, req):
-        source, node = self._get_source_and_node()
-        it = None
-        if source:
-            it = self._get_response_parts_iter(req, node, source)
-        return it
+        # populated when finding source
+        self.statuses = []
+        self.reasons = []
+        self.bodies = []
+        self.source_headers = []
+        self.sources = []
 
-    def _get_response_parts_iter(self, req, node, source):
-        # Someday we can replace this [mess] with python 3's "nonlocal"
-        source = [source]
-        node = [node]
+        # populated from response headers
+        self.start_byte = self.end_byte = self.length = None
 
-        try:
-            client_chunk_size = self.client_chunk_size
-            node_timeout = self.app.node_timeout
-            if self.server_type == 'Object':
-                node_timeout = self.app.recoverable_node_timeout
-
-            # This is safe; it sets up a generator but does not call next()
-            # on it, so no IO is performed.
-            parts_iter = [
-                http_response_to_document_iters(
-                    source[0], read_chunk_size=self.app.object_chunk_size)]
-
-            def get_next_doc_part():
-                while True:
-                    try:
-                        # This call to next() performs IO when we have a
-                        # multipart/byteranges response; it reads the MIME
-                        # boundary and part headers.
-                        #
-                        # If we don't have a multipart/byteranges response,
-                        # but just a 200 or a single-range 206, then this
-                        # performs no IO, and either just returns source or
-                        # raises StopIteration.
-                        with ChunkReadTimeout(node_timeout):
-                            # if StopIteration is raised, it escapes and is
-                            # handled elsewhere
-                            start_byte, end_byte, length, headers, part = next(
-                                parts_iter[0])
-                        return (start_byte, end_byte, length, headers, part)
-                    except ChunkReadTimeout:
-                        new_source, new_node = self._get_source_and_node()
-                        if new_source:
-                            self.app.error_occurred(
-                                node[0], _('Trying to read object during '
-                                           'GET (retrying)'))
-                            # Close-out the connection as best as possible.
-                            if getattr(source[0], 'swift_conn', None):
-                                close_swift_conn(source[0])
-                            source[0] = new_source
-                            node[0] = new_node
-                            # This is safe; it sets up a generator but does
-                            # not call next() on it, so no IO is performed.
-                            parts_iter[0] = http_response_to_document_iters(
-                                new_source,
-                                read_chunk_size=self.app.object_chunk_size)
-                        else:
-                            raise StopIteration()
-
-            def iter_bytes_from_response_part(part_file, nbytes):
-                nchunks = 0
-                buf = b''
-                part_file = ByteCountEnforcer(part_file, nbytes)
-                while True:
+    def _iter_bytes_from_response_part(self, part_file, nbytes):
+        # yield chunks of bytes from a single response part; if an error
+        # occurs, try to resume yielding bytes from a different source
+        part_file = ByteCountEnforcer(part_file, nbytes)
+        while True:
+            try:
+                with WatchdogTimeout(self.app.watchdog, self.node_timeout,
+                                     ChunkReadTimeout):
+                    chunk = part_file.read(self.app.object_chunk_size)
+                    if nbytes is not None:
+                        nbytes -= len(chunk)
+            except (ChunkReadTimeout, ShortReadError) as e:
+                if self.newest or self.server_type != 'Object':
+                    raise
+                try:
+                    self.fast_forward(self.bytes_used_from_backend)
+                except (HTTPException, ValueError):
+                    raise e
+                except RangeAlreadyComplete:
+                    break
+                if self._replace_source(
+                        'Trying to read object during GET (retrying)'):
                     try:
-                        with ChunkReadTimeout(node_timeout):
-                            chunk = part_file.read(self.app.object_chunk_size)
-                            nchunks += 1
-                            # NB: this append must be *inside* the context
-                            # manager for test.unit.SlowBody to do its thing
-                            buf += chunk
-                            if nbytes is not None:
-                                nbytes -= len(chunk)
-                    except (ChunkReadTimeout, ShortReadError):
-                        exc_type, exc_value, exc_traceback = exc_info()
-                        if self.newest or self.server_type != 'Object':
-                            raise
-                        try:
-                            self.fast_forward(self.bytes_used_from_backend)
-                        except (HTTPException, ValueError):
-                            six.reraise(exc_type, exc_value, exc_traceback)
-                        except RangeAlreadyComplete:
-                            break
-                        buf = b''
-                        new_source, new_node = self._get_source_and_node()
-                        if new_source:
-                            self.app.error_occurred(
-                                node[0], _('Trying to read object during '
-                                           'GET (retrying)'))
-                            # Close-out the connection as best as possible.
-                            if getattr(source[0], 'swift_conn', None):
-                                close_swift_conn(source[0])
-                            source[0] = new_source
-                            node[0] = new_node
-                            # This is safe; it just sets up a generator but
-                            # does not call next() on it, so no IO is
-                            # performed.
-                            parts_iter[0] = http_response_to_document_iters(
-                                new_source,
-                                read_chunk_size=self.app.object_chunk_size)
-
-                            try:
-                                _junk, _junk, _junk, _junk, part_file = \
-                                    get_next_doc_part()
-                            except StopIteration:
-                                # Tried to find a new node from which to
-                                # finish the GET, but failed. There's
-                                # nothing more we can do here.
-                                six.reraise(exc_type, exc_value, exc_traceback)
-                            part_file = ByteCountEnforcer(part_file, nbytes)
-                        else:
-                            six.reraise(exc_type, exc_value, exc_traceback)
-                    else:
-                        if buf and self.skip_bytes:
-                            if self.skip_bytes < len(buf):
-                                buf = buf[self.skip_bytes:]
-                                self.bytes_used_from_backend += self.skip_bytes
-                                self.skip_bytes = 0
-                            else:
-                                self.skip_bytes -= len(buf)
-                                self.bytes_used_from_backend += len(buf)
-                                buf = b''
-
-                        if not chunk:
-                            if buf:
-                                with ChunkWriteTimeout(
-                                        self.app.client_timeout):
-                                    self.bytes_used_from_backend += len(buf)
-                                    yield buf
-                                buf = b''
-                            break
-
-                        if client_chunk_size is not None:
-                            while len(buf) >= client_chunk_size:
-                                client_chunk = buf[:client_chunk_size]
-                                buf = buf[client_chunk_size:]
-                                with ChunkWriteTimeout(
-                                        self.app.client_timeout):
-                                    self.bytes_used_from_backend += \
-                                        len(client_chunk)
-                                    yield client_chunk
-                        else:
-                            with ChunkWriteTimeout(self.app.client_timeout):
-                                self.bytes_used_from_backend += len(buf)
-                                yield buf
-                            buf = b''
-
-                        # This is for fairness; if the network is outpacing
-                        # the CPU, we'll always be able to read and write
-                        # data without encountering an EWOULDBLOCK, and so
-                        # eventlet will not switch greenthreads on its own.
-                        # We do it manually so that clients don't starve.
-                        #
-                        # The number 5 here was chosen by making stuff up.
-                        # It's not every single chunk, but it's not too big
-                        # either, so it seemed like it would probably be an
-                        # okay choice.
-                        #
-                        # Note that we may trampoline to other greenthreads
-                        # more often than once every 5 chunks, depending on
-                        # how blocking our network IO is; the explicit sleep
-                        # here simply provides a lower bound on the rate of
-                        # trampolining.
-                        if nchunks % 5 == 0:
-                            sleep()
-
+                        _junk, _junk, _junk, _junk, part_file = \
+                            self._get_next_response_part()
+                    except StopIteration:
+                        # Tried to find a new node from which to
+                        # finish the GET, but failed. There's
+                        # nothing more we can do here.
+                        raise e
+                    part_file = ByteCountEnforcer(part_file, nbytes)
+                else:
+                    raise e
+            else:
+                if not chunk:
+                    break
+
+                with WatchdogTimeout(self.app.watchdog,
+                                     self.app.client_timeout,
+                                     ChunkWriteTimeout):
+                    self.bytes_used_from_backend += len(chunk)
+                    yield chunk
+
+    def _iter_parts_from_response(self):
+        # iterate over potentially multiple response body parts; for each
+        # part, yield an iterator over the part's bytes
+        try:
             part_iter = None
             try:
                 while True:
                     start_byte, end_byte, length, headers, part = \
-                        get_next_doc_part()
-                    # note: learn_size_from_content_range() sets
-                    # self.skip_bytes
+                        self._get_next_response_part()
                     self.learn_size_from_content_range(
                         start_byte, end_byte, length)
                     self.bytes_used_from_backend = 0
                     # not length; that refers to the whole object, so is the
                     # wrong value to use for GET-range responses
-                    byte_count = ((end_byte - start_byte + 1) - self.skip_bytes
+                    byte_count = ((end_byte - start_byte + 1)
                                   if (end_byte is not None
                                       and start_byte is not None)
                                   else None)
-                    part_iter = iter_bytes_from_response_part(part, byte_count)
+                    part_iter = CooperativeIterator(
+                        self._iter_bytes_from_response_part(part, byte_count))
                     yield {'start_byte': start_byte, 'end_byte': end_byte,
                            'entity_length': length, 'headers': headers,
                            'part_iter': part_iter}
                     self.pop_range()
             except StopIteration:
-                req.environ['swift.non_client_disconnect'] = True
+                self.req.environ['swift.non_client_disconnect'] = True
             finally:
                 if part_iter:
                     part_iter.close()
 
-        except ChunkReadTimeout:
-            self.app.exception_occurred(node[0], _('Object'),
-                                        _('Trying to read during GET'))
-            raise
         except ChunkWriteTimeout:
-            self.app.logger.warning(
-                _('Client did not read from proxy within %ss') %
+            self.logger.info(
+                'Client did not read from proxy within %ss',
                 self.app.client_timeout)
-            self.app.logger.increment('client_timeouts')
+            self.logger.increment('%s.client_timeouts' %
+                                  self.server_type.lower())
         except GeneratorExit:
             warn = True
             req_range = self.backend_headers['Range']
@@ -1192,16 +1482,16 @@ class ResumingGetter(object):
                     if end is not None and begin is not None:
                         if end - begin + 1 == self.bytes_used_from_backend:
                             warn = False
-            if not req.environ.get('swift.non_client_disconnect') and warn:
-                self.app.logger.warning(_('Client disconnected on read'))
+            if (warn and
+                    not self.req.environ.get('swift.non_client_disconnect')):
+                self.logger.info('Client disconnected on read of %r',
+                                 self.path)
             raise
         except Exception:
-            self.app.logger.exception(_('Trying to send to client'))
+            self.logger.exception('Trying to send to client')
             raise
         finally:
-            # Close-out the connection as best as possible.
-            if getattr(source[0], 'swift_conn', None):
-                close_swift_conn(source[0])
+            self.source.close()
 
     @property
     def last_status(self):
@@ -1217,39 +1507,42 @@ class ResumingGetter(object):
         else:
             return None
 
-    def _make_node_request(self, node, node_timeout, logger_thread_locals):
-        self.app.logger.thread_locals = logger_thread_locals
+    def _make_node_request(self, node, logger_thread_locals):
+        # make a backend request; return True if the response is deemed good
+        # (has an acceptable status code), useful (matches any previously
+        # discovered etag) and sufficient (a single good response is
+        # insufficient when we're searching for the newest timestamp)
+        self.logger.thread_locals = logger_thread_locals
         if node in self.used_nodes:
             return False
+
         req_headers = dict(self.backend_headers)
-        # a request may be specialised with specific backend headers
-        if self.header_provider:
-            req_headers.update(self.header_provider())
+        ip, port = get_ip_port(node, req_headers)
         start_node_timing = time.time()
         try:
             with ConnectionTimeout(self.app.conn_timeout):
                 conn = http_connect(
-                    node['ip'], node['port'], node['device'],
-                    self.partition, self.req_method, self.path,
+                    ip, port, node['device'],
+                    self.partition, self.req.method, self.path,
                     headers=req_headers,
-                    query_string=self.req_query_string)
+                    query_string=self.req.query_string)
             self.app.set_node_timing(node, time.time() - start_node_timing)
 
-            with Timeout(node_timeout):
+            with Timeout(self.node_timeout):
                 possible_source = conn.getresponse()
                 # See NOTE: swift_conn at top of file about this.
                 possible_source.swift_conn = conn
         except (Exception, Timeout):
             self.app.exception_occurred(
                 node, self.server_type,
-                _('Trying to %(method)s %(path)s') %
-                {'method': self.req_method, 'path': self.req_path})
+                'Trying to %(method)s %(path)s' %
+                {'method': self.req.method, 'path': self.req.path})
             return False
 
         src_headers = dict(
             (k.lower(), v) for k, v in
             possible_source.getheaders())
-        if self.is_good_source(possible_source):
+        if is_good_source(possible_source.status, self.server_type):
             # 404 if we know we don't have a synced copy
             if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
                 self.statuses.append(HTTP_NOT_FOUND)
@@ -1259,9 +1552,8 @@ class ResumingGetter(object):
                 close_swift_conn(possible_source)
             else:
                 if self.used_source_etag and \
-                    self.used_source_etag != src_headers.get(
-                        'x-object-sysmeta-ec-etag',
-                        src_headers.get('etag', '')).strip('"'):
+                        self.used_source_etag != normalize_etag(
+                            src_headers.get('etag', '')):
                     self.statuses.append(HTTP_NOT_FOUND)
                     self.reasons.append('')
                     self.bodies.append('')
@@ -1280,16 +1572,25 @@ class ResumingGetter(object):
                     self.reasons.append(possible_source.reason)
                     self.bodies.append(None)
                     self.source_headers.append(possible_source.getheaders())
-                    self.sources.append((possible_source, node))
+                    self.sources.append(
+                        GetterSource(self.app, possible_source, node))
                     if not self.newest:  # one good source is enough
                         return True
         else:
             if 'handoff_index' in node and \
+                    (is_server_error(possible_source.status) or
+                     possible_source.status == HTTP_NOT_FOUND) and \
+                    not Timestamp(src_headers.get('x-backend-timestamp', 0)):
+                # throw out 5XX and 404s from handoff nodes unless the data is
+                # really on disk and had been DELETEd
+                return False
+
+            if self.rebalance_missing_suppression_count > 0 and \
                     possible_source.status == HTTP_NOT_FOUND and \
                     not Timestamp(src_headers.get('x-backend-timestamp', 0)):
-                # throw out 404s from handoff nodes unless the data is really
-                # on disk and had been DELETEd
+                self.rebalance_missing_suppression_count -= 1
                 return False
+
             self.statuses.append(possible_source.status)
             self.reasons.append(possible_source.reason)
             self.bodies.append(possible_source.read())
@@ -1305,18 +1606,12 @@ class ResumingGetter(object):
                 ts = Timestamp(hdrs.get('X-Backend-Timestamp', 0))
                 if ts > self.latest_404_timestamp:
                     self.latest_404_timestamp = ts
-            if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
-                self.app.error_limit(node, _('ERROR Insufficient Storage'))
-            elif is_server_error(possible_source.status):
-                self.app.error_occurred(
-                    node, _('ERROR %(status)d %(body)s '
-                            'From %(type)s Server') %
-                    {'status': possible_source.status,
-                     'body': self.bodies[-1][:1024],
-                     'type': self.server_type})
+            self.app.check_response(node, self.server_type, possible_source,
+                                    self.req.method, self.path,
+                                    self.bodies[-1])
         return False
 
-    def _get_source_and_node(self):
+    def _find_source(self):
         self.statuses = []
         self.reasons = []
         self.bodies = []
@@ -1325,16 +1620,13 @@ class ResumingGetter(object):
 
         nodes = GreenthreadSafeIterator(self.node_iter)
 
-        node_timeout = self.app.node_timeout
-        if self.server_type == 'Object' and not self.newest:
-            node_timeout = self.app.recoverable_node_timeout
-
         pile = GreenAsyncPile(self.concurrency)
 
         for node in nodes:
-            pile.spawn(self._make_node_request, node, node_timeout,
-                       self.app.logger.thread_locals)
-            _timeout = self.app.concurrency_timeout \
+            pile.spawn(self._make_node_request, node,
+                       self.logger.thread_locals)
+            _timeout = self.app.get_policy_options(
+                self.policy).concurrency_timeout \
                 if pile.inflight < self.concurrency else None
             if pile.waitfirst(_timeout):
                 break
@@ -1346,46 +1638,37 @@ class ResumingGetter(object):
         # and added to the list in the case of x-newest.
         if self.sources:
             self.sources = [s for s in self.sources
-                            if source_key(s[0]) >= self.latest_404_timestamp]
+                            if s.timestamp >= self.latest_404_timestamp]
 
         if self.sources:
-            self.sources.sort(key=lambda s: source_key(s[0]))
-            source, node = self.sources.pop()
-            for src, _junk in self.sources:
-                close_swift_conn(src)
-            self.used_nodes.append(node)
-            src_headers = dict(
-                (k.lower(), v) for k, v in
-                source.getheaders())
+            self.sources.sort(key=operator.attrgetter('timestamp'))
+            source = self.sources.pop()
+            for unused_source in self.sources:
+                unused_source.close()
+            self.used_nodes.append(source.node)
 
             # Save off the source etag so that, if we lose the connection
             # and have to resume from a different node, we can be sure that
-            # we have the same object (replication) or a fragment archive
-            # from the same object (EC). Otherwise, if the cluster has two
-            # versions of the same object, we might end up switching between
-            # old and new mid-stream and giving garbage to the client.
-            self.used_source_etag = src_headers.get(
-                'x-object-sysmeta-ec-etag',
-                src_headers.get('etag', '')).strip('"')
-            self.node = node
-            return source, node
-        return None, None
-
-
-class GetOrHeadHandler(ResumingGetter):
-    def _make_app_iter(self, req, node, source):
+            # we have the same object (replication). Otherwise, if the cluster
+            # has two versions of the same object, we might end up switching
+            # between old and new mid-stream and giving garbage to the client.
+            if self.used_source_etag is None:
+                self.used_source_etag = normalize_etag(
+                    source.resp.getheader('etag', ''))
+            self.source = source
+            return True
+        return False
+
+    def _make_app_iter(self):
         """
         Returns an iterator over the contents of the source (via its read
         func).  There is also quite a bit of cleanup to ensure garbage
         collection works and the underlying socket of the source is closed.
 
-        :param req: incoming request object
-        :param source: The httplib.Response object this iterator should read
-                       from.
-        :param node: The node the source is reading from, for logging purposes.
+        :return: an iterator that yields chunks of response body bytes
         """
 
-        ct = source.getheader('Content-Type')
+        ct = self.source.resp.getheader('Content-Type')
         if ct:
             content_type, content_type_attrs = parse_content_type(ct)
             is_multipart = content_type == 'multipart/byteranges'
@@ -1398,7 +1681,7 @@ class GetOrHeadHandler(ResumingGetter):
             # furnished one for us, so we'll just re-use it
             boundary = dict(content_type_attrs)["boundary"]
 
-        parts_iter = self._get_response_parts_iter(req, node, source)
+        parts_iter = self._iter_parts_from_response()
 
         def add_content_type(response_part):
             response_part["content_type"] = \
@@ -1406,29 +1689,29 @@ class GetOrHeadHandler(ResumingGetter):
             return response_part
 
         return document_iters_to_http_response_body(
-            (add_content_type(pi) for pi in parts_iter),
-            boundary, is_multipart, self.app.logger)
+            ClosingMapper(add_content_type, parts_iter),
+            boundary, is_multipart, self.logger)
 
-    def get_working_response(self, req):
-        source, node = self._get_source_and_node()
+    def get_working_response(self):
         res = None
-        if source:
-            res = Response(request=req)
-            res.status = source.status
-            update_headers(res, source.getheaders())
-            if req.method == 'GET' and \
-                    source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
-                res.app_iter = self._make_app_iter(req, node, source)
+        if self._replace_source():
+            res = Response(request=self.req)
+            res.status = self.source.resp.status
+            update_headers(res, self.source.resp.getheaders())
+            if self.req.method == 'GET' and \
+                    self.source.resp.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
+                res.app_iter = self._make_app_iter()
                 # See NOTE: swift_conn at top of file about this.
-                res.swift_conn = source.swift_conn
+                res.swift_conn = self.source.resp.swift_conn
             if not res.environ:
                 res.environ = {}
-            res.environ['swift_x_timestamp'] = source.getheader('x-timestamp')
+            res.environ['swift_x_timestamp'] = self.source.resp.getheader(
+                'x-timestamp')
             res.accept_ranges = 'bytes'
-            res.content_length = source.getheader('Content-Length')
-            if source.getheader('Content-Type'):
+            res.content_length = self.source.resp.getheader('Content-Length')
+            if self.source.resp.getheader('Content-Type'):
                 res.charset = None
-                res.content_type = source.getheader('Content-Type')
+                res.content_type = self.source.resp.getheader('Content-Type')
         return res
 
 
@@ -1446,36 +1729,48 @@ class NodeIter(object):
     may not, depending on how logging is configured, the vagaries of
     socket IO and eventlet, and the phase of the moon.)
 
+    :param server_type: one of 'account', 'container', or 'object'
     :param app: a proxy app
     :param ring: ring to get yield nodes from
     :param partition: ring partition to yield nodes for
+    :param logger: a logger instance
+    :param request: yielded nodes will be annotated with `use_replication`
+        based on the `request` headers.
     :param node_iter: optional iterable of nodes to try. Useful if you
         want to filter or reorder the nodes.
     :param policy: an instance of :class:`BaseStoragePolicy`. This should be
         None for an account or container ring.
     """
 
-    def __init__(self, app, ring, partition, node_iter=None, policy=None):
+    def __init__(self, server_type, app, ring, partition, logger, request,
+                 node_iter=None, policy=None):
+        self.server_type = server_type
         self.app = app
         self.ring = ring
         self.partition = partition
+        self.logger = logger
+        self.request = request
 
         part_nodes = ring.get_part_nodes(partition)
         if node_iter is None:
             node_iter = itertools.chain(
                 part_nodes, ring.get_more_nodes(partition))
-        num_primary_nodes = len(part_nodes)
-        self.nodes_left = self.app.request_node_count(num_primary_nodes)
-        self.expected_handoffs = self.nodes_left - num_primary_nodes
+        self.num_primary_nodes = len(part_nodes)
+        self.nodes_left = self.app.request_node_count(self.num_primary_nodes)
+        self.expected_handoffs = self.nodes_left - self.num_primary_nodes
 
         # Use of list() here forcibly yanks the first N nodes (the primary
         # nodes) from node_iter, so the rest of its values are handoffs.
         self.primary_nodes = self.app.sort_nodes(
-            list(itertools.islice(node_iter, num_primary_nodes)),
+            list(itertools.islice(node_iter, self.num_primary_nodes)),
             policy=policy)
         self.handoff_iter = node_iter
         self._node_provider = None
 
+    @property
+    def primaries_left(self):
+        return len(self.primary_nodes)
+
     def __iter__(self):
         self._node_iter = self._node_gen()
         return self
@@ -1496,12 +1791,14 @@ class NodeIter(object):
             return
         extra_handoffs = handoffs - self.expected_handoffs
         if extra_handoffs > 0:
-            self.app.logger.increment('handoff_count')
-            self.app.logger.warning(
+            self.logger.increment('%s.handoff_count' %
+                                  self.server_type.lower())
+            self.logger.warning(
                 'Handoff requested (%d)' % handoffs)
-            if (extra_handoffs == len(self.primary_nodes)):
+            if (extra_handoffs == self.num_primary_nodes):
                 # all the primaries were skipped, and handoffs didn't help
-                self.app.logger.increment('handoff_all_count')
+                self.logger.increment('%s.handoff_all_count' %
+                                      self.server_type.lower())
 
     def set_node_provider(self, callback):
         """
@@ -1515,7 +1812,8 @@ class NodeIter(object):
         self._node_provider = callback
 
     def _node_gen(self):
-        for node in self.primary_nodes:
+        while self.primary_nodes:
+            node = self.primary_nodes.pop(0)
             if not self.app.error_limited(node):
                 yield node
                 if not self.app.error_limited(node):
@@ -1533,16 +1831,27 @@ class NodeIter(object):
                     if self.nodes_left <= 0:
                         return
 
-    def next(self):
+    def _annotate_node(self, node):
+        """
+        Helper function to set use_replication dict value for a node by looking
+        up the header value for x-backend-use-replication-network.
+
+        :param node: node dictionary from the ring or node_iter.
+        :returns: node dictionary with replication network enabled/disabled
+        """
+        # nodes may have come from a ring or a node_iter passed to the
+        # constructor: be careful not to mutate them!
+        return dict(node, use_replication=is_use_replication_network(
+            self.request.headers))
+
+    def __next__(self):
+        node = None
         if self._node_provider:
             # give node provider the opportunity to inject a node
             node = self._node_provider()
-            if node:
-                return node
-        return next(self._node_iter)
-
-    def __next__(self):
-        return self.next()
+        if not node:
+            node = next(self._node_iter)
+        return self._annotate_node(node)
 
 
 class Controller(object):
@@ -1564,6 +1873,10 @@ class Controller(object):
         self._allowed_methods = None
         self._private_methods = None
 
+    @property
+    def logger(self):
+        return self.app.logger
+
     @property
     def allowed_methods(self):
         if self._allowed_methods is None:
@@ -1616,39 +1929,44 @@ class Controller(object):
     def generate_request_headers(self, orig_req=None, additional=None,
                                  transfer=False):
         """
-        Create a list of headers to be used in backend requests
+        Create a dict of headers to be used in backend requests
 
         :param orig_req: the original request sent by the client to the proxy
         :param additional: additional headers to send to the backend
         :param transfer: If True, transfer headers from original client request
         :returns: a dictionary of headers
         """
-        # Use the additional headers first so they don't overwrite the headers
-        # we require.
-        headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
-        if transfer:
-            self.transfer_headers(orig_req.headers, headers)
-        headers.setdefault('x-timestamp', Timestamp.now().internal)
+        headers = HeaderKeyDict()
         if orig_req:
+            headers.update((k.lower(), v)
+                           for k, v in orig_req.headers.items()
+                           if k.lower().startswith('x-backend-'))
             referer = orig_req.as_referer()
         else:
             referer = ''
+        # additional headers can override x-backend-* headers from orig_req
+        if additional:
+            headers.update(additional)
+        if orig_req and transfer:
+            # transfer headers from orig_req can override additional headers
+            self.transfer_headers(orig_req.headers, headers)
+        headers.setdefault('x-timestamp', Timestamp.now().internal)
+        # orig_req and additional headers cannot override the following...
         headers['x-trans-id'] = self.trans_id
         headers['connection'] = 'close'
-        headers['user-agent'] = 'proxy-server %s' % os.getpid()
+        headers['user-agent'] = self.app.backend_user_agent
         headers['referer'] = referer
         return headers
 
-    def account_info(self, account, req=None):
+    def account_info(self, account, req):
         """
         Get account information, and also verify that the account exists.
 
         :param account: native str name of the account to get the info for
-        :param req: caller's HTTP request context object (optional)
+        :param req: caller's HTTP request context object
         :returns: tuple of (account partition, account nodes, container_count)
                   or (None, None, None) if it does not exist
         """
-        partition, nodes = self.app.account_ring.get_nodes(account)
         if req:
             env = getattr(req, 'environ', {})
         else:
@@ -1663,23 +1981,23 @@ class Controller(object):
                 or not info.get('account_really_exists', True)):
             return None, None, None
         container_count = info['container_count']
+        partition, nodes = self.app.account_ring.get_nodes(account)
         return partition, nodes, container_count
 
-    def container_info(self, account, container, req=None):
+    def container_info(self, account, container, req):
         """
         Get container information and thusly verify container existence.
         This will also verify account existence.
 
         :param account: native-str account name for the container
         :param container: native-str container name to look up
-        :param req: caller's HTTP request context object (optional)
+        :param req: caller's HTTP request context object
         :returns: dict containing at least container partition ('partition'),
                   container nodes ('containers'), container read
                   acl ('read_acl'), container write acl ('write_acl'),
                   and container sync key ('sync_key').
                   Values are set to None if the container does not exist.
         """
-        part, nodes = self.app.container_ring.get_nodes(account, container)
         if req:
             env = getattr(req, 'environ', {})
         else:
@@ -1689,11 +2007,11 @@ class Controller(object):
         path_env['PATH_INFO'] = "/v1/%s/%s" % (
             str_to_wsgi(account), str_to_wsgi(container))
         info = get_container_info(path_env, self.app)
-        if not info or not is_success(info.get('status')):
-            info = headers_to_container_info({}, 0)
+        if not is_success(info.get('status')):
             info['partition'] = None
             info['nodes'] = None
         else:
+            part, nodes = self.app.container_ring.get_nodes(account, container)
             info['partition'] = part
             info['nodes'] = nodes
         return info
@@ -1716,22 +2034,23 @@ class Controller(object):
         :param body: byte string to use as the request body.
                      Try to keep it small.
         :param logger_thread_locals: The thread local values to be set on the
-                                     self.app.logger to retain transaction
+                                     self.logger to retain transaction
                                      logging information.
         :returns: a swob.Response object, or None if no responses were received
         """
-        self.app.logger.thread_locals = logger_thread_locals
+        self.logger.thread_locals = logger_thread_locals
         if body:
             if not isinstance(body, bytes):
                 raise TypeError('body must be bytes, not %s' % type(body))
             headers['Content-Length'] = str(len(body))
         for node in nodes:
             try:
+                ip, port = get_ip_port(node, headers)
                 start_node_timing = time.time()
                 with ConnectionTimeout(self.app.conn_timeout):
-                    conn = http_connect(node['ip'], node['port'],
-                                        node['device'], part, method, path,
-                                        headers=headers, query_string=query)
+                    conn = http_connect(
+                        ip, port, node['device'], part, method, path,
+                        headers=headers, query_string=query)
                     conn.node = node
                 self.app.set_node_timing(node, time.time() - start_node_timing)
                 if body:
@@ -1739,27 +2058,17 @@ class Controller(object):
                         conn.send(body)
                 with Timeout(self.app.node_timeout):
                     resp = conn.getresponse()
-                    if not is_informational(resp.status) and \
-                            not is_server_error(resp.status):
-                        return resp.status, resp.reason, resp.getheaders(), \
-                            resp.read()
-                    elif resp.status == HTTP_INSUFFICIENT_STORAGE:
-                        self.app.error_limit(node,
-                                             _('ERROR Insufficient Storage'))
-                    elif is_server_error(resp.status):
-                        self.app.error_occurred(
-                            node, _('ERROR %(status)d '
-                                    'Trying to %(method)s %(path)s'
-                                    ' From %(type)s Server') % {
-                                        'status': resp.status,
-                                        'method': method,
-                                        'path': path,
-                                        'type': self.server_type})
+                    if (self.app.check_response(node, self.server_type, resp,
+                                                method, path)
+                            and not is_informational(resp.status)):
+                        return resp, resp.read(), node
+
             except (Exception, Timeout):
                 self.app.exception_occurred(
                     node, self.server_type,
-                    _('Trying to %(method)s %(path)s') %
+                    'Trying to %(method)s %(path)s' %
                     {'method': method, 'path': path})
+        return None, None, None
 
     def make_requests(self, req, ring, part, method, path, headers,
                       query_string='', overrides=None, node_count=None,
@@ -1782,36 +2091,37 @@ class Controller(object):
                           the returned status of a request.
         :param node_count: optional number of nodes to send request to.
         :param node_iterator: optional node iterator.
+        :param body: byte string to use as the request body.
+                     Try to keep it small.
         :returns: a swob.Response object
         """
-        nodes = GreenthreadSafeIterator(
-            node_iterator or self.app.iter_nodes(ring, part)
-        )
+        nodes = GreenthreadSafeIterator(node_iterator or NodeIter(
+            self.server_type.lower(), self.app, ring, part, self.logger, req))
         node_number = node_count or len(ring.get_part_nodes(part))
         pile = GreenAsyncPile(node_number)
 
         for head in headers:
             pile.spawn(self._make_request, nodes, part, method, path,
-                       head, query_string, body, self.app.logger.thread_locals)
-        response = []
+                       head, query_string, body, self.logger.thread_locals)
+        results = []
         statuses = []
-        for resp in pile:
-            if not resp:
+        for resp, body, node in pile:
+            if not is_useful_response(resp, node):
                 continue
-            response.append(resp)
-            statuses.append(resp[0])
+            results.append((resp.status, resp.reason, resp.getheaders(), body))
+            statuses.append(resp.status)
             if self.have_quorum(statuses, node_number):
                 break
         # give any pending requests *some* chance to finish
         finished_quickly = pile.waitall(self.app.post_quorum_timeout)
-        for resp in finished_quickly:
-            if not resp:
+        for resp, body, node in finished_quickly:
+            if not is_useful_response(resp, node):
                 continue
-            response.append(resp)
-            statuses.append(resp[0])
-        while len(response) < node_number:
-            response.append((HTTP_SERVICE_UNAVAILABLE, '', '', b''))
-        statuses, reasons, resp_headers, bodies = zip(*response)
+            results.append((resp.status, resp.reason, resp.getheaders(), body))
+            statuses.append(resp.status)
+        while len(results) < node_number:
+            results.append((HTTP_SERVICE_UNAVAILABLE, '', '', b''))
+        statuses, reasons, resp_headers, bodies = zip(*results)
         return self.best_response(req, statuses, reasons, bodies,
                                   '%s %s' % (self.server_type, req.method),
                                   overrides=overrides, headers=resp_headers)
@@ -1885,8 +2195,8 @@ class Controller(object):
 
         if not resp:
             resp = HTTPServiceUnavailable(request=req)
-            self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'),
-                                  {'type': server_type, 'statuses': statuses})
+            self.logger.error('%(type)s returning 503 for %(statuses)s',
+                              {'type': server_type, 'statuses': statuses})
 
         return resp
 
@@ -1913,7 +2223,7 @@ class Controller(object):
                 if headers:
                     update_headers(resp, headers[status_index])
                 if etag:
-                    resp.headers['etag'] = etag.strip('"')
+                    resp.headers['etag'] = normalize_etag(etag)
                 return resp
         return None
 
@@ -1955,20 +2265,19 @@ class Controller(object):
         headers.update((k, v)
                        for k, v in req.headers.items()
                        if is_sys_meta('account', k))
-        resp = self.make_requests(Request.blank('/v1' + path),
+        resp = self.make_requests(Request.blank(str_to_wsgi('/v1' + path)),
                                   self.app.account_ring, partition, 'PUT',
                                   path, [headers] * len(nodes))
         if is_success(resp.status_int):
-            self.app.logger.info(_('autocreate account %r'), path)
-            clear_info_cache(self.app, req.environ, account)
+            self.logger.info('autocreate account %r', path)
+            clear_info_cache(req.environ, account)
             return True
         else:
-            self.app.logger.warning(_('Could not autocreate account %r'),
-                                    path)
+            self.logger.warning('Could not autocreate account %r', path)
             return False
 
     def GETorHEAD_base(self, req, server_type, node_iter, partition, path,
-                       concurrency=1, client_chunk_size=None):
+                       concurrency=1, policy=None):
         """
         Base handler for HTTP GET or HEAD requests.
 
@@ -1978,7 +2287,7 @@ class Controller(object):
         :param partition: partition
         :param path: path for the request
         :param concurrency: number of requests to run concurrently
-        :param client_chunk_size: chunk size for response body iterator
+        :param policy: the policy instance, or None if Account or Container
         :returns: swob.Response object
         """
         backend_headers = self.generate_request_headers(
@@ -1986,9 +2295,9 @@ class Controller(object):
 
         handler = GetOrHeadHandler(self.app, req, self.server_type, node_iter,
                                    partition, path, backend_headers,
-                                   concurrency,
-                                   client_chunk_size=client_chunk_size)
-        res = handler.get_working_response(req)
+                                   concurrency, policy=policy,
+                                   logger=self.logger)
+        res = handler.get_working_response()
 
         if not res:
             res = self.best_response(
@@ -2006,7 +2315,7 @@ class Controller(object):
             if policy:
                 res.headers['X-Storage-Policy'] = policy.name
             else:
-                self.app.logger.error(
+                self.logger.error(
                     'Could not translate %s (%r) from %r to policy',
                     'X-Backend-Storage-Policy-Index',
                     res.headers['X-Backend-Storage-Policy-Index'], path)
@@ -2116,6 +2425,26 @@ class Controller(object):
             raise ValueError(
                 "server_type can only be 'account' or 'container'")
 
+    def _parse_listing_response(self, req, response):
+        if not is_success(response.status_int):
+            record_type = req.headers.get('X-Backend-Record-Type')
+            self.logger.warning(
+                'Failed to get container %s listing from %s: %s',
+                record_type, req.path_qs, response.status_int)
+            return None
+
+        try:
+            data = json.loads(response.body)
+            if not isinstance(data, list):
+                raise ValueError('not a list')
+            return data
+        except ValueError as err:
+            record_type = response.headers.get('X-Backend-Record-Type')
+            self.logger.error(
+                'Problem with container %s listing response from %s: %r',
+                record_type, req.path_qs, err)
+            return None
+
     def _get_container_listing(self, req, account, container, headers=None,
                                params=None):
         """
@@ -2123,8 +2452,10 @@ class Controller(object):
 
         :param req: original Request instance.
         :param account: account in which `container` is stored.
-        :param container: container from listing should be fetched.
-        :param headers: headers to be included with the request
+        :param container: container from which listing should be fetched.
+        :param headers: extra headers to be included with the listing
+            sub-request; these update the headers copied from the original
+            request.
         :param params: query string parameters to be used.
         :return: a tuple of (deserialized json data structure, swob Response)
         """
@@ -2138,120 +2469,33 @@ class Controller(object):
         if headers:
             subreq.headers.update(headers)
         subreq.params = params
-        self.app.logger.debug(
+        self.logger.debug(
             'Get listing from %s %s' % (subreq.path_qs, headers))
         response = self.app.handle_request(subreq)
+        data = self._parse_listing_response(subreq, response)
+        return data, response
 
-        if not is_success(response.status_int):
-            self.app.logger.warning(
-                'Failed to get container listing from %s: %s',
-                subreq.path_qs, response.status_int)
-            return None, response
-
-        try:
-            data = json.loads(response.body)
-            if not isinstance(data, list):
-                raise ValueError('not a list')
-            return data, response
-        except ValueError as err:
-            self.app.logger.error(
-                'Problem with listing response from %s: %r',
-                subreq.path_qs, err)
-            return None, response
-
-    def _get_shard_ranges(self, req, account, container, includes=None,
-                          states=None):
-        """
-        Fetch shard ranges from given `account/container`. If `includes` is
-        given then the shard range for that object name is requested, otherwise
-        all shard ranges are requested.
-
-        :param req: original Request instance.
-        :param account: account from which shard ranges should be fetched.
-        :param container: container from which shard ranges should be fetched.
-        :param includes: (optional) restricts the list of fetched shard ranges
-            to those which include the given name.
-        :param states: (optional) the states of shard ranges to be fetched.
-        :return: a list of instances of :class:`swift.common.utils.ShardRange`,
-            or None if there was a problem fetching the shard ranges
-        """
-        params = req.params.copy()
-        params.pop('limit', None)
-        params['format'] = 'json'
-        if includes:
-            params['includes'] = includes
-        if states:
-            params['states'] = states
-        headers = {'X-Backend-Record-Type': 'shard'}
-        listing, response = self._get_container_listing(
-            req, account, container, headers=headers, params=params)
+    def _parse_namespaces(self, req, listing, response):
         if listing is None:
             return None
 
         record_type = response.headers.get('x-backend-record-type')
         if record_type != 'shard':
             err = 'unexpected record type %r' % record_type
-            self.app.logger.error("Failed to get shard ranges from %s: %s",
-                                  req.path_qs, err)
+            self.logger.error("Failed to get shard ranges from %s: %s",
+                              req.path_qs, err)
             return None
 
         try:
-            return [ShardRange.from_dict(shard_range)
-                    for shard_range in listing]
+            # Note: a legacy container-server could return a list of
+            # ShardRanges, but that's ok: namespaces just need 'name', 'lower'
+            # and 'upper' keys. If we ever need to know we can look for a
+            # 'x-backend-record-shard-format' header from newer container
+            # servers.
+            return [Namespace(data['name'], data['lower'], data['upper'])
+                    for data in listing]
         except (ValueError, TypeError, KeyError) as err:
-            self.app.logger.error(
-                "Failed to get shard ranges from %s: invalid data: %r",
+            self.logger.error(
+                "Failed to get namespaces from %s: invalid data: %r",
                 req.path_qs, err)
             return None
-
-    def _get_update_shard(self, req, account, container, obj):
-        """
-        Find the appropriate shard range for an object update.
-
-        Note that this fetches and caches (in both the per-request infocache
-        and memcache, if available) all shard ranges for the given root
-        container so we won't have to contact the container DB for every write.
-
-        :param req: original Request instance.
-        :param account: account from which shard ranges should be fetched.
-        :param container: container from which shard ranges should be fetched.
-        :param obj: object getting updated.
-        :return: an instance of :class:`swift.common.utils.ShardRange`,
-            or None if the update should go back to the root
-        """
-        if not self.app.recheck_updating_shard_ranges:
-            # caching is disabled; fall back to old behavior
-            shard_ranges = self._get_shard_ranges(
-                req, account, container, states='updating', includes=obj)
-            if not shard_ranges:
-                return None
-            return shard_ranges[0]
-
-        cache_key = get_cache_key(account, container, shard='updating')
-        infocache = req.environ.setdefault('swift.infocache', {})
-        memcache = getattr(self.app, 'memcache', None) or req.environ.get(
-            'swift.cache')
-
-        cached_ranges = infocache.get(cache_key)
-        if cached_ranges is None and memcache:
-            cached_ranges = memcache.get(cache_key)
-
-        if cached_ranges:
-            shard_ranges = [
-                ShardRange.from_dict(shard_range)
-                for shard_range in cached_ranges]
-        else:
-            shard_ranges = self._get_shard_ranges(
-                req, account, container, states='updating')
-            if shard_ranges:
-                cached_ranges = [dict(sr) for sr in shard_ranges]
-                # went to disk; cache it
-                if memcache:
-                    memcache.set(cache_key, cached_ranges,
-                                 time=self.app.recheck_updating_shard_ranges)
-
-        if not shard_ranges:
-            return None
-
-        infocache[cache_key] = tuple(cached_ranges)
-        return find_shard_range(obj, shard_ranges)