swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/container/backend.py

@@ -20,9 +20,7 @@ import errno
 import os
 from uuid import uuid4
 
-import six
-from six.moves import range
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 import sqlite3
 from eventlet import tpool
 
@@ -30,9 +28,10 @@ from swift.common.constraints import CONTAINER_LISTING_LIMIT
 from swift.common.exceptions import LockTimeout
 from swift.common.utils import Timestamp, encode_timestamps, \
     decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \
-    ShardRange, renamer, find_shard_range, MD5_OF_EMPTY_STRING, mkdirs, \
-    get_db_files, parse_db_filename, make_db_file_path, split_path
-from swift.common.db import DatabaseBroker, utf8encode, BROKER_TIMEOUT, \
+    ShardRange, renamer, MD5_OF_EMPTY_STRING, mkdirs, get_db_files, \
+    parse_db_filename, make_db_file_path, split_path, RESERVED_BYTE, \
+    ShardRangeList, Namespace
+from swift.common.db import DatabaseBroker, BROKER_TIMEOUT, \
     zero_like, DatabaseAlreadyExists, SQLITE_ARG_LIMIT
 
 DATADIR = 'containers'
@@ -53,13 +52,25 @@ SHARD_STATS_STATES = [ShardRange.ACTIVE, ShardRange.SHARDING,
 SHARD_LISTING_STATES = SHARD_STATS_STATES + [ShardRange.CLEAVED]
 SHARD_UPDATE_STATES = [ShardRange.CREATED, ShardRange.CLEAVED,
                        ShardRange.ACTIVE, ShardRange.SHARDING]
-
+# when auditing a shard gets its own shard range, which could be in any state
+# except FOUND, and any potential acceptors excluding FOUND ranges that may be
+# unwanted overlaps
+SHARD_AUDITING_STATES = [ShardRange.CREATED, ShardRange.CLEAVED,
+                         ShardRange.ACTIVE, ShardRange.SHARDING,
+                         ShardRange.SHARDED, ShardRange.SHRINKING,
+                         ShardRange.SHRUNK]
+# shard's may not be fully populated while in the FOUND and CREATED
+# state, so shards should only update their own shard range's object
+# stats when they are in the following states
+SHARD_UPDATE_STAT_STATES = [ShardRange.CLEAVED, ShardRange.ACTIVE,
+                            ShardRange.SHARDING, ShardRange.SHARDED,
+                            ShardRange.SHRINKING, ShardRange.SHRUNK]
 
 # attribute names in order used when transforming shard ranges from dicts to
 # tuples and vice-versa
 SHARD_RANGE_KEYS = ('name', 'timestamp', 'lower', 'upper', 'object_count',
                     'bytes_used', 'meta_timestamp', 'deleted', 'state',
-                    'state_timestamp', 'epoch', 'reported')
+                    'state_timestamp', 'epoch', 'reported', 'tombstones')
 
 POLICY_STAT_TABLE_CREATE = '''
     CREATE TABLE policy_stat (
@@ -280,6 +291,7 @@ def merge_shards(shard_data, existing):
     if existing['meta_timestamp'] >= shard_data['meta_timestamp']:
         for k in ('object_count', 'bytes_used', 'meta_timestamp'):
             shard_data[k] = existing[k]
+        shard_data['tombstones'] = existing.get('tombstones', -1)
     else:
         new_content = True
 
@@ -287,6 +299,7 @@ def merge_shards(shard_data, existing):
     if existing['reported'] and \
             existing['object_count'] == shard_data['object_count'] and \
             existing['bytes_used'] == shard_data['bytes_used'] and \
+            existing.get('tombstones', -1) == shard_data['tombstones'] and \
             existing['state'] == shard_data['state'] and \
             existing['epoch'] == shard_data['epoch']:
         shard_data['reported'] = 1
@@ -306,6 +319,38 @@ def merge_shards(shard_data, existing):
     return new_content
 
 
+def sift_shard_ranges(new_shard_ranges, existing_shard_ranges):
+    """
+    Compares new and existing shard ranges, updating the new shard ranges with
+    any more recent state from the existing, and returns shard ranges sorted
+    into those that need adding because they contain new or updated state and
+    those that need deleting because their state has been superseded.
+
+    :param new_shard_ranges: a list of dicts, each of which represents a shard
+        range.
+    :param existing_shard_ranges: a dict mapping shard range names to dicts
+        representing a shard range.
+    :return: a tuple (to_add, to_delete); to_add is a list of dicts, each of
+        which represents a shard range that is to be added to the existing
+        shard ranges; to_delete is a set of shard range names that are to be
+        deleted.
+    """
+    to_delete = set()
+    to_add = {}
+    for item in new_shard_ranges:
+        item_ident = item['name']
+        existing = existing_shard_ranges.get(item_ident)
+        if merge_shards(item, existing):
+            # exists with older timestamp
+            if item_ident in existing_shard_ranges:
+                to_delete.add(item_ident)
+            # duplicate entries in item_list
+            if (item_ident not in to_add or
+                    merge_shards(item, to_add[item_ident])):
+                to_add[item_ident] = item
+    return to_add.values(), to_delete
+
+
 class ContainerBroker(DatabaseBroker):
     """
     Encapsulates working with a container database.
@@ -313,34 +358,34 @@ class ContainerBroker(DatabaseBroker):
     Note that this may involve multiple on-disk DB files if the container
     becomes sharded:
 
-      * :attr:`_db_file` is the path to the legacy container DB name, i.e.
-        ``<hash>.db``. This file should exist for an initialised broker that
-        has never been sharded, but will not exist once a container has been
-        sharded.
-      * :attr:`db_files` is a list of existing db files for the broker. This
-        list should have at least one entry for an initialised broker, and
-        should have two entries while a broker is in SHARDING state.
-      * :attr:`db_file` is the path to whichever db is currently authoritative
-        for the container. Depending on the container's state, this may not be
-        the same as the ``db_file`` argument given to :meth:`~__init__`, unless
-        ``force_db_file`` is True in which case :attr:`db_file` is always equal
-        to the ``db_file`` argument given to :meth:`~__init__`.
-      * :attr:`pending_file` is always equal to :attr:`_db_file` extended with
-        ``.pending``, i.e. ``<hash>.db.pending``.
+    * :attr:`_db_file` is the path to the legacy container DB name, i.e.
+      ``<hash>.db``. This file should exist for an initialised broker that
+      has never been sharded, but will not exist once a container has been
+      sharded.
+    * :attr:`db_files` is a list of existing db files for the broker. This
+      list should have at least one entry for an initialised broker, and
+      should have two entries while a broker is in SHARDING state.
+    * :attr:`db_file` is the path to whichever db is currently authoritative
+      for the container. Depending on the container's state, this may not be
+      the same as the ``db_file`` argument given to :meth:`~__init__`, unless
+      ``force_db_file`` is True in which case :attr:`db_file` is always equal
+      to the ``db_file`` argument given to :meth:`~__init__`.
+    * :attr:`pending_file` is always equal to :attr:`_db_file` extended with
+      ``.pending``, i.e. ``<hash>.db.pending``.
     """
     db_type = 'container'
     db_contains_type = 'object'
     db_reclaim_timestamp = 'created_at'
+    delete_meta_whitelist = ['x-container-sysmeta-shard-quoted-root',
+                             'x-container-sysmeta-shard-root',
+                             'x-container-sysmeta-sharding']
 
     def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None,
                  account=None, container=None, pending_timeout=None,
                  stale_reads_ok=False, skip_commits=False,
                  force_db_file=False):
         self._init_db_file = db_file
-        if db_file == ':memory:':
-            base_db_file = db_file
-        else:
-            base_db_file = make_db_file_path(db_file, None)
+        base_db_file = make_db_file_path(db_file, None)
         super(ContainerBroker, self).__init__(
             base_db_file, timeout, logger, account, container, pending_timeout,
             stale_reads_ok, skip_commits=skip_commits)
@@ -366,7 +411,10 @@ class ContainerBroker(DatabaseBroker):
         :param put_timestamp: initial timestamp if broker needs to be
             initialized
         :param storage_policy_index: the storage policy index
-        :return: a :class:`swift.container.backend.ContainerBroker` instance
+        :return: a tuple of (``broker``, ``initialized``) where ``broker`` is
+            an instance of :class:`swift.container.backend.ContainerBroker` and
+            ``initialized`` is True if the db file was initialized, False
+            otherwise.
         """
         hsh = hash_path(account, container)
         db_dir = storage_directory(DATADIR, part, hsh)
@@ -374,19 +422,19 @@ class ContainerBroker(DatabaseBroker):
             os.path.join(device_path, db_dir, hsh + '.db'), epoch)
         broker = ContainerBroker(db_path, account=account, container=container,
                                  logger=logger)
+        initialized = False
         if not os.path.exists(broker.db_file):
             try:
                 broker.initialize(put_timestamp, storage_policy_index)
+                initialized = True
             except DatabaseAlreadyExists:
                 pass
-        return broker
+        return broker, initialized
 
     def get_db_state(self):
         """
         Returns the current state of on disk db files.
         """
-        if self._db_file == ':memory:':
-            return UNSHARDED
         if not self.db_files:
             return NOTFOUND
         if len(self.db_files) > 1:
@@ -394,9 +442,9 @@ class ContainerBroker(DatabaseBroker):
         if self.db_epoch is None:
             # never been sharded
             return UNSHARDED
-        if self.db_epoch != self._own_shard_range().epoch:
+        if self.db_epoch != self.get_own_shard_range().epoch:
             return UNSHARDED
-        if not self.get_shard_ranges():
+        if not self.has_other_shard_ranges():
             return COLLAPSED
         return SHARDED
 
@@ -406,11 +454,8 @@ class ContainerBroker(DatabaseBroker):
         for sharding to have been initiated, False otherwise.
         """
         own_shard_range = self.get_own_shard_range()
-        if own_shard_range.state in (ShardRange.SHARDING,
-                                     ShardRange.SHRINKING,
-                                     ShardRange.SHARDED,
-                                     ShardRange.SHRUNK):
-            return bool(self.get_shard_ranges())
+        if own_shard_range.state in ShardRange.CLEAVING_STATES:
+            return self.has_other_shard_ranges()
         return False
 
     def sharding_required(self):
@@ -430,8 +475,6 @@ class ContainerBroker(DatabaseBroker):
         """
         Reloads the cached list of valid on disk db files for this broker.
         """
-        if self._db_file == ':memory:':
-            return
         # reset connection so the next access will use the correct DB file
         self.conn = None
         self._db_files = get_db_files(self._init_db_file)
@@ -478,7 +521,7 @@ class ContainerBroker(DatabaseBroker):
     def storage_policy_index(self):
         if not hasattr(self, '_storage_policy_index'):
             self._storage_policy_index = \
-                self.get_info()['storage_policy_index']
+                self._get_info()['storage_policy_index']
         return self._storage_policy_index
 
     @property
@@ -568,7 +611,7 @@ class ContainerBroker(DatabaseBroker):
                 put_timestamp, status_changed_at, storage_policy_index)
             VALUES (?, ?, ?, ?, ?, ?, ?);
         """, (self.account, self.container, Timestamp.now().internal,
-              str(uuid4()), put_timestamp, put_timestamp,
+              self._new_db_id(), put_timestamp, put_timestamp,
               storage_policy_index))
 
     def create_policy_stat_table(self, conn, storage_policy_index=0):
@@ -609,7 +652,8 @@ class ContainerBroker(DatabaseBroker):
                 state INTEGER,
                 state_timestamp TEXT,
                 epoch TEXT,
-                reported INTEGER DEFAULT 0
+                reported INTEGER DEFAULT 0,
+                tombstones INTEGER DEFAULT -1
             );
         """ % SHARD_RANGE_TABLE)
 
@@ -640,20 +684,6 @@ class ContainerBroker(DatabaseBroker):
             SET reported_put_timestamp = 0, reported_delete_timestamp = 0,
                 reported_object_count = 0, reported_bytes_used = 0''')
 
-    def _delete_db(self, conn, timestamp):
-        """
-        Mark the DB as deleted
-
-        :param conn: DB connection object
-        :param timestamp: timestamp to mark as deleted
-        """
-        conn.execute("""
-            UPDATE container_stat
-            SET delete_timestamp = ?,
-                status = 'DELETED',
-                status_changed_at = ?
-            WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp))
-
     def _commit_puts_load(self, item_list, entry):
         """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`"""
         (name, timestamp, size, content_type, etag, deleted) = entry[:6]
@@ -825,16 +855,24 @@ class ContainerBroker(DatabaseBroker):
         info.update(self._get_alternate_object_stats()[1])
         return self._is_deleted_info(**info)
 
-    def is_reclaimable(self, now, reclaim_age):
+    def is_old_enough_to_reclaim(self, now, reclaim_age):
         with self.get() as conn:
             info = conn.execute('''
                 SELECT put_timestamp, delete_timestamp
                 FROM container_stat''').fetchone()
-        if (Timestamp(now - reclaim_age) >
-            Timestamp(info['delete_timestamp']) >
-                Timestamp(info['put_timestamp'])):
-            return self.empty()
-        return False
+        return (Timestamp(now - reclaim_age) >
+                Timestamp(info['delete_timestamp']) >
+                Timestamp(info['put_timestamp']))
+
+    def is_empty_enough_to_reclaim(self):
+        if self.is_root_container() and (self.has_other_shard_ranges() or
+                                         self.get_db_state() == SHARDING):
+            return False
+        return self.empty()
+
+    def is_reclaimable(self, now, reclaim_age):
+        return self.is_old_enough_to_reclaim(now, reclaim_age) and \
+            self.is_empty_enough_to_reclaim()
 
     def get_info_is_deleted(self):
         """
@@ -843,7 +881,7 @@ class ContainerBroker(DatabaseBroker):
         :returns: a tuple, in the form (info, is_deleted) info is a dict as
                   returned by get_info and is_deleted is a boolean.
         """
-        if self.db_file != ':memory:' and not os.path.exists(self.db_file):
+        if not os.path.exists(self.db_file):
             return {}, True
         info = self.get_info()
         return info, self._is_deleted_info(**info)
@@ -862,7 +900,7 @@ class ContainerBroker(DatabaseBroker):
             try:
                 data = conn.execute(('''
                     SELECT account, container, created_at, put_timestamp,
-                        delete_timestamp, status_changed_at,
+                        delete_timestamp, status, status_changed_at,
                         object_count, bytes_used,
                         reported_put_timestamp, reported_delete_timestamp,
                         reported_object_count, reported_bytes_used, hash,
@@ -901,23 +939,23 @@ class ContainerBroker(DatabaseBroker):
                 self._do_get_info_query(conn)
 
     def _get_alternate_object_stats(self):
-        state = self.get_db_state()
-        if state == SHARDING:
+        db_state = self.get_db_state()
+        if db_state == SHARDING:
             other_info = self.get_brokers()[0]._get_info()
             stats = {'object_count': other_info['object_count'],
                      'bytes_used': other_info['bytes_used']}
-        elif state == SHARDED and self.is_root_container():
+        elif db_state == SHARDED and self.is_root_container():
             stats = self.get_shard_usage()
         else:
             stats = {}
-        return state, stats
+        return db_state, stats
 
     def get_info(self):
         """
         Get global data for the container.
 
         :returns: dict with keys: account, container, created_at,
-                  put_timestamp, delete_timestamp, status_changed_at,
+                  put_timestamp, delete_timestamp, status, status_changed_at,
                   object_count, bytes_used, reported_put_timestamp,
                   reported_delete_timestamp, reported_object_count,
                   reported_bytes_used, hash, id, x_container_sync_point1,
@@ -1056,7 +1094,8 @@ class ContainerBroker(DatabaseBroker):
     def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
                           path=None, storage_policy_index=0, reverse=False,
                           include_deleted=False, since_row=None,
-                          transform_func=None, all_policies=False):
+                          transform_func=None, all_policies=False,
+                          allow_reserved=False):
         """
         Get a list of objects sorted by name starting at marker onward, up
         to limit entries.  Entries will begin with the prefix and will not
@@ -1082,6 +1121,8 @@ class ContainerBroker(DatabaseBroker):
             :meth:`~_transform_record`; defaults to :meth:`~_transform_record`.
         :param all_policies: if True, include objects for all storage policies
             ignoring any value given for ``storage_policy_index``
+        :param allow_reserved: exclude names with reserved-byte by default
+
         :returns: list of tuples of (name, created_at, size, content_type,
                   etag, deleted)
         """
@@ -1095,9 +1136,6 @@ class ContainerBroker(DatabaseBroker):
         if transform_func is None:
             transform_func = self._transform_record
         delim_force_gte = False
-        if six.PY2:
-            (marker, end_marker, prefix, delimiter, path) = utf8encode(
-                marker, end_marker, prefix, delimiter, path)
         self._commit_puts_stale_ok()
         if reverse:
             # Reverse the markers if we are reversing the listing.
@@ -1138,6 +1176,9 @@ class ContainerBroker(DatabaseBroker):
                 elif prefix:
                     query_conditions.append('name >= ?')
                     query_args.append(prefix)
+                if not allow_reserved:
+                    query_conditions.append('name >= ?')
+                    query_args.append(chr(ord(RESERVED_BYTE) + 1))
                 query_conditions.append(deleted_key + deleted_arg)
                 if since_row:
                     query_conditions.append('ROWID > ?')
@@ -1256,7 +1297,7 @@ class ContainerBroker(DatabaseBroker):
             limit, marker, end_marker, prefix=None, delimiter=None, path=None,
             reverse=False, include_deleted=include_deleted,
             transform_func=self._record_to_dict, since_row=since_row,
-            all_policies=True
+            all_policies=True, allow_reserved=True
         )
 
     def _transform_record(self, record):
@@ -1289,9 +1330,7 @@ class ContainerBroker(DatabaseBroker):
         :param source: if defined, update incoming_sync with the source
         """
         for item in item_list:
-            if six.PY2 and isinstance(item['name'], six.text_type):
-                item['name'] = item['name'].encode('utf-8')
-            elif not six.PY2 and isinstance(item['name'], six.binary_type):
+            if isinstance(item['name'], bytes):
                 item['name'] = item['name'].decode('utf-8')
 
         def _really_really_merge_items(conn):
@@ -1379,7 +1418,7 @@ class ContainerBroker(DatabaseBroker):
         """
         if not shard_ranges:
             return
-        if not isinstance(shard_ranges, list):
+        if not isinstance(shard_ranges, (list, ShardRangeList)):
             shard_ranges = [shard_ranges]
 
         item_list = []
@@ -1387,9 +1426,7 @@ class ContainerBroker(DatabaseBroker):
             if isinstance(item, ShardRange):
                 item = dict(item)
             for col in ('name', 'lower', 'upper'):
-                if six.PY2 and isinstance(item[col], six.text_type):
-                    item[col] = item[col].encode('utf-8')
-                elif not six.PY2 and isinstance(item[col], six.binary_type):
+                if isinstance(item[col], bytes):
                     item[col] = item[col].decode('utf-8')
             item_list.append(item)
 
@@ -1404,28 +1441,14 @@ class ContainerBroker(DatabaseBroker):
                 chunk = [record['name'] for record
                          in item_list[offset:offset + SQLITE_ARG_LIMIT]]
                 records.update(
-                    (rec[0], rec) for rec in curs.execute(
+                    (rec[0], dict(zip(SHARD_RANGE_KEYS, rec)))
+                    for rec in curs.execute(
                         'SELECT %s FROM %s '
                         'WHERE deleted IN (0, 1) AND name IN (%s)' %
                         (', '.join(SHARD_RANGE_KEYS), SHARD_RANGE_TABLE,
                          ','.join('?' * len(chunk))), chunk))
 
-            # Sort item_list into things that need adding and deleting
-            to_delete = set()
-            to_add = {}
-            for item in item_list:
-                item_ident = item['name']
-                existing = records.get(item_ident)
-                if existing:
-                    existing = dict(zip(SHARD_RANGE_KEYS, existing))
-                if merge_shards(item, existing):
-                    # exists with older timestamp
-                    if item_ident in records:
-                        to_delete.add(item_ident)
-                    # duplicate entries in item_list
-                    if (item_ident not in to_add or
-                            merge_shards(item, to_add[item_ident])):
-                        to_add[item_ident] = item
+            to_add, to_delete = sift_shard_ranges(item_list, records)
 
             if to_delete:
                 curs.executemany(
@@ -1438,25 +1461,37 @@ class ContainerBroker(DatabaseBroker):
                     'INSERT INTO %s (%s) VALUES (%s)' %
                     (SHARD_RANGE_TABLE, ','.join(SHARD_RANGE_KEYS), vals),
                     tuple([item[k] for k in SHARD_RANGE_KEYS]
-                          for item in to_add.values()))
+                          for item in to_add))
             conn.commit()
 
+        migrations = {
+            'no such column: reported':
+                self._migrate_add_shard_range_reported,
+            'no such column: tombstones':
+                self._migrate_add_shard_range_tombstones,
+            ('no such table: %s' % SHARD_RANGE_TABLE):
+                self.create_shard_range_table,
+        }
+        migrations_done = set()
         with self.get() as conn:
-            try:
-                return _really_merge_items(conn)
-            except sqlite3.OperationalError as err:
-                # Without the rollback, new enough (>= py37) python/sqlite3
-                # will panic:
-                #   sqlite3.OperationalError: cannot start a transaction
-                #   within a transaction
-                conn.rollback()
-                if 'no such column: reported' in str(err):
-                    self._migrate_add_shard_range_reported(conn)
-                    return _really_merge_items(conn)
-                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
-                    self.create_shard_range_table(conn)
+            while True:
+                try:
                     return _really_merge_items(conn)
-                raise
+                except sqlite3.OperationalError as err:
+                    # Without the rollback, new enough (>= py37) python/sqlite3
+                    # will panic:
+                    #   sqlite3.OperationalError: cannot start a transaction
+                    #   within a transaction
+                    conn.rollback()
+                    for err_str, migration in migrations.items():
+                        if err_str in migrations_done:
+                            continue
+                        if err_str in str(err):
+                            migration(conn)
+                            migrations_done.add(err_str)
+                            break
+                    else:
+                        raise
 
     def get_reconciler_sync(self):
         with self.get() as conn:
@@ -1615,6 +1650,17 @@ class ContainerBroker(DatabaseBroker):
             COMMIT;
         ''' % SHARD_RANGE_TABLE)
 
+    def _migrate_add_shard_range_tombstones(self, conn):
+        """
+        Add the tombstones column to the 'shard_range' table.
+        """
+        conn.executescript('''
+            BEGIN;
+            ALTER TABLE %s
+            ADD COLUMN tombstones INTEGER DEFAULT -1;
+            COMMIT;
+        ''' % SHARD_RANGE_TABLE)
+
     def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):
         super(ContainerBroker, self)._reclaim_other_stuff(
             conn, age_timestamp, sync_timestamp)
@@ -1630,9 +1676,130 @@ class ContainerBroker(DatabaseBroker):
             if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err):
                 raise
 
-    def _get_shard_range_rows(self, connection=None, include_deleted=False,
-                              states=None, include_own=False,
-                              exclude_others=False):
+    def _make_filler_shard_range(self, namespaces, marker, end_marker):
+        if namespaces and namespaces[-1].upper == Namespace.MAX:
+            return None
+
+        # Insert a modified copy of own shard range to fill any gap between the
+        # end of any found and the upper bound of own shard range. Gaps
+        # enclosed within the found shard ranges are not filled.
+        own_shard_range = self.get_own_shard_range()
+        if namespaces:
+            last_upper = namespaces[-1].upper
+        else:
+            last_upper = max(marker or own_shard_range.lower,
+                             own_shard_range.lower)
+        required_upper = min(end_marker or own_shard_range.upper,
+                             own_shard_range.upper)
+        if required_upper > last_upper:
+            filler_sr = own_shard_range
+            filler_sr.lower = last_upper
+            filler_sr.upper = required_upper
+            return filler_sr
+        else:
+            return None
+
+    def get_namespaces(self, marker=None, end_marker=None, includes=None,
+                       reverse=False, states=None, fill_gaps=False):
+        """
+        Returns a list of persisted namespaces per input parameters.
+
+        :param marker: restricts the returned list to shard ranges whose
+            namespace includes or is greater than the marker value. If
+            ``reverse=True`` then ``marker`` is treated as ``end_marker``.
+            ``marker`` is ignored if ``includes`` is specified.
+        :param end_marker: restricts the returned list to shard ranges whose
+            namespace includes or is less than the end_marker value. If
+            ``reverse=True`` then ``end_marker`` is treated as ``marker``.
+            ``end_marker`` is ignored if ``includes`` is specified.
+        :param includes: restricts the returned list to the shard range that
+            includes the given value; if ``includes`` is specified then
+            ``fill_gaps``, ``marker`` and ``end_marker`` are ignored.
+        :param reverse: reverse the result order.
+        :param states: if specified, restricts the returned list to namespaces
+            that have one of the given states; should be a list of ints.
+        :param fill_gaps: if True, insert a modified copy of own shard range to
+            fill any gap between the end of any found shard ranges and the
+            upper bound of own shard range. Gaps enclosed within the found
+            shard ranges are not filled.
+        :return: a list of Namespace objects.
+        """
+        if includes is None and (marker == Namespace.MAX
+                                 or end_marker == Namespace.MIN):
+            return []
+
+        if reverse:
+            marker, end_marker = end_marker, marker
+        if marker and end_marker and marker >= end_marker:
+            return []
+
+        included_states = set(states) if states else None
+        with self.get() as conn:
+            # Namespace only needs 'name', 'lower' and 'upper', but the query
+            # also need to include 'state' to be used when subesequently
+            # sorting the rows. And the sorting can't be done within SQLite
+            # since the value for maximum upper bound is an empty string.
+
+            conditions = ['deleted = 0', 'name != ?']
+            params = [self.path]
+            if included_states:
+                conditions.append('state in (%s)' % ','.join(
+                    '?' * len(included_states)))
+                params.extend(included_states)
+            if includes is None:
+                if end_marker:
+                    conditions.append('lower < ?')
+                    params.append(end_marker)
+                if marker:
+                    conditions.append("(upper = '' OR upper > ?)")
+                    params.append(marker)
+            else:
+                conditions.extend(('lower < ?', "(upper = '' OR upper >= ?)"))
+                params.extend((includes, includes))
+            condition = ' WHERE ' + ' AND '.join(conditions)
+            sql = '''
+            SELECT name, lower, upper, state FROM %s%s
+            ''' % (SHARD_RANGE_TABLE, condition)
+            try:
+                data = conn.execute(sql, params)
+                data.row_factory = None
+                namespaces = [row for row in data]
+            except sqlite3.OperationalError as err:
+                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
+                    return []
+                else:
+                    raise
+
+        # Sort those namespaces in order, note that each namespace record also
+        # include additional attribute 'state'.
+        def sort_key(namespace):
+            return ShardRange.sort_key_order(name=namespace[0],
+                                             lower=namespace[1],
+                                             upper=namespace[2],
+                                             state=namespace[3])
+        namespaces.sort(key=sort_key)
+        # Convert the record tuples to Namespace objects.
+        namespaces = [Namespace(row[0], row[1], row[2]) for row in namespaces]
+        if includes:
+            return namespaces[:1] if namespaces else []
+
+        if fill_gaps:
+            filler_sr = self._make_filler_shard_range(
+                namespaces, marker, end_marker)
+            if filler_sr:
+                namespaces.append(Namespace(filler_sr.name,
+                                            filler_sr.lower,
+                                            filler_sr.upper))
+        if reverse:
+            namespaces.reverse()
+
+        return namespaces
+
+    def _get_shard_range_rows(self, connection=None, marker=None,
+                              end_marker=None, includes=None,
+                              include_deleted=False, states=None,
+                              include_own=False, exclude_others=False,
+                              limit=None):
         """
         Returns a list of shard range rows.
 
@@ -1641,30 +1808,49 @@ class ContainerBroker(DatabaseBroker):
         ``exclude_others=True``.
 
         :param connection: db connection
-        :param include_deleted: include rows marked as deleted
-        :param states: include only rows matching the given state(s); can be an
-            int or a list of ints.
+        :param marker: restricts the returned list to rows whose namespace
+            includes or is greater than the marker value. ``marker`` is ignored
+             if ``includes`` is specified.
+        :param end_marker: restricts the returned list to rows whose namespace
+            includes or is less than the end_marker value. ``end_marker`` is
+            ignored if ``includes`` is specified.
+        :param includes: restricts the returned list to the shard range that
+            includes the given value; if ``includes`` is specified then
+            ``marker`` and ``end_marker`` are ignored, but other constraints
+            are applied (e.g. ``exclude_others`` and ``include_deleted``).
+        :param include_deleted: include rows marked as deleted.
+        :param states: include only rows matching the given states; should be
+            a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
-            True, that row is included, otherwise it is not included. Default
-            is False.
+            True, that row is included unless it is excluded by other
+            constraints (e.g. ``marker``, ``end_marker``, ``includes``). If
+            False, that row is not included. Default is False.
         :param exclude_others: boolean that governs whether the rows whose
             names do not match the broker's path are included in the returned
             list. If True, those rows are not included, otherwise they are
             included. Default is False.
+        :param limit: restricts the returned list to the given number of rows.
+            Should be a whole number; negative values will be ignored.
+            The ``limit`` parameter is useful to optimise a search
+            when the maximum number of expected matching rows is known, and
+            particularly when that maximum number is much less than the total
+            number of rows in the DB. However, the DB search is not ordered and
+            the subset of rows returned when ``limit`` is less than all
+            possible matching rows is therefore unpredictable.
         :return: a list of tuples.
         """
 
         if exclude_others and not include_own:
             return []
 
-        included_states = set()
-        if isinstance(states, (list, tuple, set)):
-            included_states.update(states)
-        elif states is not None:
-            included_states.add(states)
+        included_states = set(states) if states else None
+
+        # defaults to be used when legacy db's are missing columns
+        default_values = {'reported': 0,
+                          'tombstones': -1}
 
-        def do_query(conn, use_reported_column=True):
+        def do_query(conn, defaults=None):
             condition = ''
             conditions = []
             params = []
@@ -1680,12 +1866,27 @@ class ContainerBroker(DatabaseBroker):
             if exclude_others:
                 conditions.append('name = ?')
                 params.append(self.path)
+            if includes is None:
+                if end_marker:
+                    conditions.append('lower < ?')
+                    params.append(end_marker)
+                if marker:
+                    conditions.append("(upper = '' OR upper > ?)")
+                    params.append(marker)
+            else:
+                conditions.extend(('lower < ?', "(upper = '' OR upper >= ?)"))
+                params.extend((includes, includes))
             if conditions:
                 condition = ' WHERE ' + ' AND '.join(conditions)
-            if use_reported_column:
-                columns = SHARD_RANGE_KEYS
-            else:
-                columns = SHARD_RANGE_KEYS[:-1] + ('0 as reported', )
+            if limit is not None and limit >= 0:
+                condition += ' LIMIT %d' % limit
+            columns = SHARD_RANGE_KEYS[:-2]
+            for column in SHARD_RANGE_KEYS[-2:]:
+                if column in defaults:
+                    columns += (('%s as %s' %
+                                 (default_values[column], column)),)
+                else:
+                    columns += (column,)
             sql = '''
             SELECT %s
             FROM %s%s;
@@ -1695,14 +1896,26 @@ class ContainerBroker(DatabaseBroker):
             return [row for row in data]
 
         with self.maybe_get(connection) as conn:
-            try:
-                return do_query(conn)
-            except sqlite3.OperationalError as err:
-                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
-                    return []
-                if 'no such column: reported' in str(err):
-                    return do_query(conn, use_reported_column=False)
-                raise
+            defaults = set()
+            attempts = len(default_values) + 1
+            while attempts:
+                attempts -= 1
+                try:
+                    return do_query(conn, defaults)
+                except sqlite3.OperationalError as err:
+                    if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
+                        return []
+                    if not attempts:
+                        raise
+                    new_defaults = set()
+                    for column in default_values.keys():
+                        if 'no such column: %s' % column in str(err):
+                            new_defaults.add(column)
+                    if not new_defaults:
+                        raise
+                    if new_defaults.intersection(defaults):
+                        raise
+                    defaults.update(new_defaults)
 
     @classmethod
     def resolve_shard_range_states(cls, states):
@@ -1713,7 +1926,10 @@ class ContainerBroker(DatabaseBroker):
 
         The following alias values are supported: 'listing' maps to all states
         that are considered valid when listing objects; 'updating' maps to all
-        states that are considered valid for redirecting an object update.
+        states that are considered valid for redirecting an object update;
+        'auditing' maps to all states that are considered valid for a shard
+        container that is updating its own shard range table from a root (this
+        currently maps to all states except FOUND).
 
         :param states: a list of values each of which may be the name of a
             state, the number of a state, or an alias
@@ -1728,6 +1944,8 @@ class ContainerBroker(DatabaseBroker):
                     resolved_states.update(SHARD_LISTING_STATES)
                 elif state == 'updating':
                     resolved_states.update(SHARD_UPDATE_STATES)
+                elif state == 'auditing':
+                    resolved_states.update(SHARD_AUDITING_STATES)
                 else:
                     resolved_states.add(ShardRange.resolve_state(state)[0])
             return resolved_states
@@ -1735,42 +1953,47 @@ class ContainerBroker(DatabaseBroker):
 
     def get_shard_ranges(self, marker=None, end_marker=None, includes=None,
                          reverse=False, include_deleted=False, states=None,
-                         include_own=False,
-                         exclude_others=False, fill_gaps=False):
+                         include_own=False, exclude_others=False,
+                         fill_gaps=False):
         """
         Returns a list of persisted shard ranges.
 
         :param marker: restricts the returned list to shard ranges whose
-            namespace includes or is greater than the marker value.
+            namespace includes or is greater than the marker value. If
+            ``reverse=True`` then ``marker`` is treated as ``end_marker``.
+            ``marker`` is ignored if ``includes`` is specified.
         :param end_marker: restricts the returned list to shard ranges whose
-            namespace includes or is less than the end_marker value.
+            namespace includes or is less than the end_marker value. If
+            ``reverse=True`` then ``end_marker`` is treated as ``marker``.
+            ``end_marker`` is ignored if ``includes`` is specified.
         :param includes: restricts the returned list to the shard range that
             includes the given value; if ``includes`` is specified then
-            ``marker`` and ``end_marker`` are ignored.
+            ``fill_gaps``, ``marker`` and ``end_marker`` are ignored, but other
+            constraints are applied (e.g. ``exclude_others`` and
+            ``include_deleted``).
         :param reverse: reverse the result order.
-        :param include_deleted: include items that have the delete marker set
+        :param include_deleted: include items that have the delete marker set.
         :param states: if specified, restricts the returned list to shard
-            ranges that have the given state(s); can be a list of ints or a
-            single int.
+            ranges that have one of the given states; should be a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
-            True, that row is included, otherwise it is not included. Default
-            is False.
+            True, that row is included unless it is excluded by other
+            constraints (e.g. ``marker``, ``end_marker``, ``includes``). If
+            False, that row is not included. Default is False.
         :param exclude_others: boolean that governs whether the rows whose
             names do not match the broker's path are included in the returned
             list. If True, those rows are not included, otherwise they are
             included. Default is False.
-        :param fill_gaps: if True, insert own shard range to fill any gaps in
-            at the tail of other shard ranges.
-        :return: a list of instances of :class:`swift.common.utils.ShardRange`
-        """
-        def shard_range_filter(sr):
-            end = start = True
-            if end_marker:
-                end = end_marker > sr.lower
-            if marker:
-                start = marker < sr.upper
-            return start and end
+        :param fill_gaps: if True, insert a modified copy of own shard range to
+            fill any gap between the end of any found shard ranges and the
+            upper bound of own shard range. Gaps enclosed within the found
+            shard ranges are not filled. ``fill_gaps`` is ignored if
+            ``includes`` is specified.
+        :return: a list of instances of :class:`swift.common.utils.ShardRange`.
+        """
+        if includes is None and (marker == Namespace.MAX
+                                 or end_marker == Namespace.MIN):
+            return []
 
         if reverse:
             marker, end_marker = end_marker, marker
@@ -1780,26 +2003,17 @@ class ContainerBroker(DatabaseBroker):
         shard_ranges = [
             ShardRange(*row)
             for row in self._get_shard_range_rows(
+                marker=marker, end_marker=end_marker, includes=includes,
                 include_deleted=include_deleted, states=states,
-                include_own=include_own,
-                exclude_others=exclude_others)]
+                include_own=include_own, exclude_others=exclude_others)]
         shard_ranges.sort(key=ShardRange.sort_key)
         if includes:
-            shard_range = find_shard_range(includes, shard_ranges)
-            return [shard_range] if shard_range else []
+            return shard_ranges[:1] if shard_ranges else []
 
-        if marker or end_marker:
-            shard_ranges = list(filter(shard_range_filter, shard_ranges))
         if fill_gaps:
-            if shard_ranges:
-                last_upper = shard_ranges[-1].upper
-            else:
-                last_upper = marker or ShardRange.MIN
-            required_upper = end_marker or ShardRange.MAX
-            if required_upper > last_upper:
-                filler_sr = self.get_own_shard_range()
-                filler_sr.lower = last_upper
-                filler_sr.upper = required_upper
+            filler_sr = self._make_filler_shard_range(
+                shard_ranges, marker, end_marker)
+            if filler_sr:
                 shard_ranges.append(filler_sr)
 
         if reverse:
@@ -1807,40 +2021,33 @@ class ContainerBroker(DatabaseBroker):
 
         return shard_ranges
 
-    def _own_shard_range(self, no_default=False):
-        shard_ranges = self.get_shard_ranges(include_own=True,
-                                             include_deleted=True,
-                                             exclude_others=True)
-        if shard_ranges:
-            own_shard_range = shard_ranges[0]
-        elif no_default:
-            return None
-        else:
-            own_shard_range = ShardRange(
-                self.path, Timestamp.now(), ShardRange.MIN, ShardRange.MAX,
-                state=ShardRange.ACTIVE)
-        return own_shard_range
-
     def get_own_shard_range(self, no_default=False):
         """
         Returns a shard range representing this broker's own shard range. If no
         such range has been persisted in the broker's shard ranges table then a
         default shard range representing the entire namespace will be returned.
 
-        The returned shard range will be updated with the current object stats
-        for this broker and a meta timestamp set to the current time. For these
-        values to be persisted the caller must merge the shard range.
+        The ``object_count`` and ``bytes_used`` of the returned shard range are
+        not guaranteed to be up-to-date with the current object stats for this
+        broker. Callers that require up-to-date stats should use the
+        ``get_info`` method.
 
         :param no_default: if True and the broker's own shard range is not
             found in the shard ranges table then None is returned, otherwise a
             default shard range is returned.
         :return: an instance of :class:`~swift.common.utils.ShardRange`
         """
-        own_shard_range = self._own_shard_range(no_default=no_default)
-        if own_shard_range:
-            info = self.get_info()
-            own_shard_range.update_meta(
-                info['object_count'], info['bytes_used'])
+        rows = self._get_shard_range_rows(
+            include_own=True, include_deleted=True, exclude_others=True,
+            limit=1)
+        if rows:
+            own_shard_range = ShardRange(*rows[0])
+        elif no_default:
+            own_shard_range = None
+        else:
+            own_shard_range = ShardRange(
+                self.path, Timestamp.now(), ShardRange.MIN, ShardRange.MAX,
+                state=ShardRange.ACTIVE)
         return own_shard_range
 
     def is_own_shard_range(self, shard_range):
@@ -1854,7 +2061,7 @@ class ContainerBroker(DatabaseBroker):
         :param epoch: a :class:`~swift.utils.common.Timestamp`
         :return: the broker's updated own shard range.
         """
-        own_shard_range = self._own_shard_range()
+        own_shard_range = self.get_own_shard_range()
         own_shard_range.update_state(ShardRange.SHARDING, epoch)
         own_shard_range.epoch = epoch
         self.merge_shard_ranges(own_shard_range)
@@ -1867,9 +2074,41 @@ class ContainerBroker(DatabaseBroker):
 
         :return: a dict with keys {bytes_used, object_count}
         """
-        shard_ranges = self.get_shard_ranges(states=SHARD_STATS_STATES)
-        return {'bytes_used': sum(sr.bytes_used for sr in shard_ranges),
-                'object_count': sum(sr.object_count for sr in shard_ranges)}
+        with self.get() as conn:
+            sql = '''
+            SELECT COALESCE(SUM(bytes_used), 0),
+                   COALESCE(SUM(object_count), 0)
+            FROM %s
+            WHERE state in (%s)
+              AND deleted = 0
+              AND name != ?
+            ''' % (SHARD_RANGE_TABLE, ','.join('?' * len(SHARD_STATS_STATES)))
+            cur = conn.execute(sql, SHARD_STATS_STATES + [self.path])
+            bytes_used, object_count = cur.fetchone()
+        return {'bytes_used': bytes_used,
+                'object_count': object_count}
+
+    def has_other_shard_ranges(self):
+        """
+        This function tells if there is any shard range other than the
+        broker's own shard range, that is not marked as deleted.
+
+        :return: A boolean value as described above.
+        """
+        with self.get() as conn:
+            sql = '''
+            SELECT 1 FROM %s
+            WHERE deleted = 0 AND name != ? LIMIT 1
+            ''' % (SHARD_RANGE_TABLE)
+            try:
+                data = conn.execute(sql, [self.path])
+                data.row_factory = None
+                return True if [row for row in data] else False
+            except sqlite3.OperationalError as err:
+                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
+                    return False
+                else:
+                    raise
 
     def get_all_shard_range_data(self):
         """
@@ -1896,10 +2135,10 @@ class ContainerBroker(DatabaseBroker):
             self.logger.warning("Container '%s' cannot be set to sharding "
                                 "state: missing epoch", self.path)
             return False
-        state = self.get_db_state()
-        if not state == UNSHARDED:
+        db_state = self.get_db_state()
+        if not db_state == UNSHARDED:
             self.logger.warning("Container '%s' cannot be set to sharding "
-                                "state while in %s state", self.path, state)
+                                "state while in %s state", self.path, db_state)
             return False
 
         info = self.get_info()
@@ -1943,17 +2182,25 @@ class ContainerBroker(DatabaseBroker):
                     self.path, err)
                 return False
 
-            # Set the created_at and hash in the container_info table the same
-            # in both brokers
+            # sync the retiring container stat into the fresh db. At least the
+            # things that either aren't covered through the normal
+            # broker api, and things that wont just be regenerated.
             try:
-                fresh_broker_conn.execute(
-                    'UPDATE container_stat SET created_at=?',
-                    (info['created_at'],))
+                sql = 'UPDATE container_stat SET created_at=?, '
+                sql += 'delete_timestamp=?, status=?, status_changed_at=?'
+                sql_data = (info['created_at'], info['delete_timestamp'],
+                            info['status'], info['status_changed_at'])
+                # 'reported_*' items are not sync'd because this is consistent
+                # with when a new DB is created after rsync'ing to another
+                # node (see _newid()). 'hash' should not be sync'd because
+                # this DB has no object rows.
+                fresh_broker_conn.execute(sql, sql_data)
                 fresh_broker_conn.commit()
             except sqlite3.OperationalError as err:
-                self.logger.error('Failed to set matching created_at time in '
-                                  'the fresh database for %s: %s',
-                                  self.path, err)
+                self.logger.error(
+                    'Failed to sync the container_stat table/view with the '
+                    'fresh database for %s: %s',
+                    self.path, err)
                 return False
 
         # Rename to the new database
@@ -1969,11 +2216,11 @@ class ContainerBroker(DatabaseBroker):
         :return: True if the retiring DB was successfully unlinked, False
             otherwise.
         """
-        state = self.get_db_state()
-        if not state == SHARDING:
+        db_state = self.get_db_state()
+        if not db_state == SHARDING:
             self.logger.warning("Container %r cannot be set to sharded "
                                 "state while in %s state",
-                                self.path, state)
+                                self.path, db_state)
             return False
 
         self.reload_db_files()
@@ -2027,7 +2274,7 @@ class ContainerBroker(DatabaseBroker):
 
     def set_sharding_sysmeta(self, key, value):
         """
-        Updates the broker's metadata metadata stored under the given key
+        Updates the broker's metadata stored under the given key
         prefixed with a sharding specific namespace.
 
         :param key: metadata key in the sharding metadata namespace.
@@ -2135,13 +2382,14 @@ class ContainerBroker(DatabaseBroker):
         """
         _, path = self._get_root_meta()
         if path is not None:
-            # We have metadata telling us where the root is; it's authoritative
+            # We have metadata telling us where the root is; it's
+            # authoritative; shards should always have this metadata even when
+            # deleted
             return self.path == path
 
-        # Else, we're either a root or a deleted shard.
-
-        # Use internal method so we don't try to update stats.
-        own_shard_range = self._own_shard_range(no_default=True)
+        # Else, we're either a root or a legacy deleted shard whose sharding
+        # sysmeta was deleted
+        own_shard_range = self.get_own_shard_range(no_default=True)
         if not own_shard_range:
             return True  # Never been sharded
 
@@ -2176,7 +2424,8 @@ class ContainerBroker(DatabaseBroker):
             row = connection.execute(sql, args).fetchone()
             return row['name'] if row else None
 
-    def find_shard_ranges(self, shard_size, limit=-1, existing_ranges=None):
+    def find_shard_ranges(self, shard_size, limit=-1, existing_ranges=None,
+                          minimum_shard_size=1):
         """
         Scans the container db for shard ranges. Scanning will start at the
         upper bound of the any ``existing_ranges`` that are given, otherwise
@@ -2195,6 +2444,10 @@ class ContainerBroker(DatabaseBroker):
             given, this list should be sorted in order of upper bounds; the
             scan for new shard ranges will start at the upper bound of the last
             existing ShardRange.
+        :param minimum_shard_size: Minimum size of the final shard range. If
+            this is greater than one then the final shard range may be extended
+            to more than shard_size in order to avoid a further shard range
+            with less minimum_shard_size rows.
         :return:  a tuple; the first value in the tuple is a list of
             dicts each having keys {'index', 'lower', 'upper', 'object_count'}
             in order of ascending 'upper'; the second value in the tuple is a
@@ -2202,8 +2455,9 @@ class ContainerBroker(DatabaseBroker):
             otherwise.
         """
         existing_ranges = existing_ranges or []
+        minimum_shard_size = max(minimum_shard_size, 1)
         object_count = self.get_info().get('object_count', 0)
-        if shard_size >= object_count:
+        if shard_size + minimum_shard_size > object_count:
             # container not big enough to shard
             return [], False
 
@@ -2234,9 +2488,10 @@ class ContainerBroker(DatabaseBroker):
         sub_broker = self.get_brokers()[0]
         index = len(existing_ranges)
         while limit is None or limit < 0 or len(found_ranges) < limit:
-            if progress + shard_size >= object_count:
-                # next shard point is at or beyond final object name so don't
-                # bother with db query
+            if progress + shard_size + minimum_shard_size > object_count:
+                # next shard point is within minimum_size rows of the final
+                # object name, or beyond it, so don't bother with db query.
+                # This shard will have <= shard_size + (minimum_size - 1) rows.
                 next_shard_upper = None
             else:
                 try:
@@ -2260,10 +2515,14 @@ class ContainerBroker(DatabaseBroker):
                     # object_count
                     shard_size = object_count - progress
 
-            # NB shard ranges are created with a non-zero object count so that
-            # the apparent container object count remains constant, and the
-            # container is non-deletable while shards have been found but not
-            # yet cleaved
+            # NB shard ranges are created with a non-zero object count for a
+            # few reasons:
+            #  1. so that the apparent container object count remains
+            #     consistent;
+            #  2. the container is non-deletable while shards have been found
+            #     but not yet cleaved; and
+            #  3. So we have a rough idea of size of the shards should be
+            #     while cleaving.
             found_ranges.append(
                 {'index': index,
                  'lower': str(last_shard_upper),