swift 2.31.1__py2.py3-none-any.whl → 2.32.1__py2.py3-none-any.whl

swift/container/backend.py

@@ -32,7 +32,7 @@ from swift.common.utils import Timestamp, encode_timestamps, \
     decode_timestamps, extract_swift_bytes, storage_directory, hash_path, \
     ShardRange, renamer, MD5_OF_EMPTY_STRING, mkdirs, get_db_files, \
     parse_db_filename, make_db_file_path, split_path, RESERVED_BYTE, \
-    filter_shard_ranges, ShardRangeList
+    ShardRangeList, Namespace
 from swift.common.db import DatabaseBroker, utf8encode, BROKER_TIMEOUT, \
     zero_like, DatabaseAlreadyExists, SQLITE_ARG_LIMIT
 
@@ -446,7 +446,7 @@ class ContainerBroker(DatabaseBroker):
             return UNSHARDED
         if self.db_epoch != self.get_own_shard_range().epoch:
             return UNSHARDED
-        if not self.get_shard_ranges():
+        if not self.has_other_shard_ranges():
             return COLLAPSED
         return SHARDED
 
@@ -457,7 +457,7 @@ class ContainerBroker(DatabaseBroker):
         """
         own_shard_range = self.get_own_shard_range()
         if own_shard_range.state in ShardRange.CLEAVING_STATES:
-            return bool(self.get_shard_ranges())
+            return self.has_other_shard_ranges()
         return False
 
     def sharding_required(self):
@@ -523,7 +523,7 @@ class ContainerBroker(DatabaseBroker):
     def storage_policy_index(self):
         if not hasattr(self, '_storage_policy_index'):
             self._storage_policy_index = \
-                self.get_info()['storage_policy_index']
+                self._get_info()['storage_policy_index']
         return self._storage_policy_index
 
     @property
@@ -867,7 +867,7 @@ class ContainerBroker(DatabaseBroker):
                 Timestamp(info['put_timestamp']))
 
     def is_empty_enough_to_reclaim(self):
-        if self.is_root_container() and (self.get_shard_ranges() or
+        if self.is_root_container() and (self.has_other_shard_ranges() or
                                          self.get_db_state() == SHARDING):
             return False
         return self.empty()
@@ -1685,9 +1685,11 @@ class ContainerBroker(DatabaseBroker):
             if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err):
                 raise
 
-    def _get_shard_range_rows(self, connection=None, includes=None,
+    def _get_shard_range_rows(self, connection=None, marker=None,
+                              end_marker=None, includes=None,
                               include_deleted=False, states=None,
-                              include_own=False, exclude_others=False):
+                              include_own=False, exclude_others=False,
+                              limit=None):
         """
         Returns a list of shard range rows.
 
@@ -1696,19 +1698,36 @@ class ContainerBroker(DatabaseBroker):
         ``exclude_others=True``.
 
         :param connection: db connection
+        :param marker: restricts the returned list to rows whose namespace
+            includes or is greater than the marker value. ``marker`` is ignored
+             if ``includes`` is specified.
+        :param end_marker: restricts the returned list to rows whose namespace
+            includes or is less than the end_marker value. ``end_marker`` is
+            ignored if ``includes`` is specified.
         :param includes: restricts the returned list to the shard range that
-            includes the given value
-        :param include_deleted: include rows marked as deleted
+            includes the given value; if ``includes`` is specified then
+            ``marker`` and ``end_marker`` are ignored, but other constraints
+            are applied (e.g. ``exclude_others`` and ``include_deleted``).
+        :param include_deleted: include rows marked as deleted.
         :param states: include only rows matching the given state(s); can be an
             int or a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
-            True, that row is included, otherwise it is not included. Default
-            is False.
+            True, that row is included unless it is excluded by other
+            constraints (e.g. ``marker``, ``end_marker``, ``includes``). If
+            False, that row is not included. Default is False.
         :param exclude_others: boolean that governs whether the rows whose
             names do not match the broker's path are included in the returned
             list. If True, those rows are not included, otherwise they are
             included. Default is False.
+        :param limit: restricts the returned list to the given number of rows.
+            Should be a whole number; negative values will be ignored.
+            The ``limit`` parameter is useful to optimise a search
+            when the maximum number of expected matching rows is known, and
+            particularly when that maximum number is much less than the total
+            number of rows in the DB. However, the DB search is not ordered and
+            the subset of rows returned when ``limit`` is less than all
+            possible matching rows is therefore unpredictable.
         :return: a list of tuples.
         """
 
@@ -1741,11 +1760,20 @@ class ContainerBroker(DatabaseBroker):
             if exclude_others:
                 conditions.append('name = ?')
                 params.append(self.path)
-            if includes is not None:
+            if includes is None:
+                if end_marker:
+                    conditions.append('lower < ?')
+                    params.append(end_marker)
+                if marker:
+                    conditions.append("(upper = '' OR upper > ?)")
+                    params.append(marker)
+            else:
                 conditions.extend(('lower < ?', "(upper = '' OR upper >= ?)"))
                 params.extend((includes, includes))
             if conditions:
                 condition = ' WHERE ' + ' AND '.join(conditions)
+            if limit is not None and limit >= 0:
+                condition += ' LIMIT %d' % limit
             columns = SHARD_RANGE_KEYS[:-2]
             for column in SHARD_RANGE_KEYS[-2:]:
                 if column in defaults:
@@ -1819,27 +1847,34 @@ class ContainerBroker(DatabaseBroker):
 
     def get_shard_ranges(self, marker=None, end_marker=None, includes=None,
                          reverse=False, include_deleted=False, states=None,
-                         include_own=False,
-                         exclude_others=False, fill_gaps=False):
+                         include_own=False, exclude_others=False,
+                         fill_gaps=False):
         """
         Returns a list of persisted shard ranges.
 
         :param marker: restricts the returned list to shard ranges whose
-            namespace includes or is greater than the marker value.
+            namespace includes or is greater than the marker value. If
+            ``reverse=True`` then ``marker`` is treated as ``end_marker``.
+            ``marker`` is ignored if ``includes`` is specified.
         :param end_marker: restricts the returned list to shard ranges whose
-            namespace includes or is less than the end_marker value.
+            namespace includes or is less than the end_marker value. If
+            ``reverse=True`` then ``end_marker`` is treated as ``marker``.
+            ``end_marker`` is ignored if ``includes`` is specified.
         :param includes: restricts the returned list to the shard range that
             includes the given value; if ``includes`` is specified then
-            ``marker`` and ``end_marker`` are ignored.
+            ``fill_gaps``, ``marker`` and ``end_marker`` are ignored, but other
+            constraints are applied (e.g. ``exclude_others`` and
+            ``include_deleted``).
         :param reverse: reverse the result order.
-        :param include_deleted: include items that have the delete marker set
+        :param include_deleted: include items that have the delete marker set.
         :param states: if specified, restricts the returned list to shard
             ranges that have the given state(s); can be a list of ints or a
             single int.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
-            True, that row is included, otherwise it is not included. Default
-            is False.
+            True, that row is included unless it is excluded by other
+            constraints (e.g. ``marker``, ``end_marker``, ``includes``). If
+            False, that row is not included. Default is False.
         :param exclude_others: boolean that governs whether the rows whose
             names do not match the broker's path are included in the returned
             list. If True, those rows are not included, otherwise they are
@@ -1847,9 +1882,14 @@ class ContainerBroker(DatabaseBroker):
         :param fill_gaps: if True, insert a modified copy of own shard range to
             fill any gap between the end of any found shard ranges and the
             upper bound of own shard range. Gaps enclosed within the found
-            shard ranges are not filled.
-        :return: a list of instances of :class:`swift.common.utils.ShardRange`
+            shard ranges are not filled. ``fill_gaps`` is ignored if
+            ``includes`` is specified.
+        :return: a list of instances of :class:`swift.common.utils.ShardRange`.
         """
+        if includes is None and (marker == Namespace.MAX
+                                 or end_marker == Namespace.MIN):
+            return []
+
         if reverse:
             marker, end_marker = end_marker, marker
         if marker and end_marker and marker >= end_marker:
@@ -1858,17 +1898,13 @@ class ContainerBroker(DatabaseBroker):
         shard_ranges = [
             ShardRange(*row)
             for row in self._get_shard_range_rows(
-                includes=includes, include_deleted=include_deleted,
-                states=states, include_own=include_own,
-                exclude_others=exclude_others)]
-
+                marker=marker, end_marker=end_marker, includes=includes,
+                include_deleted=include_deleted, states=states,
+                include_own=include_own, exclude_others=exclude_others)]
         shard_ranges.sort(key=ShardRange.sort_key)
         if includes:
             return shard_ranges[:1] if shard_ranges else []
 
-        shard_ranges = filter_shard_ranges(shard_ranges, includes,
-                                           marker, end_marker)
-
         if fill_gaps:
             own_shard_range = self.get_own_shard_range()
             if shard_ranges:
@@ -1905,13 +1941,13 @@ class ContainerBroker(DatabaseBroker):
             default shard range is returned.
         :return: an instance of :class:`~swift.common.utils.ShardRange`
         """
-        shard_ranges = self.get_shard_ranges(include_own=True,
-                                             include_deleted=True,
-                                             exclude_others=True)
-        if shard_ranges:
-            own_shard_range = shard_ranges[0]
+        rows = self._get_shard_range_rows(
+            include_own=True, include_deleted=True, exclude_others=True,
+            limit=1)
+        if rows:
+            own_shard_range = ShardRange(*rows[0])
         elif no_default:
-            return None
+            own_shard_range = None
         else:
             own_shard_range = ShardRange(
                 self.path, Timestamp.now(), ShardRange.MIN, ShardRange.MAX,
@@ -1942,9 +1978,41 @@ class ContainerBroker(DatabaseBroker):
 
         :return: a dict with keys {bytes_used, object_count}
         """
-        shard_ranges = self.get_shard_ranges(states=SHARD_STATS_STATES)
-        return {'bytes_used': sum(sr.bytes_used for sr in shard_ranges),
-                'object_count': sum(sr.object_count for sr in shard_ranges)}
+        with self.get() as conn:
+            sql = '''
+            SELECT COALESCE(SUM(bytes_used), 0),
+                   COALESCE(SUM(object_count), 0)
+            FROM %s
+            WHERE state in (%s)
+              AND deleted = 0
+              AND name != ?
+            ''' % (SHARD_RANGE_TABLE, ','.join('?' * len(SHARD_STATS_STATES)))
+            cur = conn.execute(sql, SHARD_STATS_STATES + [self.path])
+            bytes_used, object_count = cur.fetchone()
+        return {'bytes_used': bytes_used,
+                'object_count': object_count}
+
+    def has_other_shard_ranges(self):
+        """
+        This function tells if there is any shard range other than the
+        broker's own shard range, that is not marked as deleted.
+
+        :return: A boolean value as described above.
+        """
+        with self.get() as conn:
+            sql = '''
+            SELECT 1 FROM %s
+            WHERE deleted = 0 AND name != ? LIMIT 1
+            ''' % (SHARD_RANGE_TABLE)
+            try:
+                data = conn.execute(sql, [self.path])
+                data.row_factory = None
+                return True if [row for row in data] else False
+            except sqlite3.OperationalError as err:
+                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
+                    return False
+                else:
+                    raise
 
     def get_all_shard_range_data(self):
         """

swift/container/server.py

@@ -860,7 +860,14 @@ class ContainerController(BaseStorageServer):
     @public
     @timing_stats()
     def POST(self, req):
-        """Handle HTTP POST request."""
+        """
+        Handle HTTP POST request.
+
+        A POST request will update the container's ``put_timestamp``, unless
+        it has an ``X-Backend-No-Timestamp-Update`` header with a truthy value.
+
+        :param req: an instance of :class:`~swift.common.swob.Request`.
+        """
         drive, part, account, container = get_container_name_and_placement(req)
         req_timestamp = valid_timestamp(req)
         if 'x-container-sync-to' in req.headers:
@@ -878,7 +885,9 @@ class ContainerController(BaseStorageServer):
         broker = self._get_container_broker(drive, part, account, container)
         if broker.is_deleted():
             return HTTPNotFound(request=req)
-        broker.update_put_timestamp(req_timestamp.internal)
+        if not config_true_value(
+                req.headers.get('x-backend-no-timestamp-update', False)):
+            broker.update_put_timestamp(req_timestamp.internal)
         self._update_metadata(req, broker, req_timestamp, 'POST')
         return HTTPNoContent(request=req)
 

swift/container/sharder.py

@@ -38,7 +38,8 @@ from swift.common.swob import str_to_wsgi
 from swift.common.utils import get_logger, config_true_value, \
     dump_recon_cache, whataremyips, Timestamp, ShardRange, GreenAsyncPile, \
     config_positive_int_value, quorum_size, parse_override_options, \
-    Everything, config_auto_int_value, ShardRangeList, config_percent_value
+    Everything, config_auto_int_value, ShardRangeList, config_percent_value, \
+    node_to_string
 from swift.container.backend import ContainerBroker, \
     RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \
     SHARD_UPDATE_STATES, sift_shard_ranges, SHARD_UPDATE_STAT_STATES
@@ -60,7 +61,7 @@ def sharding_enabled(broker):
     # if broker has been marked deleted it will have lost sysmeta, but we still
     # need to process the broker (for example, to shrink any shard ranges) so
     # fallback to checking if it has any shard ranges
-    if broker.get_shard_ranges():
+    if broker.has_other_shard_ranges():
         return True
     return False
 
@@ -895,7 +896,6 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                 internal_client_conf_path,
                 'Swift Container Sharder',
                 request_tries,
-                allow_modify_pipeline=False,
                 use_replication_network=True,
                 global_conf={'log_name': '%s-ic' % conf.get(
                     'log_name', self.log_route)})
@@ -1079,13 +1079,13 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             # container DB, which predicates sharding starting. But s-m-s-r and
             # auto-sharding do set epoch and then merge, so we use it to tell
             # whether sharding has been taking too long or not.
-            self.warning(broker,
-                         'Cleaving has not completed in %.2f seconds since %s.'
-                         'DB state: %s, own_shard_range state: %s, '
-                         'state count of shard ranges: %s' %
-                         (time.time() - float(own_shard_range.epoch),
-                          own_shard_range.epoch.isoformat, db_state,
-                          own_shard_range.state_text, str(state_count)))
+            self.warning(
+                broker, 'Cleaving has not completed in %.2f seconds since %s. '
+                'DB state: %s, own_shard_range state: %s, state count of '
+                'shard ranges: %s' %
+                (time.time() - float(own_shard_range.epoch),
+                 own_shard_range.epoch.isoformat, db_state,
+                 own_shard_range.state_text, str(state_count)))
 
     def _report_stats(self):
         # report accumulated stats since start of one sharder cycle
@@ -1196,13 +1196,13 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                                  headers=headers, contents=body)
         except DirectClientException as err:
             self.warning(broker,
-                         'Failed to put shard ranges to %s:%s/%s %s/%s: %s',
-                         node['ip'], node['port'], node['device'],
+                         'Failed to put shard ranges to %s %s/%s: %s',
+                         node_to_string(node, replication=True),
                          quote(account), quote(container), err.http_status)
         except (Exception, Timeout) as err:
             self.exception(broker,
-                           'Failed to put shard ranges to %s:%s/%s %s/%s: %s',
-                           node['ip'], node['port'], node['device'],
+                           'Failed to put shard ranges to %s %s/%s: %s',
+                           node_to_string(node, replication=True),
                            quote(account), quote(container), err)
         else:
             return True
@@ -2331,9 +2331,13 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             return
 
         # now look and deal with misplaced objects.
+        move_start_ts = time.time()
         self._move_misplaced_objects(broker)
+        self.logger.timing_since(
+            'sharder.sharding.move_misplaced', move_start_ts)
 
         is_leader = node['index'] == 0 and self.auto_shard and not is_deleted
+
         if state in (UNSHARDED, COLLAPSED):
             if is_leader and broker.is_root_container():
                 # bootstrap sharding of root container
@@ -2344,15 +2348,18 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
             own_shard_range = broker.get_own_shard_range()
             if own_shard_range.state in ShardRange.CLEAVING_STATES:
-                if broker.get_shard_ranges():
+                if broker.has_other_shard_ranges():
                     # container has been given shard ranges rather than
                     # found them e.g. via replication or a shrink event,
                     # or manually triggered cleaving.
+                    db_start_ts = time.time()
                     if broker.set_sharding_state():
                         state = SHARDING
                         self.info(broker, 'Kick off container cleaving, '
                                           'own shard range in state %r',
                                   own_shard_range.state_text)
+                    self.logger.timing_since(
+                        'sharder.sharding.set_state', db_start_ts)
                 elif is_leader:
                     if broker.set_sharding_state():
                         state = SHARDING
@@ -2363,6 +2370,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                                own_shard_range.state_text)
 
         if state == SHARDING:
+            cleave_start_ts = time.time()
             if is_leader:
                 num_found = self._find_shard_ranges(broker)
             else:
@@ -2377,6 +2385,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
             # always try to cleave any pending shard ranges
             cleave_complete = self._cleave(broker)
+            self.logger.timing_since(
+                'sharder.sharding.cleave', cleave_start_ts)
 
             if cleave_complete:
                 if self._complete_sharding(broker):
@@ -2384,6 +2394,9 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                     self._increment_stat('visited', 'completed', statsd=True)
                     self.info(broker, 'Completed cleaving, DB set to sharded '
                                       'state')
+                    self.logger.timing_since(
+                        'sharder.sharding.completed',
+                        float(broker.get_own_shard_range().epoch))
                 else:
                     self.info(broker, 'Completed cleaving, DB remaining in '
                                       'sharding state')
@@ -2391,6 +2404,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         if not broker.is_deleted():
             if state == SHARDED and broker.is_root_container():
                 # look for shrink stats
+                send_start_ts = time.time()
                 self._identify_shrinking_candidate(broker, node)
                 if is_leader:
                     self._find_and_enable_shrinking_candidates(broker)
@@ -2400,6 +2414,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                     self._send_shard_ranges(broker, shard_range.account,
                                             shard_range.container,
                                             [shard_range])
+                self.logger.timing_since(
+                    'sharder.sharding.send_sr', send_start_ts)
 
             if not broker.is_root_container():
                 # Update the root container with this container's shard range
@@ -2408,7 +2424,10 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                 # sharding a shard, this is when the root will see the new
                 # shards move to ACTIVE state and the sharded shard
                 # simultaneously become deleted.
+                update_start_ts = time.time()
                 self._update_root_container(broker)
+                self.logger.timing_since(
+                    'sharder.sharding.update_root', update_start_ts)
 
         self.debug(broker,
                    'Finished processing, state %s%s',