swift 2.32.1__py2.py3-none-any.whl → 2.33.1__py2.py3-none-any.whl

swift/common/wsgi.py

@@ -487,6 +487,8 @@ class StrategyBase(object):
             capture_stdio(self.logger)
         drop_privileges(self.conf.get('user', 'swift'))
         del self.tracking_data  # children don't need to track siblings
+        # only MAINPID should be sending systemd notifications
+        os.environ.pop('NOTIFY_SOCKET', None)
 
     def shutdown_sockets(self):
         """
@@ -888,6 +890,7 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
         run_server(conf, logger, no_fork_sock, global_conf=global_conf,
                    ready_callback=strategy.signal_ready,
                    allow_modify_pipeline=allow_modify_pipeline)
+        systemd_notify(logger, "STOPPING=1")
         return 0
 
     def stop_with_signal(signum, *args):
@@ -981,8 +984,10 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
         else:
             logger.notice('%s received (%s)', signame, os.getpid())
     if running_context[1] == signal.SIGTERM:
+        systemd_notify(logger, "STOPPING=1")
         os.killpg(0, signal.SIGTERM)
     elif running_context[1] == signal.SIGUSR1:
+        systemd_notify(logger, "RELOADING=1")
         # set up a pipe, fork off a child to handle cleanup later,
         # and rexec ourselves with an environment variable set which will
         # indicate which fd (one of the pipe ends) to write a byte to
@@ -1041,6 +1046,9 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
                 os.close(read_fd)
             except Exception:
                 pass
+    else:
+        # SIGHUP or, less likely, run in "once" mode
+        systemd_notify(logger, "STOPPING=1")
 
     strategy.shutdown_sockets()
     signal.signal(signal.SIGTERM, signal.SIG_IGN)

swift/container/backend.py

@@ -1685,6 +1685,125 @@ class ContainerBroker(DatabaseBroker):
             if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err):
                 raise
 
+    def _make_filler_shard_range(self, namespaces, marker, end_marker):
+        if namespaces and namespaces[-1].upper == Namespace.MAX:
+            return None
+
+        # Insert a modified copy of own shard range to fill any gap between the
+        # end of any found and the upper bound of own shard range. Gaps
+        # enclosed within the found shard ranges are not filled.
+        own_shard_range = self.get_own_shard_range()
+        if namespaces:
+            last_upper = namespaces[-1].upper
+        else:
+            last_upper = max(marker or own_shard_range.lower,
+                             own_shard_range.lower)
+        required_upper = min(end_marker or own_shard_range.upper,
+                             own_shard_range.upper)
+        if required_upper > last_upper:
+            filler_sr = own_shard_range
+            filler_sr.lower = last_upper
+            filler_sr.upper = required_upper
+            return filler_sr
+        else:
+            return None
+
+    def get_namespaces(self, marker=None, end_marker=None, includes=None,
+                       reverse=False, states=None, fill_gaps=False):
+        """
+        Returns a list of persisted namespaces per input parameters.
+
+        :param marker: restricts the returned list to shard ranges whose
+            namespace includes or is greater than the marker value. If
+            ``reverse=True`` then ``marker`` is treated as ``end_marker``.
+            ``marker`` is ignored if ``includes`` is specified.
+        :param end_marker: restricts the returned list to shard ranges whose
+            namespace includes or is less than the end_marker value. If
+            ``reverse=True`` then ``end_marker`` is treated as ``marker``.
+            ``end_marker`` is ignored if ``includes`` is specified.
+        :param includes: restricts the returned list to the shard range that
+            includes the given value; if ``includes`` is specified then
+            ``fill_gaps``, ``marker`` and ``end_marker`` are ignored.
+        :param reverse: reverse the result order.
+        :param states: if specified, restricts the returned list to namespaces
+            that have one of the given states; should be a list of ints.
+        :param fill_gaps: if True, insert a modified copy of own shard range to
+            fill any gap between the end of any found shard ranges and the
+            upper bound of own shard range. Gaps enclosed within the found
+            shard ranges are not filled.
+        :return: a list of Namespace objects.
+        """
+        if includes is None and (marker == Namespace.MAX
+                                 or end_marker == Namespace.MIN):
+            return []
+
+        if reverse:
+            marker, end_marker = end_marker, marker
+        if marker and end_marker and marker >= end_marker:
+            return []
+
+        included_states = set(states) if states else None
+        with self.get() as conn:
+            # Namespace only needs 'name', 'lower' and 'upper', but the query
+            # also need to include 'state' to be used when subesequently
+            # sorting the rows. And the sorting can't be done within SQLite
+            # since the value for maximum upper bound is an empty string.
+
+            conditions = ['deleted = 0', 'name != ?']
+            params = [self.path]
+            if included_states:
+                conditions.append('state in (%s)' % ','.join(
+                    '?' * len(included_states)))
+                params.extend(included_states)
+            if includes is None:
+                if end_marker:
+                    conditions.append('lower < ?')
+                    params.append(end_marker)
+                if marker:
+                    conditions.append("(upper = '' OR upper > ?)")
+                    params.append(marker)
+            else:
+                conditions.extend(('lower < ?', "(upper = '' OR upper >= ?)"))
+                params.extend((includes, includes))
+            condition = ' WHERE ' + ' AND '.join(conditions)
+            sql = '''
+            SELECT name, lower, upper, state FROM %s%s
+            ''' % (SHARD_RANGE_TABLE, condition)
+            try:
+                data = conn.execute(sql, params)
+                data.row_factory = None
+                namespaces = [row for row in data]
+            except sqlite3.OperationalError as err:
+                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
+                    return []
+                else:
+                    raise
+
+        # Sort those namespaces in order, note that each namespace record also
+        # include additional attribute 'state'.
+        def sort_key(namespace):
+            return ShardRange.sort_key_order(name=namespace[0],
+                                             lower=namespace[1],
+                                             upper=namespace[2],
+                                             state=namespace[3])
+        namespaces.sort(key=sort_key)
+        # Convert the record tuples to Namespace objects.
+        namespaces = [Namespace(row[0], row[1], row[2]) for row in namespaces]
+        if includes:
+            return namespaces[:1] if namespaces else []
+
+        if fill_gaps:
+            filler_sr = self._make_filler_shard_range(
+                namespaces, marker, end_marker)
+            if filler_sr:
+                namespaces.append(Namespace(filler_sr.name,
+                                            filler_sr.lower,
+                                            filler_sr.upper))
+        if reverse:
+            namespaces.reverse()
+
+        return namespaces
+
     def _get_shard_range_rows(self, connection=None, marker=None,
                               end_marker=None, includes=None,
                               include_deleted=False, states=None,
@@ -1709,8 +1828,8 @@ class ContainerBroker(DatabaseBroker):
             ``marker`` and ``end_marker`` are ignored, but other constraints
             are applied (e.g. ``exclude_others`` and ``include_deleted``).
         :param include_deleted: include rows marked as deleted.
-        :param states: include only rows matching the given state(s); can be an
-            int or a list of ints.
+        :param states: include only rows matching the given states; should be
+            a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
             True, that row is included unless it is excluded by other
@@ -1734,11 +1853,7 @@ class ContainerBroker(DatabaseBroker):
         if exclude_others and not include_own:
             return []
 
-        included_states = set()
-        if isinstance(states, (list, tuple, set)):
-            included_states.update(states)
-        elif states is not None:
-            included_states.add(states)
+        included_states = set(states) if states else None
 
         # defaults to be used when legacy db's are missing columns
         default_values = {'reported': 0,
@@ -1868,8 +1983,7 @@ class ContainerBroker(DatabaseBroker):
         :param reverse: reverse the result order.
         :param include_deleted: include items that have the delete marker set.
         :param states: if specified, restricts the returned list to shard
-            ranges that have the given state(s); can be a list of ints or a
-            single int.
+            ranges that have one of the given states; should be a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
             True, that row is included unless it is excluded by other
@@ -1906,18 +2020,9 @@ class ContainerBroker(DatabaseBroker):
             return shard_ranges[:1] if shard_ranges else []
 
         if fill_gaps:
-            own_shard_range = self.get_own_shard_range()
-            if shard_ranges:
-                last_upper = shard_ranges[-1].upper
-            else:
-                last_upper = max(marker or own_shard_range.lower,
-                                 own_shard_range.lower)
-            required_upper = min(end_marker or own_shard_range.upper,
-                                 own_shard_range.upper)
-            if required_upper > last_upper:
-                filler_sr = own_shard_range
-                filler_sr.lower = last_upper
-                filler_sr.upper = required_upper
+            filler_sr = self._make_filler_shard_range(
+                shard_ranges, marker, end_marker)
+            if filler_sr:
                 shard_ranges.append(filler_sr)
 
         if reverse:

swift/container/replicator.py

@@ -20,7 +20,8 @@ from eventlet import Timeout
 from random import choice
 
 from swift.container.sync_store import ContainerSyncStore
-from swift.container.backend import ContainerBroker, DATADIR, SHARDED
+from swift.container.backend import ContainerBroker, DATADIR, SHARDED, \
+    merge_shards
 from swift.container.reconciler import (
     MISPLACED_OBJECTS_ACCOUNT, incorrect_policy_index,
     get_reconciler_container_name, get_row_to_q_entry_translator)
@@ -31,6 +32,35 @@ from swift.common.http import is_success
 from swift.common.utils import Timestamp, majority_size, get_db_files
 
 
+def check_merge_own_shard_range(shards, broker, logger, source):
+    """
+    If broker has own_shard_range *with an epoch* then filter out an
+    own_shard_range *without an epoch*, and log a warning about it.
+
+    :param shards: a list of candidate ShardRanges to merge
+    :param broker: a ContainerBroker
+    :param logger: a logger
+    :param source: string to log as source of shards
+    :return: a list of ShardRanges to actually merge
+    """
+    # work-around for https://bugs.launchpad.net/swift/+bug/1980451
+    own_sr = broker.get_own_shard_range()
+    if own_sr.epoch is None:
+        return shards
+    to_merge = []
+    for shard in shards:
+        if shard['name'] == own_sr.name and not shard['epoch']:
+            shard_copy = dict(shard)
+            new_content = merge_shards(shard_copy, dict(own_sr))
+            if new_content and shard_copy['epoch'] is None:
+                logger.warning(
+                    'Ignoring remote osr w/o epoch, own_sr: %r, remote_sr: %r,'
+                    ' source: %s', dict(own_sr), shard, source)
+            continue
+        to_merge.append(shard)
+    return to_merge
+
+
 class ContainerReplicator(db_replicator.Replicator):
     server_type = 'container'
     brokerclass = ContainerBroker
@@ -138,8 +168,10 @@ class ContainerReplicator(db_replicator.Replicator):
         with Timeout(self.node_timeout):
             response = http.replicate('get_shard_ranges')
         if response and is_success(response.status):
-            broker.merge_shard_ranges(json.loads(
-                response.data.decode('ascii')))
+            shards = json.loads(response.data.decode('ascii'))
+            shards = check_merge_own_shard_range(
+                shards, broker, self.logger, '%s%s' % (http.host, http.path))
+            broker.merge_shard_ranges(shards)
 
     def find_local_handoff_for_part(self, part):
         """
@@ -394,11 +426,15 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):
     def _post_rsync_then_merge_hook(self, existing_broker, new_broker):
         # Note the following hook will need to change to using a pointer and
         # limit in the future.
-        new_broker.merge_shard_ranges(
-            existing_broker.get_all_shard_range_data())
+        shards = existing_broker.get_all_shard_range_data()
+        shards = check_merge_own_shard_range(
+            shards, new_broker, self.logger, 'rsync')
+        new_broker.merge_shard_ranges(shards)
 
     def merge_shard_ranges(self, broker, args):
-        broker.merge_shard_ranges(args[0])
+        shards = check_merge_own_shard_range(
+            args[0], broker, self.logger, 'repl_req')
+        broker.merge_shard_ranges(shards)
         return HTTPAccepted()
 
     def get_shard_ranges(self, broker, args):