swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/common/utils/timestamp.py

@@ -18,6 +18,7 @@
 import datetime
 import functools
 import math
+import sys
 import time
 
 import six
@@ -189,12 +190,14 @@ class Timestamp(object):
             elif us < 0:
                 t -= 1
                 us += 1000000
-            dt = datetime.datetime.utcfromtimestamp(t)
+            dt = datetime.datetime.fromtimestamp(t, UTC)
             dt = dt.replace(microsecond=us)
         else:
-            dt = datetime.datetime.utcfromtimestamp(t)
+            dt = datetime.datetime.fromtimestamp(t, UTC)
 
         isoformat = dt.isoformat()
+        # need to drop tzinfo
+        isoformat = isoformat[:isoformat.index('+')]
         # python isoformat() doesn't include msecs when zero
         if len(isoformat) < len("1970-01-01T00:00:00.000000"):
             isoformat += ".000000"
@@ -397,3 +400,21 @@ def normalize_delete_at_timestamp(timestamp, high_precision=False):
     """
     fmt = '%016.5f' if high_precision else '%010d'
     return fmt % min(max(0, float(timestamp)), 9999999999.99999)
+
+
+if sys.version_info < (3, 11):
+    class _UTC(datetime.tzinfo):
+        """
+        A tzinfo class for datetimes that returns a 0 timedelta (UTC time)
+        """
+
+        def dst(self, dt):
+            return datetime.timedelta(0)
+        utcoffset = dst
+
+        def tzname(self, dt):
+            return 'UTC'
+
+    UTC = _UTC()
+else:
+    from datetime import UTC

swift/common/wsgi.py

@@ -198,9 +198,14 @@ def get_socket(conf):
             sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)),
                           family=address_family)
             if 'cert_file' in conf:
+                if six.PY2:
+                    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+                else:
+                    context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+                context.verify_mode = ssl.CERT_NONE
+                context.load_cert_chain(conf['cert_file'], conf['key_file'])
                 warn_ssl = True
-                sock = ssl.wrap_socket(sock, certfile=conf['cert_file'],
-                                       keyfile=conf['key_file'])
+                sock = context.wrap_socket(sock, server_side=True)
         except socket.error as err:
             if err.args[0] != errno.EADDRINUSE:
                 raise
@@ -460,7 +465,10 @@ def run_server(conf, logger, sock, global_conf=None, ready_callback=None,
     except socket.error as err:
         if err.errno != errno.EINVAL:
             raise
-    pool.waitall()
+    finally:
+        pool.waitall()
+        if hasattr(app._pipeline_final_app, 'watchdog'):
+            app._pipeline_final_app.watchdog.kill()
 
 
 class StrategyBase(object):
@@ -487,6 +495,8 @@ class StrategyBase(object):
             capture_stdio(self.logger)
         drop_privileges(self.conf.get('user', 'swift'))
         del self.tracking_data  # children don't need to track siblings
+        # only MAINPID should be sending systemd notifications
+        os.environ.pop('NOTIFY_SOCKET', None)
 
     def shutdown_sockets(self):
         """
@@ -888,6 +898,7 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
         run_server(conf, logger, no_fork_sock, global_conf=global_conf,
                    ready_callback=strategy.signal_ready,
                    allow_modify_pipeline=allow_modify_pipeline)
+        systemd_notify(logger, "STOPPING=1")
         return 0
 
     def stop_with_signal(signum, *args):
@@ -981,8 +992,10 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
         else:
             logger.notice('%s received (%s)', signame, os.getpid())
     if running_context[1] == signal.SIGTERM:
+        systemd_notify(logger, "STOPPING=1")
         os.killpg(0, signal.SIGTERM)
     elif running_context[1] == signal.SIGUSR1:
+        systemd_notify(logger, "RELOADING=1")
         # set up a pipe, fork off a child to handle cleanup later,
         # and rexec ourselves with an environment variable set which will
         # indicate which fd (one of the pipe ends) to write a byte to
@@ -1041,6 +1054,9 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
                 os.close(read_fd)
             except Exception:
                 pass
+    else:
+        # SIGHUP or, less likely, run in "once" mode
+        systemd_notify(logger, "STOPPING=1")
 
     strategy.shutdown_sockets()
     signal.signal(signal.SIGTERM, signal.SIG_IGN)

swift/container/auditor.py

@@ -15,7 +15,9 @@
 
 
 from swift.container.backend import ContainerBroker
+from swift.common.daemon import run_daemon
 from swift.common.db_auditor import DatabaseAuditor
+from swift.common.utils import parse_options
 
 
 class ContainerAuditor(DatabaseAuditor):
@@ -26,3 +28,12 @@ class ContainerAuditor(DatabaseAuditor):
 
     def _audit(self, job, broker):
         return None
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerAuditor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/backend.py

@@ -941,16 +941,16 @@ class ContainerBroker(DatabaseBroker):
                 self._do_get_info_query(conn)
 
     def _get_alternate_object_stats(self):
-        state = self.get_db_state()
-        if state == SHARDING:
+        db_state = self.get_db_state()
+        if db_state == SHARDING:
             other_info = self.get_brokers()[0]._get_info()
             stats = {'object_count': other_info['object_count'],
                      'bytes_used': other_info['bytes_used']}
-        elif state == SHARDED and self.is_root_container():
+        elif db_state == SHARDED and self.is_root_container():
             stats = self.get_shard_usage()
         else:
             stats = {}
-        return state, stats
+        return db_state, stats
 
     def get_info(self):
         """
@@ -1685,6 +1685,125 @@ class ContainerBroker(DatabaseBroker):
             if ('no such table: %s' % SHARD_RANGE_TABLE) not in str(err):
                 raise
 
+    def _make_filler_shard_range(self, namespaces, marker, end_marker):
+        if namespaces and namespaces[-1].upper == Namespace.MAX:
+            return None
+
+        # Insert a modified copy of own shard range to fill any gap between the
+        # end of any found and the upper bound of own shard range. Gaps
+        # enclosed within the found shard ranges are not filled.
+        own_shard_range = self.get_own_shard_range()
+        if namespaces:
+            last_upper = namespaces[-1].upper
+        else:
+            last_upper = max(marker or own_shard_range.lower,
+                             own_shard_range.lower)
+        required_upper = min(end_marker or own_shard_range.upper,
+                             own_shard_range.upper)
+        if required_upper > last_upper:
+            filler_sr = own_shard_range
+            filler_sr.lower = last_upper
+            filler_sr.upper = required_upper
+            return filler_sr
+        else:
+            return None
+
+    def get_namespaces(self, marker=None, end_marker=None, includes=None,
+                       reverse=False, states=None, fill_gaps=False):
+        """
+        Returns a list of persisted namespaces per input parameters.
+
+        :param marker: restricts the returned list to shard ranges whose
+            namespace includes or is greater than the marker value. If
+            ``reverse=True`` then ``marker`` is treated as ``end_marker``.
+            ``marker`` is ignored if ``includes`` is specified.
+        :param end_marker: restricts the returned list to shard ranges whose
+            namespace includes or is less than the end_marker value. If
+            ``reverse=True`` then ``end_marker`` is treated as ``marker``.
+            ``end_marker`` is ignored if ``includes`` is specified.
+        :param includes: restricts the returned list to the shard range that
+            includes the given value; if ``includes`` is specified then
+            ``fill_gaps``, ``marker`` and ``end_marker`` are ignored.
+        :param reverse: reverse the result order.
+        :param states: if specified, restricts the returned list to namespaces
+            that have one of the given states; should be a list of ints.
+        :param fill_gaps: if True, insert a modified copy of own shard range to
+            fill any gap between the end of any found shard ranges and the
+            upper bound of own shard range. Gaps enclosed within the found
+            shard ranges are not filled.
+        :return: a list of Namespace objects.
+        """
+        if includes is None and (marker == Namespace.MAX
+                                 or end_marker == Namespace.MIN):
+            return []
+
+        if reverse:
+            marker, end_marker = end_marker, marker
+        if marker and end_marker and marker >= end_marker:
+            return []
+
+        included_states = set(states) if states else None
+        with self.get() as conn:
+            # Namespace only needs 'name', 'lower' and 'upper', but the query
+            # also need to include 'state' to be used when subesequently
+            # sorting the rows. And the sorting can't be done within SQLite
+            # since the value for maximum upper bound is an empty string.
+
+            conditions = ['deleted = 0', 'name != ?']
+            params = [self.path]
+            if included_states:
+                conditions.append('state in (%s)' % ','.join(
+                    '?' * len(included_states)))
+                params.extend(included_states)
+            if includes is None:
+                if end_marker:
+                    conditions.append('lower < ?')
+                    params.append(end_marker)
+                if marker:
+                    conditions.append("(upper = '' OR upper > ?)")
+                    params.append(marker)
+            else:
+                conditions.extend(('lower < ?', "(upper = '' OR upper >= ?)"))
+                params.extend((includes, includes))
+            condition = ' WHERE ' + ' AND '.join(conditions)
+            sql = '''
+            SELECT name, lower, upper, state FROM %s%s
+            ''' % (SHARD_RANGE_TABLE, condition)
+            try:
+                data = conn.execute(sql, params)
+                data.row_factory = None
+                namespaces = [row for row in data]
+            except sqlite3.OperationalError as err:
+                if ('no such table: %s' % SHARD_RANGE_TABLE) in str(err):
+                    return []
+                else:
+                    raise
+
+        # Sort those namespaces in order, note that each namespace record also
+        # include additional attribute 'state'.
+        def sort_key(namespace):
+            return ShardRange.sort_key_order(name=namespace[0],
+                                             lower=namespace[1],
+                                             upper=namespace[2],
+                                             state=namespace[3])
+        namespaces.sort(key=sort_key)
+        # Convert the record tuples to Namespace objects.
+        namespaces = [Namespace(row[0], row[1], row[2]) for row in namespaces]
+        if includes:
+            return namespaces[:1] if namespaces else []
+
+        if fill_gaps:
+            filler_sr = self._make_filler_shard_range(
+                namespaces, marker, end_marker)
+            if filler_sr:
+                namespaces.append(Namespace(filler_sr.name,
+                                            filler_sr.lower,
+                                            filler_sr.upper))
+        if reverse:
+            namespaces.reverse()
+
+        return namespaces
+
     def _get_shard_range_rows(self, connection=None, marker=None,
                               end_marker=None, includes=None,
                               include_deleted=False, states=None,
@@ -1709,8 +1828,8 @@ class ContainerBroker(DatabaseBroker):
             ``marker`` and ``end_marker`` are ignored, but other constraints
             are applied (e.g. ``exclude_others`` and ``include_deleted``).
         :param include_deleted: include rows marked as deleted.
-        :param states: include only rows matching the given state(s); can be an
-            int or a list of ints.
+        :param states: include only rows matching the given states; should be
+            a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
             True, that row is included unless it is excluded by other
@@ -1734,11 +1853,7 @@ class ContainerBroker(DatabaseBroker):
         if exclude_others and not include_own:
             return []
 
-        included_states = set()
-        if isinstance(states, (list, tuple, set)):
-            included_states.update(states)
-        elif states is not None:
-            included_states.add(states)
+        included_states = set(states) if states else None
 
         # defaults to be used when legacy db's are missing columns
         default_values = {'reported': 0,
@@ -1868,8 +1983,7 @@ class ContainerBroker(DatabaseBroker):
         :param reverse: reverse the result order.
         :param include_deleted: include items that have the delete marker set.
         :param states: if specified, restricts the returned list to shard
-            ranges that have the given state(s); can be a list of ints or a
-            single int.
+            ranges that have one of the given states; should be a list of ints.
         :param include_own: boolean that governs whether the row whose name
             matches the broker's path is included in the returned list. If
             True, that row is included unless it is excluded by other
@@ -1906,18 +2020,9 @@ class ContainerBroker(DatabaseBroker):
             return shard_ranges[:1] if shard_ranges else []
 
         if fill_gaps:
-            own_shard_range = self.get_own_shard_range()
-            if shard_ranges:
-                last_upper = shard_ranges[-1].upper
-            else:
-                last_upper = max(marker or own_shard_range.lower,
-                                 own_shard_range.lower)
-            required_upper = min(end_marker or own_shard_range.upper,
-                                 own_shard_range.upper)
-            if required_upper > last_upper:
-                filler_sr = own_shard_range
-                filler_sr.lower = last_upper
-                filler_sr.upper = required_upper
+            filler_sr = self._make_filler_shard_range(
+                shard_ranges, marker, end_marker)
+            if filler_sr:
                 shard_ranges.append(filler_sr)
 
         if reverse:
@@ -2039,10 +2144,10 @@ class ContainerBroker(DatabaseBroker):
             self.logger.warning("Container '%s' cannot be set to sharding "
                                 "state: missing epoch", self.path)
             return False
-        state = self.get_db_state()
-        if not state == UNSHARDED:
+        db_state = self.get_db_state()
+        if not db_state == UNSHARDED:
             self.logger.warning("Container '%s' cannot be set to sharding "
-                                "state while in %s state", self.path, state)
+                                "state while in %s state", self.path, db_state)
             return False
 
         info = self.get_info()
@@ -2120,11 +2225,11 @@ class ContainerBroker(DatabaseBroker):
         :return: True if the retiring DB was successfully unlinked, False
             otherwise.
         """
-        state = self.get_db_state()
-        if not state == SHARDING:
+        db_state = self.get_db_state()
+        if not db_state == SHARDING:
             self.logger.warning("Container %r cannot be set to sharded "
                                 "state while in %s state",
-                                self.path, state)
+                                self.path, db_state)
             return False
 
         self.reload_db_files()

swift/container/reconciler.py

@@ -22,7 +22,7 @@ from eventlet import GreenPile, GreenPool, Timeout
 import six
 
 from swift.common import constraints
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.direct_client import (
     direct_head_container, direct_delete_container_object,
     direct_put_container_object, ClientException)
@@ -31,7 +31,7 @@ from swift.common.request_helpers import MISPLACED_OBJECTS_ACCOUNT, \
     USE_REPLICATION_NETWORK_HEADER
 from swift.common.utils import get_logger, split_path, majority_size, \
     FileLikeIter, Timestamp, last_modified_date_to_timestamp, \
-    LRUCache, decode_timestamps, hash_path
+    LRUCache, decode_timestamps, hash_path, parse_options
 from swift.common.storage_policy import POLICIES
 
 MISPLACED_OBJECTS_CONTAINER_DIVISOR = 3600  # 1 hour
@@ -860,3 +860,12 @@ class ContainerReconciler(Daemon):
             self.stats = defaultdict(int)
             self.logger.info('sleeping between intervals (%ss)', self.interval)
             time.sleep(self.interval)
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerReconciler, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/replicator.py

@@ -17,18 +17,51 @@ import os
 import json
 from collections import defaultdict
 from eventlet import Timeout
+import optparse
 from random import choice
 
 from swift.container.sync_store import ContainerSyncStore
-from swift.container.backend import ContainerBroker, DATADIR, SHARDED
+from swift.container.backend import ContainerBroker, DATADIR, SHARDED, \
+    merge_shards
 from swift.container.reconciler import (
     MISPLACED_OBJECTS_ACCOUNT, incorrect_policy_index,
     get_reconciler_container_name, get_row_to_q_entry_translator)
 from swift.common import db_replicator
+from swift.common.daemon import run_daemon
 from swift.common.storage_policy import POLICIES
 from swift.common.swob import HTTPOk, HTTPAccepted
 from swift.common.http import is_success
-from swift.common.utils import Timestamp, majority_size, get_db_files
+from swift.common.utils import Timestamp, majority_size, get_db_files, \
+    parse_options
+
+
+def check_merge_own_shard_range(shards, broker, logger, source):
+    """
+    If broker has own_shard_range *with an epoch* then filter out an
+    own_shard_range *without an epoch*, and log a warning about it.
+
+    :param shards: a list of candidate ShardRanges to merge
+    :param broker: a ContainerBroker
+    :param logger: a logger
+    :param source: string to log as source of shards
+    :return: a list of ShardRanges to actually merge
+    """
+    # work-around for https://bugs.launchpad.net/swift/+bug/1980451
+    own_sr = broker.get_own_shard_range()
+    if own_sr.epoch is None:
+        return shards
+    to_merge = []
+    for shard in shards:
+        if shard['name'] == own_sr.name and not shard['epoch']:
+            shard_copy = dict(shard)
+            new_content = merge_shards(shard_copy, dict(own_sr))
+            if new_content and shard_copy['epoch'] is None:
+                logger.warning(
+                    'Ignoring remote osr w/o epoch, own_sr: %r, remote_sr: %r,'
+                    ' source: %s', dict(own_sr), shard, source)
+            continue
+        to_merge.append(shard)
+    return to_merge
 
 
 class ContainerReplicator(db_replicator.Replicator):
@@ -138,8 +171,10 @@ class ContainerReplicator(db_replicator.Replicator):
         with Timeout(self.node_timeout):
             response = http.replicate('get_shard_ranges')
         if response and is_success(response.status):
-            broker.merge_shard_ranges(json.loads(
-                response.data.decode('ascii')))
+            shards = json.loads(response.data.decode('ascii'))
+            shards = check_merge_own_shard_range(
+                shards, broker, self.logger, '%s%s' % (http.host, http.path))
+            broker.merge_shard_ranges(shards)
 
     def find_local_handoff_for_part(self, part):
         """
@@ -394,13 +429,35 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):
     def _post_rsync_then_merge_hook(self, existing_broker, new_broker):
         # Note the following hook will need to change to using a pointer and
         # limit in the future.
-        new_broker.merge_shard_ranges(
-            existing_broker.get_all_shard_range_data())
+        shards = existing_broker.get_all_shard_range_data()
+        shards = check_merge_own_shard_range(
+            shards, new_broker, self.logger, 'rsync')
+        new_broker.merge_shard_ranges(shards)
 
     def merge_shard_ranges(self, broker, args):
-        broker.merge_shard_ranges(args[0])
+        shards = check_merge_own_shard_range(
+            args[0], broker, self.logger, 'repl_req')
+        broker.merge_shard_ranges(shards)
         return HTTPAccepted()
 
     def get_shard_ranges(self, broker, args):
         return HTTPOk(headers={'Content-Type': 'application/json'},
                       body=json.dumps(broker.get_all_shard_range_data()))
+
+
+def main():
+    parser = optparse.OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help=('Replicate only given devices. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    parser.add_option('-p', '--partitions',
+                      help=('Replicate only given partitions. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ContainerReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()