swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/container/sharder.py

@@ -17,6 +17,7 @@ import errno
 import json
 import logging
 import operator
+from optparse import OptionParser
 import time
 from collections import defaultdict
 from operator import itemgetter
@@ -32,6 +33,7 @@ from swift.common import internal_client
 from swift.common.constraints import check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.direct_client import (direct_put_container,
                                         DirectClientException)
+from swift.common.daemon import run_daemon
 from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.ring.utils import is_local_device
 from swift.common.swob import str_to_wsgi
@@ -39,7 +41,7 @@ from swift.common.utils import get_logger, config_true_value, \
     dump_recon_cache, whataremyips, Timestamp, ShardRange, GreenAsyncPile, \
     config_positive_int_value, quorum_size, parse_override_options, \
     Everything, config_auto_int_value, ShardRangeList, config_percent_value, \
-    node_to_string
+    node_to_string, parse_options
 from swift.container.backend import ContainerBroker, \
     RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \
     SHARD_UPDATE_STATES, sift_shard_ranges, SHARD_UPDATE_STAT_STATES
@@ -50,6 +52,8 @@ CLEAVE_SUCCESS = 0
 CLEAVE_FAILED = 1
 CLEAVE_EMPTY = 2
 
+DEFAULT_PERIODIC_WARNINGS_INTERVAL = 24 * 3600
+
 
 def sharding_enabled(broker):
     # NB all shards will by default have been created with
@@ -852,18 +856,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         ContainerReplicator.__init__(self, conf, logger=logger)
         ContainerSharderConf.__init__(self, conf)
         ContainerSharderConf.validate_conf(self)
-        if conf.get('auto_create_account_prefix'):
-            self.logger.warning('Option auto_create_account_prefix is '
-                                'deprecated. Configure '
-                                'auto_create_account_prefix under the '
-                                'swift-constraints section of '
-                                'swift.conf. This option will '
-                                'be ignored in a future release.')
-            auto_create_account_prefix = \
-                self.conf['auto_create_account_prefix']
-        else:
-            auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
-        self.shards_account_prefix = (auto_create_account_prefix + 'shards_')
+        self.shards_account_prefix = (AUTO_CREATE_ACCOUNT_PREFIX + 'shards_')
         self.sharding_candidates = []
         self.shrinking_candidates = []
         replica_count = self.ring.replica_count
@@ -908,9 +901,13 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                 (internal_client_conf_path, err))
         self.stats_interval = float(conf.get('stats_interval', '3600'))
         self.reported = 0
+        self.periodic_warnings_interval = float(
+            conf.get('periodic_warnings_interval',
+                     DEFAULT_PERIODIC_WARNINGS_INTERVAL))
+        self.periodic_warnings_start = time.time()
+        self.periodic_warnings = set()
 
-    def _format_log_msg(self, broker, msg, *args):
-        # make best effort to include broker properties...
+    def _get_broker_details(self, broker):
         try:
             db_file = broker.db_file
         except Exception:  # noqa
@@ -919,7 +916,11 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             path = broker.path
         except Exception:  # noqa
             path = ''
+        return db_file, path
 
+    def _format_log_msg(self, broker, msg, *args):
+        # make best effort to include broker properties...
+        db_file, path = self._get_broker_details(broker)
         if args:
             msg = msg % args
         return '%s, path: %s, db: %s' % (msg, quote(path), db_file)
@@ -939,6 +940,19 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
     def warning(self, broker, msg, *args, **kwargs):
         self._log(logging.WARNING, broker, msg, *args, **kwargs)
 
+    def periodic_warning(self, broker, msg, *args, **kwargs):
+        now = time.time()
+        if now - self.periodic_warnings_start >= \
+                self.periodic_warnings_interval:
+            self.periodic_warnings.clear()
+            self.periodic_warnings_start = now
+
+        db_file, path = self._get_broker_details(broker)
+        key = (db_file, msg)
+        if key not in self.periodic_warnings:
+            self.periodic_warnings.add(key)
+            self._log(logging.WARNING, broker, msg, *args, **kwargs)
+
     def error(self, broker, msg, *args, **kwargs):
         self._log(logging.ERROR, broker, msg, *args, **kwargs)
 
@@ -1156,6 +1170,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         params = params or {}
         params.setdefault('format', 'json')
         headers = {'X-Backend-Record-Type': 'shard',
+                   'X-Backend-Record-Shard-Format': 'full',
                    'X-Backend-Override-Deleted': 'true',
                    'X-Backend-Include-Deleted': str(include_deleted)}
         if newest:
@@ -1296,7 +1311,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                 # Shrinking is how we resolve overlaps; we've got to
                 # allow multiple shards in that state
                 continue
-            shard_ranges = broker.get_shard_ranges(states=state)
+            shard_ranges = broker.get_shard_ranges(states=[state])
             # Transient overlaps can occur during the period immediately after
             # sharding if a root learns about new child shards before it learns
             # that the parent has sharded. These overlaps are normally
@@ -1549,8 +1564,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         if broker.is_deleted():
             if broker.is_old_enough_to_reclaim(time.time(), self.reclaim_age) \
                     and not broker.is_empty_enough_to_reclaim():
-                self.warning(broker,
-                             'Reclaimable db stuck waiting for shrinking')
+                self.periodic_warning(
+                    broker, 'Reclaimable db stuck waiting for shrinking')
             # if the container has been marked as deleted, all metadata will
             # have been erased so no point auditing. But we want it to pass, in
             # case any objects exist inside it.
@@ -1678,11 +1693,15 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                               dest_broker, node_id, info):
         success, responses = self._replicate_object(
             part, dest_broker.db_file, node_id)
+        replication_successes = responses.count(True)
         quorum = quorum_size(self.ring.replica_count)
-        if not success and responses.count(True) < quorum:
-            self.warning(broker, 'Failed to sufficiently replicate misplaced '
-                                 'objects to %s (not removing)',
-                         dest_shard_range)
+        if not success and replication_successes < quorum:
+            self.warning(
+                broker, 'Failed to sufficiently replicate misplaced objects '
+                        'shard %s in state %s: %s successes, %s required '
+                        '(not removing objects), shard db: %s',
+                dest_shard_range.name, dest_shard_range.state_text,
+                replication_successes, quorum, dest_broker.db_file)
             return False
 
         if broker.get_info()['id'] != info['id']:
@@ -1700,9 +1719,9 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             success = True
 
         if not success:
-            self.warning(broker,
-                         'Refused to remove misplaced objects for dest %s',
-                         dest_shard_range)
+            self.warning(broker, 'Refused to remove misplaced objects for '
+                                 'dest %s in state %s',
+                         dest_shard_range.name, dest_shard_range.state_text)
         return success
 
     def _move_objects(self, src_broker, src_shard_range, policy_index,
@@ -1800,12 +1819,12 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
     def _make_misplaced_object_bounds(self, broker):
         bounds = []
-        state = broker.get_db_state()
-        if state == SHARDED:
+        db_state = broker.get_db_state()
+        if db_state == SHARDED:
             # Anything in the object table is treated as a misplaced object.
             bounds.append(('', ''))
 
-        if not bounds and state == SHARDING:
+        if not bounds and db_state == SHARDING:
             # Objects outside of this container's own range are misplaced.
             # Objects in already cleaved shard ranges are also misplaced.
             cleave_context = CleavingContext.load(broker)
@@ -1919,7 +1938,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         # Create shard containers that are ready to receive redirected object
         # updates. Do this now, so that redirection can begin immediately
         # without waiting for cleaving to complete.
-        found_ranges = broker.get_shard_ranges(states=ShardRange.FOUND)
+        found_ranges = broker.get_shard_ranges(states=[ShardRange.FOUND])
         created_ranges = []
         for shard_range in found_ranges:
             self._increment_stat('created', 'attempted')
@@ -2059,10 +2078,13 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                 # insufficient replication or replication not even attempted;
                 # break because we don't want to progress the cleave cursor
                 # until each shard range has been successfully cleaved
-                self.warning(broker,
-                             'Failed to sufficiently replicate cleaved shard '
-                             '%s: %s successes, %s required', shard_range,
-                             replication_successes, replication_quorum)
+                self.warning(
+                    broker, 'Failed to sufficiently replicate cleaved shard '
+                            '%s in state %s: %s successes, %s required, '
+                            'shard db: %s',
+                    shard_broker.path, shard_range.state_text,
+                    replication_successes, replication_quorum,
+                    shard_broker.db_file)
                 self._increment_stat('cleaved', 'failure', statsd=True)
                 result = CLEAVE_FAILED
             else:
@@ -2214,7 +2236,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             else:
                 own_shard_range.update_state(ShardRange.SHARDED)
                 modified_shard_ranges = broker.get_shard_ranges(
-                    states=ShardRange.CLEAVED)
+                    states=[ShardRange.CLEAVED])
                 for sr in modified_shard_ranges:
                     sr.update_state(ShardRange.ACTIVE)
             if (not broker.is_root_container() and not
@@ -2322,9 +2344,9 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
     def _process_broker(self, broker, node, part):
         broker.get_info()  # make sure account/container are populated
-        state = broker.get_db_state()
+        db_state = broker.get_db_state()
         is_deleted = broker.is_deleted()
-        self.debug(broker, 'Starting processing, state %s%s', state,
+        self.debug(broker, 'Starting processing, state %s%s', db_state,
                    ' (deleted)' if is_deleted else '')
 
         if not self._audit_container(broker):
@@ -2338,7 +2360,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
         is_leader = node['index'] == 0 and self.auto_shard and not is_deleted
 
-        if state in (UNSHARDED, COLLAPSED):
+        if db_state in (UNSHARDED, COLLAPSED):
             if is_leader and broker.is_root_container():
                 # bootstrap sharding of root container
                 own_shard_range = broker.get_own_shard_range()
@@ -2354,7 +2376,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                     # or manually triggered cleaving.
                     db_start_ts = time.time()
                     if broker.set_sharding_state():
-                        state = SHARDING
+                        db_state = SHARDING
                         self.info(broker, 'Kick off container cleaving, '
                                           'own shard range in state %r',
                                   own_shard_range.state_text)
@@ -2362,14 +2384,14 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                         'sharder.sharding.set_state', db_start_ts)
                 elif is_leader:
                     if broker.set_sharding_state():
-                        state = SHARDING
+                        db_state = SHARDING
                 else:
                     self.debug(broker,
                                'Own shard range in state %r but no shard '
                                'ranges and not leader; remaining unsharded',
                                own_shard_range.state_text)
 
-        if state == SHARDING:
+        if db_state == SHARDING:
             cleave_start_ts = time.time()
             if is_leader:
                 num_found = self._find_shard_ranges(broker)
@@ -2390,7 +2412,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
             if cleave_complete:
                 if self._complete_sharding(broker):
-                    state = SHARDED
+                    db_state = SHARDED
                     self._increment_stat('visited', 'completed', statsd=True)
                     self.info(broker, 'Completed cleaving, DB set to sharded '
                                       'state')
@@ -2402,7 +2424,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                                       'sharding state')
 
         if not broker.is_deleted():
-            if state == SHARDED and broker.is_root_container():
+            if db_state == SHARDED and broker.is_root_container():
                 # look for shrink stats
                 send_start_ts = time.time()
                 self._identify_shrinking_candidate(broker, node)
@@ -2554,3 +2576,25 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             elapsed = time.time() - begin
             self.logger.info(
                 'Container sharder "once" mode completed: %.02fs', elapsed)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help='Shard containers only on given devices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-p', '--partitions',
+                      help='Shard containers only in given partitions. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('--no-auto-shard', action='store_false',
+                      dest='auto_shard', default=None,
+                      help='Disable auto-sharding. Overrides the auto_shard '
+                           'value in the config file.')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ContainerSharder, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/sync.py

@@ -29,6 +29,7 @@ from swift.common.db import DatabaseConnectionError
 from swift.container.backend import ContainerBroker
 from swift.container.sync_store import ContainerSyncStore
 from swift.common.container_sync_realms import ContainerSyncRealms
+from swift.common.daemon import run_daemon
 from swift.common.internal_client import (
     delete_object, put_object, head_object,
     InternalClient, UnexpectedResponse)
@@ -39,7 +40,7 @@ from swift.common.swob import normalize_etag
 from swift.common.utils import (
     clean_content_type, config_true_value,
     FileLikeIter, get_logger, hash_path, quote, validate_sync_to,
-    whataremyips, Timestamp, decode_timestamps)
+    whataremyips, Timestamp, decode_timestamps, parse_options)
 from swift.common.daemon import Daemon
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
 from swift.common.wsgi import ConfigString
@@ -650,3 +651,12 @@ class ContainerSync(Daemon):
 
     def select_http_proxy(self):
         return choice(self.http_proxies) if self.http_proxies else None
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerSync, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/updater.py

@@ -32,8 +32,8 @@ from swift.common.exceptions import ConnectionTimeout, LockTimeout
 from swift.common.ring import Ring
 from swift.common.utils import get_logger, config_true_value, \
     dump_recon_cache, majority_size, Timestamp, EventletRateLimiter, \
-    eventlet_monkey_patch, node_to_string
-from swift.common.daemon import Daemon
+    eventlet_monkey_patch, node_to_string, parse_options
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
 from swift.common.recon import RECON_CONTAINER_FILE, DEFAULT_RECON_CACHE_PATH
 
@@ -158,6 +158,7 @@ class ContainerUpdater(Daemon):
                     pid2filename[pid] = tmpfilename
                 else:
                     signal.signal(signal.SIGTERM, signal.SIG_DFL)
+                    os.environ.pop('NOTIFY_SOCKET', None)
                     eventlet_monkey_patch()
                     self.no_changes = 0
                     self.successes = 0
@@ -356,3 +357,12 @@ class ContainerUpdater(Daemon):
                 return HTTP_INTERNAL_SERVER_ERROR
             finally:
                 conn.close()
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerUpdater, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/auditor.py

@@ -18,20 +18,22 @@ import os
 import sys
 import time
 import signal
+from optparse import OptionParser
 from os.path import basename, dirname, join
 from random import shuffle
 from contextlib import closing
 from eventlet import Timeout
 
 from swift.obj import diskfile, replicator
-from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist,\
+from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
     DiskFileDeleted, DiskFileExpired, QuarantineRequest
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.storage_policy import POLICIES
 from swift.common.utils import (
     config_auto_int_value, dump_recon_cache, get_logger, list_from_csv,
     listdir, load_pkg_resource, parse_prefixed_conf, EventletRateLimiter,
-    readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter)
+    readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter,
+    parse_options)
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 
 
@@ -368,6 +370,7 @@ class ObjectAuditor(Daemon):
             return pid
         else:
             signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            os.environ.pop('NOTIFY_SOCKET', None)
             if zero_byte_fps:
                 kwargs['zero_byte_fps'] = self.conf_zero_byte_fps
                 if sleep_between_zbf_scanner:
@@ -536,3 +539,17 @@ class WatcherWrapper(object):
         except (Exception, Timeout):
             self.logger.exception('Error ending watcher')
             self.watcher_in_error = True
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-z', '--zero_byte_fps',
+                      help='Audit only zero byte files at specified files/sec')
+    parser.add_option('-d', '--devices',
+                      help='Audit only given devices. Comma-separated list')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectAuditor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()