swift 2.33.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/common/wsgi.py

@@ -198,9 +198,14 @@ def get_socket(conf):
             sock = listen(bind_addr, backlog=int(conf.get('backlog', 4096)),
                           family=address_family)
             if 'cert_file' in conf:
+                if six.PY2:
+                    context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+                else:
+                    context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+                context.verify_mode = ssl.CERT_NONE
+                context.load_cert_chain(conf['cert_file'], conf['key_file'])
                 warn_ssl = True
-                sock = ssl.wrap_socket(sock, certfile=conf['cert_file'],
-                                       keyfile=conf['key_file'])
+                sock = context.wrap_socket(sock, server_side=True)
         except socket.error as err:
             if err.args[0] != errno.EADDRINUSE:
                 raise
@@ -460,7 +465,10 @@ def run_server(conf, logger, sock, global_conf=None, ready_callback=None,
     except socket.error as err:
         if err.errno != errno.EINVAL:
             raise
-    pool.waitall()
+    finally:
+        pool.waitall()
+        if hasattr(app._pipeline_final_app, 'watchdog'):
+            app._pipeline_final_app.watchdog.kill()
 
 
 class StrategyBase(object):

swift/container/auditor.py

@@ -15,7 +15,9 @@
 
 
 from swift.container.backend import ContainerBroker
+from swift.common.daemon import run_daemon
 from swift.common.db_auditor import DatabaseAuditor
+from swift.common.utils import parse_options
 
 
 class ContainerAuditor(DatabaseAuditor):
@@ -26,3 +28,12 @@ class ContainerAuditor(DatabaseAuditor):
 
     def _audit(self, job, broker):
         return None
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerAuditor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/backend.py

@@ -941,16 +941,16 @@ class ContainerBroker(DatabaseBroker):
                 self._do_get_info_query(conn)
 
     def _get_alternate_object_stats(self):
-        state = self.get_db_state()
-        if state == SHARDING:
+        db_state = self.get_db_state()
+        if db_state == SHARDING:
             other_info = self.get_brokers()[0]._get_info()
             stats = {'object_count': other_info['object_count'],
                      'bytes_used': other_info['bytes_used']}
-        elif state == SHARDED and self.is_root_container():
+        elif db_state == SHARDED and self.is_root_container():
             stats = self.get_shard_usage()
         else:
             stats = {}
-        return state, stats
+        return db_state, stats
 
     def get_info(self):
         """
@@ -2144,10 +2144,10 @@ class ContainerBroker(DatabaseBroker):
             self.logger.warning("Container '%s' cannot be set to sharding "
                                 "state: missing epoch", self.path)
             return False
-        state = self.get_db_state()
-        if not state == UNSHARDED:
+        db_state = self.get_db_state()
+        if not db_state == UNSHARDED:
             self.logger.warning("Container '%s' cannot be set to sharding "
-                                "state while in %s state", self.path, state)
+                                "state while in %s state", self.path, db_state)
             return False
 
         info = self.get_info()
@@ -2225,11 +2225,11 @@ class ContainerBroker(DatabaseBroker):
         :return: True if the retiring DB was successfully unlinked, False
             otherwise.
         """
-        state = self.get_db_state()
-        if not state == SHARDING:
+        db_state = self.get_db_state()
+        if not db_state == SHARDING:
             self.logger.warning("Container %r cannot be set to sharded "
                                 "state while in %s state",
-                                self.path, state)
+                                self.path, db_state)
             return False
 
         self.reload_db_files()

swift/container/reconciler.py

@@ -22,7 +22,7 @@ from eventlet import GreenPile, GreenPool, Timeout
 import six
 
 from swift.common import constraints
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.direct_client import (
     direct_head_container, direct_delete_container_object,
     direct_put_container_object, ClientException)
@@ -31,7 +31,7 @@ from swift.common.request_helpers import MISPLACED_OBJECTS_ACCOUNT, \
     USE_REPLICATION_NETWORK_HEADER
 from swift.common.utils import get_logger, split_path, majority_size, \
     FileLikeIter, Timestamp, last_modified_date_to_timestamp, \
-    LRUCache, decode_timestamps, hash_path
+    LRUCache, decode_timestamps, hash_path, parse_options
 from swift.common.storage_policy import POLICIES
 
 MISPLACED_OBJECTS_CONTAINER_DIVISOR = 3600  # 1 hour
@@ -860,3 +860,12 @@ class ContainerReconciler(Daemon):
             self.stats = defaultdict(int)
             self.logger.info('sleeping between intervals (%ss)', self.interval)
             time.sleep(self.interval)
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerReconciler, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/replicator.py

@@ -17,6 +17,7 @@ import os
 import json
 from collections import defaultdict
 from eventlet import Timeout
+import optparse
 from random import choice
 
 from swift.container.sync_store import ContainerSyncStore
@@ -26,10 +27,12 @@ from swift.container.reconciler import (
     MISPLACED_OBJECTS_ACCOUNT, incorrect_policy_index,
     get_reconciler_container_name, get_row_to_q_entry_translator)
 from swift.common import db_replicator
+from swift.common.daemon import run_daemon
 from swift.common.storage_policy import POLICIES
 from swift.common.swob import HTTPOk, HTTPAccepted
 from swift.common.http import is_success
-from swift.common.utils import Timestamp, majority_size, get_db_files
+from swift.common.utils import Timestamp, majority_size, get_db_files, \
+    parse_options
 
 
 def check_merge_own_shard_range(shards, broker, logger, source):
@@ -440,3 +443,21 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):
     def get_shard_ranges(self, broker, args):
         return HTTPOk(headers={'Content-Type': 'application/json'},
                       body=json.dumps(broker.get_all_shard_range_data()))
+
+
+def main():
+    parser = optparse.OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help=('Replicate only given devices. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    parser.add_option('-p', '--partitions',
+                      help=('Replicate only given partitions. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ContainerReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/server.py

@@ -15,6 +15,7 @@
 
 import json
 import os
+import sys
 import time
 import traceback
 
@@ -38,7 +39,7 @@ from swift.common.utils import get_logger, hash_path, public, \
     config_true_value, timing_stats, replication, \
     override_bytes_from_content_type, get_log_line, \
     config_fallocate_value, fs_has_free_space, list_from_csv, \
-    ShardRange
+    ShardRange, parse_options
 from swift.common.constraints import valid_timestamp, check_utf8, \
     check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.bufferedhttp import http_connect
@@ -53,6 +54,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPConflict, \
     HTTPPreconditionFailed, HTTPMethodNotAllowed, Request, Response, \
     HTTPInsufficientStorage, HTTPException, HTTPMovedPermanently, \
     wsgi_to_str, str_to_wsgi
+from swift.common.wsgi import run_wsgi
 
 
 def gen_resp_headers(info, is_deleted=False):
@@ -1048,3 +1050,12 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return ContainerController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'container-server', **options))
+
+
+if __name__ == '__main__':
+    main()

swift/container/sharder.py

@@ -17,6 +17,7 @@ import errno
 import json
 import logging
 import operator
+from optparse import OptionParser
 import time
 from collections import defaultdict
 from operator import itemgetter
@@ -32,6 +33,7 @@ from swift.common import internal_client
 from swift.common.constraints import check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.direct_client import (direct_put_container,
                                         DirectClientException)
+from swift.common.daemon import run_daemon
 from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.ring.utils import is_local_device
 from swift.common.swob import str_to_wsgi
@@ -39,7 +41,7 @@ from swift.common.utils import get_logger, config_true_value, \
     dump_recon_cache, whataremyips, Timestamp, ShardRange, GreenAsyncPile, \
     config_positive_int_value, quorum_size, parse_override_options, \
     Everything, config_auto_int_value, ShardRangeList, config_percent_value, \
-    node_to_string
+    node_to_string, parse_options
 from swift.container.backend import ContainerBroker, \
     RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \
     SHARD_UPDATE_STATES, sift_shard_ranges, SHARD_UPDATE_STAT_STATES
@@ -1817,12 +1819,12 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
     def _make_misplaced_object_bounds(self, broker):
         bounds = []
-        state = broker.get_db_state()
-        if state == SHARDED:
+        db_state = broker.get_db_state()
+        if db_state == SHARDED:
             # Anything in the object table is treated as a misplaced object.
             bounds.append(('', ''))
 
-        if not bounds and state == SHARDING:
+        if not bounds and db_state == SHARDING:
             # Objects outside of this container's own range are misplaced.
             # Objects in already cleaved shard ranges are also misplaced.
             cleave_context = CleavingContext.load(broker)
@@ -2342,9 +2344,9 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
     def _process_broker(self, broker, node, part):
         broker.get_info()  # make sure account/container are populated
-        state = broker.get_db_state()
+        db_state = broker.get_db_state()
         is_deleted = broker.is_deleted()
-        self.debug(broker, 'Starting processing, state %s%s', state,
+        self.debug(broker, 'Starting processing, state %s%s', db_state,
                    ' (deleted)' if is_deleted else '')
 
         if not self._audit_container(broker):
@@ -2358,7 +2360,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
         is_leader = node['index'] == 0 and self.auto_shard and not is_deleted
 
-        if state in (UNSHARDED, COLLAPSED):
+        if db_state in (UNSHARDED, COLLAPSED):
             if is_leader and broker.is_root_container():
                 # bootstrap sharding of root container
                 own_shard_range = broker.get_own_shard_range()
@@ -2374,7 +2376,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                     # or manually triggered cleaving.
                     db_start_ts = time.time()
                     if broker.set_sharding_state():
-                        state = SHARDING
+                        db_state = SHARDING
                         self.info(broker, 'Kick off container cleaving, '
                                           'own shard range in state %r',
                                   own_shard_range.state_text)
@@ -2382,14 +2384,14 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                         'sharder.sharding.set_state', db_start_ts)
                 elif is_leader:
                     if broker.set_sharding_state():
-                        state = SHARDING
+                        db_state = SHARDING
                 else:
                     self.debug(broker,
                                'Own shard range in state %r but no shard '
                                'ranges and not leader; remaining unsharded',
                                own_shard_range.state_text)
 
-        if state == SHARDING:
+        if db_state == SHARDING:
             cleave_start_ts = time.time()
             if is_leader:
                 num_found = self._find_shard_ranges(broker)
@@ -2410,7 +2412,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
 
             if cleave_complete:
                 if self._complete_sharding(broker):
-                    state = SHARDED
+                    db_state = SHARDED
                     self._increment_stat('visited', 'completed', statsd=True)
                     self.info(broker, 'Completed cleaving, DB set to sharded '
                                       'state')
@@ -2422,7 +2424,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                                       'sharding state')
 
         if not broker.is_deleted():
-            if state == SHARDED and broker.is_root_container():
+            if db_state == SHARDED and broker.is_root_container():
                 # look for shrink stats
                 send_start_ts = time.time()
                 self._identify_shrinking_candidate(broker, node)
@@ -2574,3 +2576,25 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
             elapsed = time.time() - begin
             self.logger.info(
                 'Container sharder "once" mode completed: %.02fs', elapsed)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help='Shard containers only on given devices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-p', '--partitions',
+                      help='Shard containers only in given partitions. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('--no-auto-shard', action='store_false',
+                      dest='auto_shard', default=None,
+                      help='Disable auto-sharding. Overrides the auto_shard '
+                           'value in the config file.')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ContainerSharder, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/sync.py

@@ -29,6 +29,7 @@ from swift.common.db import DatabaseConnectionError
 from swift.container.backend import ContainerBroker
 from swift.container.sync_store import ContainerSyncStore
 from swift.common.container_sync_realms import ContainerSyncRealms
+from swift.common.daemon import run_daemon
 from swift.common.internal_client import (
     delete_object, put_object, head_object,
     InternalClient, UnexpectedResponse)
@@ -39,7 +40,7 @@ from swift.common.swob import normalize_etag
 from swift.common.utils import (
     clean_content_type, config_true_value,
     FileLikeIter, get_logger, hash_path, quote, validate_sync_to,
-    whataremyips, Timestamp, decode_timestamps)
+    whataremyips, Timestamp, decode_timestamps, parse_options)
 from swift.common.daemon import Daemon
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
 from swift.common.wsgi import ConfigString
@@ -650,3 +651,12 @@ class ContainerSync(Daemon):
 
     def select_http_proxy(self):
         return choice(self.http_proxies) if self.http_proxies else None
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerSync, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/updater.py

@@ -32,8 +32,8 @@ from swift.common.exceptions import ConnectionTimeout, LockTimeout
 from swift.common.ring import Ring
 from swift.common.utils import get_logger, config_true_value, \
     dump_recon_cache, majority_size, Timestamp, EventletRateLimiter, \
-    eventlet_monkey_patch, node_to_string
-from swift.common.daemon import Daemon
+    eventlet_monkey_patch, node_to_string, parse_options
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.http import is_success, HTTP_INTERNAL_SERVER_ERROR
 from swift.common.recon import RECON_CONTAINER_FILE, DEFAULT_RECON_CACHE_PATH
 
@@ -357,3 +357,12 @@ class ContainerUpdater(Daemon):
                 return HTTP_INTERNAL_SERVER_ERROR
             finally:
                 conn.close()
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerUpdater, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/auditor.py

@@ -18,6 +18,7 @@ import os
 import sys
 import time
 import signal
+from optparse import OptionParser
 from os.path import basename, dirname, join
 from random import shuffle
 from contextlib import closing
@@ -26,12 +27,13 @@ from eventlet import Timeout
 from swift.obj import diskfile, replicator
 from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
     DiskFileDeleted, DiskFileExpired, QuarantineRequest
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.storage_policy import POLICIES
 from swift.common.utils import (
     config_auto_int_value, dump_recon_cache, get_logger, list_from_csv,
     listdir, load_pkg_resource, parse_prefixed_conf, EventletRateLimiter,
-    readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter)
+    readconf, round_robin_iter, unlink_paths_older_than, PrefixLoggerAdapter,
+    parse_options)
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 
 
@@ -537,3 +539,17 @@ class WatcherWrapper(object):
         except (Exception, Timeout):
             self.logger.exception('Error ending watcher')
             self.watcher_in_error = True
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-z', '--zero_byte_fps',
+                      help='Audit only zero byte files at specified files/sec')
+    parser.add_option('-d', '--devices',
+                      help='Audit only given devices. Comma-separated list')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectAuditor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/diskfile.py

@@ -600,7 +600,8 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
             try:
                 suffixes = listdir(part_path)
             except OSError as e:
-                if e.errno not in (errno.ENOTDIR, errno.ENODATA):
+                if e.errno not in (errno.ENOTDIR, errno.ENODATA,
+                                   errno.EUCLEAN):
                     raise
                 continue
             for asuffix in suffixes:
@@ -608,7 +609,8 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
                 try:
                     hashes = listdir(suff_path)
                 except OSError as e:
-                    if e.errno not in (errno.ENOTDIR, errno.ENODATA):
+                    if e.errno not in (errno.ENOTDIR, errno.ENODATA,
+                                       errno.EUCLEAN):
                         raise
                     continue
                 for hsh in hashes:
@@ -1214,7 +1216,7 @@ class BaseDiskFileManager(object):
                         'it is not a directory', {'hsh_path': hsh_path,
                                                   'quar_path': quar_path})
                     continue
-                elif err.errno == errno.ENODATA:
+                elif err.errno in (errno.ENODATA, errno.EUCLEAN):
                     try:
                         # We've seen cases where bad sectors lead to ENODATA
                         # here; use a similar hack as above
@@ -1569,7 +1571,7 @@ class BaseDiskFileManager(object):
                     'it is not a directory', {'object_path': object_path,
                                               'quar_path': quar_path})
                 raise DiskFileNotExist()
-            elif err.errno == errno.ENODATA:
+            elif err.errno in (errno.ENODATA, errno.EUCLEAN):
                 try:
                     # We've seen cases where bad sectors lead to ENODATA here;
                     # use a similar hack as above
@@ -2610,7 +2612,7 @@ class BaseDiskFile(object):
                     # want this one file and not its parent.
                     os.path.join(self._datadir, "made-up-filename"),
                     "Expected directory, found file at %s" % self._datadir)
-            elif err.errno == errno.ENODATA:
+            elif err.errno in (errno.ENODATA, errno.EUCLEAN):
                 try:
                     # We've seen cases where bad sectors lead to ENODATA here
                     raise self._quarantine(
@@ -2640,7 +2642,7 @@ class BaseDiskFile(object):
             self._fp = self._construct_from_data_file(
                 current_time=current_time, modernize=modernize, **file_info)
         except IOError as e:
-            if e.errno == errno.ENODATA:
+            if e.errno in (errno.ENODATA, errno.EUCLEAN):
                 raise self._quarantine(
                     file_info['data_file'],
                     "Failed to open %s: %s" % (file_info['data_file'], e))