swift 2.33.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/account/auditor.py

@@ -16,7 +16,9 @@
 
 from swift.account.backend import AccountBroker
 from swift.common.exceptions import InvalidAccountInfo
+from swift.common.daemon import run_daemon
 from swift.common.db_auditor import DatabaseAuditor
+from swift.common.utils import parse_options
 
 
 class AccountAuditor(DatabaseAuditor):
@@ -45,3 +47,12 @@ class AccountAuditor(DatabaseAuditor):
                 'does not match the sum of %(key)s across policies (%(sum)s)'
                 % {'key': key, 'account': info.get('account'),
                    'total': info[key], 'sum': policy_totals[key]})
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountAuditor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/reaper.py

@@ -27,6 +27,7 @@ import six
 import swift.common.db
 from swift.account.backend import AccountBroker, DATADIR
 from swift.common.constraints import check_drive
+from swift.common.daemon import run_daemon
 from swift.common.direct_client import direct_delete_container, \
     direct_delete_object, direct_get_container
 from swift.common.exceptions import ClientException
@@ -34,7 +35,7 @@ from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.ring import Ring
 from swift.common.ring.utils import is_local_device
 from swift.common.utils import get_logger, whataremyips, config_true_value, \
-    Timestamp, md5, node_to_string
+    Timestamp, md5, node_to_string, parse_options
 from swift.common.daemon import Daemon
 from swift.common.storage_policy import POLICIES, PolicyError
 
@@ -525,3 +526,12 @@ class AccountReaper(Daemon):
             else:
                 self.stats_objects_possibly_remaining += 1
                 self.logger.increment('objects_possibly_remaining')
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountReaper, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/replicator.py

@@ -13,8 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
+
 from swift.account.backend import AccountBroker, DATADIR
 from swift.common import db_replicator
+from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 
 class AccountReplicator(db_replicator.Replicator):
@@ -22,3 +26,21 @@ class AccountReplicator(db_replicator.Replicator):
     brokerclass = AccountBroker
     datadir = DATADIR
     default_port = 6202
+
+
+def main():
+    parser = optparse.OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help=('Replicate only given devices. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    parser.add_option('-p', '--partitions',
+                      help=('Replicate only given partitions. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(AccountReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/server.py

@@ -15,6 +15,7 @@
 
 import json
 import os
+import sys
 import time
 import traceback
 
@@ -30,7 +31,7 @@ from swift.common.request_helpers import get_param, \
 from swift.common.utils import get_logger, hash_path, public, \
     Timestamp, storage_directory, config_true_value, \
     timing_stats, replication, get_log_line, \
-    config_fallocate_value, fs_has_free_space
+    config_fallocate_value, fs_has_free_space, parse_options
 from swift.common.constraints import valid_timestamp, check_utf8, \
     check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common import constraints
@@ -43,6 +44,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
     HTTPPreconditionFailed, HTTPConflict, Request, \
     HTTPInsufficientStorage, HTTPException, wsgi_to_str
 from swift.common.request_helpers import is_sys_or_user_meta
+from swift.common.wsgi import run_wsgi
 
 
 def get_account_name_and_placement(req):
@@ -339,3 +341,12 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return AccountController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'account-server', **options))
+
+
+if __name__ == '__main__':
+    main()

swift-2.33.0.data/scripts/swift-account-audit → swift/cli/account_audit.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -359,7 +359,7 @@ class Auditor(object):
             _print_stat("MD5 Mismatch", self.object_checksum_mismatch)
 
 
-if __name__ == '__main__':
+def main():
     try:
         optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
     except getopt.GetoptError as err:
@@ -384,3 +384,7 @@ if __name__ == '__main__':
         auditor.audit(*split_path(path, 1, 3, True))
     auditor.wait()
     auditor.print_stats()
+
+
+if __name__ == '__main__':
+    main()

swift-2.33.0.data/scripts/swift-config → swift/cli/config.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

swift-2.33.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -81,7 +81,7 @@ def report(success):
     stdout.flush()
 
 
-if __name__ == '__main__':
+def main():
     global begun, created, item_type, next_report, need_to_create, retries_done
     patcher.monkey_patch()
     try:
@@ -281,3 +281,7 @@ Usage: %%prog [options] [conf_file]
             print('\r\x1B[KTotal object coverage is now %.2f%%.' %
                   ((float(obj_coverage) / object_ring.partition_count * 100)))
         stdout.flush()
+
+
+if __name__ == '__main__':
+    main()

swift-2.33.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -147,7 +147,12 @@ def get_errors(error_re, log_file_pattern, minutes, logger,
                 log_time = datetime.datetime.strptime(
                     log_time_string, '%Y %b %d %H:%M:%S')
             except ValueError:
-                continue
+                # Some versions use ISO timestamps instead
+                try:
+                    log_time = datetime.datetime.strptime(
+                        line[0:19], '%Y-%m-%dT%H:%M:%S')
+                except ValueError:
+                    continue
             if log_time > end_time:
                 for err in error_re:
                     for device in err.findall(line):
@@ -174,7 +179,7 @@ def comment_fstab(mount_point):
     os.rename('/etc/fstab.new', '/etc/fstab')
 
 
-if __name__ == '__main__':
+def main():
     c = ConfigParser()
     try:
         conf_path = sys.argv[1]
@@ -257,3 +262,7 @@ if __name__ == '__main__':
     elif os.path.isdir("/run/systemd/system"):
         logger.debug("fstab updated, calling systemctl daemon-reload")
         subprocess.call(["/usr/bin/systemctl", "daemon-reload"])
+
+
+if __name__ == '__main__':
+    main()

swift-2.33.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,7 +25,7 @@ from swift.cli.info import (parse_get_node_args, print_item_locations,
                             InfoSystemExit)
 
 
-if __name__ == '__main__':
+def main():
 
     usage = '''
     Shows the nodes responsible for the item specified.
@@ -74,3 +74,7 @@ if __name__ == '__main__':
         print_item_locations(ring, ring_name, *args, **vars(options))
     except InfoSystemExit:
         sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

swift/cli/info.py

@@ -10,15 +10,21 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+
 from __future__ import print_function
+import codecs
 import itertools
 import json
+from optparse import OptionParser
 import os
 import sqlite3
+import sys
 from collections import defaultdict
 
+import six
 from six.moves import urllib
 
+from swift.common.exceptions import LockTimeout
 from swift.common.utils import hash_path, storage_directory, \
     Timestamp, is_valid_ipv6
 from swift.common.ring import Ring
@@ -29,10 +35,10 @@ from swift.account.backend import AccountBroker, DATADIR as ABDATADIR
 from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
 from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
     extract_policy
-from swift.common.storage_policy import POLICIES
+from swift.common.storage_policy import POLICIES, reload_storage_policies
 from swift.common.swob import wsgi_to_str
 from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
-from swift.common.utils import md5
+from swift.common.utils import md5, set_swift_dir
 
 
 class InfoSystemExit(Exception):
@@ -713,3 +719,98 @@ def print_item_locations(ring, ring_name=None, account=None, container=None,
         print('Object   \t%s\n\n' % urllib.parse.quote(obj))
     print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
                          policy_index=policy_index)
+
+
+def obj_main():
+    if not six.PY2:
+        # Make stdout able to write escaped bytes
+        sys.stdout = codecs.getwriter("utf-8")(
+            sys.stdout.detach(), errors='surrogateescape')
+
+    parser = OptionParser('%prog [options] OBJECT_FILE')
+    parser.add_option(
+        '-n', '--no-check-etag', default=True,
+        action="store_false", dest="check_etag",
+        help="Don't verify file contents against stored etag")
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift', dest='swift_dir',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-P', '--policy-name', dest='policy_name',
+        help="Specify storage policy name")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    if set_swift_dir(options.swift_dir):
+        reload_storage_policies()
+
+    try:
+        print_obj(*args, **vars(options))
+    except InfoSystemExit:
+        sys.exit(1)
+
+
+def run_print_info(db_type, args, opts):
+    try:
+        print_info(db_type, *args, **opts)
+    except InfoSystemExit:
+        sys.exit(1)
+    except (sqlite3.OperationalError, LockTimeout) as e:
+        if not opts.get('stale_reads_ok'):
+            opts['stale_reads_ok'] = True
+            print('Warning: Possibly Stale Data')
+            run_print_info(db_type, args, opts)
+            sys.exit(2)
+        else:
+            print('%s info failed: %s' % (db_type.title(), e))
+            sys.exit(1)
+
+
+def container_main():
+    parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-v', '--verbose', default=False, action="store_true",
+        help="Show all shard ranges. By default, only the number of shard "
+             "ranges is displayed if there are many shards.")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    run_print_info('container', args, vars(options))
+
+
+def account_main():
+    parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    run_print_info('account', args, vars(options))

swift-2.33.0.data/scripts/swift-oldies → swift/cli/oldies.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import optparse
 import subprocess
 import sys
 
 
-if __name__ == '__main__':
+def main():
     parser = optparse.OptionParser(usage='''%prog [options]
 
 Lists old Swift processes.
@@ -88,3 +87,7 @@ Lists old Swift processes.
         for hours, pid, args in listing:
             print('%*s %*s %s' % (hours_len, hours, pid_len,
                                   pid, args[:args_len]))
+
+
+if __name__ == '__main__':
+    main()

swift-2.33.0.data/scripts/swift-orphans → swift/cli/orphans.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -22,7 +22,8 @@ import sys
 
 from swift.common.manager import RUN_DIR
 
-if __name__ == '__main__':
+
+def main():
     parser = optparse.OptionParser(usage='''%prog [options]
 
 Lists and optionally kills orphaned Swift processes. This is done by scanning
@@ -130,3 +131,7 @@ Example (sends SIGTERM to all orphaned Swift processes older than two hours):
         for hours, pid, args in listing:
             os.kill(int(pid), signum)
         print('Done.')
+
+
+if __name__ == '__main__':
+    main()

swift/cli/recon_cron.py

@@ -18,18 +18,18 @@ import time
 from eventlet import Timeout
 
 from swift.common.utils import get_logger, dump_recon_cache, readconf, \
-    lock_path
+    lock_path, listdir
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj.diskfile import ASYNCDIR_BASE
 
 
 def get_async_count(device_dir):
     async_count = 0
-    for i in os.listdir(device_dir):
+    for i in listdir(device_dir):
         device = os.path.join(device_dir, i)
         if not os.path.isdir(device):
             continue
-        for asyncdir in os.listdir(device):
+        for asyncdir in listdir(device):
             # skip stuff like "accounts", "containers", etc.
             if not (asyncdir == ASYNCDIR_BASE or
                     asyncdir.startswith(ASYNCDIR_BASE + '-')):
@@ -37,10 +37,10 @@ def get_async_count(device_dir):
             async_pending = os.path.join(device, asyncdir)
 
             if os.path.isdir(async_pending):
-                for entry in os.listdir(async_pending):
+                for entry in listdir(async_pending):
                     if os.path.isdir(os.path.join(async_pending, entry)):
                         async_hdir = os.path.join(async_pending, entry)
-                        async_count += len(os.listdir(async_hdir))
+                        async_count += len(listdir(async_hdir))
     return async_count
 
 

swift-2.33.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,12 +11,10 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import print_function
 import sys
 from optparse import OptionParser
 
 import eventlet.debug
-eventlet.debug.hub_exceptions(True)
 
 from swift.common.ring import Ring
 from swift.common.utils import split_path
@@ -50,6 +48,7 @@ parser.add_option('-f', '--force', action='store_true',
 
 
 def main():
+    eventlet.debug.hub_exceptions(True)
     options, args = parser.parse_args()
     try:
         policy_index, path, timestamp = args

swift/cli/relinker.py

@@ -743,7 +743,7 @@ def auto_or_int(value):
     return config_auto_int_value(value, default='auto')
 
 
-def main(args):
+def main(args=None):
     parser = argparse.ArgumentParser(
         description='Relink and cleanup objects to increase partition power')
     parser.add_argument('action', choices=['relink', 'cleanup'])

swift/cli/ringbuilder.py

@@ -22,9 +22,11 @@ from itertools import islice
 from operator import itemgetter
 from os import mkdir
 from os.path import basename, abspath, dirname, exists, join as pathjoin
+import sys
 from sys import argv as sys_argv, exit, stderr, stdout
 from textwrap import wrap
 from time import time
+import traceback
 from datetime import timedelta
 import optparse
 import math
@@ -1698,3 +1700,25 @@ def main(arguments=None):
             exit(2)
     else:
         getattr(Commands, command, Commands.unknown)()
+
+
+def error_handling_main():
+    # We exit code 1 on WARNING statuses, 2 on ERROR. This means we need
+    # to handle any uncaught exceptions by printing the usual backtrace,
+    # but then exiting 2 (not 1 as is usual for a python
+    # exception).
+
+    # We *don't* want to do this in main(), however, because we don't want to
+    # pollute the test environment or cause a bunch of test churn to mock out
+    # sys.excepthook
+
+    def exit_with_status_two(tp, val, tb):
+        traceback.print_exception(tp, val, tb)
+        exit(2)
+
+    sys.excepthook = exit_with_status_two
+    main()
+
+
+if __name__ == '__main__':
+    error_handling_main()

swift/common/db.py

@@ -138,7 +138,8 @@ class GreenDBConnection(sqlite3.Connection):
             timeout = BROKER_TIMEOUT
         self.timeout = timeout
         self.db_file = database
-        super(GreenDBConnection, self).__init__(database, 0, *args, **kwargs)
+        super(GreenDBConnection, self).__init__(
+            database, timeout=0, *args, **kwargs)
 
     def cursor(self, cls=None):
         if cls is None:

swift/common/db_auditor.py

@@ -96,7 +96,7 @@ class DatabaseAuditor(Daemon):
         time.sleep(random() * self.interval)
         while True:
             self.logger.info(
-                'Begin {} audit pass.'.format(self.server_type))
+                'Begin %s audit pass.', self.server_type)
             begin = time.time()
             try:
                 reported = self._one_audit_pass(reported)
@@ -116,7 +116,7 @@ class DatabaseAuditor(Daemon):
     def run_once(self, *args, **kwargs):
         """Run the database audit once."""
         self.logger.info(
-            'Begin {} audit "once" mode'.format(self.server_type))
+            'Begin %s audit "once" mode', self.server_type)
         begin = reported = time.time()
         self._one_audit_pass(reported)
         elapsed = time.time() - begin

swift/common/db_replicator.py

@@ -240,6 +240,12 @@ class Replicator(Daemon):
         self.handoffs_only = config_true_value(conf.get('handoffs_only', 'no'))
         self.handoff_delete = config_auto_int_value(
             conf.get('handoff_delete', 'auto'), 0)
+        if self.handoff_delete >= self.ring.replica_count:
+            self.logger.warning(
+                'handoff_delete=%d is too high to have an effect on a ring '
+                'with replica count %d. Disabling.',
+                self.handoff_delete, self.ring.replica_count)
+            self.handoff_delete = 0
 
     def _zero_stats(self):
         """Zero out the stats."""

swift/common/exceptions.py

@@ -243,6 +243,18 @@ class QuarantineRequest(SwiftException):
     pass
 
 
+class MemcacheConnectionError(Exception):
+    pass
+
+
+class MemcacheIncrNotFoundError(MemcacheConnectionError):
+    pass
+
+
+class MemcachePoolTimeout(Timeout):
+    pass
+
+
 class ClientException(Exception):
 
     def __init__(self, msg, http_scheme='', http_host='', http_port='',