swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/account/auditor.py

@@ -16,7 +16,9 @@
 
 from swift.account.backend import AccountBroker
 from swift.common.exceptions import InvalidAccountInfo
+from swift.common.daemon import run_daemon
 from swift.common.db_auditor import DatabaseAuditor
+from swift.common.utils import parse_options
 
 
 class AccountAuditor(DatabaseAuditor):
@@ -45,3 +47,12 @@ class AccountAuditor(DatabaseAuditor):
                 'does not match the sum of %(key)s across policies (%(sum)s)'
                 % {'key': key, 'account': info.get('account'),
                    'total': info[key], 'sum': policy_totals[key]})
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountAuditor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/reaper.py

@@ -27,6 +27,7 @@ import six
 import swift.common.db
 from swift.account.backend import AccountBroker, DATADIR
 from swift.common.constraints import check_drive
+from swift.common.daemon import run_daemon
 from swift.common.direct_client import direct_delete_container, \
     direct_delete_object, direct_get_container
 from swift.common.exceptions import ClientException
@@ -34,7 +35,7 @@ from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.ring import Ring
 from swift.common.ring.utils import is_local_device
 from swift.common.utils import get_logger, whataremyips, config_true_value, \
-    Timestamp, md5, node_to_string
+    Timestamp, md5, node_to_string, parse_options
 from swift.common.daemon import Daemon
 from swift.common.storage_policy import POLICIES, PolicyError
 
@@ -525,3 +526,12 @@ class AccountReaper(Daemon):
             else:
                 self.stats_objects_possibly_remaining += 1
                 self.logger.increment('objects_possibly_remaining')
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountReaper, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/replicator.py

@@ -13,8 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
+
 from swift.account.backend import AccountBroker, DATADIR
 from swift.common import db_replicator
+from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 
 class AccountReplicator(db_replicator.Replicator):
@@ -22,3 +26,21 @@ class AccountReplicator(db_replicator.Replicator):
     brokerclass = AccountBroker
     datadir = DATADIR
     default_port = 6202
+
+
+def main():
+    parser = optparse.OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help=('Replicate only given devices. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    parser.add_option('-p', '--partitions',
+                      help=('Replicate only given partitions. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(AccountReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/server.py

@@ -15,6 +15,7 @@
 
 import json
 import os
+import sys
 import time
 import traceback
 
@@ -30,7 +31,7 @@ from swift.common.request_helpers import get_param, \
 from swift.common.utils import get_logger, hash_path, public, \
     Timestamp, storage_directory, config_true_value, \
     timing_stats, replication, get_log_line, \
-    config_fallocate_value, fs_has_free_space
+    config_fallocate_value, fs_has_free_space, parse_options
 from swift.common.constraints import valid_timestamp, check_utf8, \
     check_drive, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common import constraints
@@ -43,6 +44,7 @@ from swift.common.swob import HTTPAccepted, HTTPBadRequest, \
     HTTPPreconditionFailed, HTTPConflict, Request, \
     HTTPInsufficientStorage, HTTPException, wsgi_to_str
 from swift.common.request_helpers import is_sys_or_user_meta
+from swift.common.wsgi import run_wsgi
 
 
 def get_account_name_and_placement(req):
@@ -85,17 +87,7 @@ class AccountController(BaseStorageServer):
         self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                             self.mount_check,
                                             logger=self.logger)
-        if conf.get('auto_create_account_prefix'):
-            self.logger.warning('Option auto_create_account_prefix is '
-                                'deprecated. Configure '
-                                'auto_create_account_prefix under the '
-                                'swift-constraints section of '
-                                'swift.conf. This option will '
-                                'be ignored in a future release.')
-            self.auto_create_account_prefix = \
-                conf['auto_create_account_prefix']
-        else:
-            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+        self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
 
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))
@@ -349,3 +341,12 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return AccountController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'account-server', **options))
+
+
+if __name__ == '__main__':
+    main()

swift-2.32.0.data/scripts/swift-account-audit → swift/cli/account_audit.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -359,7 +359,7 @@ class Auditor(object):
             _print_stat("MD5 Mismatch", self.object_checksum_mismatch)
 
 
-if __name__ == '__main__':
+def main():
     try:
         optlist, args = getopt.getopt(sys.argv[1:], 'c:r:e:d')
     except getopt.GetoptError as err:
@@ -384,3 +384,7 @@ if __name__ == '__main__':
         auditor.audit(*split_path(path, 1, 3, True))
     auditor.wait()
     auditor.print_stats()
+
+
+if __name__ == '__main__':
+    main()

swift-2.32.0.data/scripts/swift-config → swift/cli/config.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

swift-2.32.0.data/scripts/swift-dispersion-populate → swift/cli/dispersion_populate.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -81,7 +81,7 @@ def report(success):
     stdout.flush()
 
 
-if __name__ == '__main__':
+def main():
     global begun, created, item_type, next_report, need_to_create, retries_done
     patcher.monkey_patch()
     try:
@@ -281,3 +281,7 @@ Usage: %%prog [options] [conf_file]
             print('\r\x1B[KTotal object coverage is now %.2f%%.' %
                   ((float(obj_coverage) / object_ring.partition_count * 100)))
         stdout.flush()
+
+
+if __name__ == '__main__':
+    main()

swift-2.32.0.data/scripts/swift-drive-audit → swift/cli/drive_audit.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -147,7 +147,12 @@ def get_errors(error_re, log_file_pattern, minutes, logger,
                 log_time = datetime.datetime.strptime(
                     log_time_string, '%Y %b %d %H:%M:%S')
             except ValueError:
-                continue
+                # Some versions use ISO timestamps instead
+                try:
+                    log_time = datetime.datetime.strptime(
+                        line[0:19], '%Y-%m-%dT%H:%M:%S')
+                except ValueError:
+                    continue
             if log_time > end_time:
                 for err in error_re:
                     for device in err.findall(line):
@@ -174,7 +179,7 @@ def comment_fstab(mount_point):
     os.rename('/etc/fstab.new', '/etc/fstab')
 
 
-if __name__ == '__main__':
+def main():
     c = ConfigParser()
     try:
         conf_path = sys.argv[1]
@@ -257,3 +262,7 @@ if __name__ == '__main__':
     elif os.path.isdir("/run/systemd/system"):
         logger.debug("fstab updated, calling systemctl daemon-reload")
         subprocess.call(["/usr/bin/systemctl", "daemon-reload"])
+
+
+if __name__ == '__main__':
+    main()

swift-2.32.0.data/scripts/swift-get-nodes → swift/cli/get_nodes.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,7 +25,7 @@ from swift.cli.info import (parse_get_node_args, print_item_locations,
                             InfoSystemExit)
 
 
-if __name__ == '__main__':
+def main():
 
     usage = '''
     Shows the nodes responsible for the item specified.
@@ -74,3 +74,7 @@ if __name__ == '__main__':
         print_item_locations(ring, ring_name, *args, **vars(options))
     except InfoSystemExit:
         sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

swift/cli/info.py

@@ -10,15 +10,21 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+
 from __future__ import print_function
+import codecs
 import itertools
 import json
+from optparse import OptionParser
 import os
 import sqlite3
+import sys
 from collections import defaultdict
 
+import six
 from six.moves import urllib
 
+from swift.common.exceptions import LockTimeout
 from swift.common.utils import hash_path, storage_directory, \
     Timestamp, is_valid_ipv6
 from swift.common.ring import Ring
@@ -29,10 +35,10 @@ from swift.account.backend import AccountBroker, DATADIR as ABDATADIR
 from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
 from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
     extract_policy
-from swift.common.storage_policy import POLICIES
+from swift.common.storage_policy import POLICIES, reload_storage_policies
 from swift.common.swob import wsgi_to_str
 from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
-from swift.common.utils import md5
+from swift.common.utils import md5, set_swift_dir
 
 
 class InfoSystemExit(Exception):
@@ -198,6 +204,28 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
           'real value is set in the config file on each storage node.')
 
 
+def get_max_len_sync_item(syncs, item, title):
+    def map_func(element):
+        return str(element[item])
+    return max(list(map(len, map(map_func, syncs))) + [len(title)])
+
+
+def print_db_syncs(incoming, syncs):
+    max_sync_point_len = get_max_len_sync_item(syncs, 'sync_point',
+                                               "Sync Point")
+    max_remote_len = get_max_len_sync_item(syncs, 'remote_id', "Remote ID")
+    print('%s Syncs:' % ('Incoming' if incoming else 'Outgoing'))
+    print('  %s\t%s\t%s' % ("Sync Point".ljust(max_sync_point_len),
+                            "Remote ID".ljust(max_remote_len),
+                            "Updated At"))
+    for sync in syncs:
+        print('  %s\t%s\t%s (%s)' % (
+            str(sync['sync_point']).ljust(max_sync_point_len),
+            sync['remote_id'].ljust(max_remote_len),
+            Timestamp(sync['updated_at']).isoformat,
+            sync['updated_at']))
+
+
 def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
                            verbose=False):
     """
@@ -439,7 +467,7 @@ def print_obj_metadata(metadata, drop_prefixes=False):
 
 
 def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
-               drop_prefixes=False, verbose=False):
+               drop_prefixes=False, verbose=False, sync=False):
     if db_type not in ('account', 'container'):
         print("Unrecognized DB type: internal error")
         raise InfoSystemExit()
@@ -473,6 +501,11 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
             info['shard_ranges'] = sranges
     print_db_info_metadata(
         db_type, info, broker.metadata, drop_prefixes, verbose)
+    if sync:
+        # Print incoming / outgoing sync tables.
+        for incoming in (True, False):
+            print_db_syncs(incoming, broker.get_syncs(incoming,
+                                                      include_timestamp=True))
     try:
         ring = Ring(swift_dir, ring_name=db_type)
     except Exception:
@@ -686,3 +719,98 @@ def print_item_locations(ring, ring_name=None, account=None, container=None,
         print('Object   \t%s\n\n' % urllib.parse.quote(obj))
     print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
                          policy_index=policy_index)
+
+
+def obj_main():
+    if not six.PY2:
+        # Make stdout able to write escaped bytes
+        sys.stdout = codecs.getwriter("utf-8")(
+            sys.stdout.detach(), errors='surrogateescape')
+
+    parser = OptionParser('%prog [options] OBJECT_FILE')
+    parser.add_option(
+        '-n', '--no-check-etag', default=True,
+        action="store_false", dest="check_etag",
+        help="Don't verify file contents against stored etag")
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift', dest='swift_dir',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-P', '--policy-name', dest='policy_name',
+        help="Specify storage policy name")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    if set_swift_dir(options.swift_dir):
+        reload_storage_policies()
+
+    try:
+        print_obj(*args, **vars(options))
+    except InfoSystemExit:
+        sys.exit(1)
+
+
+def run_print_info(db_type, args, opts):
+    try:
+        print_info(db_type, *args, **opts)
+    except InfoSystemExit:
+        sys.exit(1)
+    except (sqlite3.OperationalError, LockTimeout) as e:
+        if not opts.get('stale_reads_ok'):
+            opts['stale_reads_ok'] = True
+            print('Warning: Possibly Stale Data')
+            run_print_info(db_type, args, opts)
+            sys.exit(2)
+        else:
+            print('%s info failed: %s' % (db_type.title(), e))
+            sys.exit(1)
+
+
+def container_main():
+    parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-v', '--verbose', default=False, action="store_true",
+        help="Show all shard ranges. By default, only the number of shard "
+             "ranges is displayed if there are many shards.")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    run_print_info('container', args, vars(options))
+
+
+def account_main():
+    parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    run_print_info('account', args, vars(options))

swift-2.32.0.data/scripts/swift-oldies → swift/cli/oldies.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import optparse
 import subprocess
 import sys
 
 
-if __name__ == '__main__':
+def main():
     parser = optparse.OptionParser(usage='''%prog [options]
 
 Lists old Swift processes.
@@ -88,3 +87,7 @@ Lists old Swift processes.
         for hours, pid, args in listing:
             print('%*s %*s %s' % (hours_len, hours, pid_len,
                                   pid, args[:args_len]))
+
+
+if __name__ == '__main__':
+    main()

swift-2.32.0.data/scripts/swift-orphans → swift/cli/orphans.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -22,7 +22,8 @@ import sys
 
 from swift.common.manager import RUN_DIR
 
-if __name__ == '__main__':
+
+def main():
     parser = optparse.OptionParser(usage='''%prog [options]
 
 Lists and optionally kills orphaned Swift processes. This is done by scanning
@@ -130,3 +131,7 @@ Example (sends SIGTERM to all orphaned Swift processes older than two hours):
         for hours, pid, args in listing:
             os.kill(int(pid), signum)
         print('Done.')
+
+
+if __name__ == '__main__':
+    main()

swift-2.32.0.data/scripts/swift-recon-cron → swift/cli/recon_cron.py

@@ -1,4 +1,3 @@
-#!python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,10 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-swift-recon-cron.py
-"""
-
 import os
 import sys
 import time
@@ -23,18 +18,18 @@ import time
 from eventlet import Timeout
 
 from swift.common.utils import get_logger, dump_recon_cache, readconf, \
-    lock_path
+    lock_path, listdir
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj.diskfile import ASYNCDIR_BASE
 
 
-def get_async_count(device_dir, logger):
+def get_async_count(device_dir):
     async_count = 0
-    for i in os.listdir(device_dir):
+    for i in listdir(device_dir):
         device = os.path.join(device_dir, i)
         if not os.path.isdir(device):
             continue
-        for asyncdir in os.listdir(device):
+        for asyncdir in listdir(device):
             # skip stuff like "accounts", "containers", etc.
             if not (asyncdir == ASYNCDIR_BASE or
                     asyncdir.startswith(ASYNCDIR_BASE + '-')):
@@ -42,10 +37,10 @@ def get_async_count(device_dir, logger):
             async_pending = os.path.join(device, asyncdir)
 
             if os.path.isdir(async_pending):
-                for entry in os.listdir(async_pending):
+                for entry in listdir(async_pending):
                     if os.path.isdir(os.path.join(async_pending, entry)):
                         async_hdir = os.path.join(async_pending, entry)
-                        async_count += len(os.listdir(async_hdir))
+                        async_count += len(listdir(async_hdir))
     return async_count
 
 
@@ -55,7 +50,7 @@ def main():
     except Exception:
         print("Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1])
         print("ex: swift-recon-cron /etc/swift/object-server.conf")
-        sys.exit(1)
+        return 1
     conf = readconf(conf_path, 'filter:recon')
     device_dir = conf.get('devices', '/srv/node')
     recon_cache_path = conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH)
@@ -66,7 +61,7 @@ def main():
     logger = get_logger(conf, log_route='recon-cron')
     try:
         with lock_path(lock_dir):
-            asyncs = get_async_count(device_dir, logger)
+            asyncs = get_async_count(device_dir)
             dump_recon_cache({
                 'async_pending': asyncs,
                 'async_pending_last': time.time(),
@@ -75,8 +70,4 @@ def main():
         msg = 'Exception during recon-cron while accessing devices'
         logger.exception(msg)
         print('%s: %s' % (msg, err))
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()
+        return 1

swift-2.32.0.data/scripts/swift-reconciler-enqueue → swift/cli/reconciler_enqueue.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,12 +11,10 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import print_function
 import sys
 from optparse import OptionParser
 
 import eventlet.debug
-eventlet.debug.hub_exceptions(True)
 
 from swift.common.ring import Ring
 from swift.common.utils import split_path
@@ -50,6 +48,7 @@ parser.add_option('-f', '--force', action='store_true',
 
 
 def main():
+    eventlet.debug.hub_exceptions(True)
     options, args = parser.parse_args()
     try:
         policy_index, path, timestamp = args

swift/cli/relinker.py

@@ -743,7 +743,7 @@ def auto_or_int(value):
     return config_auto_int_value(value, default='auto')
 
 
-def main(args):
+def main(args=None):
     parser = argparse.ArgumentParser(
         description='Relink and cleanup objects to increase partition power')
     parser.add_argument('action', choices=['relink', 'cleanup'])