swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift-2.23.2.data/scripts/swift-drive-audit → swift/cli/drive_audit.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,13 +18,13 @@ import datetime
 import glob
 import locale
 import os
+import os.path
 import re
 import subprocess
 import sys
 
 
-import six
-from six.moves.configparser import ConfigParser
+from configparser import ConfigParser
 
 from swift.common.utils import backward, get_logger, dump_recon_cache, \
     config_true_value
@@ -32,6 +32,27 @@ from swift.common.utils import backward, get_logger, dump_recon_cache, \
 
 def get_devices(device_dir, logger):
     devices = []
+    majmin_devices = {}
+
+    # List /dev/block
+    # Using os.scandir on recent versions of python, else os.listdir
+    if 'scandir' in dir(os):
+        with os.scandir("/dev/block") as it:
+            for ent in it:
+                if ent.is_symlink():
+                    dev_name = os.path.basename(os.readlink(ent.path))
+                    majmin = os.path.basename(ent.path).split(':')
+                    majmin_devices[dev_name] = {'major': majmin[0],
+                                                'minor': majmin[1]}
+    else:
+        for ent in os.listdir("/dev/block"):
+            ent_path = os.path.join("/dev/block", ent)
+            if os.path.is_symlink(ent_path):
+                dev_name = os.path.basename(os.readlink(ent_path))
+                majmin = os.path.basename(ent_path).split(':')
+                majmin_devices[dev_name] = {'major': majmin[0],
+                                            'minor': majmin[1]}
+
     for line in open('/proc/mounts').readlines():
         data = line.strip().split()
         block_device = data[0]
@@ -40,15 +61,25 @@ def get_devices(device_dir, logger):
             device = {}
             device['mount_point'] = mount_point
             device['block_device'] = block_device
-            try:
-                device_num = os.stat(block_device).st_rdev
-            except OSError:
-                # If we can't stat the device, then something weird is going on
-                logger.error("Error: Could not stat %s!" %
-                             block_device)
-                continue
-            device['major'] = str(os.major(device_num))
-            device['minor'] = str(os.minor(device_num))
+            dev_name = os.path.basename(block_device)
+            if dev_name in majmin_devices:
+                # If symlink is in /dev/block
+                device['major'] = majmin_devices[dev_name]['major']
+                device['minor'] = majmin_devices[dev_name]['minor']
+            else:
+                # Else we try to stat block_device
+                try:
+                    device_num = os.stat(block_device).st_rdev
+                except OSError:
+                    # If we can't stat the device,
+                    # then something weird is going on
+                    logger.error(
+                        'Could not determine major:minor numbers for %s '
+                        '(mounted at %s)! Skipping...',
+                        block_device, mount_point)
+                    continue
+                device['major'] = str(os.major(device_num))
+                device['minor'] = str(os.minor(device_num))
             devices.append(device)
     for line in open('/proc/partitions').readlines()[2:]:
         major, minor, blocks, kernel_device = line.strip().split()
@@ -84,7 +115,7 @@ def get_errors(error_re, log_file_pattern, minutes, logger,
     # track of the year and month in case the year recently
     # ticked over
     year = now_time.year
-    prev_entry_month = now_time.month
+    prev_ent_month = now_time.strftime('%b')
     errors = {}
 
     reached_old_logs = False
@@ -96,8 +127,7 @@ def get_errors(error_re, log_file_pattern, minutes, logger,
             print("Unable to open " + path)
             sys.exit(1)
         for line in backward(f):
-            if not six.PY2:
-                line = line.decode(log_file_encoding, 'surrogateescape')
+            line = line.decode(log_file_encoding, 'surrogateescape')
             if '[    0.000000]' in line \
                 or 'KERNEL supported cpus:' in line \
                     or 'BIOS-provided physical RAM map:' in line:
@@ -106,16 +136,21 @@ def get_errors(error_re, log_file_pattern, minutes, logger,
                 break
             # Solves the problem with year change - kern.log does not
             # keep track of the year.
-            log_time_entry = line.split()[:3]
-            if log_time_entry[0] == 'Dec' and prev_entry_month == 'Jan':
+            log_time_ent = line.split()[:3]
+            if log_time_ent[0] == 'Dec' and prev_ent_month == 'Jan':
                 year -= 1
-            prev_entry_month = log_time_entry[0]
-            log_time_string = '%d %s' % (year, ' '.join(log_time_entry))
+            prev_ent_month = log_time_ent[0]
+            log_time_string = '%d %s' % (year, ' '.join(log_time_ent))
             try:
                 log_time = datetime.datetime.strptime(
                     log_time_string, '%Y %b %d %H:%M:%S')
             except ValueError:
-                continue
+                # Some versions use ISO timestamps instead
+                try:
+                    log_time = datetime.datetime.strptime(
+                        line[0:19], '%Y-%m-%dT%H:%M:%S')
+                except ValueError:
+                    continue
             if log_time > end_time:
                 for err in error_re:
                     for device in err.findall(line):
@@ -142,7 +177,7 @@ def comment_fstab(mount_point):
     os.rename('/etc/fstab.new', '/etc/fstab')
 
 
-if __name__ == '__main__':
+def main():
     c = ConfigParser()
     try:
         conf_path = sys.argv[1]
@@ -222,3 +257,10 @@ if __name__ == '__main__':
 
     if unmounts == 0:
         logger.info("No drives were unmounted")
+    elif os.path.isdir("/run/systemd/system"):
+        logger.debug("fstab updated, calling systemctl daemon-reload")
+        subprocess.call(["/usr/bin/systemctl", "daemon-reload"])
+
+
+if __name__ == '__main__':
+    main()

swift/cli/form_signature.py

@@ -15,9 +15,7 @@
 """
 Script for generating a form signature for use with FormPost middleware.
 """
-from __future__ import print_function
 import hmac
-import six
 from hashlib import sha1
 from os.path import basename
 from time import time
@@ -95,11 +93,9 @@ def main(argv):
         return 1
     data = '%s\n%s\n%s\n%s\n%s' % (path, redirect, max_file_size,
                                    max_file_count, expires)
-    if six.PY3:
-        data = data if isinstance(data, six.binary_type) else \
-            data.encode('utf8')
-        key = key if isinstance(key, six.binary_type) else \
-            key.encode('utf8')
+    data = data.encode('utf8')
+    key = key if isinstance(key, bytes) else \
+        key.encode('utf8')
     sig = hmac.new(key, data,
                    sha1).hexdigest()
     print('  Expires:', expires)

swift-2.23.2.data/scripts/swift-get-nodes → swift/cli/get_nodes.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Copyright (c) 2010-2012 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,7 +25,7 @@ from swift.cli.info import (parse_get_node_args, print_item_locations,
                             InfoSystemExit)
 
 
-if __name__ == '__main__':
+def main():
 
     usage = '''
     Shows the nodes responsible for the item specified.
@@ -52,6 +52,8 @@ if __name__ == '__main__':
                       help='Specify which policy to use')
     parser.add_option('-d', '--swift-dir', default='/etc/swift',
                       dest='swift_dir', help='Path to swift directory')
+    parser.add_option('-Q', '--quoted', action='store_true',
+                      help='Assume swift paths are quoted')
     options, args = parser.parse_args()
 
     if set_swift_dir(options.swift_dir):
@@ -72,3 +74,7 @@ if __name__ == '__main__':
         print_item_locations(ring, ring_name, *args, **vars(options))
     except InfoSystemExit:
         sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

swift/cli/info.py

@@ -10,15 +10,19 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from __future__ import print_function
+
+import codecs
 import itertools
 import json
+from optparse import OptionParser
 import os
 import sqlite3
-from hashlib import md5
+import sys
+from collections import defaultdict
 
-from six.moves import urllib
+import urllib
 
+from swift.common.exceptions import LockTimeout
 from swift.common.utils import hash_path, storage_directory, \
     Timestamp, is_valid_ipv6
 from swift.common.ring import Ring
@@ -29,8 +33,10 @@ from swift.account.backend import AccountBroker, DATADIR as ABDATADIR
 from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
 from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
     extract_policy
-from swift.common.storage_policy import POLICIES
+from swift.common.storage_policy import POLICIES, reload_storage_policies
+from swift.common.swob import wsgi_to_str
 from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
+from swift.common.utils import md5, set_swift_dir
 
 
 class InfoSystemExit(Exception):
@@ -57,6 +63,8 @@ def parse_get_node_args(options, args):
         else:
             raise InfoSystemExit('Ring file does not exist')
 
+    if options.quoted:
+        args = [urllib.parse.unquote(arg) for arg in args]
     if len(args) == 1:
         args = args[0].strip('/').split('/', 2)
 
@@ -95,6 +103,7 @@ def curl_head_command(ip, port, device, part, target, policy_index):
     if policy_index is not None:
         cmd += ' -H "%s: %s"' % ('X-Backend-Storage-Policy-Index',
                                  policy_index)
+    cmd += ' --path-as-is'
     return cmd
 
 
@@ -193,7 +202,30 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
           'real value is set in the config file on each storage node.')
 
 
-def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
+def get_max_len_sync_item(syncs, item, title):
+    def map_func(element):
+        return str(element[item])
+    return max(list(map(len, map(map_func, syncs))) + [len(title)])
+
+
+def print_db_syncs(incoming, syncs):
+    max_sync_point_len = get_max_len_sync_item(syncs, 'sync_point',
+                                               "Sync Point")
+    max_remote_len = get_max_len_sync_item(syncs, 'remote_id', "Remote ID")
+    print('%s Syncs:' % ('Incoming' if incoming else 'Outgoing'))
+    print('  %s\t%s\t%s' % ("Sync Point".ljust(max_sync_point_len),
+                            "Remote ID".ljust(max_remote_len),
+                            "Updated At"))
+    for sync in syncs:
+        print('  %s\t%s\t%s (%s)' % (
+            str(sync['sync_point']).ljust(max_sync_point_len),
+            sync['remote_id'].ljust(max_remote_len),
+            Timestamp(sync['updated_at']).isoformat,
+            sync['updated_at']))
+
+
+def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
+                           verbose=False):
     """
     print out data base info/metadata based on its type
 
@@ -227,6 +259,7 @@ def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
         if db_type == 'container':
             print('  Container: %s' % container)
 
+        print('  Deleted: %s' % info['is_deleted'])
         path_hash = hash_path(account, container)
         if db_type == 'container':
             print('  Container Hash: %s' % path_hash)
@@ -291,12 +324,16 @@ def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
             title = key.replace('_', '-').title()
             print('  %s: %s' % (title, value))
     if sys_metadata:
-        print('  System Metadata: %s' % sys_metadata)
+        print('  System Metadata:')
+        for key, value in sys_metadata.items():
+            print('    %s: %s' % (key, value))
     else:
         print('No system metadata found in db file')
 
     if user_metadata:
-        print('  User Metadata: %s' % user_metadata)
+        print('  User Metadata:')
+        for key, value in user_metadata.items():
+            print('    %s: %s' % (key, value))
     else:
         print('No user metadata found in db file')
 
@@ -306,20 +343,31 @@ def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False):
         print('  Type: %s' % shard_type)
         print('  State: %s' % info['db_state'])
     if info.get('shard_ranges'):
-        print('Shard Ranges (%d):' % len(info['shard_ranges']))
+        num_shards = len(info['shard_ranges'])
+        print('Shard Ranges (%d):' % num_shards)
+        count_by_state = defaultdict(int)
         for srange in info['shard_ranges']:
-            srange = dict(srange, state_text=srange.state_text)
-            print('  Name: %(name)s' % srange)
-            print('    lower: %(lower)r, upper: %(upper)r' % srange)
-            print('    Object Count: %(object_count)d, Bytes Used: '
-                  '%(bytes_used)d, State: %(state_text)s (%(state)d)'
-                  % srange)
-            print('    Created at: %s (%s)'
-                  % (Timestamp(srange['timestamp']).isoformat,
-                     srange['timestamp']))
-            print('    Meta Timestamp: %s (%s)'
-                  % (Timestamp(srange['meta_timestamp']).isoformat,
-                     srange['meta_timestamp']))
+            count_by_state[(srange.state, srange.state_text)] += 1
+        print('  States:')
+        for key_state, count in sorted(count_by_state.items()):
+            key, state = key_state
+            print('    %9s: %s' % (state, count))
+        if verbose:
+            for srange in info['shard_ranges']:
+                srange = dict(srange, state_text=srange.state_text)
+                print('  Name: %(name)s' % srange)
+                print('    lower: %(lower)r, upper: %(upper)r' % srange)
+                print('    Object Count: %(object_count)d, Bytes Used: '
+                      '%(bytes_used)d, State: %(state_text)s (%(state)d)'
+                      % srange)
+                print('    Created at: %s (%s)'
+                      % (Timestamp(srange['timestamp']).isoformat,
+                         srange['timestamp']))
+                print('    Meta Timestamp: %s (%s)'
+                      % (Timestamp(srange['meta_timestamp']).isoformat,
+                         srange['meta_timestamp']))
+        else:
+            print('(Use -v/--verbose to show more Shard Ranges details)')
 
 
 def print_obj_metadata(metadata, drop_prefixes=False):
@@ -417,7 +465,7 @@ def print_obj_metadata(metadata, drop_prefixes=False):
 
 
 def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
-               drop_prefixes=False):
+               drop_prefixes=False, verbose=False, sync=False):
     if db_type not in ('account', 'container'):
         print("Unrecognized DB type: internal error")
         raise InfoSystemExit()
@@ -442,13 +490,20 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
         raise
     account = info['account']
     container = None
+    info['is_deleted'] = broker.is_deleted()
     if db_type == 'container':
         container = info['container']
         info['is_root'] = broker.is_root_container()
         sranges = broker.get_shard_ranges()
         if sranges:
             info['shard_ranges'] = sranges
-    print_db_info_metadata(db_type, info, broker.metadata, drop_prefixes)
+    print_db_info_metadata(
+        db_type, info, broker.metadata, drop_prefixes, verbose)
+    if sync:
+        # Print incoming / outgoing sync tables.
+        for incoming in (True, False):
+            print_db_syncs(incoming, broker.get_syncs(incoming,
+                                                      include_timestamp=True))
     try:
         ring = Ring(swift_dir, ring_name=db_type)
     except Exception:
@@ -518,6 +573,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
         except EOFError:
             print("Invalid metadata")
             raise InfoSystemExit()
+        metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v)
+                    for k, v in metadata.items()}
 
         etag = metadata.pop('ETag', '')
         length = metadata.pop('Content-Length', '')
@@ -527,7 +584,7 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
         # Optional integrity check; it's useful, but slow.
         file_len = None
         if check_etag:
-            h = md5()
+            h = md5(usedforsecurity=False)
             file_len = 0
             while True:
                 data = fp.read(64 * 1024)
@@ -612,15 +669,15 @@ def print_item_locations(ring, ring_name=None, account=None, container=None,
         ring = POLICIES.get_object_ring(policy_index, swift_dir)
         ring_name = (POLICIES.get_by_name(policy_name)).ring_name
 
-    if account is None and (container is not None or obj is not None):
+    if (container or obj) and not account:
         print('No account specified')
         raise InfoSystemExit()
 
-    if container is None and obj is not None:
+    if obj and not container:
         print('No container specified')
         raise InfoSystemExit()
 
-    if account is None and part is None:
+    if not account and not part:
         print('No target specified')
         raise InfoSystemExit()
 
@@ -652,8 +709,105 @@ def print_item_locations(ring, ring_name=None, account=None, container=None,
                 print('Warning: account specified ' +
                       'but ring not named "account"')
 
-    print('\nAccount  \t%s' % account)
-    print('Container\t%s' % container)
-    print('Object   \t%s\n\n' % obj)
+    if account:
+        print('\nAccount  \t%s' % urllib.parse.quote(account))
+    if container:
+        print('Container\t%s' % urllib.parse.quote(container))
+    if obj:
+        print('Object   \t%s\n\n' % urllib.parse.quote(obj))
     print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
                          policy_index=policy_index)
+
+
+def obj_main():
+    # Make stdout able to write escaped bytes
+    sys.stdout = codecs.getwriter("utf-8")(
+        sys.stdout.detach(), errors='surrogateescape')
+
+    parser = OptionParser('%prog [options] OBJECT_FILE')
+    parser.add_option(
+        '-n', '--no-check-etag', default=True,
+        action="store_false", dest="check_etag",
+        help="Don't verify file contents against stored etag")
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift', dest='swift_dir',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-P', '--policy-name', dest='policy_name',
+        help="Specify storage policy name")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    if set_swift_dir(options.swift_dir):
+        reload_storage_policies()
+
+    try:
+        print_obj(*args, **vars(options))
+    except InfoSystemExit:
+        sys.exit(1)
+
+
+def run_print_info(db_type, args, opts):
+    try:
+        print_info(db_type, *args, **opts)
+    except InfoSystemExit:
+        sys.exit(1)
+    except (sqlite3.OperationalError, LockTimeout) as e:
+        if not opts.get('stale_reads_ok'):
+            opts['stale_reads_ok'] = True
+            print('Warning: Possibly Stale Data')
+            run_print_info(db_type, args, opts)
+            sys.exit(2)
+        else:
+            print('%s info failed: %s' % (db_type.title(), e))
+            sys.exit(1)
+
+
+def container_main():
+    parser = OptionParser('%prog [options] CONTAINER_DB_FILE')
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '-v', '--verbose', default=False, action="store_true",
+        help="Show all shard ranges. By default, only the number of shard "
+             "ranges is displayed if there are many shards.")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    run_print_info('container', args, vars(options))
+
+
+def account_main():
+    parser = OptionParser('%prog [options] ACCOUNT_DB_FILE')
+    parser.add_option(
+        '-d', '--swift-dir', default='/etc/swift',
+        help="Pass location of swift directory")
+    parser.add_option(
+        '--drop-prefixes', default=False, action="store_true",
+        help="When outputting metadata, drop the per-section common prefixes")
+    parser.add_option(
+        '--sync', '-s', default=False, action="store_true",
+        help="Output the contents of the incoming/outging sync tables")
+
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        sys.exit(parser.print_help())
+
+    run_print_info('account', args, vars(options))