swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift-2.23.2.data/scripts/swift-oldies → swift/cli/oldies.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,13 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import optparse
 import subprocess
 import sys
 
 
-if __name__ == '__main__':
+def main():
     parser = optparse.OptionParser(usage='''%prog [options]
 
 Lists old Swift processes.
@@ -26,6 +25,9 @@ Lists old Swift processes.
     parser.add_option('-a', '--age', dest='hours', type='int', default=720,
                       help='look for processes at least HOURS old; '
                       'default: 720 (30 days)')
+    parser.add_option('-p', '--pids', action='store_true',
+                      help='only print the pids found; for example, to pipe '
+                      'to xargs kill')
     (options, args) = parser.parse_args()
 
     listing = []
@@ -68,15 +70,24 @@ Lists old Swift processes.
     if not listing:
         sys.exit()
 
-    hours_len = len('Hours')
-    pid_len = len('PID')
-    args_len = len('Command')
-    for hours, pid, args in listing:
-        hours_len = max(hours_len, len(hours))
-        pid_len = max(pid_len, len(pid))
-        args_len = max(args_len, len(args))
-    args_len = min(args_len, 78 - hours_len - pid_len)
+    if options.pids:
+        for hours, pid, args in listing:
+            print(pid)
+    else:
+        hours_len = len('Hours')
+        pid_len = len('PID')
+        args_len = len('Command')
+        for hours, pid, args in listing:
+            hours_len = max(hours_len, len(hours))
+            pid_len = max(pid_len, len(pid))
+            args_len = max(args_len, len(args))
+        args_len = min(args_len, 78 - hours_len - pid_len)
+
+        print('%*s %*s %s' % (hours_len, 'Hours', pid_len, 'PID', 'Command'))
+        for hours, pid, args in listing:
+            print('%*s %*s %s' % (hours_len, hours, pid_len,
+                                  pid, args[:args_len]))
 
-    print('%*s %*s %s' % (hours_len, 'Hours', pid_len, 'PID', 'Command'))
-    for hours, pid, args in listing:
-        print('%*s %*s %s' % (hours_len, hours, pid_len, pid, args[:args_len]))
+
+if __name__ == '__main__':
+    main()

swift-2.23.2.data/scripts/swift-orphans → swift/cli/orphans.py

@@ -1,4 +1,4 @@
-#!python
+#!/usr/bin/env python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import optparse
 import os
 import re
@@ -22,7 +21,8 @@ import sys
 
 from swift.common.manager import RUN_DIR
 
-if __name__ == '__main__':
+
+def main():
     parser = optparse.OptionParser(usage='''%prog [options]
 
 Lists and optionally kills orphaned Swift processes. This is done by scanning
@@ -130,3 +130,7 @@ Example (sends SIGTERM to all orphaned Swift processes older than two hours):
         for hours, pid, args in listing:
             os.kill(int(pid), signum)
         print('Done.')
+
+
+if __name__ == '__main__':
+    main()

swift/cli/recon.py

@@ -15,11 +15,9 @@
     cmdline utility to perform cluster reconnaissance
 """
 
-from __future__ import print_function
 
 from eventlet.green import socket
-from six import string_types
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 from swift.common.utils import (
     SWIFT_CONF_FILE, md5_hash_for_file, set_swift_dir)
@@ -30,13 +28,9 @@ import json
 import optparse
 import time
 import sys
-import six
 import os
 
-if six.PY3:
-    from eventlet.green.urllib import request as urllib2
-else:
-    from eventlet.green import urllib2
+from eventlet.green.urllib import request as urllib_request
 
 
 def seconds2timeunit(seconds):
@@ -86,19 +80,19 @@ class Scout(object):
         """
         url = base_url + recon_type
         try:
-            body = urllib2.urlopen(url, timeout=self.timeout).read()
-            if six.PY3 and isinstance(body, six.binary_type):
+            body = urllib_request.urlopen(url, timeout=self.timeout).read()
+            if isinstance(body, bytes):
                 body = body.decode('utf8')
             content = json.loads(body)
             if self.verbose:
                 print("-> %s: %s" % (url, content))
             status = 200
-        except urllib2.HTTPError as err:
+        except urllib_request.HTTPError as err:
             if not self.suppress_errors or self.verbose:
                 print("-> %s: %s" % (url, err))
             content = err
             status = err.code
-        except (urllib2.URLError, socket.timeout) as err:
+        except (urllib_request.URLError, socket.timeout) as err:
             if not self.suppress_errors or self.verbose:
                 print("-> %s: %s" % (url, err))
             content = err
@@ -128,19 +122,19 @@ class Scout(object):
         """
         try:
             url = "http://%s:%s/" % (host[0], host[1])
-            req = urllib2.Request(url)
+            req = urllib_request.Request(url)
             req.get_method = lambda: 'OPTIONS'
-            conn = urllib2.urlopen(req)
+            conn = urllib_request.urlopen(req)
             header = conn.info().get('Server')
             server_header = header.split('/')
             content = server_header[0]
             status = 200
-        except urllib2.HTTPError as err:
+        except urllib_request.HTTPError as err:
             if not self.suppress_errors or self.verbose:
                 print("-> %s: %s" % (url, err))
             content = err
             status = err.code
-        except (urllib2.URLError, socket.timeout) as err:
+        except (urllib_request.URLError, socket.timeout) as err:
             if not self.suppress_errors or self.verbose:
                 print("-> %s: %s" % (url, err))
             content = err
@@ -221,7 +215,7 @@ class SwiftRecon(object):
         Compare ring md5sum's with those on remote host
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         :param swift_dir: The local directory with the ring files.
         """
         matches = 0
@@ -275,7 +269,7 @@ class SwiftRecon(object):
         Compare swift.conf md5sum with that on remote hosts
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         :param printfn: function to print text; defaults to print()
         """
         matches = 0
@@ -307,7 +301,7 @@ class SwiftRecon(object):
         Obtain and print async pending statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         scan = {}
         recon = Scout("async", self.verbose, self.suppress_errors,
@@ -329,7 +323,7 @@ class SwiftRecon(object):
         Obtain and print drive audit error statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)]
         """
         scan = {}
         recon = Scout("driveaudit", self.verbose, self.suppress_errors,
@@ -351,7 +345,7 @@ class SwiftRecon(object):
         Check for and print unmounted drives
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         unmounted = {}
         errors = {}
@@ -384,7 +378,7 @@ class SwiftRecon(object):
         Check for server types on the ring
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         errors = {}
         recon = Scout("server_type_check", self.verbose, self.suppress_errors,
@@ -407,7 +401,7 @@ class SwiftRecon(object):
         Obtain and print expirer statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         stats = {'object_expiration_pass': [], 'expired_last_pass': []}
         recon = Scout("expirer/%s" % self.server_type, self.verbose,
@@ -431,22 +425,91 @@ class SwiftRecon(object):
                 print("[%s] - No hosts returned valid data." % k)
         print("=" * 79)
 
+    def _calculate_least_and_most_recent(self, url_time_data):
+        """calulate and print the least and most recent urls
+
+        Given a list of url and time tuples calulate the most and least
+        recent timings and print it out.
+        :param url_time_data: list of url and time tuples: [(url, time_), ..]
+        """
+        least_recent_time = 9999999999
+        least_recent_url = None
+        most_recent_time = 0
+        most_recent_url = None
+
+        for url, last in url_time_data:
+            if last is None:
+                continue
+            if last < least_recent_time:
+                least_recent_time = last
+                least_recent_url = url
+            if last > most_recent_time:
+                most_recent_time = last
+                most_recent_url = url
+
+        if least_recent_url is not None:
+            host = urlparse(least_recent_url).netloc
+            if not least_recent_time:
+                print('Oldest completion was NEVER by %s.' % host)
+            else:
+                elapsed = time.time() - least_recent_time
+                elapsed, elapsed_unit = seconds2timeunit(elapsed)
+                print('Oldest completion was %s (%d %s ago) by %s.' % (
+                    self._ptime(least_recent_time),
+                    elapsed, elapsed_unit, host))
+        if most_recent_url is not None:
+            host = urlparse(most_recent_url).netloc
+            elapsed = time.time() - most_recent_time
+            elapsed, elapsed_unit = seconds2timeunit(elapsed)
+            print('Most recent completion was %s (%d %s ago) by %s.' % (
+                self._ptime(most_recent_time),
+                elapsed, elapsed_unit, host))
+
+    def reconstruction_check(self, hosts):
+        """
+        Obtain and print reconstructon statistics
+
+        :param hosts: set of hosts to check. in the format of:
+            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+        """
+        stats = []
+        last_stats = []
+        recon = Scout("reconstruction/%s" % self.server_type, self.verbose,
+                      self.suppress_errors, self.timeout)
+        print("[%s] Checking on reconstructors" % self._ptime())
+        for url, response, status, ts_start, ts_end in self.pool.imap(
+                recon.scout, hosts):
+            if status == 200:
+                stats.append(response.get('object_reconstruction_time'))
+                last = response.get('object_reconstruction_last', 0)
+                last_stats.append((url, last))
+        if stats:
+            computed = self._gen_stats(stats,
+                                       name='object_reconstruction_time')
+            if computed['reported'] > 0:
+                self._print_stats(computed)
+            else:
+                print("[object_reconstruction_time] - No hosts returned "
+                      "valid data.")
+        else:
+            print("[object_reconstruction_time] - No hosts returned "
+                  "valid data.")
+        self._calculate_least_and_most_recent(last_stats)
+        print("=" * 79)
+
     def replication_check(self, hosts):
         """
         Obtain and print replication statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         stats = {'replication_time': [], 'failure': [], 'success': [],
                  'attempted': []}
+        last_stats = []
         recon = Scout("replication/%s" % self.server_type, self.verbose,
                       self.suppress_errors, self.timeout)
         print("[%s] Checking on replication" % self._ptime())
-        least_recent_time = 9999999999
-        least_recent_url = None
-        most_recent_time = 0
-        most_recent_url = None
         for url, response, status, ts_start, ts_end in self.pool.imap(
                 recon.scout, hosts):
             if status == 200:
@@ -459,14 +522,7 @@ class SwiftRecon(object):
                         stats[stat_key].append(repl_stats.get(stat_key))
                 last = response.get('replication_last',
                                     response.get('object_replication_last', 0))
-                if last is None:
-                    continue
-                if last < least_recent_time:
-                    least_recent_time = last
-                    least_recent_url = url
-                if last > most_recent_time:
-                    most_recent_time = last
-                    most_recent_url = url
+                last_stats.append((url, last))
         for k in stats:
             if stats[k]:
                 if k != 'replication_time':
@@ -480,23 +536,7 @@ class SwiftRecon(object):
                     print("[%s] - No hosts returned valid data." % k)
             else:
                 print("[%s] - No hosts returned valid data." % k)
-        if least_recent_url is not None:
-            host = urlparse(least_recent_url).netloc
-            if not least_recent_time:
-                print('Oldest completion was NEVER by %s.' % host)
-            else:
-                elapsed = time.time() - least_recent_time
-                elapsed, elapsed_unit = seconds2timeunit(elapsed)
-                print('Oldest completion was %s (%d %s ago) by %s.' % (
-                    self._ptime(least_recent_time),
-                    elapsed, elapsed_unit, host))
-        if most_recent_url is not None:
-            host = urlparse(most_recent_url).netloc
-            elapsed = time.time() - most_recent_time
-            elapsed, elapsed_unit = seconds2timeunit(elapsed)
-            print('Most recent completion was %s (%d %s ago) by %s.' % (
-                self._ptime(most_recent_time),
-                elapsed, elapsed_unit, host))
+        self._calculate_least_and_most_recent(last_stats)
         print("=" * 79)
 
     def updater_check(self, hosts):
@@ -504,7 +544,7 @@ class SwiftRecon(object):
         Obtain and print updater statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         stats = []
         recon = Scout("updater/%s" % self.server_type, self.verbose,
@@ -531,7 +571,7 @@ class SwiftRecon(object):
         Obtain and print obj auditor statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         scan = {}
         adone = '%s_auditor_pass_completed' % self.server_type
@@ -603,7 +643,7 @@ class SwiftRecon(object):
         Obtain and print obj auditor statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         all_scan = {}
         zbf_scan = {}
@@ -674,12 +714,75 @@ class SwiftRecon(object):
             print("[ZBF_auditor] - No hosts returned valid data.")
         print("=" * 79)
 
+    def sharding_check(self, hosts):
+        """
+        Obtain and print sharding statistics
+
+        :param hosts: set of hosts to check. in the format of:
+            set([('127.0.0.1', 6221), ('127.0.0.2', 6231)])
+        """
+        stats = {'sharding_time': [],
+                 'attempted': [], 'failure': [], 'success': []}
+        recon = Scout("sharding", self.verbose,
+                      self.suppress_errors, self.timeout)
+        print("[%s] Checking on sharders" % self._ptime())
+        least_recent_time = 9999999999
+        least_recent_url = None
+        most_recent_time = 0
+        most_recent_url = None
+        for url, response, status, ts_start, ts_end in self.pool.imap(
+                recon.scout, hosts):
+            if status == 200:
+                stats['sharding_time'].append(response.get('sharding_time', 0))
+                shard_stats = response.get('sharding_stats')
+                if shard_stats:
+                    # Sharding has a ton more stats, like "no_change".
+                    # Not sure if we need them at all, or maybe for -v.
+                    for stat_key in ['attempted', 'failure', 'success']:
+                        stats[stat_key].append(shard_stats.get(stat_key))
+                last = response.get('sharding_last', 0)
+                if last is None:
+                    continue
+                if last < least_recent_time:
+                    least_recent_time = last
+                    least_recent_url = url
+                if last > most_recent_time:
+                    most_recent_time = last
+                    most_recent_url = url
+        for k in stats:
+            if stats[k]:
+                computed = self._gen_stats(stats[k], name=k)
+                if computed['reported'] > 0:
+                    self._print_stats(computed)
+                else:
+                    print("[%s] - No hosts returned valid data." % k)
+            else:
+                print("[%s] - No hosts returned valid data." % k)
+        if least_recent_url is not None:
+            host = urlparse(least_recent_url).netloc
+            if not least_recent_time:
+                print('Oldest completion was NEVER by %s.' % host)
+            else:
+                elapsed = time.time() - least_recent_time
+                elapsed, elapsed_unit = seconds2timeunit(elapsed)
+                print('Oldest completion was %s (%d %s ago) by %s.' % (
+                    self._ptime(least_recent_time),
+                    elapsed, elapsed_unit, host))
+        if most_recent_url is not None:
+            host = urlparse(most_recent_url).netloc
+            elapsed = time.time() - most_recent_time
+            elapsed, elapsed_unit = seconds2timeunit(elapsed)
+            print('Most recent completion was %s (%d %s ago) by %s.' % (
+                self._ptime(most_recent_time),
+                elapsed, elapsed_unit, host))
+        print("=" * 79)
+
     def load_check(self, hosts):
         """
         Obtain and print load average statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         load1 = {}
         load5 = {}
@@ -708,7 +811,7 @@ class SwiftRecon(object):
         Obtain and print quarantine statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         objq = {}
         conq = {}
@@ -742,7 +845,7 @@ class SwiftRecon(object):
         Obtain and print /proc/net/sockstat statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         inuse4 = {}
         mem = {}
@@ -776,7 +879,7 @@ class SwiftRecon(object):
         Obtain and print disk usage statistics
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         stats = {}
         highs = []
@@ -788,9 +891,15 @@ class SwiftRecon(object):
         low_percents = [(None, 100)] * lowest
         recon = Scout("diskusage", self.verbose, self.suppress_errors,
                       self.timeout)
+        # We want to only query each host once, but we don't care
+        # which of the available ports we use. So we filter hosts by
+        # constructing a host->port dictionary, since the dict
+        # constructor ensures each key is unique, thus each host
+        # appears only once in filtered_hosts.
+        filtered_hosts = set(dict(hosts).items())
         print("[%s] Checking disk usage now" % self._ptime())
         for url, response, status, ts_start, ts_end in self.pool.imap(
-                recon.scout, hosts):
+                recon.scout, filtered_hosts):
             if status == 200:
                 hostusage = []
                 for entry in response:
@@ -875,7 +984,7 @@ class SwiftRecon(object):
         Check a time synchronization of hosts with current time
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         :param jitter: Maximal allowed time jitter
         """
 
@@ -914,7 +1023,7 @@ class SwiftRecon(object):
         Check OS Swift version of hosts. Inform if differs.
 
         :param hosts: set of hosts to check. in the format of:
-            set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
+            set([('127.0.0.1', 6220), ('127.0.0.2', 6230)])
         """
         versions = set()
         errors = 0
@@ -959,7 +1068,7 @@ class SwiftRecon(object):
             ring_names = [p.ring_name for p in POLICIES if (
                 p.name == policy or not policy or (
                     policy.isdigit() and int(policy) == int(p) or
-                    (isinstance(policy, string_types)
+                    (isinstance(policy, str)
                      and policy in p.aliases)))]
         else:
             ring_names = [self.server_type]
@@ -973,7 +1082,7 @@ class SwiftRecon(object):
         print("=" * 79)
         usage = '''
         usage: %prog <server_type> [<server_type> [<server_type>]]
-        [-v] [--suppress] [-a] [-r] [-u] [-d]
+        [-v] [--suppress] [-a] [-r] [-u] [-d] [-R]
         [-l] [-T] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
         [--human-readable]
 
@@ -992,12 +1101,16 @@ class SwiftRecon(object):
                         help="Get async stats")
         args.add_option('--replication', '-r', action="store_true",
                         help="Get replication stats")
+        args.add_option('--reconstruction', '-R', action="store_true",
+                        help="Get reconstruction stats")
         args.add_option('--auditor', action="store_true",
                         help="Get auditor stats")
         args.add_option('--updater', action="store_true",
                         help="Get updater stats")
         args.add_option('--expirer', action="store_true",
                         help="Get expirer stats")
+        args.add_option('--sharding', action="store_true",
+                        help="Get sharding stats")
         args.add_option('--unmounted', '-u', action="store_true",
                         help="Check cluster for unmounted devices")
         args.add_option('--diskusage', '-d', action="store_true",
@@ -1029,7 +1142,7 @@ class SwiftRecon(object):
                         help='Also show the lowest COUNT entries in rank \
                         order.')
         args.add_option('--all', action="store_true",
-                        help="Perform all checks. Equal to \t\t\t-arudlqT "
+                        help="Perform all checks. Equal to \t\t\t-arRudlqT "
                         "--md5 --sockstat --auditor --updater --expirer "
                         "--driveaudit --validate-servers --swift-versions")
         args.add_option('--region', type="int",
@@ -1087,9 +1200,11 @@ class SwiftRecon(object):
                     self.object_auditor_check(hosts)
                     self.updater_check(hosts)
                     self.expirer_check(hosts)
+                    self.reconstruction_check(hosts)
                 elif self.server_type == 'container':
                     self.auditor_check(hosts)
                     self.updater_check(hosts)
+                    self.sharding_check(hosts)
                 elif self.server_type == 'account':
                     self.auditor_check(hosts)
                 self.replication_check(hosts)
@@ -1133,9 +1248,23 @@ class SwiftRecon(object):
                     if self.server_type == 'object':
                         self.expirer_check(hosts)
                     else:
-                        print("Error: Can't check expired on non object "
+                        print("Error: Can't check expirer on non object "
                               "servers.")
                         print("=" * 79)
+                if options.sharding:
+                    if self.server_type == 'container':
+                        self.sharding_check(hosts)
+                    else:
+                        print("Error: Can't check sharding on non container "
+                              "servers.")
+                        print("=" * 79)
+                if options.reconstruction:
+                    if self.server_type == 'object':
+                        self.reconstruction_check(hosts)
+                    else:
+                        print("Error: Can't check reconstruction stats on "
+                              "non object servers.")
+                        print("=" * 79)
                 if options.validate_servers:
                     self.server_type_check(hosts)
                 if options.loadstats: