swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/cli/reload.py

@@ -0,0 +1,135 @@
+# Copyright (c) 2022 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Safely reload WSGI servers while minimizing client downtime and errors by
+
+   * validating that the process is a Swift WSGI server manager,
+   * checking that the configuration file used is valid,
+   * sending the "seamless reload" signal, and
+   * waiting for the reload to complete.
+"""
+
+import argparse
+import errno
+import os
+import os.path
+import signal
+import socket
+import subprocess
+import sys
+
+from swift.common.utils import NotificationServer
+
+
+EXIT_BAD_PID = 2  # similar to argparse exiting 2 on an unknown arg
+EXIT_RELOAD_FAILED = 1
+EXIT_RELOAD_TIMEOUT = 128 + errno.ETIMEDOUT
+
+
+def validate_manager_pid(pid):
+    try:
+        with open('/proc/%d/cmdline' % pid, 'r') as fp:
+            cmd = fp.read().strip('\x00').split('\x00')
+        sid = os.getsid(pid)
+    except (IOError, OSError):
+        print("Failed to get process information for %s" % pid,
+              file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    scripts = [os.path.basename(c) for c in cmd
+               if '/bin/' in c and '/bin/python' not in c]
+
+    if len(scripts) != 1 or not scripts[0].startswith("swift-"):
+        print("Non-swift process: %r" % ' '.join(cmd), file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    if scripts[0] not in {"swift-proxy-server", "swift-account-server",
+                          "swift-container-server", "swift-object-server"}:
+        print("Process does not support config checks: %s" % scripts[0],
+              file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    if sid != pid:
+        print("Process appears to be a %s worker, not a manager. "
+              "Did you mean %s?" % (scripts[0], sid), file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    return cmd, scripts[0]
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument("pid", type=int,
+                        help="server PID which should be reloaded")
+    wait_group = parser.add_mutually_exclusive_group()
+    wait_group.add_argument("-t", "--timeout", type=float, default=300.0,
+                            help="max time to wait for reload to complete")
+    wait_group.add_argument("-w", "--no-wait",
+                            action="store_false", dest="wait",
+                            help="skip waiting for reload to complete")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="display more information as the process reloads")
+    args = parser.parse_args(args)
+
+    cmd, script = validate_manager_pid(args.pid)
+
+    if args.verbose:
+        print("Checking config for %s" % script)
+    try:
+        subprocess.check_call(cmd + ["--test-config"])
+    except subprocess.CalledProcessError:
+        print("Failed to validate config", file=sys.stderr)
+        exit(EXIT_RELOAD_FAILED)
+
+    if args.wait:
+        try:
+            with NotificationServer(args.pid, args.timeout) as notifications:
+                if args.verbose:
+                    print("Sending USR1 signal")
+                os.kill(args.pid, signal.SIGUSR1)
+
+                try:
+                    ready = False
+                    while not ready:
+                        data = notifications.receive()
+                        for data in data.split(b"\n"):
+                            if args.verbose:
+                                if data in (b"READY=1", b"RELOADING=1",
+                                            b"STOPPING=1"):
+                                    print("Process is %s" %
+                                          data.decode("ascii")[:-2])
+                                else:
+                                    print("Received notification %r" % data)
+
+                            if data == b"READY=1":
+                                ready = True
+                except socket.timeout:
+                    print("Timed out reloading %s" % script, file=sys.stderr)
+                    exit(EXIT_RELOAD_TIMEOUT)
+        except OSError as e:
+            print("Could not bind notification socket: %s" % e,
+                  file=sys.stderr)
+            exit(EXIT_RELOAD_FAILED)
+    else:  # --no-wait
+        if args.verbose:
+            print("Sending USR1 signal")
+        os.kill(args.pid, signal.SIGUSR1)
+
+    print("Reloaded %s" % script)
+
+
+if __name__ == "__main__":
+    main()

swift/cli/ringbuilder.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 import logging
 
 from collections import defaultdict
@@ -22,16 +21,15 @@ from itertools import islice
 from operator import itemgetter
 from os import mkdir
 from os.path import basename, abspath, dirname, exists, join as pathjoin
+import sys
 from sys import argv as sys_argv, exit, stderr, stdout
 from textwrap import wrap
 from time import time
+import traceback
 from datetime import timedelta
 import optparse
 import math
 
-from six.moves import zip as izip
-from six.moves import input
-
 from swift.common import exceptions
 from swift.common.ring import RingBuilder, Ring, RingData
 from swift.common.ring.builder import MAX_BALANCE
@@ -154,8 +152,8 @@ def _parse_add_values(argvish):
             print(Commands.add.__doc__.strip())
             exit(EXIT_ERROR)
 
-        devs_and_weights = izip(islice(args, 0, len(args), 2),
-                                islice(args, 1, len(args), 2))
+        devs_and_weights = zip(islice(args, 0, len(args), 2),
+                               islice(args, 1, len(args), 2))
 
         for devstr, weightstr in devs_and_weights:
             dev_dict = parse_add_value(devstr)
@@ -194,7 +192,11 @@ def check_devs(devs, input_question, opts, abort_msg):
         print('Matched more than one device:')
         for dev in devs:
             print('    %s' % format_device(dev))
-        if not opts.yes and input(input_question) != 'y':
+        try:
+            abort = not opts.yes and input(input_question) != 'y'
+        except (EOFError, KeyboardInterrupt):
+            abort = True
+        if abort:
             print(abort_msg)
             exit(EXIT_ERROR)
 
@@ -212,6 +214,32 @@ def _set_weight_values(devs, weight, opts):
                                        dev['weight']))
 
 
+def _set_region_values(devs, region, opts):
+
+    input_question = 'Are you sure you want to update the region for these ' \
+                     '%s devices? (y/N) ' % len(devs)
+    abort_msg = 'Aborting device modifications'
+    check_devs(devs, input_question, opts, abort_msg)
+
+    for dev in devs:
+        builder.set_dev_region(dev['id'], region)
+        print('%s region set to %s' % (format_device(dev),
+                                       dev['region']))
+
+
+def _set_zone_values(devs, zone, opts):
+
+    input_question = 'Are you sure you want to update the zone for these ' \
+                     '%s devices? (y/N) ' % len(devs)
+    abort_msg = 'Aborting device modifications'
+    check_devs(devs, input_question, opts, abort_msg)
+
+    for dev in devs:
+        builder.set_dev_zone(dev['id'], zone)
+        print('%s zone set to %s' % (format_device(dev),
+                                     dev['zone']))
+
+
 def _parse_set_weight_values(argvish):
 
     new_cmd_format, opts, args = validate_args(argvish)
@@ -225,8 +253,8 @@ def _parse_set_weight_values(argvish):
                 print(Commands.set_weight.__doc__.strip())
                 exit(EXIT_ERROR)
 
-            devs_and_weights = izip(islice(argvish, 0, len(argvish), 2),
-                                    islice(argvish, 1, len(argvish), 2))
+            devs_and_weights = zip(islice(argvish, 0, len(argvish), 2),
+                                   islice(argvish, 1, len(argvish), 2))
             for devstr, weightstr in devs_and_weights:
                 devs = (builder.search_devs(
                     parse_search_value(devstr)) or [])
@@ -301,6 +329,76 @@ def calculate_change_value(change_value, change, v_name, v_name_port):
     return change_value
 
 
+def _parse_set_region_values(argvish):
+
+    new_cmd_format, opts, args = validate_args(argvish)
+
+    # We'll either parse the all-in-one-string format or the
+    # --options format,
+    # but not both. If both are specified, raise an error.
+    try:
+        devs = []
+        if not new_cmd_format:
+            if len(args) % 2 != 0:
+                print(Commands.set_region.__doc__.strip())
+                exit(EXIT_ERROR)
+
+            devs_and_regions = zip(islice(argvish, 0, len(argvish), 2),
+                                   islice(argvish, 1, len(argvish), 2))
+            for devstr, regionstr in devs_and_regions:
+                devs.extend(builder.search_devs(
+                    parse_search_value(devstr)) or [])
+                region = int(regionstr)
+                _set_region_values(devs, region, opts)
+        else:
+            if len(args) != 1:
+                print(Commands.set_region.__doc__.strip())
+                exit(EXIT_ERROR)
+
+            devs.extend(builder.search_devs(
+                parse_search_values_from_opts(opts)) or [])
+            region = int(args[0])
+            _set_region_values(devs, region, opts)
+    except ValueError as e:
+        print(e)
+        exit(EXIT_ERROR)
+
+
+def _parse_set_zone_values(argvish):
+
+    new_cmd_format, opts, args = validate_args(argvish)
+
+    # We'll either parse the all-in-one-string format or the
+    # --options format,
+    # but not both. If both are specified, raise an error.
+    try:
+        devs = []
+        if not new_cmd_format:
+            if len(args) % 2 != 0:
+                print(Commands.set_zone.__doc__.strip())
+                exit(EXIT_ERROR)
+
+            devs_and_zones = zip(islice(argvish, 0, len(argvish), 2),
+                                 islice(argvish, 1, len(argvish), 2))
+            for devstr, zonestr in devs_and_zones:
+                devs.extend(builder.search_devs(
+                    parse_search_value(devstr)) or [])
+                zone = int(zonestr)
+                _set_zone_values(devs, zone, opts)
+        else:
+            if len(args) != 1:
+                print(Commands.set_zone.__doc__.strip())
+                exit(EXIT_ERROR)
+
+            devs.extend(builder.search_devs(
+                parse_search_values_from_opts(opts)) or [])
+            zone = int(args[0])
+            _set_zone_values(devs, zone, opts)
+    except ValueError as e:
+        print(e)
+        exit(EXIT_ERROR)
+
+
 def _parse_set_info_values(argvish):
 
     new_cmd_format, opts, args = validate_args(argvish)
@@ -313,8 +411,8 @@ def _parse_set_info_values(argvish):
             print(Commands.search.__doc__.strip())
             exit(EXIT_ERROR)
 
-        searches_and_changes = izip(islice(argvish, 0, len(argvish), 2),
-                                    islice(argvish, 1, len(argvish), 2))
+        searches_and_changes = zip(islice(argvish, 0, len(argvish), 2),
+                                   islice(argvish, 1, len(argvish), 2))
 
         for search_value, change_value in searches_and_changes:
             devs = builder.search_devs(parse_search_value(search_value))
@@ -380,6 +478,7 @@ def _make_display_device_table(builder):
     rep_ip_width = 14
     rep_port_width = 4
     ip_ipv6 = rep_ipv6 = False
+    weight_width = 6
     for dev in builder._iter_devs():
         if is_valid_ipv6(dev['ip']):
             ip_ipv6 = True
@@ -390,6 +489,8 @@ def _make_display_device_table(builder):
         port_width = max(len(str(dev['port'])), port_width)
         rep_port_width = max(len(str(dev['replication_port'])),
                              rep_port_width)
+        weight_width = max(len('%6.02f' % dev['weight']),
+                           weight_width)
     if ip_ipv6:
         ip_width += 2
     if rep_ipv6:
@@ -397,7 +498,7 @@ def _make_display_device_table(builder):
     header_line = ('Devices:%5s %6s %4s %' + str(ip_width)
                    + 's:%-' + str(port_width) + 's %' +
                    str(rep_ip_width) + 's:%-' + str(rep_port_width) +
-                   's %5s %6s %10s %7s %5s %s') % (
+                   's %5s %' + str(weight_width) + 's %10s %7s %5s %s') % (
                        'id', 'region', 'zone', 'ip address',
                        'port', 'replication ip', 'port', 'name',
                        'weight', 'partitions', 'balance', 'flags',
@@ -415,7 +516,8 @@ def _make_display_device_table(builder):
                                  '%', str(ip_width), 's:%-',
                                  str(port_width), 'd ', '%',
                                  str(rep_ip_width), 's', ':%-',
-                                 str(rep_port_width), 'd %5s %6.02f'
+                                 str(rep_port_width), 'd %5s %',
+                                 str(weight_width), '.02f'
                                  ' %10s %7.02f %5s %s'])
         args = (dev['id'], dev['region'], dev['zone'], dev_ip, dev['port'],
                 dev_replication_ip, dev['replication_port'], dev['device'],
@@ -444,7 +546,11 @@ swift-ring-builder <builder_file> create <part_power> <replicas>
         if len(argv) < 6:
             print(Commands.create.__doc__.strip())
             exit(EXIT_ERROR)
-        builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
+        try:
+            builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
+        except ValueError as e:
+            print(e)
+            exit(EXIT_ERROR)
         backup_dir = pathjoin(dirname(builder_file), 'backups')
         try:
             mkdir(backup_dir)
@@ -681,7 +787,7 @@ swift-ring-builder <builder_file> add
         if builder.next_part_power:
             print('Partition power increase in progress. You need ')
             print('to finish the increase first before adding devices.')
-            exit(EXIT_WARNING)
+            exit(EXIT_ERROR)
 
         try:
             for new_dev in _parse_add_values(argv[3:]):
@@ -745,6 +851,75 @@ swift-ring-builder <builder_file> set_weight
         builder.save(builder_file)
         exit(EXIT_SUCCESS)
 
+    @staticmethod
+    def set_region():
+        """
+swift-ring-builder <builder_file> set_region <search-value> <region>
+    [<search-value> <region] ...
+
+or
+
+swift-ring-builder <builder_file> set_region
+    --region <region> --zone <zone> --ip <ip or hostname> --port <port>
+    --replication-ip <r_ip or r_hostname> --replication-port <r_port>
+    --device <device_name> --meta <meta> <new region> [--yes]
+
+    Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
+    and port.
+    Any of the options are optional in both cases.
+
+    Resets the devices' regions. No partitions will be reassigned to or from
+    the device until after running 'rebalance'. This is so you can make
+    multiple device changes and rebalance them all just once.
+
+    Option --yes assume a yes response to all questions.
+        """
+        if len(argv) < 5:
+            print(Commands.set_region.__doc__.strip())
+            print()
+            print(parse_search_value.__doc__.strip())
+            exit(EXIT_ERROR)
+
+        _parse_set_region_values(argv[3:])
+
+        builder.save(builder_file)
+        exit(EXIT_SUCCESS)
+
+    @staticmethod
+    def set_zone():
+        """
+swift-ring-builder <builder_file> set_zone <search-value> <zone>
+    [<search-value> <zone] ...
+
+or
+
+swift-ring-builder <builder_file> set_zone
+    --region <region> --zone <zone> --ip <ip or hostname> --port <port>
+    --replication-ip <r_ip or r_hostname> --replication-port <r_port>
+    --device <device_name> --meta <meta> <new zone> [--yes]
+
+    Where <r_ip>, <r_hostname> and <r_port> are replication ip, hostname
+    and port.
+    Any of the options are optional in both cases.
+
+    Resets the devices' zones. No partitions will be reassigned to or from
+    the device until after running 'rebalance'. This is so you can make
+    multiple device changes and rebalance them all just once.
+
+    Option --yes assume a yes response to all questions.
+        """
+        # if len(argv) < 5 or len(argv) % 2 != 1:
+        if len(argv) < 5:
+            print(Commands.set_zone.__doc__.strip())
+            print()
+            print(parse_search_value.__doc__.strip())
+            exit(EXIT_ERROR)
+
+        _parse_set_zone_values(argv[3:])
+
+        builder.save(builder_file)
+        exit(EXIT_SUCCESS)
+
     @staticmethod
     def set_info():
         """
@@ -831,7 +1006,7 @@ swift-ring-builder <builder_file> remove
         if builder.next_part_power:
             print('Partition power increase in progress. You need ')
             print('to finish the increase first before removing devices.')
-            exit(EXIT_WARNING)
+            exit(EXIT_ERROR)
 
         devs, opts = _parse_remove_values(argv[3:])
 
@@ -898,7 +1073,7 @@ swift-ring-builder <builder_file> rebalance [options]
         if builder.next_part_power:
             print('Partition power increase in progress.')
             print('You need to finish the increase first before rebalancing.')
-            exit(EXIT_WARNING)
+            exit(EXIT_ERROR)
 
         devs_changed = builder.devs_changed
         min_part_seconds_left = builder.min_part_seconds_left
@@ -999,7 +1174,7 @@ swift-ring-builder <builder_file> rebalance [options]
 
     @staticmethod
     def dispersion():
-        """
+        r"""
 swift-ring-builder <builder_file> dispersion <search_filter> [options]
 
     Output report on dispersion.
@@ -1167,7 +1342,7 @@ swift-ring-builder <ring_file> write_builder [min_part_hours]
             'parts': ring.partition_count,
             'devs': ring.devs,
             'devs_changed': False,
-            'version': 0,
+            'version': ring.version or 0,
             '_replica2part2dev': ring._replica2part2dev_id,
             '_last_part_moves_epoch': None,
             '_last_part_moves': None,
@@ -1493,7 +1668,7 @@ def main(arguments=None):
         print(msg)
         exit(EXIT_ERROR)
     except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
-        if len(argv) < 3 or argv[2] not in('create', 'write_builder'):
+        if len(argv) < 3 or argv[2] not in ('create', 'write_builder'):
             print(e)
             exit(EXIT_ERROR)
     except Exception as e:
@@ -1521,3 +1696,25 @@ def main(arguments=None):
             exit(2)
     else:
         getattr(Commands, command, Commands.unknown)()
+
+
+def error_handling_main():
+    # We exit code 1 on WARNING statuses, 2 on ERROR. This means we need
+    # to handle any uncaught exceptions by printing the usual backtrace,
+    # but then exiting 2 (not 1 as is usual for a python
+    # exception).
+
+    # We *don't* want to do this in main(), however, because we don't want to
+    # pollute the test environment or cause a bunch of test churn to mock out
+    # sys.excepthook
+
+    def exit_with_status_two(tp, val, tb):
+        traceback.print_exception(tp, val, tb)
+        exit(2)
+
+    sys.excepthook = exit_with_status_two
+    main()
+
+
+if __name__ == '__main__':
+    error_handling_main()

swift/cli/ringcomposer.py

@@ -65,7 +65,6 @@ For further details use::
 
   swift-ring-composer -h
 """
-from __future__ import print_function
 import argparse
 import json
 import os

swift/cli/shard-info.py

@@ -89,7 +89,7 @@ def print_own_shard_range(node, sr, indent_level):
     indent = indent_level * TAB
     range = '%r - %r' % (sr.lower, sr.upper)
     print('%s(%s) %23s, objs: %3s, bytes: %3s, timestamp: %s (%s), '
-          'modified: %s (%s), %7s: %s (%s), deleted: %s epoch: %s' %
+          'modified: %s (%s), %7s: %s (%s), deleted: %s, epoch: %s' %
           (indent, node[1][0], range, sr.object_count, sr.bytes_used,
            sr.timestamp.isoformat, sr.timestamp.internal,
            sr.meta_timestamp.isoformat, sr.meta_timestamp.internal,
@@ -108,12 +108,13 @@ def print_shard_range(node, sr, indent_level):
     indent = indent_level * TAB
     range = '%r - %r' % (sr.lower, sr.upper)
     print('%s(%s) %23s, objs: %3s, bytes: %3s, timestamp: %s (%s), '
-          'modified: %s (%s), %7s: %s (%s), deleted: %s %s' %
+          'modified: %s (%s), %7s: %s (%s), deleted: %s, epoch: %s %s' %
           (indent, node[1][0], range, sr.object_count, sr.bytes_used,
            sr.timestamp.isoformat, sr.timestamp.internal,
            sr.meta_timestamp.isoformat, sr.meta_timestamp.internal,
            sr.state_text, sr.state_timestamp.isoformat,
-           sr.state_timestamp.internal, sr.deleted, sr.name))
+           sr.state_timestamp.internal, sr.deleted,
+           sr.epoch.internal if sr.epoch else None, sr.name))
 
 
 def print_shard_range_info(node, shard_ranges, indent_level=0):

swift/common/base_storage_server.py

@@ -27,10 +27,8 @@ class BaseStorageServer(object):
 
     def __init__(self, conf, **kwargs):
         self._allowed_methods = None
-        replication_server = conf.get('replication_server', None)
-        if replication_server is not None:
-            replication_server = config_true_value(replication_server)
-        self.replication_server = replication_server
+        self.replication_server = config_true_value(
+            conf.get('replication_server', 'true'))
         self.log_format = conf.get('log_format', LOG_LINE_DEFAULT_FORMAT)
         self.anonymization_method = conf.get('log_anonymization_method', 'md5')
         self.anonymization_salt = conf.get('log_anonymization_salt', '')
@@ -45,22 +43,13 @@ class BaseStorageServer(object):
         if self._allowed_methods is None:
             self._allowed_methods = []
             all_methods = inspect.getmembers(self, predicate=callable)
-
-            if self.replication_server is True:
-                for name, m in all_methods:
-                    if (getattr(m, 'publicly_accessible', False) and
-                            getattr(m, 'replication', False)):
-                        self._allowed_methods.append(name)
-            elif self.replication_server is False:
-                for name, m in all_methods:
-                    if (getattr(m, 'publicly_accessible', False) and not
-                            getattr(m, 'replication', False)):
-                        self._allowed_methods.append(name)
-            elif self.replication_server is None:
-                for name, m in all_methods:
-                    if getattr(m, 'publicly_accessible', False):
-                        self._allowed_methods.append(name)
-
+            for name, m in all_methods:
+                if not getattr(m, 'publicly_accessible', False):
+                    continue
+                if getattr(m, 'replication', False) and \
+                        not self.replication_server:
+                    continue
+                self._allowed_methods.append(name)
             self._allowed_methods.sort()
         return self._allowed_methods