swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/cli/reload.py

@@ -0,0 +1,141 @@
+# Copyright (c) 2022 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Safely reload WSGI servers while minimizing client downtime and errors by
+
+   * validating that the process is a Swift WSGI server manager,
+   * checking that the configuration file used is valid,
+   * sending the "seamless reload" signal, and
+   * waiting for the reload to complete.
+"""
+
+from __future__ import print_function
+import argparse
+import errno
+import os
+import os.path
+import signal
+import subprocess
+import sys
+import time
+
+from swift.common.manager import get_child_pids
+
+
+EXIT_BAD_PID = 2  # similar to argparse exiting 2 on an unknown arg
+EXIT_RELOAD_FAILED = 1
+EXIT_RELOAD_TIMEOUT = 128 + errno.ETIMEDOUT
+
+
+def validate_manager_pid(pid):
+    try:
+        with open('/proc/%d/cmdline' % pid, 'r') as fp:
+            cmd = fp.read().strip('\x00').split('\x00')
+        sid = os.getsid(pid)
+    except (IOError, OSError):
+        print("Failed to get process information for %s" % pid,
+              file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    scripts = [os.path.basename(c) for c in cmd
+               if '/bin/' in c and '/bin/python' not in c]
+
+    if len(scripts) != 1 or not scripts[0].startswith("swift-"):
+        print("Non-swift process: %r" % ' '.join(cmd), file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    if scripts[0] not in {"swift-proxy-server", "swift-account-server",
+                          "swift-container-server", "swift-object-server"}:
+        print("Process does not support config checks: %s" % scripts[0],
+              file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    if sid != pid:
+        print("Process appears to be a %s worker, not a manager. "
+              "Did you mean %s?" % (scripts[0], sid), file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    return cmd, scripts[0]
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument("pid", type=int,
+                        help="server PID which should be reloaded")
+    wait_group = parser.add_mutually_exclusive_group()
+    wait_group.add_argument("-t", "--timeout", type=float, default=300.0,
+                            help="max time to wait for reload to complete")
+    wait_group.add_argument("-w", "--no-wait",
+                            action="store_false", dest="wait",
+                            help="skip waiting for reload to complete")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="display more information as the process reloads")
+    args = parser.parse_args(args)
+
+    cmd, script = validate_manager_pid(args.pid)
+
+    if args.verbose:
+        print("Checking config for %s" % script)
+    try:
+        subprocess.check_call(cmd + ["--test-config"])
+    except subprocess.CalledProcessError:
+        print("Failed to validate config", file=sys.stderr)
+        exit(EXIT_RELOAD_FAILED)
+
+    if args.wait:
+        try:
+            original_children = get_child_pids(args.pid)
+            children_since_reload = set()
+
+            if args.verbose:
+                print("Sending USR1 signal")
+            os.kill(args.pid, signal.SIGUSR1)
+
+            start = time.time()
+            while time.time() - start < args.timeout:
+                children = get_child_pids(args.pid)
+                new_children = (children - original_children
+                                - children_since_reload)
+                if new_children:
+                    if args.verbose:
+                        print("Found new children: %s" % ", ".join(
+                            str(pid) for pid in new_children))
+                    children_since_reload |= new_children
+                if children_since_reload - children:
+                    # At least one new child exited; presumably, it was
+                    # the temporary child waiting to shutdown sockets
+                    break
+                # We want this to be fairly low, since the temporary child
+                # may not hang around very long
+                time.sleep(0.1)
+            else:
+                print("Timed out reloading %s" % script, file=sys.stderr)
+                exit(EXIT_RELOAD_TIMEOUT)
+
+        except subprocess.CalledProcessError:
+            # This could pop during any of the calls to get_child_pids
+            print("Process seems to have died!", file=sys.stderr)
+            exit(EXIT_RELOAD_FAILED)
+    else:  # --no-wait
+        if args.verbose:
+            print("Sending USR1 signal")
+        os.kill(args.pid, signal.SIGUSR1)
+
+    print("Reloaded %s" % script)
+
+
+if __name__ == "__main__":
+    main()

swift/cli/ringbuilder.py

@@ -22,9 +22,11 @@ from itertools import islice
 from operator import itemgetter
 from os import mkdir
 from os.path import basename, abspath, dirname, exists, join as pathjoin
+import sys
 from sys import argv as sys_argv, exit, stderr, stdout
 from textwrap import wrap
 from time import time
+import traceback
 from datetime import timedelta
 import optparse
 import math
@@ -1698,3 +1700,25 @@ def main(arguments=None):
             exit(2)
     else:
         getattr(Commands, command, Commands.unknown)()
+
+
+def error_handling_main():
+    # We exit code 1 on WARNING statuses, 2 on ERROR. This means we need
+    # to handle any uncaught exceptions by printing the usual backtrace,
+    # but then exiting 2 (not 1 as is usual for a python
+    # exception).
+
+    # We *don't* want to do this in main(), however, because we don't want to
+    # pollute the test environment or cause a bunch of test churn to mock out
+    # sys.excepthook
+
+    def exit_with_status_two(tp, val, tb):
+        traceback.print_exception(tp, val, tb)
+        exit(2)
+
+    sys.excepthook = exit_with_status_two
+    main()
+
+
+if __name__ == '__main__':
+    error_handling_main()

swift/common/daemon.py

@@ -159,7 +159,7 @@ class DaemonStrategy(object):
         except KeyboardInterrupt:
             self.logger.notice('User quit')
         finally:
-            self.cleanup()
+            self.cleanup(stopping=True)
         self.running = False
 
     def _fork(self, once, **kwargs):
@@ -167,6 +167,8 @@ class DaemonStrategy(object):
         if pid == 0:
             signal.signal(signal.SIGHUP, signal.SIG_DFL)
             signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            # only MAINPID should be sending notifications
+            os.environ.pop('NOTIFY_SOCKET', None)
 
             self.daemon.run(once, **kwargs)
 
@@ -245,7 +247,15 @@ class DaemonStrategy(object):
         self.daemon.post_multiprocess_run()
         return 0
 
-    def cleanup(self):
+    def cleanup(self, stopping=False):
+        """
+        Cleanup worker processes
+
+        :param stopping: if set, tell systemd we're stopping
+        """
+
+        if stopping:
+            utils.systemd_notify(self.logger, "STOPPING=1")
         for p in self.spawned_pids():
             try:
                 os.kill(p, signal.SIGTERM)

swift/common/db.py

@@ -138,7 +138,8 @@ class GreenDBConnection(sqlite3.Connection):
             timeout = BROKER_TIMEOUT
         self.timeout = timeout
         self.db_file = database
-        super(GreenDBConnection, self).__init__(database, 0, *args, **kwargs)
+        super(GreenDBConnection, self).__init__(
+            database, timeout=0, *args, **kwargs)
 
     def cursor(self, cls=None):
         if cls is None:
@@ -724,22 +725,26 @@ class DatabaseBroker(object):
                 return -1
             return row['sync_point']
 
-    def get_syncs(self, incoming=True):
+    def get_syncs(self, incoming=True, include_timestamp=False):
         """
         Get a serialized copy of the sync table.
 
         :param incoming: if True, get the last incoming sync, otherwise get
                          the last outgoing sync
-        :returns: list of {'remote_id', 'sync_point'}
+        :param include_timestamp: If True include the updated_at timestamp
+        :returns: list of {'remote_id', 'sync_point'} or
+                 {'remote_id', 'sync_point', 'updated_at'}
+                 if include_timestamp is True.
         """
         with self.get() as conn:
+            columns = 'remote_id, sync_point'
+            if include_timestamp:
+                columns += ', updated_at'
             curs = conn.execute('''
-                SELECT remote_id, sync_point FROM %s_sync
-            ''' % ('incoming' if incoming else 'outgoing'))
-            result = []
-            for row in curs:
-                result.append({'remote_id': row[0], 'sync_point': row[1]})
-            return result
+                SELECT %s FROM %s_sync
+            ''' % (columns, 'incoming' if incoming else 'outgoing'))
+            curs.row_factory = dict_factory
+            return [r for r in curs]
 
     def get_max_row(self, table=None):
         if not table:

swift/common/db_auditor.py

@@ -96,7 +96,7 @@ class DatabaseAuditor(Daemon):
         time.sleep(random() * self.interval)
         while True:
             self.logger.info(
-                'Begin {} audit pass.'.format(self.server_type))
+                'Begin %s audit pass.', self.server_type)
             begin = time.time()
             try:
                 reported = self._one_audit_pass(reported)
@@ -116,7 +116,7 @@ class DatabaseAuditor(Daemon):
     def run_once(self, *args, **kwargs):
         """Run the database audit once."""
         self.logger.info(
-            'Begin {} audit "once" mode'.format(self.server_type))
+            'Begin %s audit "once" mode', self.server_type)
         begin = reported = time.time()
         self._one_audit_pass(reported)
         elapsed = time.time() - begin

swift/common/db_replicator.py

@@ -240,6 +240,12 @@ class Replicator(Daemon):
         self.handoffs_only = config_true_value(conf.get('handoffs_only', 'no'))
         self.handoff_delete = config_auto_int_value(
             conf.get('handoff_delete', 'auto'), 0)
+        if self.handoff_delete >= self.ring.replica_count:
+            self.logger.warning(
+                'handoff_delete=%d is too high to have an effect on a ring '
+                'with replica count %d. Disabling.',
+                self.handoff_delete, self.ring.replica_count)
+            self.handoff_delete = 0
 
     def _zero_stats(self):
         """Zero out the stats."""

swift/common/exceptions.py

@@ -243,6 +243,18 @@ class QuarantineRequest(SwiftException):
     pass
 
 
+class MemcacheConnectionError(Exception):
+    pass
+
+
+class MemcacheIncrNotFoundError(MemcacheConnectionError):
+    pass
+
+
+class MemcachePoolTimeout(Timeout):
+    pass
+
+
 class ClientException(Exception):
 
     def __init__(self, msg, http_scheme='', http_host='', http_port='',

swift/common/http_protocol.py

@@ -16,15 +16,21 @@
 from eventlet import wsgi, websocket
 import six
 
+from swift.common.utils import generate_trans_id
+from swift.common.http import HTTP_NO_CONTENT, HTTP_RESET_CONTENT, \
+    HTTP_NOT_MODIFIED
 
 if six.PY2:
     from eventlet.green import httplib as http_client
+    from cgi import escape
 else:
     from eventlet.green.http import client as http_client
+    from html import escape
 
 
 class SwiftHttpProtocol(wsgi.HttpProtocol):
     default_request_version = "HTTP/1.0"
+    reject_bad_requests = False
 
     def __init__(self, *args, **kwargs):
         # See https://github.com/eventlet/eventlet/pull/590
@@ -52,7 +58,7 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
             self.server.log.info('ERROR WSGI: ' + f, *a)
 
     class MessageClass(wsgi.HttpProtocol.MessageClass):
-        '''Subclass to see when the client didn't provide a Content-Type'''
+        """Subclass to see when the client didn't provide a Content-Type"""
         # for py2:
         def parsetype(self):
             if self.typeheader is None:
@@ -61,7 +67,7 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
 
         # for py3:
         def get_default_type(self):
-            '''If the client didn't provide a content type, leave it blank.'''
+            """If the client didn't provide a content type, leave it blank."""
             return ''
 
     def parse_request(self):
@@ -241,6 +247,74 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
                 self.conn_state[2] = wsgi.STATE_IDLE
         return got
 
+    def send_error(self, code, message=None, explain=None):
+        """Send and log an error reply, we are overriding the cpython parent
+        class method, so we can have logger generate txn_id's for error
+        response from wsgi since we are at the edge of the proxy server.
+        This sends an error response (so it must be called before any output
+        has been generated), logs the error, and finally sends a piece of HTML
+        explaining the error to the user.
+
+        :param code:    an HTTP error code
+                   3 digits
+        :param message: a simple optional 1 line reason phrase.
+                   *( HTAB / SP / VCHAR / %x80-FF )
+                   defaults to short entry matching the response code
+        :param explain: a detailed message defaults to the long entry
+                   matching the response code.
+        """
+
+        try:
+            shortmsg, longmsg = self.responses[code]
+        except KeyError:
+            shortmsg, longmsg = '???', '???'
+        if message is None:
+            message = shortmsg
+        if explain is None:
+            explain = longmsg
+
+        try:
+            # assume we have a LogAdapter
+            txn_id = self.server.app.logger.txn_id  # just in case it was set
+        except AttributeError:
+            # turns out we don't have a LogAdapter, so go direct
+            txn_id = generate_trans_id('')
+            self.log_error("code %d, message %s, (txn: %s)", code,
+                           message, txn_id)
+        else:
+            # we do have a LogAdapter, but likely not yet a txn_id
+            txn_id = txn_id or generate_trans_id('')
+            self.server.app.logger.txn_id = txn_id
+            self.log_error("code %d, message %s", code, message)
+        self.send_response(code, message)
+        self.send_header('Connection', 'close')
+
+        # Message body is omitted for cases described in:
+        #  - RFC7230: 3.3. 1xx, 204(No Content), 304(Not Modified)
+        #  - RFC7231: 6.3.6. 205(Reset Content)
+        body = None
+        exclude_status = (HTTP_NO_CONTENT,
+                          HTTP_RESET_CONTENT,
+                          HTTP_NOT_MODIFIED)
+        if (code >= 200 and
+                code not in exclude_status):
+            # HTML encode to prevent Cross Site Scripting attacks
+            # (see bug https://bugs.python.org/issue1100201)
+            content = (self.error_message_format % {
+                'code': code,
+                'message': escape(message, quote=False),
+                'explain': escape(explain, quote=False)
+            })
+            body = content.encode('UTF-8', 'replace')
+            self.send_header("Content-Type", self.error_content_type)
+            self.send_header('Content-Length', str(len(body)))
+            self.send_header('X-Trans-Id', txn_id)
+            self.send_header('X-Openstack-Request-Id', txn_id)
+        self.end_headers()
+
+        if self.command != 'HEAD' and body:
+            self.wfile.write(body)
+
 
 class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
     """
@@ -271,7 +345,6 @@ class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
         # ourselves and our gateway proxy before processing the client
         # protocol request.  Hopefully the operator will know what to do!
         msg = 'Invalid PROXY line %r' % connection_line
-        self.log_message(msg)
         # Even assuming HTTP we don't even known what version of HTTP the
         # client is sending?  This entire endeavor seems questionable.
         self.request_version = self.default_request_version

swift/common/manager.py

@@ -13,9 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 from __future__ import print_function
 import functools
 import errno
+from optparse import OptionParser
 import os
 import resource
 import signal
@@ -23,8 +25,13 @@ import time
 import subprocess
 import re
 import six
+import sys
 import tempfile
-from distutils.spawn import find_executable
+try:
+    from shutil import which
+except ImportError:
+    # py2
+    from distutils.spawn import find_executable as which
 
 from swift.common.utils import search_tree, remove_file, write_file, readconf
 from swift.common.exceptions import InvalidPidFileException
@@ -176,6 +183,17 @@ def kill_group(pid, sig):
     os.kill(-pid, sig)
 
 
+def get_child_pids(pid):
+    """
+    Get the current set of all child PIDs for a PID.
+
+    :param pid: process id
+    """
+    output = subprocess.check_output(
+        ["ps", "--ppid", str(pid), "--no-headers", "-o", "pid"])
+    return {int(pid) for pid in output.split()}
+
+
 def format_server_name(servername):
     """
     Formats server name as swift compatible server names
@@ -204,7 +222,7 @@ def verify_server(server):
     if not server:
         return False
     _, cmd = format_server_name(server)
-    if find_executable(cmd) is None:
+    if which(cmd) is None:
         return False
     return True
 
@@ -696,9 +714,7 @@ class Server(object):
                 print('Removing pid file %s with invalid pid' % pid_file)
                 remove_file(pid_file)
                 continue
-            ps_cmd = ['ps', '--ppid', str(pid), '--no-headers', '-o', 'pid']
-            for pid in subprocess.check_output(ps_cmd).split():
-                pid = int(pid)
+            for pid in get_child_pids(pid):
                 if self._signal_pid(sig, pid, pid_file, kwargs.get('verbose')):
                     pids[pid] = pid_file
         return pids
@@ -923,3 +939,102 @@ class Server(object):
 
         """
         return self.kill_running_pids(**kwargs)
+
+
+USAGE = \
+    """%prog <server>[.<config>] [<server>[.<config>] ...] <command> [options]
+
+where:
+    <server>  is the name of a swift service e.g. proxy-server.
+              The '-server' part of the name may be omitted.
+              'all', 'main' and 'rest' are reserved words that represent a
+              group of services.
+              all: Expands to all swift daemons.
+              main: Expands to main swift daemons.
+                    (proxy, container, account, object)
+              rest: Expands to all remaining background daemons (beyond
+                    "main").
+                    (updater, replicator, auditor, etc)
+    <config>  is an explicit configuration filename without the
+              .conf extension. If <config> is specified then <server> should
+              refer to a directory containing the configuration file, e.g.:
+
+                  swift-init object.1 start
+
+              will start an object-server using the configuration file
+              /etc/swift/object-server/1.conf
+    <command> is a command from the list below.
+
+Commands:
+""" + '\n'.join(["%16s: %s" % x for x in Manager.list_commands()])
+
+
+def main():
+    parser = OptionParser(USAGE)
+    parser.add_option('-v', '--verbose', action="store_true",
+                      default=False, help="display verbose output")
+    parser.add_option('-w', '--no-wait', action="store_false", dest="wait",
+                      default=True, help="won't wait for server to start "
+                      "before returning")
+    parser.add_option('-o', '--once', action="store_true",
+                      default=False, help="only run one pass of daemon")
+    # this is a negative option, default is options.daemon = True
+    parser.add_option('-n', '--no-daemon', action="store_false", dest="daemon",
+                      default=True, help="start server interactively")
+    parser.add_option('-g', '--graceful', action="store_true",
+                      default=False, help="send SIGHUP to supporting servers")
+    parser.add_option('-c', '--config-num', metavar="N", type="int",
+                      dest="number", default=0,
+                      help="send command to the Nth server only")
+    parser.add_option('-k', '--kill-wait', metavar="N", type="int",
+                      dest="kill_wait", default=KILL_WAIT,
+                      help="wait N seconds for processes to die (default 15)")
+    parser.add_option('-r', '--run-dir', type="str",
+                      dest="run_dir", default=RUN_DIR,
+                      help="alternative directory to store running pid files "
+                      "default: %s" % RUN_DIR)
+    # Changing behaviour if missing config
+    parser.add_option('--strict', dest='strict', action='store_true',
+                      help="Return non-zero status code if some config is "
+                           "missing. Default mode if all servers are "
+                           "explicitly named.")
+    # a negative option for strict
+    parser.add_option('--non-strict', dest='strict', action='store_false',
+                      help="Return zero status code even if some config is "
+                           "missing. Default mode if any server is a glob or "
+                           "one of aliases `all`, `main` or `rest`.")
+    # SIGKILL daemon after kill_wait period
+    parser.add_option('--kill-after-timeout', dest='kill_after_timeout',
+                      action='store_true',
+                      help="Kill daemon and all children after kill-wait "
+                           "period.")
+
+    options, args = parser.parse_args()
+
+    if len(args) < 2:
+        parser.print_help()
+        print('ERROR: specify server(s) and command')
+        return 1
+
+    command = args[-1]
+    servers = args[:-1]
+
+    # this is just a silly swap for me cause I always try to "start main"
+    commands = dict(Manager.list_commands()).keys()
+    if command not in commands and servers[0] in commands:
+        servers.append(command)
+        command = servers.pop(0)
+
+    manager = Manager(servers, run_dir=options.run_dir)
+    try:
+        status = manager.run_command(command, **options.__dict__)
+    except UnknownCommandError:
+        parser.print_help()
+        print('ERROR: unknown command, %s' % command)
+        status = 1
+
+    return 1 if status else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())