swift 2.32.1__py2.py3-none-any.whl → 2.33.1__py2.py3-none-any.whl

swift/account/server.py

@@ -85,17 +85,7 @@ class AccountController(BaseStorageServer):
         self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                             self.mount_check,
                                             logger=self.logger)
-        if conf.get('auto_create_account_prefix'):
-            self.logger.warning('Option auto_create_account_prefix is '
-                                'deprecated. Configure '
-                                'auto_create_account_prefix under the '
-                                'swift-constraints section of '
-                                'swift.conf. This option will '
-                                'be ignored in a future release.')
-            self.auto_create_account_prefix = \
-                conf['auto_create_account_prefix']
-        else:
-            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+        self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
 
         swift.common.db.DB_PREALLOCATION = \
             config_true_value(conf.get('db_preallocation', 'f'))

swift/cli/info.py

@@ -198,6 +198,28 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
           'real value is set in the config file on each storage node.')
 
 
+def get_max_len_sync_item(syncs, item, title):
+    def map_func(element):
+        return str(element[item])
+    return max(list(map(len, map(map_func, syncs))) + [len(title)])
+
+
+def print_db_syncs(incoming, syncs):
+    max_sync_point_len = get_max_len_sync_item(syncs, 'sync_point',
+                                               "Sync Point")
+    max_remote_len = get_max_len_sync_item(syncs, 'remote_id', "Remote ID")
+    print('%s Syncs:' % ('Incoming' if incoming else 'Outgoing'))
+    print('  %s\t%s\t%s' % ("Sync Point".ljust(max_sync_point_len),
+                            "Remote ID".ljust(max_remote_len),
+                            "Updated At"))
+    for sync in syncs:
+        print('  %s\t%s\t%s (%s)' % (
+            str(sync['sync_point']).ljust(max_sync_point_len),
+            sync['remote_id'].ljust(max_remote_len),
+            Timestamp(sync['updated_at']).isoformat,
+            sync['updated_at']))
+
+
 def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
                            verbose=False):
     """
@@ -439,7 +461,7 @@ def print_obj_metadata(metadata, drop_prefixes=False):
 
 
 def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
-               drop_prefixes=False, verbose=False):
+               drop_prefixes=False, verbose=False, sync=False):
     if db_type not in ('account', 'container'):
         print("Unrecognized DB type: internal error")
         raise InfoSystemExit()
@@ -473,6 +495,11 @@ def print_info(db_type, db_file, swift_dir='/etc/swift', stale_reads_ok=False,
             info['shard_ranges'] = sranges
     print_db_info_metadata(
         db_type, info, broker.metadata, drop_prefixes, verbose)
+    if sync:
+        # Print incoming / outgoing sync tables.
+        for incoming in (True, False):
+            print_db_syncs(incoming, broker.get_syncs(incoming,
+                                                      include_timestamp=True))
     try:
         ring = Ring(swift_dir, ring_name=db_type)
     except Exception:

swift-2.32.1.data/scripts/swift-recon-cron → swift/cli/recon_cron.py

@@ -1,4 +1,3 @@
-#!python
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,10 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-swift-recon-cron.py
-"""
-
 import os
 import sys
 import time
@@ -28,7 +23,7 @@ from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj.diskfile import ASYNCDIR_BASE
 
 
-def get_async_count(device_dir, logger):
+def get_async_count(device_dir):
     async_count = 0
     for i in os.listdir(device_dir):
         device = os.path.join(device_dir, i)
@@ -55,7 +50,7 @@ def main():
     except Exception:
         print("Usage: %s CONF_FILE" % sys.argv[0].split('/')[-1])
         print("ex: swift-recon-cron /etc/swift/object-server.conf")
-        sys.exit(1)
+        return 1
     conf = readconf(conf_path, 'filter:recon')
     device_dir = conf.get('devices', '/srv/node')
     recon_cache_path = conf.get('recon_cache_path', DEFAULT_RECON_CACHE_PATH)
@@ -66,7 +61,7 @@ def main():
     logger = get_logger(conf, log_route='recon-cron')
     try:
         with lock_path(lock_dir):
-            asyncs = get_async_count(device_dir, logger)
+            asyncs = get_async_count(device_dir)
             dump_recon_cache({
                 'async_pending': asyncs,
                 'async_pending_last': time.time(),
@@ -75,8 +70,4 @@ def main():
         msg = 'Exception during recon-cron while accessing devices'
         logger.exception(msg)
         print('%s: %s' % (msg, err))
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()
+        return 1

swift/cli/reload.py

@@ -0,0 +1,141 @@
+# Copyright (c) 2022 NVIDIA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Safely reload WSGI servers while minimizing client downtime and errors by
+
+   * validating that the process is a Swift WSGI server manager,
+   * checking that the configuration file used is valid,
+   * sending the "seamless reload" signal, and
+   * waiting for the reload to complete.
+"""
+
+from __future__ import print_function
+import argparse
+import errno
+import os
+import os.path
+import signal
+import subprocess
+import sys
+import time
+
+from swift.common.manager import get_child_pids
+
+
+EXIT_BAD_PID = 2  # similar to argparse exiting 2 on an unknown arg
+EXIT_RELOAD_FAILED = 1
+EXIT_RELOAD_TIMEOUT = 128 + errno.ETIMEDOUT
+
+
+def validate_manager_pid(pid):
+    try:
+        with open('/proc/%d/cmdline' % pid, 'r') as fp:
+            cmd = fp.read().strip('\x00').split('\x00')
+        sid = os.getsid(pid)
+    except (IOError, OSError):
+        print("Failed to get process information for %s" % pid,
+              file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    scripts = [os.path.basename(c) for c in cmd
+               if '/bin/' in c and '/bin/python' not in c]
+
+    if len(scripts) != 1 or not scripts[0].startswith("swift-"):
+        print("Non-swift process: %r" % ' '.join(cmd), file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    if scripts[0] not in {"swift-proxy-server", "swift-account-server",
+                          "swift-container-server", "swift-object-server"}:
+        print("Process does not support config checks: %s" % scripts[0],
+              file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    if sid != pid:
+        print("Process appears to be a %s worker, not a manager. "
+              "Did you mean %s?" % (scripts[0], sid), file=sys.stderr)
+        exit(EXIT_BAD_PID)
+
+    return cmd, scripts[0]
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument("pid", type=int,
+                        help="server PID which should be reloaded")
+    wait_group = parser.add_mutually_exclusive_group()
+    wait_group.add_argument("-t", "--timeout", type=float, default=300.0,
+                            help="max time to wait for reload to complete")
+    wait_group.add_argument("-w", "--no-wait",
+                            action="store_false", dest="wait",
+                            help="skip waiting for reload to complete")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="display more information as the process reloads")
+    args = parser.parse_args(args)
+
+    cmd, script = validate_manager_pid(args.pid)
+
+    if args.verbose:
+        print("Checking config for %s" % script)
+    try:
+        subprocess.check_call(cmd + ["--test-config"])
+    except subprocess.CalledProcessError:
+        print("Failed to validate config", file=sys.stderr)
+        exit(EXIT_RELOAD_FAILED)
+
+    if args.wait:
+        try:
+            original_children = get_child_pids(args.pid)
+            children_since_reload = set()
+
+            if args.verbose:
+                print("Sending USR1 signal")
+            os.kill(args.pid, signal.SIGUSR1)
+
+            start = time.time()
+            while time.time() - start < args.timeout:
+                children = get_child_pids(args.pid)
+                new_children = (children - original_children
+                                - children_since_reload)
+                if new_children:
+                    if args.verbose:
+                        print("Found new children: %s" % ", ".join(
+                            str(pid) for pid in new_children))
+                    children_since_reload |= new_children
+                if children_since_reload - children:
+                    # At least one new child exited; presumably, it was
+                    # the temporary child waiting to shutdown sockets
+                    break
+                # We want this to be fairly low, since the temporary child
+                # may not hang around very long
+                time.sleep(0.1)
+            else:
+                print("Timed out reloading %s" % script, file=sys.stderr)
+                exit(EXIT_RELOAD_TIMEOUT)
+
+        except subprocess.CalledProcessError:
+            # This could pop during any of the calls to get_child_pids
+            print("Process seems to have died!", file=sys.stderr)
+            exit(EXIT_RELOAD_FAILED)
+    else:  # --no-wait
+        if args.verbose:
+            print("Sending USR1 signal")
+        os.kill(args.pid, signal.SIGUSR1)
+
+    print("Reloaded %s" % script)
+
+
+if __name__ == "__main__":
+    main()

swift/common/daemon.py

@@ -159,7 +159,7 @@ class DaemonStrategy(object):
         except KeyboardInterrupt:
             self.logger.notice('User quit')
         finally:
-            self.cleanup()
+            self.cleanup(stopping=True)
         self.running = False
 
     def _fork(self, once, **kwargs):
@@ -167,6 +167,8 @@ class DaemonStrategy(object):
         if pid == 0:
             signal.signal(signal.SIGHUP, signal.SIG_DFL)
             signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            # only MAINPID should be sending notifications
+            os.environ.pop('NOTIFY_SOCKET', None)
 
             self.daemon.run(once, **kwargs)
 
@@ -245,7 +247,15 @@ class DaemonStrategy(object):
         self.daemon.post_multiprocess_run()
         return 0
 
-    def cleanup(self):
+    def cleanup(self, stopping=False):
+        """
+        Cleanup worker processes
+
+        :param stopping: if set, tell systemd we're stopping
+        """
+
+        if stopping:
+            utils.systemd_notify(self.logger, "STOPPING=1")
         for p in self.spawned_pids():
             try:
                 os.kill(p, signal.SIGTERM)

swift/common/db.py

@@ -724,22 +724,26 @@ class DatabaseBroker(object):
                 return -1
             return row['sync_point']
 
-    def get_syncs(self, incoming=True):
+    def get_syncs(self, incoming=True, include_timestamp=False):
         """
         Get a serialized copy of the sync table.
 
         :param incoming: if True, get the last incoming sync, otherwise get
                          the last outgoing sync
-        :returns: list of {'remote_id', 'sync_point'}
+        :param include_timestamp: If True include the updated_at timestamp
+        :returns: list of {'remote_id', 'sync_point'} or
+                 {'remote_id', 'sync_point', 'updated_at'}
+                 if include_timestamp is True.
         """
         with self.get() as conn:
+            columns = 'remote_id, sync_point'
+            if include_timestamp:
+                columns += ', updated_at'
             curs = conn.execute('''
-                SELECT remote_id, sync_point FROM %s_sync
-            ''' % ('incoming' if incoming else 'outgoing'))
-            result = []
-            for row in curs:
-                result.append({'remote_id': row[0], 'sync_point': row[1]})
-            return result
+                SELECT %s FROM %s_sync
+            ''' % (columns, 'incoming' if incoming else 'outgoing'))
+            curs.row_factory = dict_factory
+            return [r for r in curs]
 
     def get_max_row(self, table=None):
         if not table:

swift/common/http_protocol.py

@@ -16,15 +16,21 @@
 from eventlet import wsgi, websocket
 import six
 
+from swift.common.utils import generate_trans_id
+from swift.common.http import HTTP_NO_CONTENT, HTTP_RESET_CONTENT, \
+    HTTP_NOT_MODIFIED
 
 if six.PY2:
     from eventlet.green import httplib as http_client
+    from cgi import escape
 else:
     from eventlet.green.http import client as http_client
+    from html import escape
 
 
 class SwiftHttpProtocol(wsgi.HttpProtocol):
     default_request_version = "HTTP/1.0"
+    reject_bad_requests = False
 
     def __init__(self, *args, **kwargs):
         # See https://github.com/eventlet/eventlet/pull/590
@@ -52,7 +58,7 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
             self.server.log.info('ERROR WSGI: ' + f, *a)
 
     class MessageClass(wsgi.HttpProtocol.MessageClass):
-        '''Subclass to see when the client didn't provide a Content-Type'''
+        """Subclass to see when the client didn't provide a Content-Type"""
         # for py2:
         def parsetype(self):
             if self.typeheader is None:
@@ -61,7 +67,7 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
 
         # for py3:
         def get_default_type(self):
-            '''If the client didn't provide a content type, leave it blank.'''
+            """If the client didn't provide a content type, leave it blank."""
             return ''
 
     def parse_request(self):
@@ -241,6 +247,74 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
                 self.conn_state[2] = wsgi.STATE_IDLE
         return got
 
+    def send_error(self, code, message=None, explain=None):
+        """Send and log an error reply, we are overriding the cpython parent
+        class method, so we can have logger generate txn_id's for error
+        response from wsgi since we are at the edge of the proxy server.
+        This sends an error response (so it must be called before any output
+        has been generated), logs the error, and finally sends a piece of HTML
+        explaining the error to the user.
+
+        :param code:    an HTTP error code
+                   3 digits
+        :param message: a simple optional 1 line reason phrase.
+                   *( HTAB / SP / VCHAR / %x80-FF )
+                   defaults to short entry matching the response code
+        :param explain: a detailed message defaults to the long entry
+                   matching the response code.
+        """
+
+        try:
+            shortmsg, longmsg = self.responses[code]
+        except KeyError:
+            shortmsg, longmsg = '???', '???'
+        if message is None:
+            message = shortmsg
+        if explain is None:
+            explain = longmsg
+
+        try:
+            # assume we have a LogAdapter
+            txn_id = self.server.app.logger.txn_id  # just in case it was set
+        except AttributeError:
+            # turns out we don't have a LogAdapter, so go direct
+            txn_id = generate_trans_id('')
+            self.log_error("code %d, message %s, (txn: %s)", code,
+                           message, txn_id)
+        else:
+            # we do have a LogAdapter, but likely not yet a txn_id
+            txn_id = txn_id or generate_trans_id('')
+            self.server.app.logger.txn_id = txn_id
+            self.log_error("code %d, message %s", code, message)
+        self.send_response(code, message)
+        self.send_header('Connection', 'close')
+
+        # Message body is omitted for cases described in:
+        #  - RFC7230: 3.3. 1xx, 204(No Content), 304(Not Modified)
+        #  - RFC7231: 6.3.6. 205(Reset Content)
+        body = None
+        exclude_status = (HTTP_NO_CONTENT,
+                          HTTP_RESET_CONTENT,
+                          HTTP_NOT_MODIFIED)
+        if (code >= 200 and
+                code not in exclude_status):
+            # HTML encode to prevent Cross Site Scripting attacks
+            # (see bug https://bugs.python.org/issue1100201)
+            content = (self.error_message_format % {
+                'code': code,
+                'message': escape(message, quote=False),
+                'explain': escape(explain, quote=False)
+            })
+            body = content.encode('UTF-8', 'replace')
+            self.send_header("Content-Type", self.error_content_type)
+            self.send_header('Content-Length', str(len(body)))
+            self.send_header('X-Trans-Id', txn_id)
+            self.send_header('X-Openstack-Request-Id', txn_id)
+        self.end_headers()
+
+        if self.command != 'HEAD' and body:
+            self.wfile.write(body)
+
 
 class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
     """
@@ -271,7 +345,6 @@ class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
         # ourselves and our gateway proxy before processing the client
         # protocol request.  Hopefully the operator will know what to do!
         msg = 'Invalid PROXY line %r' % connection_line
-        self.log_message(msg)
         # Even assuming HTTP we don't even known what version of HTTP the
         # client is sending?  This entire endeavor seems questionable.
         self.request_version = self.default_request_version

swift/common/manager.py

@@ -24,7 +24,11 @@ import subprocess
 import re
 import six
 import tempfile
-from distutils.spawn import find_executable
+try:
+    from shutil import which
+except ImportError:
+    # py2
+    from distutils.spawn import find_executable as which
 
 from swift.common.utils import search_tree, remove_file, write_file, readconf
 from swift.common.exceptions import InvalidPidFileException
@@ -176,6 +180,17 @@ def kill_group(pid, sig):
     os.kill(-pid, sig)
 
 
+def get_child_pids(pid):
+    """
+    Get the current set of all child PIDs for a PID.
+
+    :param pid: process id
+    """
+    output = subprocess.check_output(
+        ["ps", "--ppid", str(pid), "--no-headers", "-o", "pid"])
+    return {int(pid) for pid in output.split()}
+
+
 def format_server_name(servername):
     """
     Formats server name as swift compatible server names
@@ -204,7 +219,7 @@ def verify_server(server):
     if not server:
         return False
     _, cmd = format_server_name(server)
-    if find_executable(cmd) is None:
+    if which(cmd) is None:
         return False
     return True
 
@@ -696,9 +711,7 @@ class Server(object):
                 print('Removing pid file %s with invalid pid' % pid_file)
                 remove_file(pid_file)
                 continue
-            ps_cmd = ['ps', '--ppid', str(pid), '--no-headers', '-o', 'pid']
-            for pid in subprocess.check_output(ps_cmd).split():
-                pid = int(pid)
+            for pid in get_child_pids(pid):
                 if self._signal_pid(sig, pid, pid_file, kwargs.get('verbose')):
                     pids[pid] = pid_file
         return pids

swift/common/memcached.py

@@ -245,6 +245,13 @@ class MemcacheRing(object):
     def memcache_servers(self):
         return list(self._client_cache.keys())
 
+    def _log_error(self, server, cmd, action, msg):
+        self.logger.error(
+            "Error %(action)s to memcached: %(server)s"
+            ": with key_prefix %(key_prefix)s, method %(method)s: %(msg)s",
+            {'action': action, 'server': server, 'key_prefix': cmd.key_prefix,
+             'method': cmd.method, 'msg': msg})
+
     """
     Handles exceptions.
 
@@ -367,7 +374,8 @@ class MemcacheRing(object):
                 self._exception_occurred(server, e, cmd, pool_start_time,
                                          action='connecting', sock=sock)
         if not any_yielded:
-            self.logger.error('All memcached servers error-limited')
+            self._log_error('ALL', cmd, 'connecting',
+                            'No more memcached servers to try')
 
     def _return_conn(self, server, fp, sock):
         """Returns a server connection to the pool."""
@@ -404,6 +412,14 @@ class MemcacheRing(object):
         elif not isinstance(value, bytes):
             value = str(value).encode('utf-8')
 
+        if 0 <= self.item_size_warning_threshold <= len(value):
+            self.logger.warning(
+                "Item size larger than warning threshold: "
+                "%d (%s) >= %d (%s)", len(value),
+                human_readable(len(value)),
+                self.item_size_warning_threshold,
+                human_readable(self.item_size_warning_threshold))
+
         for (server, fp, sock) in self._get_conns(cmd):
             conn_start_time = tm.time()
             try:
@@ -414,17 +430,7 @@ class MemcacheRing(object):
                     if msg != b'STORED':
                         if not six.PY2:
                             msg = msg.decode('ascii')
-                        self.logger.error(
-                            "Error setting value in memcached: "
-                            "%(server)s: %(msg)s",
-                            {'server': server, 'msg': msg})
-                    if 0 <= self.item_size_warning_threshold <= len(value):
-                        self.logger.warning(
-                            "Item size larger than warning threshold: "
-                            "%d (%s) >= %d (%s)", len(value),
-                            human_readable(len(value)),
-                            self.item_size_warning_threshold,
-                            human_readable(self.item_size_warning_threshold))
+                        raise MemcacheConnectionError('failed set: %s' % msg)
                     self._return_conn(server, fp, sock)
                     return
             except (Exception, Timeout) as e: