swift 2.31.1__py2.py3-none-any.whl → 2.32.1__py2.py3-none-any.whl

swift/cli/info.py

@@ -30,6 +30,7 @@ from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
 from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
     extract_policy
 from swift.common.storage_policy import POLICIES
+from swift.common.swob import wsgi_to_str
 from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
 from swift.common.utils import md5
 
@@ -297,12 +298,16 @@ def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False,
             title = key.replace('_', '-').title()
             print('  %s: %s' % (title, value))
     if sys_metadata:
-        print('  System Metadata: %s' % sys_metadata)
+        print('  System Metadata:')
+        for key, value in sys_metadata.items():
+            print('    %s: %s' % (key, value))
     else:
         print('No system metadata found in db file')
 
     if user_metadata:
-        print('  User Metadata: %s' % user_metadata)
+        print('  User Metadata:')
+        for key, value in user_metadata.items():
+            print('    %s: %s' % (key, value))
     else:
         print('No user metadata found in db file')
 
@@ -537,6 +542,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
         except EOFError:
             print("Invalid metadata")
             raise InfoSystemExit()
+        metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v)
+                    for k, v in metadata.items()}
 
         etag = metadata.pop('ETag', '')
         length = metadata.pop('Content-Length', '')

swift/cli/ringbuilder.py

@@ -194,7 +194,11 @@ def check_devs(devs, input_question, opts, abort_msg):
         print('Matched more than one device:')
         for dev in devs:
             print('    %s' % format_device(dev))
-        if not opts.yes and input(input_question) != 'y':
+        try:
+            abort = not opts.yes and input(input_question) != 'y'
+        except (EOFError, KeyboardInterrupt):
+            abort = True
+        if abort:
             print(abort_msg)
             exit(EXIT_ERROR)
 

swift/common/container_sync_realms.py

@@ -22,7 +22,6 @@ import time
 import six
 from six.moves import configparser
 
-from swift import gettext_ as _
 from swift.common.utils import get_valid_utf8_str
 
 
@@ -58,7 +57,7 @@ class ContainerSyncRealms(object):
                     log_func = self.logger.debug
                 else:
                     log_func = self.logger.error
-                log_func(_('Could not load %(conf)r: %(error)s') % {
+                log_func('Could not load %(conf)r: %(error)s', {
                          'conf': self.conf_path, 'error': err})
             else:
                 if mtime != self.conf_path_mtime:
@@ -68,8 +67,8 @@ class ContainerSyncRealms(object):
                         conf.read(self.conf_path)
                     except configparser.ParsingError as err:
                         self.logger.error(
-                            _('Could not load %(conf)r: %(error)s')
-                            % {'conf': self.conf_path, 'error': err})
+                            'Could not load %(conf)r: %(error)s',
+                            {'conf': self.conf_path, 'error': err})
                     else:
                         try:
                             self.mtime_check_interval = conf.getfloat(
@@ -82,9 +81,9 @@ class ContainerSyncRealms(object):
                                 now + self.mtime_check_interval
                         except (configparser.ParsingError, ValueError) as err:
                             self.logger.error(
-                                _('Error in %(conf)r with '
-                                  'mtime_check_interval: %(error)s')
-                                % {'conf': self.conf_path, 'error': err})
+                                'Error in %(conf)r with '
+                                'mtime_check_interval: %(error)s',
+                                {'conf': self.conf_path, 'error': err})
                         realms = {}
                         for section in conf.sections():
                             realm = {}

swift/common/daemon.py

@@ -20,8 +20,8 @@ import time
 import signal
 from re import sub
 
+import eventlet
 import eventlet.debug
-from eventlet.hubs import use_hub
 
 from swift.common import utils
 
@@ -281,7 +281,9 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
         # and results in an exit code of 1.
         sys.exit(e)
 
-    use_hub(utils.get_hub())
+    # patch eventlet/logging early
+    utils.monkey_patch()
+    eventlet.hubs.use_hub(utils.get_hub())
 
     # once on command line (i.e. daemonize=false) will over-ride config
     once = once or not utils.config_true_value(conf.get('daemonize', 'true'))
@@ -315,7 +317,9 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs):
 
     logger.notice('Starting %s', os.getpid())
     try:
-        DaemonStrategy(klass(conf), logger).run(once=once, **kwargs)
+        d = klass(conf)
+        DaemonStrategy(d, logger).run(once=once, **kwargs)
     except KeyboardInterrupt:
         logger.info('User quit')
     logger.notice('Exited %s', os.getpid())
+    return d

swift/common/db.py

@@ -26,7 +26,6 @@ import time
 import errno
 import six
 import six.moves.cPickle as pickle
-from swift import gettext_ as _
 from tempfile import mkstemp
 
 from eventlet import sleep, Timeout
@@ -130,6 +129,9 @@ class DatabaseAlreadyExists(sqlite3.DatabaseError):
 
 class GreenDBConnection(sqlite3.Connection):
     """SQLite DB Connection handler that plays well with eventlet."""
+    # slots are needed for python 3.11.0 (there's an issue fixed in 3.11.1,
+    # see https://github.com/python/cpython/issues/99886)
+    __slots__ = ('timeout', 'db_file')
 
     def __init__(self, database, timeout=None, *args, **kwargs):
         if timeout is None:
@@ -143,6 +145,13 @@ class GreenDBConnection(sqlite3.Connection):
             cls = GreenDBCursor
         return sqlite3.Connection.cursor(self, cls)
 
+    def execute(self, *args, **kwargs):
+        # py311 stopped calling self.cursor() to get the cursor;
+        # see https://github.com/python/cpython/pull/31351
+        curs = self.cursor()
+        curs.execute(*args, **kwargs)
+        return curs
+
     def commit(self):
         return _db_timeout(
             self.timeout, self.db_file,
@@ -151,6 +160,9 @@ class GreenDBConnection(sqlite3.Connection):
 
 class GreenDBCursor(sqlite3.Cursor):
     """SQLite Cursor handler that plays well with eventlet."""
+    # slots are needed for python 3.11.0 (there's an issue fixed in 3.11.1,
+    # see https://github.com/python/cpython/issues/99886)
+    __slots__ = ('timeout', 'db_file')
 
     def __init__(self, *args, **kwargs):
         self.timeout = args[0].timeout
@@ -162,6 +174,9 @@ class GreenDBCursor(sqlite3.Cursor):
             self.timeout, self.db_file, lambda: sqlite3.Cursor.execute(
                 self, *args, **kwargs))
 
+    # NB: executemany and executescript are *not* greened, and never have been
+    # (as far as I can tell)
+
 
 def dict_factory(crs, row):
     """
@@ -482,10 +497,10 @@ class DatabaseBroker(object):
                 raise
             quar_path = "%s-%s" % (quar_path, uuid4().hex)
             renamer(self.db_dir, quar_path, fsync=False)
-        detail = _('Quarantined %(db_dir)s to %(quar_path)s due to '
-                   '%(reason)s') % {'db_dir': self.db_dir,
-                                    'quar_path': quar_path,
-                                    'reason': reason}
+        detail = ('Quarantined %(db_dir)s to %(quar_path)s due to '
+                  '%(reason)s') % {'db_dir': self.db_dir,
+                                   'quar_path': quar_path,
+                                   'reason': reason}
         self.logger.error(detail)
         raise sqlite3.DatabaseError(detail)
 
@@ -584,7 +599,7 @@ class DatabaseBroker(object):
                 self.conn = conn
             except (Exception, Timeout):
                 logging.exception(
-                    _('Broker error trying to rollback locked connection'))
+                    'Broker error trying to rollback locked connection')
                 conn.close()
 
     def _new_db_id(self):
@@ -836,7 +851,7 @@ class DatabaseBroker(object):
                         self._commit_puts_load(item_list, data)
                     except Exception:
                         self.logger.exception(
-                            _('Invalid pending entry %(file)s: %(entry)s'),
+                            'Invalid pending entry %(file)s: %(entry)s',
                             {'file': self.pending_file, 'entry': entry})
             if item_list:
                 self.merge_items(item_list)

swift/common/db_replicator.py

@@ -23,7 +23,6 @@ import uuid
 import errno
 import re
 from contextlib import contextmanager
-from swift import gettext_ as _
 
 from eventlet import GreenPool, sleep, Timeout
 from eventlet.green import subprocess
@@ -176,7 +175,7 @@ class ReplConnection(BufferedHTTPConnection):
         except (Exception, Timeout):
             self.close()
             self.logger.exception(
-                _('ERROR reading HTTP response from %s'), self.node)
+                'ERROR reading HTTP response from %s', self.node)
             return None
 
 
@@ -254,15 +253,15 @@ class Replicator(Daemon):
         """Report the current stats to the logs."""
         now = time.time()
         self.logger.info(
-            _('Attempted to replicate %(count)d dbs in %(time).5f seconds '
-              '(%(rate).5f/s)'),
+            'Attempted to replicate %(count)d dbs in %(time).5f seconds '
+            '(%(rate).5f/s)',
             {'count': self.stats['attempted'],
              'time': now - self.stats['start'],
              'rate': self.stats['attempted'] /
                 (now - self.stats['start'] + 0.0000001)})
-        self.logger.info(_('Removed %(remove)d dbs') % self.stats)
-        self.logger.info(_('%(success)s successes, %(failure)s failures')
-                         % self.stats)
+        self.logger.info('Removed %(remove)d dbs', self.stats)
+        self.logger.info('%(success)s successes, %(failure)s failures',
+                         self.stats)
         dump_recon_cache(
             {'replication_stats': self.stats,
              'replication_time': now - self.stats['start'],
@@ -308,7 +307,7 @@ class Replicator(Daemon):
         proc = subprocess.Popen(popen_args)
         proc.communicate()
         if proc.returncode != 0:
-            self.logger.error(_('ERROR rsync failed with %(code)s: %(args)s'),
+            self.logger.error('ERROR rsync failed with %(code)s: %(args)s',
                               {'code': proc.returncode, 'args': popen_args})
         return proc.returncode == 0
 
@@ -625,10 +624,10 @@ class Replicator(Daemon):
                     'replicate out and remove.' % (object_file, name, bpart))
         except (Exception, Timeout) as e:
             if 'no such table' in str(e):
-                self.logger.error(_('Quarantining DB %s'), object_file)
+                self.logger.error('Quarantining DB %s', object_file)
                 quarantine_db(broker.db_file, broker.db_type)
             else:
-                self.logger.exception(_('ERROR reading db %s'), object_file)
+                self.logger.exception('ERROR reading db %s', object_file)
             nodes = self.ring.get_part_nodes(int(partition))
             self._add_failure_stats([(failure_dev['replication_ip'],
                                       failure_dev['device'])
@@ -680,13 +679,13 @@ class Replicator(Daemon):
                     repl_nodes.append(next(more_nodes))
                 except StopIteration:
                     self.logger.error(
-                        _('ERROR There are not enough handoff nodes to reach '
-                          'replica count for partition %s'),
+                        'ERROR There are not enough handoff nodes to reach '
+                        'replica count for partition %s',
                         partition)
-                self.logger.error(_('ERROR Remote drive not mounted %s'), node)
+                self.logger.error('ERROR Remote drive not mounted %s', node)
             except (Exception, Timeout):
-                self.logger.exception(_('ERROR syncing %(file)s with node'
-                                        ' %(node)s'),
+                self.logger.exception('ERROR syncing %(file)s with node'
+                                      ' %(node)s',
                                       {'file': object_file, 'node': node})
             if not success:
                 failure_devs_info.add((node['replication_ip'], node['device']))
@@ -785,7 +784,7 @@ class Replicator(Daemon):
         dirs = []
         ips = whataremyips(self.bind_ip)
         if not ips:
-            self.logger.error(_('ERROR Failed to get my own IPs?'))
+            self.logger.error('ERROR Failed to get my own IPs?')
             return
 
         if self.handoffs_only or self.handoff_delete:
@@ -830,12 +829,12 @@ class Replicator(Daemon):
             self.logger.error("Can't find itself %s with port %s in ring "
                               "file, not replicating",
                               ", ".join(ips), self.port)
-        self.logger.info(_('Beginning replication run'))
+        self.logger.info('Beginning replication run')
         for part, object_file, node_id in self.roundrobin_datadirs(dirs):
             self.cpool.spawn_n(
                 self._replicate_object, part, object_file, node_id)
         self.cpool.waitall()
-        self.logger.info(_('Replication run OVER'))
+        self.logger.info('Replication run OVER')
         if self.handoffs_only or self.handoff_delete:
             self.logger.warning(
                 'Finished replication pass with handoffs_only and/or '
@@ -853,7 +852,7 @@ class Replicator(Daemon):
             try:
                 self.run_once()
             except (Exception, Timeout):
-                self.logger.exception(_('ERROR trying to replicate'))
+                self.logger.exception('ERROR trying to replicate')
             elapsed = time.time() - begin
             if elapsed < self.interval:
                 sleep(self.interval - elapsed)
@@ -957,7 +956,7 @@ class ReplicatorRpc(object):
                 info = self._get_synced_replication_info(broker, remote_info)
             except (Exception, Timeout) as e:
                 if 'no such table' in str(e):
-                    self.logger.error(_("Quarantining DB %s"), broker)
+                    self.logger.error("Quarantining DB %s", broker)
                     quarantine_db(broker.db_file, broker.db_type)
                     return HTTPNotFound()
                 raise

swift/common/direct_client.py

@@ -156,29 +156,46 @@ def _get_direct_account_container(path, stype, node, part,
                                   marker=None, limit=None,
                                   prefix=None, delimiter=None,
                                   conn_timeout=5, response_timeout=15,
-                                  end_marker=None, reverse=None, headers=None):
-    """Base class for get direct account and container.
+                                  end_marker=None, reverse=None, headers=None,
+                                  extra_params=None):
+    """Base function for get direct account and container.
 
-    Do not use directly use the get_direct_account or
-    get_direct_container instead.
+    Do not use directly use the direct_get_account or
+    direct_get_container instead.
     """
     if headers is None:
         headers = {}
 
-    params = ['format=json']
+    params = {'format': 'json'}
+    if extra_params:
+        for key, value in extra_params.items():
+            if value is not None:
+                params[key] = value
     if marker:
-        params.append('marker=%s' % quote(marker))
+        if 'marker' in params:
+            raise TypeError('duplicate values for keyword arg: marker')
+        params['marker'] = quote(marker)
     if limit:
-        params.append('limit=%d' % limit)
+        if 'limit' in params:
+            raise TypeError('duplicate values for keyword arg: limit')
+        params['limit'] = '%d' % limit
     if prefix:
-        params.append('prefix=%s' % quote(prefix))
+        if 'prefix' in params:
+            raise TypeError('duplicate values for keyword arg: prefix')
+        params['prefix'] = quote(prefix)
     if delimiter:
-        params.append('delimiter=%s' % quote(delimiter))
+        if 'delimiter' in params:
+            raise TypeError('duplicate values for keyword arg: delimiter')
+        params['delimiter'] = quote(delimiter)
     if end_marker:
-        params.append('end_marker=%s' % quote(end_marker))
+        if 'end_marker' in params:
+            raise TypeError('duplicate values for keyword arg: end_marker')
+        params['end_marker'] = quote(end_marker)
     if reverse:
-        params.append('reverse=%s' % quote(reverse))
-    qs = '&'.join(params)
+        if 'reverse' in params:
+            raise TypeError('duplicate values for keyword arg: reverse')
+        params['reverse'] = quote(reverse)
+    qs = '&'.join('%s=%s' % (k, v) for k, v in params.items())
 
     ip, port = get_ip_port(node, headers)
     with Timeout(conn_timeout):
@@ -293,7 +310,7 @@ def direct_head_container(node, part, account, container, conn_timeout=5,
 def direct_get_container(node, part, account, container, marker=None,
                          limit=None, prefix=None, delimiter=None,
                          conn_timeout=5, response_timeout=15, end_marker=None,
-                         reverse=None, headers=None):
+                         reverse=None, headers=None, extra_params=None):
     """
     Get container listings directly from the container server.
 
@@ -310,6 +327,12 @@ def direct_get_container(node, part, account, container, marker=None,
     :param end_marker: end_marker query
     :param reverse: reverse the returned listing
     :param headers: headers to be included in the request
+    :param extra_params: a dict of extra parameters to be included in the
+        request. It can be used to pass additional parameters, e.g,
+        {'states':'updating'} can be used with shard_range/namespace listing.
+        It can also be used to pass the existing keyword args, like 'marker' or
+        'limit', but if the same parameter appears twice in both keyword arg
+        (not None) and extra_params, this function will raise TypeError.
     :returns: a tuple of (response headers, a list of objects) The response
               headers will be a HeaderKeyDict.
     """
@@ -322,7 +345,8 @@ def direct_get_container(node, part, account, container, marker=None,
                                          reverse=reverse,
                                          conn_timeout=conn_timeout,
                                          response_timeout=response_timeout,
-                                         headers=headers)
+                                         headers=headers,
+                                         extra_params=extra_params)
 
 
 def direct_delete_container(node, part, account, container, conn_timeout=5,
@@ -378,6 +402,31 @@ def direct_put_container(node, part, account, container, conn_timeout=5,
               content_length=content_length, chunk_size=chunk_size)
 
 
+def direct_post_container(node, part, account, container, conn_timeout=5,
+                          response_timeout=15, headers=None):
+    """
+    Make a POST request to a container server.
+
+    :param node: node dictionary from the ring
+    :param part: partition the container is on
+    :param account: account name
+    :param container: container name
+    :param conn_timeout: timeout in seconds for establishing the connection
+    :param response_timeout: timeout in seconds for getting the response
+    :param headers: additional headers to include in the request
+    :raises ClientException: HTTP PUT request failed
+    """
+    if headers is None:
+        headers = {}
+
+    lower_headers = set(k.lower() for k in headers)
+    headers_out = gen_headers(headers,
+                              add_ts='x-timestamp' not in lower_headers)
+    path = _make_path(account, container)
+    return _make_req(node, part, 'POST', path, headers_out, 'Container',
+                     conn_timeout, response_timeout)
+
+
 def direct_put_container_object(node, part, account, container, obj,
                                 conn_timeout=5, response_timeout=15,
                                 headers=None):

swift/common/internal_client.py

@@ -28,6 +28,7 @@ from zlib import compressobj
 from swift.common.exceptions import ClientException
 from swift.common.http import (HTTP_NOT_FOUND, HTTP_MULTIPLE_CHOICES,
                                is_client_error, is_server_error)
+from swift.common.middleware.gatekeeper import GatekeeperMiddleware
 from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.swob import Request, bytes_to_wsgi
 from swift.common.utils import quote, close_if_possible, drain_and_close
@@ -144,6 +145,8 @@ class InternalClient(object):
     :param user_agent: User agent to be sent to requests to Swift.
     :param request_tries: Number of tries before InternalClient.make_request()
                           gives up.
+    :param use_replication_network: Force the client to use the replication
+        network over the cluster.
     :param global_conf: a dict of options to update the loaded proxy config.
         Options in ``global_conf`` will override those in ``conf_path`` except
         where the ``conf_path`` option is preceded by ``set``.
@@ -151,12 +154,17 @@ class InternalClient(object):
     """
 
     def __init__(self, conf_path, user_agent, request_tries,
-                 allow_modify_pipeline=False, use_replication_network=False,
-                 global_conf=None, app=None):
+                 use_replication_network=False, global_conf=None, app=None,
+                 **kwargs):
         if request_tries < 1:
             raise ValueError('request_tries must be positive')
+        # Internal clients don't use the gatekeeper and the pipeline remains
+        # static so we never allow anything to modify the proxy pipeline.
+        if kwargs.get('allow_modify_pipeline'):
+            raise ValueError("'allow_modify_pipeline' is no longer supported")
         self.app = app or loadapp(conf_path, global_conf=global_conf,
-                                  allow_modify_pipeline=allow_modify_pipeline,)
+                                  allow_modify_pipeline=False,)
+        self.check_gatekeeper_not_loaded(self.app)
         self.user_agent = \
             self.app._pipeline_final_app.backend_user_agent = user_agent
         self.request_tries = request_tries
@@ -167,6 +175,19 @@ class InternalClient(object):
         self.auto_create_account_prefix = \
             self.app._pipeline_final_app.auto_create_account_prefix
 
+    @staticmethod
+    def check_gatekeeper_not_loaded(app):
+        # the Gatekeeper middleware would prevent an InternalClient passing
+        # X-Backend-* headers to the proxy app, so ensure it's not present
+        try:
+            for app in app._pipeline:
+                if isinstance(app, GatekeeperMiddleware):
+                    raise ValueError(
+                        "Gatekeeper middleware is not allowed in the "
+                        "InternalClient proxy pipeline")
+        except AttributeError:
+            pass
+
     def make_request(
             self, method, path, headers, acceptable_statuses, body_file=None,
             params=None):