swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/__init__.py

@@ -13,20 +13,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import sys
-import gettext
+import warnings
 
-import pkg_resources
+__version__ = None
 
+# First, try to get our version out of PKG-INFO. If we're installed,
+# this'll let us find our version without pulling in pbr. After all, if
+# we're installed on a system, we're not in a Git-managed source tree, so
+# pbr doesn't really buy us anything.
 try:
-    # First, try to get our version out of PKG-INFO. If we're installed,
-    # this'll let us find our version without pulling in pbr. After all, if
-    # we're installed on a system, we're not in a Git-managed source tree, so
-    # pbr doesn't really buy us anything.
-    __version__ = __canonical_version__ = pkg_resources.get_provider(
-        pkg_resources.Requirement.parse('swift')).version
-except pkg_resources.DistributionNotFound:
+    import importlib.metadata
+except ImportError:
+    # python < 3.8
+    import pkg_resources
+    try:
+        __version__ = __canonical_version__ = pkg_resources.get_provider(
+            pkg_resources.Requirement.parse('swift')).version
+    except pkg_resources.DistributionNotFound:
+        pass
+else:
+    try:
+        __version__ = __canonical_version__ = importlib.metadata.distribution(
+            'swift').version
+    except importlib.metadata.PackageNotFoundError:
+        pass
+
+if __version__ is None:
     # No PKG-INFO? We're probably running from a checkout, then. Let pbr do
     # its thing to figure out a version number.
     import pbr.version
@@ -34,43 +46,10 @@ except pkg_resources.DistributionNotFound:
     __version__ = _version_info.release_string()
     __canonical_version__ = _version_info.version_string()
 
-_localedir = os.environ.get('SWIFT_LOCALEDIR')
-_t = gettext.translation('swift', localedir=_localedir, fallback=True)
 
-
-def gettext_(msg):
-    return _t.gettext(msg)
-
-
-if (3, 0) <= sys.version_info[:2] <= (3, 5):
-    # In the development of py3, json.loads() stopped accepting byte strings
-    # for a while. https://bugs.python.org/issue17909 got fixed for py36, but
-    # since it was termed an enhancement and not a regression, we don't expect
-    # any backports. At the same time, it'd be better if we could avoid
-    # leaving a whole bunch of json.loads(resp.body.decode(...)) scars in the
-    # code that'd probably persist even *after* we drop support for 3.5 and
-    # earlier. So, monkey patch stdlib.
-    import json
-    if not getattr(json.loads, 'patched_to_decode', False):
-        class JsonLoadsPatcher(object):
-            def __init__(self, orig):
-                self._orig = orig
-
-            def __call__(self, s, **kw):
-                if isinstance(s, bytes):
-                    # No fancy byte-order mark detection for us; just assume
-                    # UTF-8 and raise a UnicodeDecodeError if appropriate.
-                    s = s.decode('utf8')
-                return self._orig(s, **kw)
-
-            def __getattribute__(self, attr):
-                if attr == 'patched_to_decode':
-                    return True
-                if attr == '_orig':
-                    return super().__getattribute__(attr)
-                # Pass through all other attrs to the original; among other
-                # things, this preserves doc strings, etc.
-                return getattr(self._orig, attr)
-
-        json.loads = JsonLoadsPatcher(json.loads)
-        del JsonLoadsPatcher
+warnings.filterwarnings('ignore', module='cryptography|OpenSSL', message=(
+    'Python 2 is no longer supported by the Python core team. '
+    'Support for it is now deprecated in cryptography'))
+warnings.filterwarnings('ignore', message=(
+    'Python 3.6 is no longer supported by the Python core team. '
+    'Therefore, support for it is deprecated in cryptography'))

swift/account/auditor.py

@@ -13,100 +13,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import time
-from swift import gettext_ as _
-from random import random
 
-import swift.common.db
-from swift.account.backend import AccountBroker, DATADIR
+from swift.account.backend import AccountBroker
 from swift.common.exceptions import InvalidAccountInfo
-from swift.common.utils import get_logger, audit_location_generator, \
-    config_true_value, dump_recon_cache, ratelimit_sleep
-from swift.common.daemon import Daemon
+from swift.common.daemon import run_daemon
+from swift.common.db_auditor import DatabaseAuditor
+from swift.common.utils import parse_options
 
-from eventlet import Timeout
 
-
-class AccountAuditor(Daemon):
+class AccountAuditor(DatabaseAuditor):
     """Audit accounts."""
 
-    def __init__(self, conf, logger=None):
-        self.conf = conf
-        self.logger = logger or get_logger(conf, log_route='account-auditor')
-        self.devices = conf.get('devices', '/srv/node')
-        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
-        self.interval = int(conf.get('interval', 1800))
-        self.logging_interval = 3600  # once an hour
-        self.account_passes = 0
-        self.account_failures = 0
-        self.accounts_running_time = 0
-        self.max_accounts_per_second = \
-            float(conf.get('accounts_per_second', 200))
-        swift.common.db.DB_PREALLOCATION = \
-            config_true_value(conf.get('db_preallocation', 'f'))
-        self.recon_cache_path = conf.get('recon_cache_path',
-                                         '/var/cache/swift')
-        self.rcache = os.path.join(self.recon_cache_path, "account.recon")
-
-    def _one_audit_pass(self, reported):
-        all_locs = audit_location_generator(self.devices, DATADIR, '.db',
-                                            mount_check=self.mount_check,
-                                            logger=self.logger)
-        for path, device, partition in all_locs:
-            self.account_audit(path)
-            if time.time() - reported >= self.logging_interval:
-                self.logger.info(_('Since %(time)s: Account audits: '
-                                   '%(passed)s passed audit,'
-                                   '%(failed)s failed audit'),
-                                 {'time': time.ctime(reported),
-                                  'passed': self.account_passes,
-                                  'failed': self.account_failures})
-                dump_recon_cache({'account_audits_since': reported,
-                                  'account_audits_passed': self.account_passes,
-                                  'account_audits_failed':
-                                  self.account_failures},
-                                 self.rcache, self.logger)
-                reported = time.time()
-                self.account_passes = 0
-                self.account_failures = 0
-            self.accounts_running_time = ratelimit_sleep(
-                self.accounts_running_time, self.max_accounts_per_second)
-        return reported
-
-    def run_forever(self, *args, **kwargs):
-        """Run the account audit until stopped."""
-        reported = time.time()
-        time.sleep(random() * self.interval)
-        while True:
-            self.logger.info(_('Begin account audit pass.'))
-            begin = time.time()
-            try:
-                reported = self._one_audit_pass(reported)
-            except (Exception, Timeout):
-                self.logger.increment('errors')
-                self.logger.exception(_('ERROR auditing'))
-            elapsed = time.time() - begin
-            if elapsed < self.interval:
-                time.sleep(self.interval - elapsed)
-            self.logger.info(
-                _('Account audit pass completed: %.02fs'), elapsed)
-            dump_recon_cache({'account_auditor_pass_completed': elapsed},
-                             self.rcache, self.logger)
+    server_type = "account"
+    broker_class = AccountBroker
 
-    def run_once(self, *args, **kwargs):
-        """Run the account audit once."""
-        self.logger.info(_('Begin account audit "once" mode'))
-        begin = reported = time.time()
-        self._one_audit_pass(reported)
-        elapsed = time.time() - begin
-        self.logger.info(
-            _('Account audit "once" mode completed: %.02fs'), elapsed)
-        dump_recon_cache({'account_auditor_pass_completed': elapsed},
-                         self.rcache, self.logger)
-
-    def validate_per_policy_counts(self, broker):
-        info = broker.get_info()
+    def _audit(self, info, broker):
+        # Validate per policy counts
         policy_stats = broker.get_policy_stats(do_migrations=True)
         policy_totals = {
             'container_count': 0,
@@ -120,36 +42,17 @@ class AccountAuditor(Daemon):
         for key in policy_totals:
             if policy_totals[key] == info[key]:
                 continue
-            raise InvalidAccountInfo(_(
-                'The total %(key)s for the container (%(total)s) does not '
-                'match the sum of %(key)s across policies (%(sum)s)')
-                % {'key': key,
-                   'total': info[key],
-                   'sum': policy_totals[key]})
+            return InvalidAccountInfo(
+                'The total %(key)s for the account %(account)s (%(total)s) '
+                'does not match the sum of %(key)s across policies (%(sum)s)'
+                % {'key': key, 'account': info.get('account'),
+                   'total': info[key], 'sum': policy_totals[key]})
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountAuditor, conf_file, **options)
 
-    def account_audit(self, path):
-        """
-        Audits the given account path
 
-        :param path: the path to an account db
-        """
-        start_time = time.time()
-        try:
-            broker = AccountBroker(path, logger=self.logger)
-            if not broker.is_deleted():
-                self.validate_per_policy_counts(broker)
-                self.logger.increment('passes')
-                self.account_passes += 1
-                self.logger.debug(_('Audit passed for %s'), broker)
-        except InvalidAccountInfo as e:
-            self.logger.increment('failures')
-            self.account_failures += 1
-            self.logger.error(
-                _('Audit Failed for %(path)s: %(err)s'),
-                {'path': path, 'err': str(e)})
-        except (Exception, Timeout):
-            self.logger.increment('failures')
-            self.account_failures += 1
-            self.logger.exception(_('ERROR Could not get account info %s'),
-                                  path)
-        self.logger.timing_since('timing', start_time)
+if __name__ == '__main__':
+    main()

swift/account/backend.py

@@ -16,14 +16,11 @@
 Pluggable Back-end for Account Server
 """
 
-from uuid import uuid4
 
 import sqlite3
 
-import six
-
-from swift.common.utils import Timestamp
-from swift.common.db import DatabaseBroker, utf8encode, zero_like
+from swift.common.utils import Timestamp, RESERVED_BYTE
+from swift.common.db import DatabaseBroker, zero_like
 
 DATADIR = 'accounts'
 
@@ -154,7 +151,7 @@ class AccountBroker(DatabaseBroker):
         conn.execute('''
             UPDATE account_stat SET account = ?, created_at = ?, id = ?,
                    put_timestamp = ?, status_changed_at = ?
-            ''', (self.account, Timestamp.now().internal, str(uuid4()),
+            ''', (self.account, Timestamp.now().internal, self._new_db_id(),
                   put_timestamp, put_timestamp))
 
     def create_policy_stat_table(self, conn):
@@ -189,20 +186,6 @@ class AccountBroker(DatabaseBroker):
                 self._db_version = 1
         return self._db_version
 
-    def _delete_db(self, conn, timestamp, force=False):
-        """
-        Mark the DB as deleted.
-
-        :param conn: DB connection object
-        :param timestamp: timestamp to mark as deleted
-        """
-        conn.execute("""
-            UPDATE account_stat
-            SET delete_timestamp = ?,
-                status = 'DELETED',
-                status_changed_at = ?
-            WHERE delete_timestamp < ? """, (timestamp, timestamp, timestamp))
-
     def _commit_puts_load(self, item_list, entry):
         """See :func:`swift.common.db.DatabaseBroker._commit_puts_load`"""
         # check to see if the update includes policy_index or not
@@ -369,7 +352,7 @@ class AccountBroker(DatabaseBroker):
             ''').fetchone())
 
     def list_containers_iter(self, limit, marker, end_marker, prefix,
-                             delimiter, reverse=False):
+                             delimiter, reverse=False, allow_reserved=False):
         """
         Get a list of containers sorted by name starting at marker onward, up
         to limit entries. Entries will begin with the prefix and will not have
@@ -381,14 +364,12 @@ class AccountBroker(DatabaseBroker):
         :param prefix: prefix query
         :param delimiter: delimiter for query
         :param reverse: reverse the result order.
+        :param allow_reserved: exclude names with reserved-byte by default
 
         :returns: list of tuples of (name, object_count, bytes_used,
-                  put_timestamp, 0)
+                  put_timestamp, storage_policy_index, is_subdir)
         """
         delim_force_gte = False
-        if six.PY2:
-            (marker, end_marker, prefix, delimiter) = utf8encode(
-                marker, end_marker, prefix, delimiter)
         if reverse:
             # Reverse the markers if we are reversing the listing.
             marker, end_marker = end_marker, marker
@@ -402,7 +383,8 @@ class AccountBroker(DatabaseBroker):
             results = []
             while len(results) < limit:
                 query = """
-                    SELECT name, object_count, bytes_used, put_timestamp, 0
+                    SELECT name, object_count, bytes_used, put_timestamp,
+                    {storage_policy_index}, 0
                     FROM container
                     WHERE """
                 query_args = []
@@ -424,6 +406,9 @@ class AccountBroker(DatabaseBroker):
                 elif prefix:
                     query += ' name >= ? AND'
                     query_args.append(prefix)
+                if not allow_reserved:
+                    query += ' name >= ? AND'
+                    query_args.append(chr(ord(RESERVED_BYTE) + 1))
                 if self.get_db_version(conn) < 1:
                     query += ' +deleted = 0'
                 else:
@@ -431,7 +416,27 @@ class AccountBroker(DatabaseBroker):
                 query += ' ORDER BY name %s LIMIT ?' % \
                          ('DESC' if reverse else '')
                 query_args.append(limit - len(results))
-                curs = conn.execute(query, query_args)
+                try:
+                    # First, try querying with the storage policy index.
+                    curs = conn.execute(
+                        query.format(
+                            storage_policy_index="storage_policy_index"),
+                        query_args)
+                except sqlite3.OperationalError as err:
+                    # If the storage policy column is not available,
+                    # the database has not been migrated to the new schema
+                    # with storage_policy_index. Re-run the query with
+                    # storage_policy_index set to 0, which is what
+                    # would be set once the database is migrated.
+                    # TODO(callumdickinson): If support for migrating
+                    # pre-storage policy versions of Swift is dropped,
+                    # then this special handling can be removed.
+                    if "no such column: storage_policy_index" in str(err):
+                        curs = conn.execute(
+                            query.format(storage_policy_index="0"),
+                            query_args)
+                    else:
+                        raise
                 curs.row_factory = None
 
                 # Delimiters without a prefix is ignored, further if there
@@ -468,7 +473,7 @@ class AccountBroker(DatabaseBroker):
                             delim_force_gte = True
                         dir_name = name[:end + len(delimiter)]
                         if dir_name != orig_marker:
-                            results.append([dir_name, 0, 0, '0', 1])
+                            results.append([dir_name, 0, 0, '0', -1, 1])
                         curs.close()
                         break
                     results.append(row)

swift/account/reaper.py

@@ -19,22 +19,22 @@ import socket
 from logging import DEBUG
 from math import sqrt
 from time import time
-from hashlib import md5
 import itertools
 
 from eventlet import GreenPool, sleep, Timeout
-import six
 
 import swift.common.db
 from swift.account.backend import AccountBroker, DATADIR
 from swift.common.constraints import check_drive
+from swift.common.daemon import run_daemon
 from swift.common.direct_client import direct_delete_container, \
     direct_delete_object, direct_get_container
 from swift.common.exceptions import ClientException
+from swift.common.request_helpers import USE_REPLICATION_NETWORK_HEADER
 from swift.common.ring import Ring
 from swift.common.ring.utils import is_local_device
 from swift.common.utils import get_logger, whataremyips, config_true_value, \
-    Timestamp
+    Timestamp, md5, node_to_string, parse_options
 from swift.common.daemon import Daemon
 from swift.common.storage_policy import POLICIES, PolicyError
 
@@ -65,7 +65,7 @@ class AccountReaper(Daemon):
         self.logger = logger or get_logger(conf, log_route='account-reaper')
         self.devices = conf.get('devices', '/srv/node')
         self.mount_check = config_true_value(conf.get('mount_check', 'true'))
-        self.interval = int(conf.get('interval', 3600))
+        self.interval = float(conf.get('interval', 3600))
         self.swift_dir = conf.get('swift_dir', '/etc/swift')
         self.account_ring = None
         self.container_ring = None
@@ -262,16 +262,15 @@ class AccountReaper(Daemon):
             container_limit *= len(nodes)
         try:
             containers = list(broker.list_containers_iter(
-                container_limit, '', None, None, None))
+                container_limit, '', None, None, None, allow_reserved=True))
             while containers:
                 try:
-                    for (container, _junk, _junk, _junk, _junk) in containers:
-                        if six.PY3:
-                            container_ = container.encode('utf-8')
-                        else:
-                            container_ = container
-                        this_shard = int(md5(container_).hexdigest(), 16) % \
-                            len(nodes)
+                    for row in containers:
+                        container = row[0]
+                        this_shard = (
+                            int(md5(container.encode('utf-8'),
+                                    usedforsecurity=False)
+                                .hexdigest(), 16) % len(nodes))
                         if container_shard not in (this_shard, None):
                             continue
 
@@ -282,7 +281,8 @@ class AccountReaper(Daemon):
                     self.logger.exception(
                         'Exception with containers for account %s', account)
                 containers = list(broker.list_containers_iter(
-                    container_limit, containers[-1][0], None, None, None))
+                    container_limit, containers[-1][0], None, None, None,
+                    allow_reserved=True))
             log_buf = ['Completed pass on account %s' % account]
         except (Exception, Timeout):
             self.logger.exception('Exception with account %s', account)
@@ -369,22 +369,22 @@ class AccountReaper(Daemon):
                     node, part, account, container,
                     marker=marker,
                     conn_timeout=self.conn_timeout,
-                    response_timeout=self.node_timeout)
+                    response_timeout=self.node_timeout,
+                    headers={USE_REPLICATION_NETWORK_HEADER: 'true'})
                 self.stats_return_codes[2] = \
                     self.stats_return_codes.get(2, 0) + 1
                 self.logger.increment('return_codes.2')
             except ClientException as err:
                 if self.logger.getEffectiveLevel() <= DEBUG:
                     self.logger.exception(
-                        'Exception with %(ip)s:%(port)s/%(device)s', node)
+                        'Exception with %s', node_to_string(node))
                 self.stats_return_codes[err.http_status // 100] = \
                     self.stats_return_codes.get(err.http_status // 100, 0) + 1
                 self.logger.increment(
                     'return_codes.%d' % (err.http_status // 100,))
-            except (Timeout, socket.error) as err:
+            except (Timeout, socket.error):
                 self.logger.error(
-                    'Timeout Exception with %(ip)s:%(port)s/%(device)s',
-                    node)
+                    'Timeout Exception with %s', node_to_string(node))
             if not objects:
                 break
             try:
@@ -417,7 +417,8 @@ class AccountReaper(Daemon):
                              'X-Account-Partition': str(account_partition),
                              'X-Account-Device': anode['device'],
                              'X-Account-Override-Deleted': 'yes',
-                             'X-Timestamp': timestamp.internal})
+                             'X-Timestamp': timestamp.internal,
+                             USE_REPLICATION_NETWORK_HEADER: 'true'})
                 successes += 1
                 self.stats_return_codes[2] = \
                     self.stats_return_codes.get(2, 0) + 1
@@ -425,17 +426,16 @@ class AccountReaper(Daemon):
             except ClientException as err:
                 if self.logger.getEffectiveLevel() <= DEBUG:
                     self.logger.exception(
-                        'Exception with %(ip)s:%(port)s/%(device)s', node)
+                        'Exception with %s', node_to_string(node))
                 failures += 1
                 self.logger.increment('containers_failures')
                 self.stats_return_codes[err.http_status // 100] = \
                     self.stats_return_codes.get(err.http_status // 100, 0) + 1
                 self.logger.increment(
                     'return_codes.%d' % (err.http_status // 100,))
-            except (Timeout, socket.error) as err:
+            except (Timeout, socket.error):
                 self.logger.error(
-                    'Timeout Exception with %(ip)s:%(port)s/%(device)s',
-                    node)
+                    'Timeout Exception with %s', node_to_string(node))
                 failures += 1
                 self.logger.increment('containers_failures')
         if successes > failures:
@@ -493,7 +493,8 @@ class AccountReaper(Daemon):
                              'X-Container-Partition': str(container_partition),
                              'X-Container-Device': cnode['device'],
                              'X-Backend-Storage-Policy-Index': policy_index,
-                             'X-Timestamp': timestamp.internal})
+                             'X-Timestamp': timestamp.internal,
+                             USE_REPLICATION_NETWORK_HEADER: 'true'})
                 successes += 1
                 self.stats_return_codes[2] = \
                     self.stats_return_codes.get(2, 0) + 1
@@ -501,19 +502,18 @@ class AccountReaper(Daemon):
             except ClientException as err:
                 if self.logger.getEffectiveLevel() <= DEBUG:
                     self.logger.exception(
-                        'Exception with %(ip)s:%(port)s/%(device)s', node)
+                        'Exception with %s', node_to_string(node))
                 failures += 1
                 self.logger.increment('objects_failures')
                 self.stats_return_codes[err.http_status // 100] = \
                     self.stats_return_codes.get(err.http_status // 100, 0) + 1
                 self.logger.increment(
                     'return_codes.%d' % (err.http_status // 100,))
-            except (Timeout, socket.error) as err:
+            except (Timeout, socket.error):
                 failures += 1
                 self.logger.increment('objects_failures')
                 self.logger.error(
-                    'Timeout Exception with %(ip)s:%(port)s/%(device)s',
-                    node)
+                    'Timeout Exception with %s', node_to_string(node))
             if successes > failures:
                 self.stats_objects_deleted += 1
                 self.logger.increment('objects_deleted')
@@ -523,3 +523,12 @@ class AccountReaper(Daemon):
             else:
                 self.stats_objects_possibly_remaining += 1
                 self.logger.increment('objects_possibly_remaining')
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountReaper, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/account/replicator.py

@@ -13,8 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import optparse
+
 from swift.account.backend import AccountBroker, DATADIR
 from swift.common import db_replicator
+from swift.common.daemon import run_daemon
+from swift.common.utils import parse_options
 
 
 class AccountReplicator(db_replicator.Replicator):
@@ -22,3 +26,21 @@ class AccountReplicator(db_replicator.Replicator):
     brokerclass = AccountBroker
     datadir = DATADIR
     default_port = 6202
+
+
+def main():
+    parser = optparse.OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help=('Replicate only given devices. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    parser.add_option('-p', '--partitions',
+                      help=('Replicate only given partitions. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(AccountReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()