swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/db.py

@@ -17,17 +17,13 @@
 
 from contextlib import contextmanager, closing
 import base64
-import hashlib
 import json
 import logging
 import os
 from uuid import uuid4
-import sys
 import time
 import errno
-import six
-import six.moves.cPickle as pickle
-from swift import gettext_ as _
+import pickle  # nosec: B403
 from tempfile import mkstemp
 
 from eventlet import sleep, Timeout
@@ -36,13 +32,15 @@ import sqlite3
 from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE, \
     check_utf8
 from swift.common.utils import Timestamp, renamer, \
-    mkdirs, lock_parent_directory, fallocate
+    mkdirs, lock_parent_directory, fallocate, md5
 from swift.common.exceptions import LockTimeout
 from swift.common.swob import HTTPBadRequest
 
 
 #: Whether calls will be made to preallocate disk space for database files.
 DB_PREALLOCATION = False
+#: Whether calls will be made to log queries (py3 only)
+QUERY_LOGGING = False
 #: Timeout for trying to connect to a DB
 BROKER_TIMEOUT = 25
 #: Pickle protocol to use
@@ -55,24 +53,14 @@ SQLITE_ARG_LIMIT = 999
 RECLAIM_PAGE_SIZE = 10000
 
 
-def utf8encode(*args):
-    return [(s.encode('utf8') if isinstance(s, six.text_type) else s)
-            for s in args]
-
-
-def native_str_keys(metadata):
-    if six.PY2:
-        uni_keys = [k for k in metadata if isinstance(k, six.text_type)]
-        for k in uni_keys:
-            sv = metadata[k]
-            del metadata[k]
-            metadata[k.encode('utf-8')] = sv
-    else:
-        bin_keys = [k for k in metadata if isinstance(k, six.binary_type)]
-        for k in bin_keys:
-            sv = metadata[k]
-            del metadata[k]
-            metadata[k.decode('utf-8')] = sv
+def native_str_keys_and_values(metadata):
+    bin_keys = [k for k in metadata if isinstance(k, bytes)]
+    for k in bin_keys:
+        sv = metadata[k]
+        del metadata[k]
+        metadata[k.decode('utf-8')] = [
+            x.decode('utf-8') if isinstance(x, bytes) else x
+            for x in sv]
 
 
 ZERO_LIKE_VALUES = {None, '', 0, '0'}
@@ -125,19 +113,30 @@ class DatabaseAlreadyExists(sqlite3.DatabaseError):
 
 class GreenDBConnection(sqlite3.Connection):
     """SQLite DB Connection handler that plays well with eventlet."""
+    # slots are needed for python 3.11.0 (there's an issue fixed in 3.11.1,
+    # see https://github.com/python/cpython/issues/99886)
+    __slots__ = ('timeout', 'db_file')
 
     def __init__(self, database, timeout=None, *args, **kwargs):
         if timeout is None:
             timeout = BROKER_TIMEOUT
         self.timeout = timeout
         self.db_file = database
-        super(GreenDBConnection, self).__init__(database, 0, *args, **kwargs)
+        super(GreenDBConnection, self).__init__(
+            database, timeout=0, *args, **kwargs)
 
     def cursor(self, cls=None):
         if cls is None:
             cls = GreenDBCursor
         return sqlite3.Connection.cursor(self, cls)
 
+    def execute(self, *args, **kwargs):
+        # py311 stopped calling self.cursor() to get the cursor;
+        # see https://github.com/python/cpython/pull/31351
+        curs = self.cursor()
+        curs.execute(*args, **kwargs)
+        return curs
+
     def commit(self):
         return _db_timeout(
             self.timeout, self.db_file,
@@ -146,6 +145,9 @@ class GreenDBConnection(sqlite3.Connection):
 
 class GreenDBCursor(sqlite3.Cursor):
     """SQLite Cursor handler that plays well with eventlet."""
+    # slots are needed for python 3.11.0 (there's an issue fixed in 3.11.1,
+    # see https://github.com/python/cpython/issues/99886)
+    __slots__ = ('timeout', 'db_file')
 
     def __init__(self, *args, **kwargs):
         self.timeout = args[0].timeout
@@ -157,6 +159,9 @@ class GreenDBCursor(sqlite3.Cursor):
             self.timeout, self.db_file, lambda: sqlite3.Cursor.execute(
                 self, *args, **kwargs))
 
+    # NB: executemany and executescript are *not* greened, and never have been
+    # (as far as I can tell)
+
 
 def dict_factory(crs, row):
     """
@@ -180,11 +185,12 @@ def chexor(old, name, timestamp):
     """
     if name is None:
         raise Exception('name is None!')
-    new = hashlib.md5(('%s-%s' % (name, timestamp)).encode('utf8')).hexdigest()
+    new = md5(('%s-%s' % (name, timestamp)).encode('utf8'),
+              usedforsecurity=False).hexdigest()
     return '%032x' % (int(old, 16) ^ int(new, 16))
 
 
-def get_db_connection(path, timeout=30, okay_to_create=False):
+def get_db_connection(path, timeout=30, logger=None, okay_to_create=False):
     """
     Returns a properly configured SQLite database connection.
 
@@ -197,7 +203,9 @@ def get_db_connection(path, timeout=30, okay_to_create=False):
         connect_time = time.time()
         conn = sqlite3.connect(path, check_same_thread=False,
                                factory=GreenDBConnection, timeout=timeout)
-        if path != ':memory:' and not okay_to_create:
+        if QUERY_LOGGING and logger:
+            conn.set_trace_callback(logger.debug)
+        if not okay_to_create:
             # attempt to detect and fail when connect creates the db file
             stat = os.stat(path)
             if stat.st_size == 0 and stat.st_ctime >= connect_time:
@@ -219,9 +227,87 @@ def get_db_connection(path, timeout=30, okay_to_create=False):
     return conn
 
 
+class TombstoneReclaimer(object):
+    """Encapsulates reclamation of deleted rows in a database."""
+    def __init__(self, broker, age_timestamp):
+        """
+        Encapsulates reclamation of deleted rows in a database.
+
+        :param broker: an instance of :class:`~swift.common.db.DatabaseBroker`.
+        :param age_timestamp: a float timestamp: tombstones older than this
+            time will be deleted.
+        """
+        self.broker = broker
+        self.age_timestamp = age_timestamp
+        self.marker = ''
+        self.remaining_tombstones = self.reclaimed = 0
+        self.finished = False
+        # limit 1 offset N gives back the N+1th matching row; that row is used
+        # as an exclusive end_marker for a batch of deletes, so a batch
+        # comprises rows satisfying self.marker <= name < end_marker.
+        self.batch_query = '''
+            SELECT name FROM %s WHERE deleted = 1
+            AND name >= ?
+            ORDER BY NAME LIMIT 1 OFFSET ?
+        ''' % self.broker.db_contains_type
+        self.clean_batch_query = '''
+            DELETE FROM %s WHERE deleted = 1
+            AND name >= ? AND %s < %s
+        ''' % (self.broker.db_contains_type, self.broker.db_reclaim_timestamp,
+               self.age_timestamp)
+
+    def _reclaim(self, conn):
+        curs = conn.execute(self.batch_query, (self.marker, RECLAIM_PAGE_SIZE))
+        row = curs.fetchone()
+        end_marker = row[0] if row else ''
+        if end_marker:
+            # do a single book-ended DELETE and bounce out
+            curs = conn.execute(self.clean_batch_query + ' AND name < ?',
+                                (self.marker, end_marker))
+            self.marker = end_marker
+            self.reclaimed += curs.rowcount
+            self.remaining_tombstones += RECLAIM_PAGE_SIZE - curs.rowcount
+        else:
+            # delete off the end
+            curs = conn.execute(self.clean_batch_query, (self.marker,))
+            self.finished = True
+            self.reclaimed += curs.rowcount
+
+    def reclaim(self):
+        """
+        Perform reclaim of deleted rows older than ``age_timestamp``.
+        """
+        while not self.finished:
+            with self.broker.get() as conn:
+                self._reclaim(conn)
+                conn.commit()
+
+    def get_tombstone_count(self):
+        """
+        Return the number of remaining tombstones newer than ``age_timestamp``.
+        Executes the ``reclaim`` method if it has not already been called on
+        this instance.
+
+        :return: The number of tombstones in the ``broker`` that are newer than
+            ``age_timestamp``.
+        """
+        if not self.finished:
+            self.reclaim()
+        with self.broker.get() as conn:
+            curs = conn.execute('''
+                SELECT COUNT(*) FROM %s WHERE deleted = 1
+                AND name >= ?
+            ''' % (self.broker.db_contains_type,), (self.marker,))
+        tombstones = curs.fetchone()[0]
+        self.remaining_tombstones += tombstones
+        return self.remaining_tombstones
+
+
 class DatabaseBroker(object):
     """Encapsulates working with a database."""
 
+    delete_meta_whitelist = []
+
     def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None,
                  account=None, container=None, pending_timeout=None,
                  stale_reads_ok=False, skip_commits=False):
@@ -273,15 +359,13 @@ class DatabaseBroker(object):
         :param put_timestamp: internalized timestamp of initial PUT request
         :param storage_policy_index: only required for containers
         """
-        if self._db_file == ':memory:':
-            tmp_db_file = None
-            conn = get_db_connection(self._db_file, self.timeout)
-        else:
-            mkdirs(self.db_dir)
-            fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
-            os.close(fd)
-            conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
-                                   factory=GreenDBConnection, timeout=0)
+        mkdirs(self.db_dir)
+        fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
+        os.close(fd)
+        conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
+                               factory=GreenDBConnection, timeout=0)
+        if QUERY_LOGGING:
+            conn.set_trace_callback(self.logger.debug)
         # creating dbs implicitly does a lot of transactions, so we
         # pick fast, unsafe options here and do a big fsync at the end.
         with closing(conn.cursor()) as cur:
@@ -342,7 +426,8 @@ class DatabaseBroker(object):
                     # of the system were "racing" each other.
                     raise DatabaseAlreadyExists(self.db_file)
                 renamer(tmp_db_file, self.db_file)
-            self.conn = get_db_connection(self.db_file, self.timeout)
+            self.conn = get_db_connection(self.db_file, self.timeout,
+                                          self.logger)
         else:
             self.conn = conn
 
@@ -355,11 +440,20 @@ class DatabaseBroker(object):
         # first, clear the metadata
         cleared_meta = {}
         for k in self.metadata:
+            if k.lower() in self.delete_meta_whitelist:
+                continue
             cleared_meta[k] = ('', timestamp)
         self.update_metadata(cleared_meta)
         # then mark the db as deleted
         with self.get() as conn:
-            self._delete_db(conn, timestamp)
+            conn.execute(
+                """
+                UPDATE %s_stat
+                SET delete_timestamp = ?,
+                    status = 'DELETED',
+                    status_changed_at = ?
+                WHERE delete_timestamp < ? """ % self.db_type,
+                (timestamp, timestamp, timestamp))
             conn.commit()
 
     @property
@@ -388,32 +482,32 @@ class DatabaseBroker(object):
                 raise
             quar_path = "%s-%s" % (quar_path, uuid4().hex)
             renamer(self.db_dir, quar_path, fsync=False)
-        detail = _('Quarantined %(db_dir)s to %(quar_path)s due to '
-                   '%(reason)s') % {'db_dir': self.db_dir,
-                                    'quar_path': quar_path,
-                                    'reason': reason}
+        detail = ('Quarantined %(db_dir)s to %(quar_path)s due to '
+                  '%(reason)s') % {'db_dir': self.db_dir,
+                                   'quar_path': quar_path,
+                                   'reason': reason}
         self.logger.error(detail)
         raise sqlite3.DatabaseError(detail)
 
-    def possibly_quarantine(self, exc_type, exc_value, exc_traceback):
+    def possibly_quarantine(self, err):
         """
         Checks the exception info to see if it indicates a quarantine situation
         (malformed or corrupted database). If not, the original exception will
         be reraised. If so, the database will be quarantined and a new
         sqlite3.DatabaseError will be raised indicating the action taken.
         """
-        if 'database disk image is malformed' in str(exc_value):
+        if 'database disk image is malformed' in str(err):
             exc_hint = 'malformed database'
-        elif 'malformed database schema' in str(exc_value):
+        elif 'malformed database schema' in str(err):
             exc_hint = 'malformed database'
-        elif ' is not a database' in str(exc_value):
+        elif ' is not a database' in str(err):
             # older versions said 'file is not a database'
             # now 'file is encrypted or is not a database'
             exc_hint = 'corrupted database'
-        elif 'disk I/O error' in str(exc_value):
+        elif 'disk I/O error' in str(err):
             exc_hint = 'disk error while accessing database'
         else:
-            six.reraise(exc_type, exc_value, exc_traceback)
+            raise err
 
         self.quarantine(exc_hint)
 
@@ -443,11 +537,12 @@ class DatabaseBroker(object):
     def get(self):
         """Use with the "with" statement; returns a database connection."""
         if not self.conn:
-            if self.db_file != ':memory:' and os.path.exists(self.db_file):
+            if os.path.exists(self.db_file):
                 try:
-                    self.conn = get_db_connection(self.db_file, self.timeout)
-                except (sqlite3.DatabaseError, DatabaseConnectionError):
-                    self.possibly_quarantine(*sys.exc_info())
+                    self.conn = get_db_connection(self.db_file, self.timeout,
+                                                  self.logger)
+                except (sqlite3.DatabaseError, DatabaseConnectionError) as e:
+                    self.possibly_quarantine(e)
             else:
                 raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
         conn = self.conn
@@ -456,12 +551,12 @@ class DatabaseBroker(object):
             yield conn
             conn.rollback()
             self.conn = conn
-        except sqlite3.DatabaseError:
+        except sqlite3.DatabaseError as e:
             try:
                 conn.close()
             except Exception:
                 pass
-            self.possibly_quarantine(*sys.exc_info())
+            self.possibly_quarantine(e)
         except (Exception, Timeout):
             conn.close()
             raise
@@ -470,8 +565,9 @@ class DatabaseBroker(object):
     def lock(self):
         """Use with the "with" statement; locks a database."""
         if not self.conn:
-            if self.db_file != ':memory:' and os.path.exists(self.db_file):
-                self.conn = get_db_connection(self.db_file, self.timeout)
+            if os.path.exists(self.db_file):
+                self.conn = get_db_connection(self.db_file, self.timeout,
+                                              self.logger)
             else:
                 raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
         conn = self.conn
@@ -481,16 +577,19 @@ class DatabaseBroker(object):
         conn.execute('BEGIN IMMEDIATE')
         try:
             yield True
-        except (Exception, Timeout):
-            pass
-        try:
-            conn.execute('ROLLBACK')
-            conn.isolation_level = orig_isolation_level
-            self.conn = conn
-        except (Exception, Timeout):
-            logging.exception(
-                _('Broker error trying to rollback locked connection'))
-            conn.close()
+        finally:
+            try:
+                conn.execute('ROLLBACK')
+                conn.isolation_level = orig_isolation_level
+                self.conn = conn
+            except (Exception, Timeout):
+                logging.exception(
+                    'Broker error trying to rollback locked connection')
+                conn.close()
+
+    def _new_db_id(self):
+        device_name = os.path.basename(self.get_device_path())
+        return "%s-%s" % (str(uuid4()), device_name)
 
     def newid(self, remote_id):
         """
@@ -501,7 +600,7 @@ class DatabaseBroker(object):
         with self.get() as conn:
             row = conn.execute('''
                 UPDATE %s_stat SET id=?
-            ''' % self.db_type, (str(uuid4()),))
+            ''' % self.db_type, (self._new_db_id(),))
             row = conn.execute('''
                 SELECT ROWID FROM %s ORDER BY ROWID DESC LIMIT 1
             ''' % self.db_contains_type).fetchone()
@@ -533,7 +632,7 @@ class DatabaseBroker(object):
 
         :returns: True if the DB is considered to be deleted, False otherwise
         """
-        if self.db_file != ':memory:' and not os.path.exists(self.db_file):
+        if not os.path.exists(self.db_file):
             return True
         self._commit_puts_stale_ok()
         with self.get() as conn:
@@ -610,22 +709,26 @@ class DatabaseBroker(object):
                 return -1
             return row['sync_point']
 
-    def get_syncs(self, incoming=True):
+    def get_syncs(self, incoming=True, include_timestamp=False):
         """
         Get a serialized copy of the sync table.
 
         :param incoming: if True, get the last incoming sync, otherwise get
                          the last outgoing sync
-        :returns: list of {'remote_id', 'sync_point'}
+        :param include_timestamp: If True include the updated_at timestamp
+        :returns: list of {'remote_id', 'sync_point'} or
+                 {'remote_id', 'sync_point', 'updated_at'}
+                 if include_timestamp is True.
         """
         with self.get() as conn:
+            columns = 'remote_id, sync_point'
+            if include_timestamp:
+                columns += ', updated_at'
             curs = conn.execute('''
-                SELECT remote_id, sync_point FROM %s_sync
-            ''' % ('incoming' if incoming else 'outgoing'))
-            result = []
-            for row in curs:
-                result.append({'remote_id': row[0], 'sync_point': row[1]})
-            return result
+                SELECT %s FROM %s_sync
+            ''' % (columns, 'incoming' if incoming else 'outgoing'))
+            curs.row_factory = dict_factory
+            return [r for r in curs]
 
     def get_max_row(self, table=None):
         if not table:
@@ -666,8 +769,8 @@ class DatabaseBroker(object):
         """
         Put a record into the DB. If the DB has an associated pending file with
         space then the record is appended to that file and a commit to the DB
-        is deferred. If the DB is in-memory or its pending file is full then
-        the record will be committed immediately.
+        is deferred. If its pending file is full then the record will be
+        committed immediately.
 
         :param record: a record to be added to the DB.
         :raises DatabaseConnectionError: if the DB file does not exist or if
@@ -675,9 +778,6 @@ class DatabaseBroker(object):
         :raises LockTimeout: if a timeout occurs while waiting to take a lock
             to write to the pending file.
         """
-        if self._db_file == ':memory:':
-            self.merge_items([record])
-            return
         if not os.path.exists(self.db_file):
             raise DatabaseConnectionError(self.db_file, "DB doesn't exist")
         if self.skip_commits:
@@ -703,8 +803,7 @@ class DatabaseBroker(object):
                     fp.flush()
 
     def _skip_commit_puts(self):
-        return (self._db_file == ':memory:' or self.skip_commits or not
-                os.path.exists(self.pending_file))
+        return self.skip_commits or not os.path.exists(self.pending_file)
 
     def _commit_puts(self, item_list=None):
         """
@@ -733,15 +832,12 @@ class DatabaseBroker(object):
             for entry in fp.read().split(b':'):
                 if entry:
                     try:
-                        if six.PY2:
-                            data = pickle.loads(base64.b64decode(entry))
-                        else:
-                            data = pickle.loads(base64.b64decode(entry),
-                                                encoding='utf8')
+                        data = pickle.loads(base64.b64decode(entry),
+                                            encoding='utf8')  # nosec: B301
                         self._commit_puts_load(item_list, data)
                     except Exception:
                         self.logger.exception(
-                            _('Invalid pending entry %(file)s: %(entry)s'),
+                            'Invalid pending entry %(file)s: %(entry)s',
                             {'file': self.pending_file, 'entry': entry})
             if item_list:
                 self.merge_items(item_list)
@@ -818,7 +914,7 @@ class DatabaseBroker(object):
         within 512k of a boundary, it allocates to the next boundary.
         Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
         """
-        if not DB_PREALLOCATION or self._db_file == ':memory:':
+        if not DB_PREALLOCATION:
             return
         MB = (1024 * 1024)
 
@@ -865,7 +961,7 @@ class DatabaseBroker(object):
         metadata = self.get_raw_metadata()
         if metadata:
             metadata = json.loads(metadata)
-            native_str_keys(metadata)
+            native_str_keys_and_values(metadata)
         else:
             metadata = {}
         return metadata
@@ -927,7 +1023,7 @@ class DatabaseBroker(object):
                                     self.db_type)
                 md = row[0]
                 md = json.loads(md) if md else {}
-                native_str_keys(md)
+                native_str_keys_and_values(md)
             except sqlite3.OperationalError as err:
                 if 'no such column: metadata' not in str(err):
                     raise
@@ -964,47 +1060,22 @@ class DatabaseBroker(object):
             with lock_parent_directory(self.pending_file,
                                        self.pending_timeout):
                 self._commit_puts()
-        marker = ''
-        finished = False
-        while not finished:
-            with self.get() as conn:
-                marker = self._reclaim(conn, age_timestamp, marker)
-                if not marker:
-                    finished = True
-                    self._reclaim_other_stuff(
-                        conn, age_timestamp, sync_timestamp)
-                conn.commit()
+
+        tombstone_reclaimer = TombstoneReclaimer(self, age_timestamp)
+        tombstone_reclaimer.reclaim()
+        with self.get() as conn:
+            self._reclaim_other_stuff(conn, age_timestamp, sync_timestamp)
+            conn.commit()
+        return tombstone_reclaimer
 
     def _reclaim_other_stuff(self, conn, age_timestamp, sync_timestamp):
         """
-        This is only called once at the end of reclaim after _reclaim has been
-        called for each page.
+        This is only called once at the end of reclaim after tombstone reclaim
+        has been completed.
         """
         self._reclaim_sync(conn, sync_timestamp)
         self._reclaim_metadata(conn, age_timestamp)
 
-    def _reclaim(self, conn, age_timestamp, marker):
-        clean_batch_qry = '''
-            DELETE FROM %s WHERE deleted = 1
-            AND name > ? AND %s < ?
-        ''' % (self.db_contains_type, self.db_reclaim_timestamp)
-        curs = conn.execute('''
-            SELECT name FROM %s WHERE deleted = 1
-            AND name > ?
-            ORDER BY NAME LIMIT 1 OFFSET ?
-        ''' % (self.db_contains_type,), (marker, RECLAIM_PAGE_SIZE))
-        row = curs.fetchone()
-        if row:
-            # do a single book-ended DELETE and bounce out
-            end_marker = row[0]
-            conn.execute(clean_batch_qry + ' AND name <= ?', (
-                marker, age_timestamp, end_marker))
-        else:
-            # delete off the end and reset marker to indicate we're done
-            end_marker = ''
-            conn.execute(clean_batch_qry, (marker, age_timestamp))
-        return end_marker
-
     def _reclaim_sync(self, conn, sync_timestamp):
         try:
             conn.execute('''