swift 2.32.1__py2.py3-none-any.whl → 2.33.1__py2.py3-none-any.whl

swift/common/utils/__init__.py

@@ -114,6 +114,7 @@ from swift.common.utils.timestamp import (  # noqa
     EPOCH,
     last_modified_date_to_timestamp,
     normalize_delete_at_timestamp,
+    UTC,
 )
 from swift.common.utils.ipaddrs import (  # noqa
     is_valid_ip,
@@ -170,6 +171,11 @@ LOG_LINE_DEFAULT_FORMAT = '{remote_addr} - - [{time.d}/{time.b}/{time.Y}' \
                           '{trans_time:.4f} "{additional_info}" {pid} ' \
                           '{policy_index}'
 DEFAULT_LOCK_TIMEOUT = 10
+# this is coupled with object-server.conf's network_chunk_size; if someone is
+# running that unreasonably small they may find this number inefficient, but in
+# the more likely case they've increased the value to optimize high througput
+# transfers this will still cut off the transfer after the first chunk.
+DEFAULT_DRAIN_LIMIT = 65536
 
 
 class InvalidHashPathConfigError(ValueError):
@@ -275,7 +281,7 @@ def backward(f, blocksize=4096):
 
 
 # Used when reading config values
-TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
+TRUE_VALUES = {'true', '1', 'yes', 'on', 't', 'y'}
 
 
 def non_negative_float(value):
@@ -516,22 +522,6 @@ def get_policy_index(req_headers, res_headers):
     return str(policy_index) if policy_index is not None else None
 
 
-class _UTC(datetime.tzinfo):
-    """
-    A tzinfo class for datetime objects that returns a 0 timedelta (UTC time)
-    """
-
-    def dst(self, dt):
-        return datetime.timedelta(0)
-    utcoffset = dst
-
-    def tzname(self, dt):
-        return 'UTC'
-
-
-UTC = _UTC()
-
-
 class LogStringFormatter(string.Formatter):
     def __init__(self, default='', quote=False):
         super(LogStringFormatter, self).__init__()
@@ -830,13 +820,14 @@ class FileLikeIter(object):
         self.closed = True
 
 
-def fs_has_free_space(fs_path, space_needed, is_percent):
+def fs_has_free_space(fs_path_or_fd, space_needed, is_percent):
     """
     Check to see whether or not a filesystem has the given amount of space
     free. Unlike fallocate(), this does not reserve any space.
 
-    :param fs_path: path to a file or directory on the filesystem; typically
-        the path to the filesystem's mount point
+    :param fs_path_or_fd: path to a file or directory on the filesystem, or an
+        open file descriptor; if a directory, typically the path to the
+        filesystem's mount point
 
     :param space_needed: minimum bytes or percentage of free space
 
@@ -849,7 +840,10 @@ def fs_has_free_space(fs_path, space_needed, is_percent):
 
     :raises OSError: if fs_path does not exist
     """
-    st = os.statvfs(fs_path)
+    if isinstance(fs_path_or_fd, int):
+        st = os.fstatvfs(fs_path_or_fd)
+    else:
+        st = os.statvfs(fs_path_or_fd)
     free_bytes = st.f_frsize * st.f_bavail
     if is_percent:
         size_bytes = st.f_frsize * st.f_blocks
@@ -1495,7 +1489,7 @@ class StatsdClient(object):
 
     def _timing(self, metric, timing_ms, sample_rate):
         # This method was added to disagregate timing metrics when testing
-        return self._send(metric, timing_ms, 'ms', sample_rate)
+        return self._send(metric, round(timing_ms, 4), 'ms', sample_rate)
 
     def timing(self, metric, timing_ms, sample_rate=None):
         return self._timing(metric, timing_ms, sample_rate)
@@ -1577,28 +1571,23 @@ class SwiftLoggerAdapter(logging.LoggerAdapter):
         # py3 does this for us already; add it for py2
         return self.logger.name
 
-    def get_metric_name(self, metric):
-        # subclasses may override this method to annotate the metric name
-        return metric
+    def update_stats(self, *a, **kw):
+        return self.logger.update_stats(*a, **kw)
 
-    def update_stats(self, metric, *a, **kw):
-        return self.logger.update_stats(self.get_metric_name(metric), *a, **kw)
+    def increment(self, *a, **kw):
+        return self.logger.increment(*a, **kw)
 
-    def increment(self, metric, *a, **kw):
-        return self.logger.increment(self.get_metric_name(metric), *a, **kw)
+    def decrement(self, *a, **kw):
+        return self.logger.decrement(*a, **kw)
 
-    def decrement(self, metric, *a, **kw):
-        return self.logger.decrement(self.get_metric_name(metric), *a, **kw)
+    def timing(self, *a, **kw):
+        return self.logger.timing(*a, **kw)
 
-    def timing(self, metric, *a, **kw):
-        return self.logger.timing(self.get_metric_name(metric), *a, **kw)
+    def timing_since(self, *a, **kw):
+        return self.logger.timing_since(*a, **kw)
 
-    def timing_since(self, metric, *a, **kw):
-        return self.logger.timing_since(self.get_metric_name(metric), *a, **kw)
-
-    def transfer_rate(self, metric, *a, **kw):
-        return self.logger.transfer_rate(
-            self.get_metric_name(metric), *a, **kw)
+    def transfer_rate(self, *a, **kw):
+        return self.logger.transfer_rate(*a, **kw)
 
     @property
     def thread_locals(self):
@@ -1635,27 +1624,6 @@ class PrefixLoggerAdapter(SwiftLoggerAdapter):
         return (msg, kwargs)
 
 
-class MetricsPrefixLoggerAdapter(SwiftLoggerAdapter):
-    """
-    Adds a prefix to all Statsd metrics' names.
-    """
-
-    def __init__(self, logger, extra, metric_prefix):
-        """
-        :param logger: an instance of logging.Logger
-        :param extra: a dict-like object
-        :param metric_prefix: A prefix that will be added to the start of each
-            metric name such that the metric name is transformed to:
-            ``<metric_prefix>.<metric name>``. Note that the logger's
-            StatsdClient also adds its configured prefix to metric names.
-        """
-        super(MetricsPrefixLoggerAdapter, self).__init__(logger, extra)
-        self.metric_prefix = metric_prefix
-
-    def get_metric_name(self, metric):
-        return '%s.%s' % (self.metric_prefix, metric)
-
-
 # double inheritance to support property with setter
 class LogAdapter(logging.LoggerAdapter, object):
     """
@@ -3425,7 +3393,7 @@ def put_recon_cache_entry(cache_entry, key, item):
 
     If ``item`` is an empty dict then any existing ``key`` in ``cache_entry``
     will be deleted. Similarly if ``item`` is a dict and any of its values are
-    empty dicts then the corrsponsing key will be deleted from the nested dict
+    empty dicts then the corresponding key will be deleted from the nested dict
     in ``cache_entry``.
 
     We use nested recon cache entries when the object auditor
@@ -3704,27 +3672,66 @@ def csv_append(csv_string, item):
         return item
 
 
-class CloseableChain(object):
+class ClosingIterator(object):
     """
-    Like itertools.chain, but with a close method that will attempt to invoke
-    its sub-iterators' close methods, if any.
+    Wrap another iterator and close it, if possible, on completion/exception.
+
+    If other closeable objects are given then they will also be closed when
+    this iterator is closed.
+
+    This is particularly useful for ensuring a generator properly closes its
+    resources, even if the generator was never started.
+
+    This class may be subclassed to override the behavior of
+    ``_get_next_item``.
+
+    :param iterable: iterator to wrap.
+    :param other_closeables: other resources to attempt to close.
     """
+    __slots__ = ('closeables', 'wrapped_iter', 'closed')
 
-    def __init__(self, *iterables):
-        self.iterables = iterables
-        self.chained_iter = itertools.chain(*self.iterables)
+    def __init__(self, iterable, other_closeables=None):
+        self.closeables = [iterable]
+        if other_closeables:
+            self.closeables.extend(other_closeables)
+        # this is usually, but not necessarily, the same object
+        self.wrapped_iter = iter(iterable)
+        self.closed = False
 
     def __iter__(self):
         return self
 
+    def _get_next_item(self):
+        return next(self.wrapped_iter)
+
     def __next__(self):
-        return next(self.chained_iter)
+        try:
+            return self._get_next_item()
+        except Exception:
+            # note: if wrapped_iter is a generator then the exception
+            # already caused it to exit (without raising a GeneratorExit)
+            # but we still need to close any other closeables.
+            self.close()
+            raise
 
     next = __next__  # py2
 
     def close(self):
-        for it in self.iterables:
-            close_if_possible(it)
+        if not self.closed:
+            for wrapped in self.closeables:
+                close_if_possible(wrapped)
+            self.closed = True
+
+
+class CloseableChain(ClosingIterator):
+    """
+    Like itertools.chain, but with a close method that will attempt to invoke
+    its sub-iterators' close methods, if any.
+    """
+
+    def __init__(self, *iterables):
+        chained_iter = itertools.chain(*iterables)
+        super(CloseableChain, self).__init__(chained_iter, iterables)
 
 
 def reiterate(iterable):
@@ -4040,7 +4047,7 @@ def closing_if_possible(maybe_closable):
         close_if_possible(maybe_closable)
 
 
-def drain_and_close(response_or_app_iter):
+def drain_and_close(response_or_app_iter, read_limit=None):
     """
     Drain and close a swob or WSGI response.
 
@@ -4050,9 +4057,26 @@ def drain_and_close(response_or_app_iter):
     app_iter = getattr(response_or_app_iter, 'app_iter', response_or_app_iter)
     if app_iter is None:  # for example, if we used the Response.body property
         return
-    for _chunk in app_iter:
-        pass
-    close_if_possible(app_iter)
+    bytes_read = 0
+    with closing_if_possible(app_iter):
+        for chunk in app_iter:
+            bytes_read += len(chunk)
+            if read_limit is not None and bytes_read >= read_limit:
+                break
+
+
+def friendly_close(resp):
+    """
+    Close a swob or WSGI response and maybe drain it.
+
+    It's basically free to "read" a HEAD or HTTPException response - the bytes
+    are probably already in our network buffers.  For a larger response we
+    could possibly burn a lot of CPU/network trying to drain an un-used
+    response.  This method will read up to DEFAULT_DRAIN_LIMIT bytes to avoid
+    logging a 499 in the proxy when it would otherwise be easy to just throw
+    away the small/empty body.
+    """
+    return drain_and_close(resp, read_limit=DEFAULT_DRAIN_LIMIT)
 
 
 _rfc_token = r'[^()<>@,;:\"/\[\]?={}\x00-\x20\x7f]+'
@@ -4396,6 +4420,47 @@ def document_iters_to_multipart_byteranges(ranges_iter, boundary):
     yield terminator
 
 
+class StringAlong(ClosingIterator):
+    """
+    This iterator wraps and iterates over a first iterator until it stops, and
+    then iterates a second iterator, expecting it to stop immediately. This
+    "stringing along" of the second iterator is useful when the exit of the
+    second iterator must be delayed until the first iterator has stopped. For
+    example, when the second iterator has already yielded its item(s) but
+    has resources that mustn't be garbage collected until the first iterator
+    has stopped.
+
+    The second iterator is expected to have no more items and raise
+    StopIteration when called. If this is not the case then
+    ``unexpected_items_func`` is called.
+
+    :param iterable: a first iterator that is wrapped and iterated.
+    :param other_iter: a second iterator that is stopped once the first
+        iterator has stopped.
+    :param unexpected_items_func: a no-arg function that will be called if the
+        second iterator is found to have remaining items.
+    """
+    __slots__ = ('other_iter', 'unexpected_items_func')
+
+    def __init__(self, iterable, other_iter, unexpected_items_func):
+        super(StringAlong, self).__init__(iterable, [other_iter])
+        self.other_iter = other_iter
+        self.unexpected_items_func = unexpected_items_func
+
+    def _get_next_item(self):
+        try:
+            return super(StringAlong, self)._get_next_item()
+        except StopIteration:
+            try:
+                next(self.other_iter)
+            except StopIteration:
+                pass
+            else:
+                self.unexpected_items_func()
+            finally:
+                raise
+
+
 def document_iters_to_http_response_body(ranges_iter, boundary, multipart,
                                          logger):
     """
@@ -4445,20 +4510,11 @@ def document_iters_to_http_response_body(ranges_iter, boundary, multipart,
         # ranges_iter has a finally block that calls close_swift_conn, and
         # so if that finally block fires before we read response_body_iter,
         # there's nothing there.
-        def string_along(useful_iter, useless_iter_iter, logger):
-            with closing_if_possible(useful_iter):
-                for x in useful_iter:
-                    yield x
-
-            try:
-                next(useless_iter_iter)
-            except StopIteration:
-                pass
-            else:
-                logger.warning(
-                    "More than one part in a single-part response?")
-
-        return string_along(response_body_iter, ranges_iter, logger)
+        result = StringAlong(
+            response_body_iter, ranges_iter,
+            lambda: logger.warning(
+                "More than one part in a single-part response?"))
+        return result
 
 
 def multipart_byteranges_to_document_iters(input_file, boundary,
@@ -4562,13 +4618,14 @@ class Namespace(object):
     A Namespace encapsulates parameters that define a range of the object
     namespace.
 
-    :param name: the name of the ``Namespace``.
+    :param name: the name of the ``Namespace``; this SHOULD take the form of a
+        path to a container i.e. <account_name>/<container_name>.
     :param lower: the lower bound of object names contained in the namespace;
         the lower bound *is not* included in the namespace.
     :param upper: the upper bound of object names contained in the namespace;
         the upper bound *is* included in the namespace.
     """
-    __slots__ = ('_lower', '_upper', 'name')
+    __slots__ = ('_lower', '_upper', '_name')
 
     @functools.total_ordering
     class MaxBound(NamespaceOuterBound):
@@ -4588,9 +4645,14 @@ class Namespace(object):
     def __init__(self, name, lower, upper):
         self._lower = Namespace.MIN
         self._upper = Namespace.MAX
+        # We deliberately do not validate that the name has the form 'a/c'
+        # because we want Namespace instantiation to be fast. Namespaces are
+        # typically created using state that has previously been serialized
+        # from a ShardRange instance, and the ShardRange will have validated
+        # the name format.
+        self._name = self._encode(name)
         self.lower = lower
         self.upper = upper
-        self.name = name
 
     def __iter__(self):
         yield 'name', str(self.name)
@@ -4618,7 +4680,7 @@ class Namespace(object):
     def __gt__(self, other):
         # a Namespace is greater than other if its entire namespace is greater
         # than other; if other is another Namespace that implies that this
-        # Namespace's lower must be less greater than or equal to the other
+        # Namespace's lower must be greater than or equal to the other
         # Namespace's upper
         if self.lower == Namespace.MIN:
             return False
@@ -4663,6 +4725,21 @@ class Namespace(object):
             raise TypeError('must be a string type')
         return self._encode(bound)
 
+    @property
+    def account(self):
+        return self._name.split('/')[0]
+
+    @property
+    def container(self):
+        # note: this may raise an IndexError if name does not have the expected
+        # form 'a/c'; that is a deliberate trade-off against the overhead of
+        # validating the name every time a Namespace is instantiated.
+        return self._name.split('/')[1]
+
+    @property
+    def name(self):
+        return self._name
+
     @property
     def lower(self):
         return self._lower
@@ -4988,7 +5065,7 @@ class ShardRange(Namespace):
     range record and the most recent version of an attribute should be
     persisted.
 
-    :param name: the name of the shard range; this should take the form of a
+    :param name: the name of the shard range; this MUST take the form of a
         path to a container i.e. <account_name>/<container_name>.
     :param timestamp: a timestamp that represents the time at which the
         shard range's ``lower``, ``upper`` or ``deleted`` attributes were
@@ -5046,7 +5123,6 @@ class ShardRange(Namespace):
     CLEAVING_STATES = SHRINKING_STATES + SHARDING_STATES
 
     __slots__ = (
-        'account', 'container',
         '_timestamp', '_meta_timestamp', '_state_timestamp', '_epoch',
         '_deleted', '_state', '_count', '_bytes',
         '_tombstones', '_reported')
@@ -5057,12 +5133,12 @@ class ShardRange(Namespace):
                  deleted=False, state=None, state_timestamp=None, epoch=None,
                  reported=False, tombstones=-1, **kwargs):
         super(ShardRange, self).__init__(name=name, lower=lower, upper=upper)
-        self.account = self.container = self._timestamp = \
-            self._meta_timestamp = self._state_timestamp = self._epoch = None
+        self._validate_name(self.name)
+        self._timestamp = self._meta_timestamp = self._state_timestamp = \
+            self._epoch = None
         self._deleted = False
         self._state = None
 
-        self.name = name
         self.timestamp = timestamp
         self.deleted = deleted
         self.object_count = object_count
@@ -5076,11 +5152,18 @@ class ShardRange(Namespace):
 
     @classmethod
     def sort_key(cls, sr):
+        return cls.sort_key_order(sr.name, sr.lower, sr.upper, sr.state)
+
+    @staticmethod
+    def sort_key_order(name, lower, upper, state):
+        # Use Namespace.MaxBound() for upper bound '', this will allow this
+        # record to be sorted correctly by upper.
+        upper = upper if upper else Namespace.MaxBound()
         # defines the sort order for shard ranges
         # note if this ever changes to *not* sort by upper first then it breaks
         # a key assumption for bisect, which is used by utils.find_namespace
         # with shard ranges.
-        return sr.upper, sr.state, sr.lower, sr.name
+        return upper, state, lower, name
 
     def is_child_of(self, parent):
         """
@@ -5221,16 +5304,24 @@ class ShardRange(Namespace):
 
     @property
     def name(self):
-        return '%s/%s' % (self.account, self.container)
-
-    @name.setter
-    def name(self, path):
-        path = self._encode(path)
-        if not path or len(path.split('/')) != 2 or not all(path.split('/')):
+        return self._name
+
+    @staticmethod
+    def _validate_name(name):
+        # Validate the name format is 'a/c'. The ShardRange class is typically
+        # used when shard state is created (e.g. by the sharder or
+        # swift-manage-shard-ranges), but it is not typically used in
+        # performance sensitive paths (e.g. listing namespaces), so we can
+        # afford the overhead of being more defensive here.
+        if not name or len(name.split('/')) != 2 or not all(name.split('/')):
             raise ValueError(
                 "Name must be of the form '<account>/<container>', got %r" %
-                path)
-        self.account, self.container = path.split('/')
+                name)
+        return name
+
+    @name.setter
+    def name(self, name):
+        self._name = self._validate_name(self._encode(name))
 
     @property
     def timestamp(self):
@@ -5533,7 +5624,7 @@ class ShardRangeList(UserList):
     def __getitem__(self, index):
         # workaround for py3 - not needed for py2.7,py3.8
         result = self.data[index]
-        return ShardRangeList(result) if type(result) == list else result
+        return ShardRangeList(result) if type(result) is list else result
 
     @property
     def lower(self):
@@ -6228,17 +6319,28 @@ def get_db_files(db_path):
     return sorted(results)
 
 
-def systemd_notify(logger=None):
+def systemd_notify(logger=None, msg=b"READY=1"):
     """
-    Notify the service manager that started this process, if it is
-    systemd-compatible, that this process correctly started. To do so,
-    it communicates through a Unix socket stored in environment variable
-    NOTIFY_SOCKET. More information can be found in systemd documentation:
+    Send systemd-compatible notifications.
+
+    Notify the service manager that started this process, if it has set the
+    NOTIFY_SOCKET environment variable. For example, systemd will set this
+    when the unit has ``Type=notify``. More information can be found in
+    systemd documentation:
     https://www.freedesktop.org/software/systemd/man/sd_notify.html
 
+    Common messages include::
+
+       READY=1
+       RELOADING=1
+       STOPPING=1
+       STATUS=<some string>
+
     :param logger: a logger object
+    :param msg: the message to send
     """
-    msg = b'READY=1'
+    if not isinstance(msg, bytes):
+        msg = msg.encode('utf8')
     notify_socket = os.getenv('NOTIFY_SOCKET')
     if notify_socket:
         if notify_socket.startswith('@'):
@@ -6249,7 +6351,6 @@ def systemd_notify(logger=None):
             try:
                 sock.connect(notify_socket)
                 sock.sendall(msg)
-                del os.environ['NOTIFY_SOCKET']
             except EnvironmentError:
                 if logger:
                     logger.debug("Systemd notification failed", exc_info=True)
@@ -6320,7 +6421,7 @@ class Watchdog(object):
         :param key: timeout id, as returned by start()
         """
         try:
-            del(self._timeouts[key])
+            del self._timeouts[key]
         except KeyError:
             pass
 
@@ -6385,7 +6486,7 @@ class WatchdogTimeout(object):
         self.watchdog.stop(self.key)
 
 
-class CooperativeIterator(object):
+class CooperativeIterator(ClosingIterator):
     """
     Wrapper to make a deliberate periodic call to ``sleep()`` while iterating
     over wrapped iterator, providing an opportunity to switch greenthreads.
@@ -6405,26 +6506,20 @@ class CooperativeIterator(object):
 
     :param iterable: iterator to wrap.
     :param period: number of items yielded from this iterator between calls to
-        ``sleep()``.
+        ``sleep()``; a negative value or 0 mean that cooperative sleep will be
+        disabled.
     """
-    __slots__ = ('period', 'count', 'wrapped_iter')
+    __slots__ = ('period', 'count')
 
     def __init__(self, iterable, period=5):
-        self.wrapped_iter = iterable
+        super(CooperativeIterator, self).__init__(iterable)
         self.count = 0
-        self.period = period
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        if self.count >= self.period:
-            self.count = 0
-            sleep()
-        self.count += 1
-        return next(self.wrapped_iter)
-
-    __next__ = next
-
-    def close(self):
-        close_if_possible(self.wrapped_iter)
+        self.period = max(0, period or 0)
+
+    def _get_next_item(self):
+        if self.period:
+            if self.count >= self.period:
+                self.count = 0
+                sleep()
+            self.count += 1
+        return super(CooperativeIterator, self)._get_next_item()

swift/common/utils/timestamp.py

@@ -18,6 +18,7 @@
 import datetime
 import functools
 import math
+import sys
 import time
 
 import six
@@ -189,12 +190,14 @@ class Timestamp(object):
             elif us < 0:
                 t -= 1
                 us += 1000000
-            dt = datetime.datetime.utcfromtimestamp(t)
+            dt = datetime.datetime.fromtimestamp(t, UTC)
             dt = dt.replace(microsecond=us)
         else:
-            dt = datetime.datetime.utcfromtimestamp(t)
+            dt = datetime.datetime.fromtimestamp(t, UTC)
 
         isoformat = dt.isoformat()
+        # need to drop tzinfo
+        isoformat = isoformat[:isoformat.index('+')]
         # python isoformat() doesn't include msecs when zero
         if len(isoformat) < len("1970-01-01T00:00:00.000000"):
             isoformat += ".000000"
@@ -397,3 +400,21 @@ def normalize_delete_at_timestamp(timestamp, high_precision=False):
     """
     fmt = '%016.5f' if high_precision else '%010d'
     return fmt % min(max(0, float(timestamp)), 9999999999.99999)
+
+
+if sys.version_info < (3, 11):
+    class _UTC(datetime.tzinfo):
+        """
+        A tzinfo class for datetimes that returns a 0 timedelta (UTC time)
+        """
+
+        def dst(self, dt):
+            return datetime.timedelta(0)
+        utcoffset = dst
+
+        def tzname(self, dt):
+            return 'UTC'
+
+    UTC = _UTC()
+else:
+    from datetime import UTC