swift 2.31.1__py2.py3-none-any.whl → 2.32.1__py2.py3-none-any.whl

swift/common/utils/timestamp.py

@@ -0,0 +1,399 @@
+# Copyright (c) 2010-2023 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Timestamp-related functions for use with Swift."""
+
+import datetime
+import functools
+import math
+import time
+
+import six
+
+
+NORMAL_FORMAT = "%016.05f"
+INTERNAL_FORMAT = NORMAL_FORMAT + '_%016x'
+SHORT_FORMAT = NORMAL_FORMAT + '_%x'
+MAX_OFFSET = (16 ** 16) - 1
+PRECISION = 1e-5
+# Setting this to True will cause the internal format to always display
+# extended digits - even when the value is equivalent to the normalized form.
+# This isn't ideal during an upgrade when some servers might not understand
+# the new time format - but flipping it to True works great for testing.
+FORCE_INTERNAL = False  # or True
+
+
+@functools.total_ordering
+class Timestamp(object):
+    """
+    Internal Representation of Swift Time.
+
+    The normalized form of the X-Timestamp header looks like a float
+    with a fixed width to ensure stable string sorting - normalized
+    timestamps look like "1402464677.04188"
+
+    To support overwrites of existing data without modifying the original
+    timestamp but still maintain consistency a second internal offset vector
+    is append to the normalized timestamp form which compares and sorts
+    greater than the fixed width float format but less than a newer timestamp.
+    The internalized format of timestamps looks like
+    "1402464677.04188_0000000000000000" - the portion after the underscore is
+    the offset and is a formatted hexadecimal integer.
+
+    The internalized form is not exposed to clients in responses from
+    Swift.  Normal client operations will not create a timestamp with an
+    offset.
+
+    The Timestamp class in common.utils supports internalized and
+    normalized formatting of timestamps and also comparison of timestamp
+    values.  When the offset value of a Timestamp is 0 - it's considered
+    insignificant and need not be represented in the string format; to
+    support backwards compatibility during a Swift upgrade the
+    internalized and normalized form of a Timestamp with an
+    insignificant offset are identical.  When a timestamp includes an
+    offset it will always be represented in the internalized form, but
+    is still excluded from the normalized form.  Timestamps with an
+    equivalent timestamp portion (the float part) will compare and order
+    by their offset.  Timestamps with a greater timestamp portion will
+    always compare and order greater than a Timestamp with a lesser
+    timestamp regardless of it's offset.  String comparison and ordering
+    is guaranteed for the internalized string format, and is backwards
+    compatible for normalized timestamps which do not include an offset.
+    """
+
+    def __init__(self, timestamp, offset=0, delta=0, check_bounds=True):
+        """
+        Create a new Timestamp.
+
+        :param timestamp: time in seconds since the Epoch, may be any of:
+
+            * a float or integer
+            * normalized/internalized string
+            * another instance of this class (offset is preserved)
+
+        :param offset: the second internal offset vector, an int
+        :param delta: deca-microsecond difference from the base timestamp
+                      param, an int
+        """
+        if isinstance(timestamp, bytes):
+            timestamp = timestamp.decode('ascii')
+        if isinstance(timestamp, six.string_types):
+            base, base_offset = timestamp.partition('_')[::2]
+            self.timestamp = float(base)
+            if '_' in base_offset:
+                raise ValueError('invalid literal for int() with base 16: '
+                                 '%r' % base_offset)
+            if base_offset:
+                self.offset = int(base_offset, 16)
+            else:
+                self.offset = 0
+        else:
+            self.timestamp = float(timestamp)
+            self.offset = getattr(timestamp, 'offset', 0)
+        # increment offset
+        if offset >= 0:
+            self.offset += offset
+        else:
+            raise ValueError('offset must be non-negative')
+        if self.offset > MAX_OFFSET:
+            raise ValueError('offset must be smaller than %d' % MAX_OFFSET)
+        self.raw = int(round(self.timestamp / PRECISION))
+        # add delta
+        if delta:
+            self.raw = self.raw + delta
+            if self.raw <= 0:
+                raise ValueError(
+                    'delta must be greater than %d' % (-1 * self.raw))
+            self.timestamp = float(self.raw * PRECISION)
+        if check_bounds:
+            if self.timestamp < 0:
+                raise ValueError('timestamp cannot be negative')
+            if self.timestamp >= 10000000000:
+                raise ValueError('timestamp too large')
+
+    @classmethod
+    def now(cls, offset=0, delta=0):
+        return cls(time.time(), offset=offset, delta=delta)
+
+    def __repr__(self):
+        return INTERNAL_FORMAT % (self.timestamp, self.offset)
+
+    def __str__(self):
+        raise TypeError('You must specify which string format is required')
+
+    def __float__(self):
+        return self.timestamp
+
+    def __int__(self):
+        return int(self.timestamp)
+
+    def __nonzero__(self):
+        return bool(self.timestamp or self.offset)
+
+    def __bool__(self):
+        return self.__nonzero__()
+
+    @property
+    def normal(self):
+        return NORMAL_FORMAT % self.timestamp
+
+    @property
+    def internal(self):
+        if self.offset or FORCE_INTERNAL:
+            return INTERNAL_FORMAT % (self.timestamp, self.offset)
+        else:
+            return self.normal
+
+    @property
+    def short(self):
+        if self.offset or FORCE_INTERNAL:
+            return SHORT_FORMAT % (self.timestamp, self.offset)
+        else:
+            return self.normal
+
+    @property
+    def isoformat(self):
+        """
+        Get an isoformat string representation of the 'normal' part of the
+        Timestamp with microsecond precision and no trailing timezone, for
+        example::
+
+            1970-01-01T00:00:00.000000
+
+        :return: an isoformat string
+        """
+        t = float(self.normal)
+        if six.PY3:
+            # On Python 3, round manually using ROUND_HALF_EVEN rounding
+            # method, to use the same rounding method than Python 2. Python 3
+            # used a different rounding method, but Python 3.4.4 and 3.5.1 use
+            # again ROUND_HALF_EVEN as Python 2.
+            # See https://bugs.python.org/issue23517
+            frac, t = math.modf(t)
+            us = round(frac * 1e6)
+            if us >= 1000000:
+                t += 1
+                us -= 1000000
+            elif us < 0:
+                t -= 1
+                us += 1000000
+            dt = datetime.datetime.utcfromtimestamp(t)
+            dt = dt.replace(microsecond=us)
+        else:
+            dt = datetime.datetime.utcfromtimestamp(t)
+
+        isoformat = dt.isoformat()
+        # python isoformat() doesn't include msecs when zero
+        if len(isoformat) < len("1970-01-01T00:00:00.000000"):
+            isoformat += ".000000"
+        return isoformat
+
+    @classmethod
+    def from_isoformat(cls, date_string):
+        """
+        Parse an isoformat string representation of time to a Timestamp object.
+
+        :param date_string: a string formatted as per an Timestamp.isoformat
+            property.
+        :return: an instance of  this class.
+        """
+        start = datetime.datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S.%f")
+        delta = start - EPOCH
+        # This calculation is based on Python 2.7's Modules/datetimemodule.c,
+        # function delta_to_microseconds(), but written in Python.
+        return cls(delta.total_seconds())
+
+    def ceil(self):
+        """
+        Return the 'normal' part of the timestamp rounded up to the nearest
+        integer number of seconds.
+
+        This value should be used whenever the second-precision Last-Modified
+        time of a resource is required.
+
+        :return: a float value with second precision.
+        """
+        return math.ceil(float(self))
+
+    def __eq__(self, other):
+        if other is None:
+            return False
+        if not isinstance(other, Timestamp):
+            try:
+                other = Timestamp(other, check_bounds=False)
+            except ValueError:
+                return False
+        return self.internal == other.internal
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __lt__(self, other):
+        if other is None:
+            return False
+        if not isinstance(other, Timestamp):
+            other = Timestamp(other, check_bounds=False)
+        if other.timestamp < 0:
+            return False
+        if other.timestamp >= 10000000000:
+            return True
+        return self.internal < other.internal
+
+    def __hash__(self):
+        return hash(self.internal)
+
+    def __invert__(self):
+        if self.offset:
+            raise ValueError('Cannot invert timestamps with offsets')
+        return Timestamp((999999999999999 - self.raw) * PRECISION)
+
+
+def encode_timestamps(t1, t2=None, t3=None, explicit=False):
+    """
+    Encode up to three timestamps into a string. Unlike a Timestamp object, the
+    encoded string does NOT used fixed width fields and consequently no
+    relative chronology of the timestamps can be inferred from lexicographic
+    sorting of encoded timestamp strings.
+
+    The format of the encoded string is:
+        <t1>[<+/-><t2 - t1>[<+/-><t3 - t2>]]
+
+    i.e. if t1 = t2 = t3 then just the string representation of t1 is returned,
+    otherwise the time offsets for t2 and t3 are appended. If explicit is True
+    then the offsets for t2 and t3 are always appended even if zero.
+
+    Note: any offset value in t1 will be preserved, but offsets on t2 and t3
+    are not preserved. In the anticipated use cases for this method (and the
+    inverse decode_timestamps method) the timestamps passed as t2 and t3 are
+    not expected to have offsets as they will be timestamps associated with a
+    POST request. In the case where the encoding is used in a container objects
+    table row, t1 could be the PUT or DELETE time but t2 and t3 represent the
+    content type and metadata times (if different from the data file) i.e.
+    correspond to POST timestamps. In the case where the encoded form is used
+    in a .meta file name, t1 and t2 both correspond to POST timestamps.
+    """
+    form = '{0}'
+    values = [t1.short]
+    if t2 is not None:
+        t2_t1_delta = t2.raw - t1.raw
+        explicit = explicit or (t2_t1_delta != 0)
+        values.append(t2_t1_delta)
+        if t3 is not None:
+            t3_t2_delta = t3.raw - t2.raw
+            explicit = explicit or (t3_t2_delta != 0)
+            values.append(t3_t2_delta)
+        if explicit:
+            form += '{1:+x}'
+            if t3 is not None:
+                form += '{2:+x}'
+    return form.format(*values)
+
+
+def decode_timestamps(encoded, explicit=False):
+    """
+    Parses a string of the form generated by encode_timestamps and returns
+    a tuple of the three component timestamps. If explicit is False, component
+    timestamps that are not explicitly encoded will be assumed to have zero
+    delta from the previous component and therefore take the value of the
+    previous component. If explicit is True, component timestamps that are
+    not explicitly encoded will be returned with value None.
+    """
+    # TODO: some tests, e.g. in test_replicator, put float timestamps values
+    # into container db's, hence this defensive check, but in real world
+    # this may never happen.
+    if not isinstance(encoded, six.string_types):
+        ts = Timestamp(encoded)
+        return ts, ts, ts
+
+    parts = []
+    signs = []
+    pos_parts = encoded.split('+')
+    for part in pos_parts:
+        # parse time components and their signs
+        # e.g. x-y+z --> parts = [x, y, z] and signs = [+1, -1, +1]
+        neg_parts = part.split('-')
+        parts = parts + neg_parts
+        signs = signs + [1] + [-1] * (len(neg_parts) - 1)
+    t1 = Timestamp(parts[0])
+    t2 = t3 = None
+    if len(parts) > 1:
+        t2 = t1
+        delta = signs[1] * int(parts[1], 16)
+        # if delta = 0 we want t2 = t3 = t1 in order to
+        # preserve any offset in t1 - only construct a distinct
+        # timestamp if there is a non-zero delta.
+        if delta:
+            t2 = Timestamp((t1.raw + delta) * PRECISION)
+    elif not explicit:
+        t2 = t1
+    if len(parts) > 2:
+        t3 = t2
+        delta = signs[2] * int(parts[2], 16)
+        if delta:
+            t3 = Timestamp((t2.raw + delta) * PRECISION)
+    elif not explicit:
+        t3 = t2
+    return t1, t2, t3
+
+
+def normalize_timestamp(timestamp):
+    """
+    Format a timestamp (string or numeric) into a standardized
+    xxxxxxxxxx.xxxxx (10.5) format.
+
+    Note that timestamps using values greater than or equal to November 20th,
+    2286 at 17:46 UTC will use 11 digits to represent the number of
+    seconds.
+
+    :param timestamp: unix timestamp
+    :returns: normalized timestamp as a string
+    """
+    return Timestamp(timestamp).normal
+
+
+EPOCH = datetime.datetime(1970, 1, 1)
+
+
+def last_modified_date_to_timestamp(last_modified_date_str):
+    """
+    Convert a last modified date (like you'd get from a container listing,
+    e.g. 2014-02-28T23:22:36.698390) to a float.
+    """
+    return Timestamp.from_isoformat(last_modified_date_str)
+
+
+def normalize_delete_at_timestamp(timestamp, high_precision=False):
+    """
+    Format a timestamp (string or numeric) into a standardized
+    xxxxxxxxxx (10) or xxxxxxxxxx.xxxxx (10.5) format.
+
+    Note that timestamps less than 0000000000 are raised to
+    0000000000 and values greater than November 20th, 2286 at
+    17:46:39 UTC will be capped at that date and time, resulting in
+    no return value exceeding 9999999999.99999 (or 9999999999 if
+    using low-precision).
+
+    This cap is because the expirer is already working through a
+    sorted list of strings that were all a length of 10. Adding
+    another digit would mess up the sort and cause the expirer to
+    break from processing early. By 2286, this problem will need to
+    be fixed, probably by creating an additional .expiring_objects
+    account to work from with 11 (or more) digit container names.
+
+    :param timestamp: unix timestamp
+    :returns: normalized timestamp as a string
+    """
+    fmt = '%016.5f' if high_precision else '%010d'
+    return fmt % min(max(0, float(timestamp)), 9999999999.99999)

swift/common/wsgi.py

@@ -21,7 +21,6 @@ import errno
 import fcntl
 import os
 import signal
-from swift import gettext_ as _
 import sys
 from textwrap import dedent
 import time
@@ -207,19 +206,19 @@ def get_socket(conf):
                 raise
             sleep(0.1)
     if not sock:
-        raise Exception(_('Could not bind to %(addr)s:%(port)s '
-                          'after trying for %(timeout)s seconds') % {
-                              'addr': bind_addr[0], 'port': bind_addr[1],
-                              'timeout': bind_timeout})
+        raise Exception('Could not bind to %(addr)s:%(port)s '
+                        'after trying for %(timeout)s seconds' % {
+                            'addr': bind_addr[0], 'port': bind_addr[1],
+                            'timeout': bind_timeout})
     # in my experience, sockets can hang around forever without keepalive
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
     sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
     if hasattr(socket, 'TCP_KEEPIDLE'):
         sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, keepidle)
     if warn_ssl:
-        ssl_warning_message = _('WARNING: SSL should only be enabled for '
-                                'testing purposes. Use external SSL '
-                                'termination for a production deployment.')
+        ssl_warning_message = ('WARNING: SSL should only be enabled for '
+                               'testing purposes. Use external SSL '
+                               'termination for a production deployment.')
         get_logger(conf).warning(ssl_warning_message)
         print(ssl_warning_message)
     return sock
@@ -361,12 +360,30 @@ def loadapp(conf_file, global_conf=None, allow_modify_pipeline=True):
         if func and allow_modify_pipeline:
             func(PipelineWrapper(ctx))
         filters = [c.create() for c in reversed(ctx.filter_contexts)]
-        app = ultimate_app
-        app._pipeline_final_app = ultimate_app
+        pipeline = [ultimate_app]
+        request_logging_app = app = ultimate_app
         for filter_app in filters:
-            app = filter_app(app)
+            app = filter_app(pipeline[0])
+            pipeline.insert(0, app)
+            if request_logging_app is ultimate_app and \
+                    app.__class__.__name__ == 'ProxyLoggingMiddleware':
+                request_logging_app = filter_app(ultimate_app)
+                # Set some separate-pipeline attrs
+                request_logging_app._pipeline = [
+                    request_logging_app, ultimate_app]
+                request_logging_app._pipeline_request_logging_app = \
+                    request_logging_app
+                request_logging_app._pipeline_final_app = ultimate_app
+
+        for app in pipeline:
+            app._pipeline = pipeline
+            # For things like making (logged) backend requests for
+            # get_account_info and get_container_info
+            app._pipeline_request_logging_app = request_logging_app
+            # For getting proxy-server options like *_existence_skip_cache_pct
             app._pipeline_final_app = ultimate_app
-        return app
+
+        return pipeline[0]
     return ctx.create()
 
 
@@ -430,6 +447,9 @@ def run_server(conf, logger, sock, global_conf=None, ready_callback=None,
         # header; "Etag" just won't do).
         'capitalize_response_headers': False,
     }
+    if conf.get('keepalive_timeout'):
+        server_kwargs['keepalive'] = float(conf['keepalive_timeout']) or False
+
     if ready_callback:
         ready_callback()
     # Yes, eventlet, we know -- we have to support bad clients, though
@@ -788,27 +808,10 @@ class ServersPerPortStrategy(StrategyBase):
                 yield sock
 
 
-def run_wsgi(conf_path, app_section, *args, **kwargs):
-    """
-    Runs the server according to some strategy.  The default strategy runs a
-    specified number of workers in pre-fork model.  The object-server (only)
-    may use a servers-per-port strategy if its config has a servers_per_port
-    setting with a value greater than zero.
-
-    :param conf_path: Path to paste.deploy style configuration file/directory
-    :param app_section: App name from conf file to load config from
-    :param allow_modify_pipeline: Boolean for whether the server should have
-                                  an opportunity to change its own pipeline.
-                                  Defaults to True
-    :returns: 0 if successful, nonzero otherwise
-    """
+def check_config(conf_path, app_section, *args, **kwargs):
     # Load configuration, Set logger and Load request processor
-    try:
-        (conf, logger, log_name) = \
-            _initrp(conf_path, app_section, *args, **kwargs)
-    except ConfigFileError as e:
-        print(e)
-        return 1
+    (conf, logger, log_name) = \
+        _initrp(conf_path, app_section, *args, **kwargs)
 
     # optional nice/ionice priority scheduling
     utils.modify_priority(conf, logger)
@@ -825,16 +828,16 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
         strategy = WorkersStrategy(conf, logger)
         try:
             # Quick sanity check
-            int(conf['bind_port'])
+            if not (1 <= int(conf['bind_port']) <= 2 ** 16 - 1):
+                raise ValueError
         except (ValueError, KeyError, TypeError):
             error_msg = 'bind_port wasn\'t properly set in the config file. ' \
-                'It must be explicitly set to a valid port number.'
+                        'It must be explicitly set to a valid port number.'
             logger.error(error_msg)
-            print(error_msg)
-            return 1
+            raise ConfigFileError(error_msg)
 
     # patch event before loadapp
-    utils.eventlet_monkey_patch()
+    utils.monkey_patch()
 
     # Ensure the configuration and application can be loaded before proceeding.
     global_conf = {'log_name': log_name}
@@ -842,16 +845,44 @@ def run_wsgi(conf_path, app_section, *args, **kwargs):
     if 'global_conf_callback' in kwargs:
         kwargs['global_conf_callback'](conf, global_conf)
 
-    allow_modify_pipeline = kwargs.get('allow_modify_pipeline', True)
-
     # set utils.FALLOCATE_RESERVE if desired
     utils.FALLOCATE_RESERVE, utils.FALLOCATE_IS_PERCENT = \
         utils.config_fallocate_value(conf.get('fallocate_reserve', '1%'))
+    return conf, logger, global_conf, strategy
+
+
+def run_wsgi(conf_path, app_section, *args, **kwargs):
+    """
+    Runs the server according to some strategy.  The default strategy runs a
+    specified number of workers in pre-fork model.  The object-server (only)
+    may use a servers-per-port strategy if its config has a servers_per_port
+    setting with a value greater than zero.
+
+    :param conf_path: Path to paste.deploy style configuration file/directory
+    :param app_section: App name from conf file to load config from
+    :param allow_modify_pipeline: Boolean for whether the server should have
+                                  an opportunity to change its own pipeline.
+                                  Defaults to True
+    :param test_config: if False (the default) then load and validate the
+        config and if successful then continue to run the server; if True then
+        load and validate the config but do not run the server.
+    :returns: 0 if successful, nonzero otherwise
+    """
+    try:
+        conf, logger, global_conf, strategy = check_config(
+            conf_path, app_section, *args, **kwargs)
+    except ConfigFileError as err:
+        print(err)
+        return 1
+
+    if kwargs.get('test_config'):
+        return 0
 
     # Do some daemonization process hygene before we fork any children or run a
     # server without forking.
     clean_up_daemon_hygiene()
 
+    allow_modify_pipeline = kwargs.get('allow_modify_pipeline', True)
     no_fork_sock = strategy.no_fork_sock()
     if no_fork_sock:
         run_server(conf, logger, no_fork_sock, global_conf=global_conf,