swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/common/request_helpers.py

@@ -36,7 +36,8 @@ from swift.common.swob import HTTPBadRequest, \
     HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \
     HTTPPreconditionFailed, wsgi_to_bytes, wsgi_unquote, wsgi_to_str
 from swift.common.utils import split_path, validate_device_partition, \
-    close_if_possible, maybe_multipart_byteranges_to_document_iters, \
+    close_if_possible, friendly_close, \
+    maybe_multipart_byteranges_to_document_iters, \
     multipart_byteranges_to_document_iters, parse_content_type, \
     parse_content_range, csv_append, list_from_csv, Spliterator, quote, \
     RESERVED, config_true_value, md5, CloseableChain, select_ip_port
@@ -94,6 +95,39 @@ def get_param(req, name, default=None):
     return value
 
 
+def get_valid_part_num(req):
+    """
+    Any non-range GET or HEAD request for a SLO object may include a
+    part-number parameter in query string.  If the passed in request
+    includes a part-number parameter it will be parsed into a valid integer
+    and returned.  If the passed in request does not include a part-number
+    param we will return None.  If the part-number parameter is invalid for
+    the given request we will raise the appropriate HTTP exception
+
+    :param req: the request object
+
+    :returns: validated part-number value or None
+    :raises HTTPBadRequest: if request or part-number param is not valid
+    """
+    part_number_param = get_param(req, 'part-number')
+    if part_number_param is None:
+        return None
+    try:
+        part_number = int(part_number_param)
+        if part_number <= 0:
+            raise ValueError
+    except ValueError:
+        raise HTTPBadRequest('Part number must be an integer greater '
+                             'than 0')
+
+    if req.range:
+        raise HTTPBadRequest(req=req,
+                             body='Range requests are not supported '
+                                  'with part number queries')
+
+    return part_number
+
+
 def validate_params(req, names):
     """
     Get list of parameters from an HTTP request, validating the encoding of
@@ -460,15 +494,17 @@ class SegmentedIterable(object):
     :param app: WSGI application from which segments will come
 
     :param listing_iter: iterable yielding the object segments to fetch,
-        along with the byte subranges to fetch, in the form of a 5-tuple
-        (object-path, object-etag, object-size, first-byte, last-byte).
+        along with the byte sub-ranges to fetch. Each yielded item should be a
+        dict with the following keys: ``path`` or ``raw_data``,
+        ``first-byte``, ``last-byte``, ``hash`` (optional), ``bytes``
+        (optional).
 
-        If object-etag is None, no MD5 verification will be done.
+        If ``hash`` is None, no MD5 verification will be done.
 
-        If object-size is None, no length verification will be done.
+        If ``bytes`` is None, no length verification will be done.
 
-        If first-byte and last-byte are None, then the entire object will be
-        fetched.
+        If ``first-byte`` and ``last-byte`` are None, then the entire object
+        will be fetched.
 
     :param max_get_time: maximum permitted duration of a GET request (seconds)
     :param logger: logger object
@@ -740,7 +776,10 @@ class SegmentedIterable(object):
             for x in mri:
                 yield x
         finally:
-            self.close()
+            # Spliterator and multi_range_iterator can't possibly know we've
+            # consumed the whole of the app_iter, but we want to read/close the
+            # final segment response
+            friendly_close(self.app_iter)
 
     def validate_first_segment(self):
         """
@@ -900,6 +939,24 @@ def update_ignore_range_header(req, name):
     req.headers[hdr] = csv_append(req.headers.get(hdr), name)
 
 
+def resolve_ignore_range_header(req, metadata):
+    """
+    Helper function to remove Range header from request if metadata matching
+    the X-Backend-Ignore-Range-If-Metadata-Present header is found.
+
+    :param req: a swob Request
+    :param metadata: dictionary of object metadata
+    """
+    ignore_range_headers = set(
+        h.strip().lower()
+        for h in req.headers.get(
+            'X-Backend-Ignore-Range-If-Metadata-Present',
+            '').split(','))
+    if ignore_range_headers.intersection(
+            h.lower() for h in metadata):
+        req.headers.pop('Range', None)
+
+
 def is_use_replication_network(headers=None):
     """
     Determine if replication network should be used.
@@ -936,3 +993,39 @@ def get_ip_port(node, headers):
     """
     return select_ip_port(
         node, use_replication=is_use_replication_network(headers))
+
+
+def is_open_expired(app, req):
+    """
+    Helper function to check if a request with the header 'x-open-expired'
+    can access an object that has not yet been reaped by the object-expirer
+    based on the allow_open_expired global config.
+
+    :param app: the application instance
+    :param req: request object
+    """
+    return (config_true_value(app.allow_open_expired) and
+            config_true_value(req.headers.get('x-open-expired')))
+
+
+def is_backend_open_expired(request):
+    """
+    Helper function to check if a request has either the headers
+    'x-backend-open-expired' or 'x-backend-replication' for the backend
+    to access expired objects.
+
+    :param request: request object
+    """
+    x_backend_open_expired = config_true_value(request.headers.get(
+        'x-backend-open-expired', 'false'))
+    x_backend_replication = config_true_value(request.headers.get(
+        'x-backend-replication', 'false'))
+    return x_backend_open_expired or x_backend_replication
+
+
+def append_log_info(environ, log_info):
+    environ.setdefault('swift.log_info', []).append(log_info)
+
+
+def get_log_info(environ):
+    return ','.join(environ.get('swift.log_info', []))

swift/common/statsd_client.py

@@ -0,0 +1,207 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Statsd Client """
+
+import time
+import warnings
+from contextlib import closing
+from random import random
+
+from eventlet.green import socket
+import six
+
+
+def get_statsd_client(conf=None, tail_prefix='', logger=None):
+    """
+    Get an instance of StatsdClient using config settings.
+
+    **config and defaults**::
+
+        log_statsd_host = (disabled)
+        log_statsd_port = 8125
+        log_statsd_default_sample_rate = 1.0
+        log_statsd_sample_rate_factor = 1.0
+        log_statsd_metric_prefix = (empty-string)
+
+    :param conf: Configuration dict to read settings from
+    :param tail_prefix: tail prefix to pass to statsd client
+    :param logger: stdlib logger instance used by statsd client for logging
+    :return: an instance of ``StatsdClient``
+
+    """
+    conf = conf or {}
+
+    host = conf.get('log_statsd_host')
+    port = int(conf.get('log_statsd_port', 8125))
+    base_prefix = conf.get('log_statsd_metric_prefix', '')
+    default_sample_rate = float(
+        conf.get('log_statsd_default_sample_rate', 1))
+    sample_rate_factor = float(
+        conf.get('log_statsd_sample_rate_factor', 1))
+
+    return StatsdClient(host, port, base_prefix=base_prefix,
+                        tail_prefix=tail_prefix,
+                        default_sample_rate=default_sample_rate,
+                        sample_rate_factor=sample_rate_factor, logger=logger)
+
+
+class StatsdClient(object):
+    def __init__(self, host, port, base_prefix='', tail_prefix='',
+                 default_sample_rate=1, sample_rate_factor=1, logger=None):
+        self._host = host
+        self._port = port
+        self._base_prefix = base_prefix
+        self._default_sample_rate = default_sample_rate
+        self._sample_rate_factor = sample_rate_factor
+        self.random = random
+        self.logger = logger
+        self._set_prefix(tail_prefix)
+        self._sock_family = self._target = None
+
+        if self._host:
+            self._set_sock_family_and_target(self._host, self._port)
+
+    def _set_sock_family_and_target(self, host, port):
+        # Determine if host is IPv4 or IPv6
+        addr_info = None
+        try:
+            addr_info = socket.getaddrinfo(host, port, socket.AF_INET)
+            self._sock_family = socket.AF_INET
+        except socket.gaierror:
+            try:
+                addr_info = socket.getaddrinfo(host, port, socket.AF_INET6)
+                self._sock_family = socket.AF_INET6
+            except socket.gaierror:
+                # Don't keep the server from starting from what could be a
+                # transient DNS failure.  Any hostname will get re-resolved as
+                # necessary in the .sendto() calls.
+                # However, we don't know if we're IPv4 or IPv6 in this case, so
+                # we assume legacy IPv4.
+                self._sock_family = socket.AF_INET
+
+        # NOTE: we use the original host value, not the DNS-resolved one
+        # because if host is a hostname, we don't want to cache the DNS
+        # resolution for the entire lifetime of this process.  Let standard
+        # name resolution caching take effect.  This should help operators use
+        # DNS trickery if they want.
+        if addr_info is not None:
+            # addr_info is a list of 5-tuples with the following structure:
+            #     (family, socktype, proto, canonname, sockaddr)
+            # where sockaddr is the only thing of interest to us, and we only
+            # use the first result.  We want to use the originally supplied
+            # host (see note above) and the remainder of the variable-length
+            # sockaddr: IPv4 has (address, port) while IPv6 has (address,
+            # port, flow info, scope id).
+            sockaddr = addr_info[0][-1]
+            self._target = (host,) + (sockaddr[1:])
+        else:
+            self._target = (host, port)
+
+    def _set_prefix(self, tail_prefix):
+        """
+        Modifies the prefix that is added to metric names. The resulting prefix
+        is the concatenation of the component parts `base_prefix` and
+        `tail_prefix`. Only truthy components are included. Each included
+        component is followed by a period, e.g.::
+
+            <base_prefix>.<tail_prefix>.
+            <tail_prefix>.
+            <base_prefix>.
+            <the empty string>
+
+        Note: this method is expected to be called from the constructor only,
+        but exists to provide backwards compatible functionality for the
+        deprecated set_prefix() method.
+
+        :param tail_prefix: The new value of tail_prefix
+        """
+        if tail_prefix and self._base_prefix:
+            self._prefix = '.'.join([self._base_prefix, tail_prefix, ''])
+        elif tail_prefix:
+            self._prefix = tail_prefix + '.'
+        elif self._base_prefix:
+            self._prefix = self._base_prefix + '.'
+        else:
+            self._prefix = ''
+
+    def set_prefix(self, tail_prefix):
+        """
+        This method is deprecated; use the ``tail_prefix`` argument of the
+        constructor when instantiating the class instead.
+        """
+        warnings.warn(
+            'set_prefix() is deprecated; use the ``tail_prefix`` argument of '
+            'the constructor when instantiating the class instead.',
+            DeprecationWarning, stacklevel=2
+        )
+        self._set_prefix(tail_prefix)
+
+    def _send(self, m_name, m_value, m_type, sample_rate):
+        if not self._host:
+            # StatsD not configured
+            return
+
+        if sample_rate is None:
+            sample_rate = self._default_sample_rate
+        sample_rate = sample_rate * self._sample_rate_factor
+
+        parts = ['%s%s:%s' % (self._prefix, m_name, m_value), m_type]
+        if sample_rate < 1:
+            if self.random() < sample_rate:
+                parts.append('@%s' % (sample_rate,))
+            else:
+                return
+        if six.PY3:
+            parts = [part.encode('utf-8') for part in parts]
+        # Ideally, we'd cache a sending socket in self, but that
+        # results in a socket getting shared by multiple green threads.
+        with closing(self._open_socket()) as sock:
+            try:
+                return sock.sendto(b'|'.join(parts), self._target)
+            except IOError as err:
+                if self.logger:
+                    self.logger.warning(
+                        'Error sending UDP message to %(target)r: %(err)s',
+                        {'target': self._target, 'err': err})
+
+    def _open_socket(self):
+        return socket.socket(self._sock_family, socket.SOCK_DGRAM)
+
+    def update_stats(self, m_name, m_value, sample_rate=None):
+        return self._send(m_name, m_value, 'c', sample_rate)
+
+    def increment(self, metric, sample_rate=None):
+        return self.update_stats(metric, 1, sample_rate)
+
+    def decrement(self, metric, sample_rate=None):
+        return self.update_stats(metric, -1, sample_rate)
+
+    def _timing(self, metric, timing_ms, sample_rate):
+        # This method was added to disagregate timing metrics when testing
+        return self._send(metric, round(timing_ms, 4), 'ms', sample_rate)
+
+    def timing(self, metric, timing_ms, sample_rate=None):
+        return self._timing(metric, timing_ms, sample_rate)
+
+    def timing_since(self, metric, orig_time, sample_rate=None):
+        return self._timing(metric, (time.time() - orig_time) * 1000,
+                            sample_rate)
+
+    def transfer_rate(self, metric, elapsed_time, byte_xfer, sample_rate=None):
+        if byte_xfer:
+            return self.timing(metric,
+                               elapsed_time * 1000 / byte_xfer * 1000,
+                               sample_rate)

swift/common/storage_policy.py

@@ -160,7 +160,7 @@ class BaseStoragePolicy(object):
                  object_ring=None, aliases='',
                  diskfile_module='egg:swift#replication.fs'):
         # do not allow BaseStoragePolicy class to be instantiated directly
-        if type(self) == BaseStoragePolicy:
+        if type(self) is BaseStoragePolicy:
             raise TypeError("Can't instantiate BaseStoragePolicy directly")
         # policy parameter validation
         try:

swift/common/swob.py

@@ -55,7 +55,7 @@ from six.moves import urllib
 
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.utils import UTC, reiterate, split_path, Timestamp, pairs, \
-    close_if_possible, closing_if_possible, config_true_value, drain_and_close
+    close_if_possible, closing_if_possible, config_true_value, friendly_close
 from swift.common.exceptions import InvalidTimestamp
 
 
@@ -1395,12 +1395,15 @@ class Response(object):
             if empty_resp is not None:
                 self.status = empty_resp
                 self.content_length = 0
+                # the existing successful response and it's app_iter have been
+                # determined to not meet the conditions of the reqeust, the
+                # response app_iter should be closed but not drained.
                 close_if_possible(app_iter)
                 return [b'']
 
         if self.request and self.request.method == 'HEAD':
             # We explicitly do NOT want to set self.content_length to 0 here
-            drain_and_close(app_iter)  # be friendly to our app_iter
+            friendly_close(app_iter)  # be friendly to our app_iter
             return [b'']
 
         if self.conditional_response and self.request and \