swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/common/ring/composite_builder.py

@@ -79,7 +79,7 @@ rebuilding of the composite ring.
 
     The ``id`` of each component RingBuilder is therefore stored in metadata of
     the composite and used to check for the component ordering when the same
-    composite ring is re-composed. RingBuilder ``id``\s are normally assigned
+    composite ring is re-composed. RingBuilder ``id``\\s are normally assigned
     when a RingBuilder instance is first saved. Older RingBuilder instances
     loaded from file may not have an ``id`` assigned and will need to be saved
     before they can be used as components of a composite ring. This can be

swift/common/ring/ring.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 
 import array
-import six.moves.cPickle as pickle
+import contextlib
+
+import pickle  # nosec: B403
 import json
 from collections import defaultdict
 from gzip import GzipFile
@@ -22,38 +24,121 @@ from os.path import getmtime
 import struct
 from time import time
 import os
-from io import BufferedReader
-from hashlib import md5
 from itertools import chain, count
 from tempfile import NamedTemporaryFile
 import sys
-
-from six.moves import range
+import zlib
 
 from swift.common.exceptions import RingLoadError
-from swift.common.utils import hash_path, validate_configuration
+from swift.common.utils import hash_path, validate_configuration, md5
 from swift.common.ring.utils import tiers_for_dev
 
 
+DEFAULT_RELOAD_TIME = 15
+
+
 def calc_replica_count(replica2part2dev_id):
+    if not replica2part2dev_id:
+        return 0
     base = len(replica2part2dev_id) - 1
     extra = 1.0 * len(replica2part2dev_id[-1]) / len(replica2part2dev_id[0])
     return base + extra
 
 
+def normalize_devices(devs):
+    # NOTE(akscram): Replication parameters like replication_ip
+    #                and replication_port are required for
+    #                replication process. An old replication
+    #                ring doesn't contain this parameters into
+    #                device. Old-style pickled rings won't have
+    #                region information.
+    for dev in devs:
+        if dev is None:
+            continue
+        dev.setdefault('region', 1)
+        if 'ip' in dev:
+            dev.setdefault('replication_ip', dev['ip'])
+        if 'port' in dev:
+            dev.setdefault('replication_port', dev['port'])
+
+
+class RingReader(object):
+    chunk_size = 2 ** 16
+
+    def __init__(self, filename):
+        self.fp = open(filename, 'rb')
+        self._reset()
+
+    def _reset(self):
+        self._buffer = b''
+        self.size = 0
+        self.raw_size = 0
+        self._md5 = md5(usedforsecurity=False)
+        self._decomp = zlib.decompressobj(32 + zlib.MAX_WBITS)
+
+    @property
+    def close(self):
+        return self.fp.close
+
+    def seek(self, pos, ref=0):
+        if (pos, ref) != (0, 0):
+            raise NotImplementedError
+        self._reset()
+        return self.fp.seek(pos, ref)
+
+    def _buffer_chunk(self):
+        chunk = self.fp.read(self.chunk_size)
+        if not chunk:
+            return False
+        self.size += len(chunk)
+        self._md5.update(chunk)
+        chunk = self._decomp.decompress(chunk)
+        self.raw_size += len(chunk)
+        self._buffer += chunk
+        return True
+
+    def read(self, amount=-1):
+        if amount < 0:
+            raise IOError("don't be greedy")
+
+        while amount > len(self._buffer):
+            if not self._buffer_chunk():
+                break
+
+        result, self._buffer = self._buffer[:amount], self._buffer[amount:]
+        return result
+
+    def readline(self):
+        # apparently pickle needs this?
+        while b'\n' not in self._buffer:
+            if not self._buffer_chunk():
+                break
+
+        line, sep, self._buffer = self._buffer.partition(b'\n')
+        return line + sep
+
+    def readinto(self, buffer):
+        chunk = self.read(len(buffer))
+        buffer[:len(chunk)] = chunk
+        return len(chunk)
+
+    @property
+    def md5(self):
+        return self._md5.hexdigest()
+
+
 class RingData(object):
     """Partitioned consistent hashing ring data (used for serialization)."""
 
     def __init__(self, replica2part2dev_id, devs, part_shift,
-                 next_part_power=None):
+                 next_part_power=None, version=None):
+        normalize_devices(devs)
         self.devs = devs
         self._replica2part2dev_id = replica2part2dev_id
         self._part_shift = part_shift
         self.next_part_power = next_part_power
-
-        for dev in self.devs:
-            if dev is not None:
-                dev.setdefault("region", 1)
+        self.version = version
+        self.md5 = self.size = self.raw_size = None
 
     @property
     def replica_count(self):
@@ -104,27 +189,32 @@ class RingData(object):
         :param bool metadata_only: If True, only load `devs` and `part_shift`.
         :returns: A RingData instance containing the loaded data.
         """
-        gz_file = BufferedReader(GzipFile(filename, 'rb'))
-
-        # See if the file is in the new format
-        magic = gz_file.read(4)
-        if magic == b'R1NG':
-            format_version, = struct.unpack('!H', gz_file.read(2))
-            if format_version == 1:
-                ring_data = cls.deserialize_v1(
-                    gz_file, metadata_only=metadata_only)
+        with contextlib.closing(RingReader(filename)) as gz_file:
+            # See if the file is in the new format
+            magic = gz_file.read(4)
+            if magic == b'R1NG':
+                format_version, = struct.unpack('!H', gz_file.read(2))
+                if format_version == 1:
+                    ring_data = cls.deserialize_v1(
+                        gz_file, metadata_only=metadata_only)
+                else:
+                    raise Exception('Unknown ring format version %d' %
+                                    format_version)
             else:
-                raise Exception('Unknown ring format version %d' %
-                                format_version)
-        else:
-            # Assume old-style pickled ring
-            gz_file.seek(0)
-            ring_data = pickle.load(gz_file)
+                # Assume old-style pickled ring
+                gz_file.seek(0)
+                ring_data = pickle.load(gz_file)  # nosec: B301
 
-        if not hasattr(ring_data, 'devs'):
+        if hasattr(ring_data, 'devs'):
+            # pickled RingData; make sure we've got region/replication info
+            normalize_devices(ring_data.devs)
+        else:
             ring_data = RingData(ring_data['replica2part2dev_id'],
                                  ring_data['devs'], ring_data['part_shift'],
-                                 ring_data.get('next_part_power'))
+                                 ring_data.get('next_part_power'),
+                                 ring_data.get('version'))
+        for attr in ('md5', 'size', 'raw_size'):
+            setattr(ring_data, attr, getattr(gz_file, attr))
         return ring_data
 
     def serialize_v1(self, file_obj):
@@ -138,6 +228,9 @@ class RingData(object):
                  'replica_count': len(ring['replica2part2dev_id']),
                  'byteorder': sys.byteorder}
 
+        if ring['version'] is not None:
+            _text['version'] = ring['version']
+
         next_part_power = ring.get('next_part_power')
         if next_part_power is not None:
             _text['next_part_power'] = next_part_power
@@ -148,7 +241,7 @@ class RingData(object):
         file_obj.write(struct.pack('!I', json_len))
         file_obj.write(json_text)
         for part2dev_id in ring['replica2part2dev_id']:
-            file_obj.write(part2dev_id.tostring())
+            part2dev_id.tofile(file_obj)
 
     def save(self, filename, mtime=1300507380.0):
         """
@@ -175,7 +268,8 @@ class RingData(object):
         return {'devs': self.devs,
                 'replica2part2dev_id': self._replica2part2dev_id,
                 'part_shift': self._part_shift,
-                'next_part_power': self.next_part_power}
+                'next_part_power': self.next_part_power,
+                'version': self.version}
 
 
 class Ring(object):
@@ -190,7 +284,7 @@ class Ring(object):
     :raises RingLoadError: if the loaded ring data violates its constraint
     """
 
-    def __init__(self, serialized_path, reload_time=15, ring_name=None,
+    def __init__(self, serialized_path, reload_time=None, ring_name=None,
                  validation_hook=lambda ring_data: None):
         # can't use the ring unless HASH_PATH_SUFFIX is set
         validate_configuration()
@@ -199,7 +293,8 @@ class Ring(object):
                                                 ring_name + '.ring.gz')
         else:
             self.serialized_path = os.path.join(serialized_path)
-        self.reload_time = reload_time
+        self.reload_time = (DEFAULT_RELOAD_TIME if reload_time is None
+                            else reload_time)
         self._validation_hook = validation_hook
         self._reload(force=True)
 
@@ -220,25 +315,15 @@ class Ring(object):
 
             self._mtime = getmtime(self.serialized_path)
             self._devs = ring_data.devs
-            # NOTE(akscram): Replication parameters like replication_ip
-            #                and replication_port are required for
-            #                replication process. An old replication
-            #                ring doesn't contain this parameters into
-            #                device. Old-style pickled rings won't have
-            #                region information.
-            for dev in self._devs:
-                if dev:
-                    dev.setdefault('region', 1)
-                    if 'ip' in dev:
-                        dev.setdefault('replication_ip', dev['ip'])
-                    if 'port' in dev:
-                        dev.setdefault('replication_port', dev['port'])
-
             self._replica2part2dev_id = ring_data._replica2part2dev_id
             self._part_shift = ring_data._part_shift
             self._rebuild_tier_data()
             self._update_bookkeeping()
             self._next_part_power = ring_data.next_part_power
+            self._version = ring_data.version
+            self._md5 = ring_data.md5
+            self._size = ring_data.size
+            self._raw_size = ring_data.raw_size
 
     def _update_bookkeeping(self):
         # Do this now, when we know the data has changed, rather than
@@ -257,24 +342,49 @@ class Ring(object):
         zones = set()
         ips = set()
         self._num_devs = 0
+        self._num_assigned_devs = 0
+        self._num_weighted_devs = 0
         for dev in self._devs:
-            if dev and dev['id'] in dev_ids_with_parts:
+            if dev is None:
+                continue
+            self._num_devs += 1
+            if dev.get('weight', 0) > 0:
+                self._num_weighted_devs += 1
+            if dev['id'] in dev_ids_with_parts:
                 regions.add(dev['region'])
                 zones.add((dev['region'], dev['zone']))
                 ips.add((dev['region'], dev['zone'], dev['ip']))
-                self._num_devs += 1
+                self._num_assigned_devs += 1
         self._num_regions = len(regions)
         self._num_zones = len(zones)
         self._num_ips = len(ips)
 
     @property
     def next_part_power(self):
+        if time() > self._rtime:
+            self._reload()
         return self._next_part_power
 
     @property
     def part_power(self):
         return 32 - self._part_shift
 
+    @property
+    def version(self):
+        return self._version
+
+    @property
+    def md5(self):
+        return self._md5
+
+    @property
+    def size(self):
+        return self._size
+
+    @property
+    def raw_size(self):
+        return self._raw_size
+
     def _rebuild_tier_data(self):
         self.tier2devs = defaultdict(list)
         for dev in self._devs:
@@ -301,6 +411,21 @@ class Ring(object):
         """Number of partitions in the ring."""
         return len(self._replica2part2dev_id[0])
 
+    @property
+    def device_count(self):
+        """Number of devices in the ring."""
+        return self._num_devs
+
+    @property
+    def weighted_device_count(self):
+        """Number of devices with weight in the ring."""
+        return self._num_weighted_devs
+
+    @property
+    def assigned_device_count(self):
+        """Number of devices with assignments in the ring."""
+        return self._num_assigned_devs
+
     @property
     def devs(self):
         """devices in the ring"""
@@ -416,7 +541,8 @@ class Ring(object):
             (d['region'], d['zone'], d['ip']) for d in primary_nodes)
 
         parts = len(self._replica2part2dev_id[0])
-        part_hash = md5(str(part).encode('ascii')).digest()
+        part_hash = md5(str(part).encode('ascii'),
+                        usedforsecurity=False).digest()
         start = struct.unpack_from('>I', part_hash)[0] >> self._part_shift
         inc = int(parts / 65536) or 1
         # Multiple loops for execution speed; the checks and bookkeeping get
@@ -490,7 +616,7 @@ class Ring(object):
                             hit_all_ips = True
                             break
 
-        hit_all_devs = len(used) == self._num_devs
+        hit_all_devs = len(used) == self._num_assigned_devs
         for handoff_part in chain(range(start, parts, inc),
                                   range(inc - ((parts - start) % inc),
                                         start, inc)):
@@ -505,6 +631,6 @@ class Ring(object):
                         dev = self._devs[dev_id]
                         yield dict(dev, handoff_index=next(index))
                         used.add(dev_id)
-                        if len(used) == self._num_devs:
+                        if len(used) == self._num_assigned_devs:
                             hit_all_devs = True
                             break

swift/common/ring/utils.py

@@ -199,7 +199,7 @@ def is_valid_hostname(hostname):
     if hostname.endswith('.'):
         # strip exactly one dot from the right, if present
         hostname = hostname[:-1]
-    allowed = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
+    allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
     return all(allowed.match(x) for x in hostname.split("."))
 
 

swift/common/splice.py

@@ -18,8 +18,6 @@ Bindings to the `tee` and `splice` system calls
 '''
 
 import os
-import operator
-import six
 import ctypes
 import ctypes.util
 
@@ -82,8 +80,10 @@ class Tee(object):
         if not self.available:
             raise EnvironmentError('tee not available')
 
-        if not isinstance(flags, six.integer_types):
-            c_flags = six.moves.reduce(operator.or_, flags, 0)
+        if not isinstance(flags, int):
+            c_flags = 0
+            for flag in flags:
+                c_flags |= flag
         else:
             c_flags = flags
 
@@ -98,6 +98,7 @@ class Tee(object):
 
         return self._c_tee is not None
 
+
 tee = Tee()
 del Tee
 
@@ -173,8 +174,10 @@ class Splice(object):
         if not self.available:
             raise EnvironmentError('splice not available')
 
-        if not isinstance(flags, six.integer_types):
-            c_flags = six.moves.reduce(operator.or_, flags, 0)
+        if not isinstance(flags, int):
+            c_flags = 0
+            for flag in flags:
+                c_flags |= flag
         else:
             c_flags = flags
 
@@ -195,5 +198,6 @@ class Splice(object):
 
         return self._c_splice is not None
 
+
 splice = Splice()
 del Splice

swift/common/statsd_client.py

@@ -0,0 +1,205 @@
+# Copyright (c) 2010-2012 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Statsd Client """
+
+import time
+import warnings
+from contextlib import closing
+from random import random
+
+from eventlet.green import socket
+
+
+def get_statsd_client(conf=None, tail_prefix='', logger=None):
+    """
+    Get an instance of StatsdClient using config settings.
+
+    **config and defaults**::
+
+        log_statsd_host = (disabled)
+        log_statsd_port = 8125
+        log_statsd_default_sample_rate = 1.0
+        log_statsd_sample_rate_factor = 1.0
+        log_statsd_metric_prefix = (empty-string)
+
+    :param conf: Configuration dict to read settings from
+    :param tail_prefix: tail prefix to pass to statsd client
+    :param logger: stdlib logger instance used by statsd client for logging
+    :return: an instance of ``StatsdClient``
+
+    """
+    conf = conf or {}
+
+    host = conf.get('log_statsd_host')
+    port = int(conf.get('log_statsd_port', 8125))
+    base_prefix = conf.get('log_statsd_metric_prefix', '')
+    default_sample_rate = float(
+        conf.get('log_statsd_default_sample_rate', 1))
+    sample_rate_factor = float(
+        conf.get('log_statsd_sample_rate_factor', 1))
+
+    return StatsdClient(host, port, base_prefix=base_prefix,
+                        tail_prefix=tail_prefix,
+                        default_sample_rate=default_sample_rate,
+                        sample_rate_factor=sample_rate_factor, logger=logger)
+
+
+class StatsdClient(object):
+    def __init__(self, host, port, base_prefix='', tail_prefix='',
+                 default_sample_rate=1, sample_rate_factor=1, logger=None):
+        self._host = host
+        self._port = port
+        self._base_prefix = base_prefix
+        self._default_sample_rate = default_sample_rate
+        self._sample_rate_factor = sample_rate_factor
+        self.random = random
+        self.logger = logger
+        self._set_prefix(tail_prefix)
+        self._sock_family = self._target = None
+
+        if self._host:
+            self._set_sock_family_and_target(self._host, self._port)
+
+    def _set_sock_family_and_target(self, host, port):
+        # Determine if host is IPv4 or IPv6
+        addr_info = None
+        try:
+            addr_info = socket.getaddrinfo(host, port, socket.AF_INET)
+            self._sock_family = socket.AF_INET
+        except socket.gaierror:
+            try:
+                addr_info = socket.getaddrinfo(host, port, socket.AF_INET6)
+                self._sock_family = socket.AF_INET6
+            except socket.gaierror:
+                # Don't keep the server from starting from what could be a
+                # transient DNS failure.  Any hostname will get re-resolved as
+                # necessary in the .sendto() calls.
+                # However, we don't know if we're IPv4 or IPv6 in this case, so
+                # we assume legacy IPv4.
+                self._sock_family = socket.AF_INET
+
+        # NOTE: we use the original host value, not the DNS-resolved one
+        # because if host is a hostname, we don't want to cache the DNS
+        # resolution for the entire lifetime of this process.  Let standard
+        # name resolution caching take effect.  This should help operators use
+        # DNS trickery if they want.
+        if addr_info is not None:
+            # addr_info is a list of 5-tuples with the following structure:
+            #     (family, socktype, proto, canonname, sockaddr)
+            # where sockaddr is the only thing of interest to us, and we only
+            # use the first result.  We want to use the originally supplied
+            # host (see note above) and the remainder of the variable-length
+            # sockaddr: IPv4 has (address, port) while IPv6 has (address,
+            # port, flow info, scope id).
+            sockaddr = addr_info[0][-1]
+            self._target = (host,) + (sockaddr[1:])
+        else:
+            self._target = (host, port)
+
+    def _set_prefix(self, tail_prefix):
+        """
+        Modifies the prefix that is added to metric names. The resulting prefix
+        is the concatenation of the component parts `base_prefix` and
+        `tail_prefix`. Only truthy components are included. Each included
+        component is followed by a period, e.g.::
+
+            <base_prefix>.<tail_prefix>.
+            <tail_prefix>.
+            <base_prefix>.
+            <the empty string>
+
+        Note: this method is expected to be called from the constructor only,
+        but exists to provide backwards compatible functionality for the
+        deprecated set_prefix() method.
+
+        :param tail_prefix: The new value of tail_prefix
+        """
+        if tail_prefix and self._base_prefix:
+            self._prefix = '.'.join([self._base_prefix, tail_prefix, ''])
+        elif tail_prefix:
+            self._prefix = tail_prefix + '.'
+        elif self._base_prefix:
+            self._prefix = self._base_prefix + '.'
+        else:
+            self._prefix = ''
+
+    def set_prefix(self, tail_prefix):
+        """
+        This method is deprecated; use the ``tail_prefix`` argument of the
+        constructor when instantiating the class instead.
+        """
+        warnings.warn(
+            'set_prefix() is deprecated; use the ``tail_prefix`` argument of '
+            'the constructor when instantiating the class instead.',
+            DeprecationWarning, stacklevel=2
+        )
+        self._set_prefix(tail_prefix)
+
+    def _send(self, m_name, m_value, m_type, sample_rate):
+        if not self._host:
+            # StatsD not configured
+            return
+
+        if sample_rate is None:
+            sample_rate = self._default_sample_rate
+        sample_rate = sample_rate * self._sample_rate_factor
+
+        parts = ['%s%s:%s' % (self._prefix, m_name, m_value), m_type]
+        if sample_rate < 1:
+            if self.random() < sample_rate:
+                parts.append('@%s' % (sample_rate,))
+            else:
+                return
+        parts = [part.encode('utf-8') for part in parts]
+        # Ideally, we'd cache a sending socket in self, but that
+        # results in a socket getting shared by multiple green threads.
+        with closing(self._open_socket()) as sock:
+            try:
+                return sock.sendto(b'|'.join(parts), self._target)
+            except IOError as err:
+                if self.logger:
+                    self.logger.warning(
+                        'Error sending UDP message to %(target)r: %(err)s',
+                        {'target': self._target, 'err': err})
+
+    def _open_socket(self):
+        return socket.socket(self._sock_family, socket.SOCK_DGRAM)
+
+    def update_stats(self, m_name, m_value, sample_rate=None):
+        return self._send(m_name, m_value, 'c', sample_rate)
+
+    def increment(self, metric, sample_rate=None):
+        return self.update_stats(metric, 1, sample_rate)
+
+    def decrement(self, metric, sample_rate=None):
+        return self.update_stats(metric, -1, sample_rate)
+
+    def _timing(self, metric, timing_ms, sample_rate):
+        # This method was added to disagregate timing metrics when testing
+        return self._send(metric, round(timing_ms, 4), 'ms', sample_rate)
+
+    def timing(self, metric, timing_ms, sample_rate=None):
+        return self._timing(metric, timing_ms, sample_rate)
+
+    def timing_since(self, metric, orig_time, sample_rate=None):
+        return self._timing(metric, (time.time() - orig_time) * 1000,
+                            sample_rate)
+
+    def transfer_rate(self, metric, elapsed_time, byte_xfer, sample_rate=None):
+        if byte_xfer:
+            return self.timing(metric,
+                               elapsed_time * 1000 / byte_xfer * 1000,
+                               sample_rate)