swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/container/reconciler.py

@@ -19,19 +19,20 @@ import itertools
 import logging
 
 from eventlet import GreenPile, GreenPool, Timeout
-import six
 
 from swift.common import constraints
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.direct_client import (
     direct_head_container, direct_delete_container_object,
     direct_put_container_object, ClientException)
 from swift.common.internal_client import InternalClient, UnexpectedResponse
+from swift.common.request_helpers import MISPLACED_OBJECTS_ACCOUNT, \
+    USE_REPLICATION_NETWORK_HEADER
 from swift.common.utils import get_logger, split_path, majority_size, \
     FileLikeIter, Timestamp, last_modified_date_to_timestamp, \
-    LRUCache, decode_timestamps
+    LRUCache, decode_timestamps, hash_path, parse_options
+from swift.common.storage_policy import POLICIES
 
-MISPLACED_OBJECTS_ACCOUNT = '.misplaced_objects'
 MISPLACED_OBJECTS_CONTAINER_DIVISOR = 3600  # 1 hour
 CONTAINER_POLICY_TTL = 30
 
@@ -85,8 +86,13 @@ def cmp_policy_info(info, remote_info):
             return 1
         elif not remote_recreated:
             return -1
-        return cmp(remote_info['status_changed_at'],
-                   info['status_changed_at'])
+        # both have been recreated, everything devoles to here eventually
+        most_recent_successful_delete = max(info['delete_timestamp'],
+                                            remote_info['delete_timestamp'])
+        if info['put_timestamp'] < most_recent_successful_delete:
+            return 1
+        elif remote_info['put_timestamp'] < most_recent_successful_delete:
+            return -1
     return cmp(info['status_changed_at'], remote_info['status_changed_at'])
 
 
@@ -224,6 +230,7 @@ def add_to_reconciler_queue(container_ring, account, container, obj,
         'X-Etag': obj_timestamp,
         'X-Timestamp': x_timestamp,
         'X-Content-Type': q_op_type,
+        USE_REPLICATION_NETWORK_HEADER: 'true',
     }
 
     def _check_success(*args, **kwargs):
@@ -264,11 +271,7 @@ def parse_raw_obj(obj_info):
     :returns: a queue entry dict with the keys: q_policy_index, account,
               container, obj, q_op, q_ts, q_record, and path
     """
-    if six.PY2:
-        raw_obj_name = obj_info['name'].encode('utf-8')
-    else:
-        raw_obj_name = obj_info['name']
-
+    raw_obj_name = obj_info['name']
     policy_index, obj_name = raw_obj_name.split(':', 1)
     q_policy_index = int(policy_index)
     account, container, obj = split_path(obj_name, 3, 3, rest_with_last=True)
@@ -307,7 +310,8 @@ def direct_get_container_policy_index(container_ring, account_name,
     """
     def _eat_client_exception(*args):
         try:
-            return direct_head_container(*args)
+            return direct_head_container(*args, headers={
+                USE_REPLICATION_NETWORK_HEADER: 'true'})
         except ClientException as err:
             if err.http_status == 404:
                 return err.http_headers
@@ -333,6 +337,10 @@ def direct_delete_container_entry(container_ring, account_name, container_name,
     object listing. Does not talk to object servers; use this only when a
     container entry does not actually have a corresponding object.
     """
+    if headers is None:
+        headers = {}
+    headers[USE_REPLICATION_NETWORK_HEADER] = 'true'
+
     pool = GreenPool()
     part, nodes = container_ring.get_nodes(account_name, container_name)
     for node in nodes:
@@ -348,23 +356,45 @@ class ContainerReconciler(Daemon):
     """
     Move objects that are in the wrong storage policy.
     """
+    log_route = 'container-reconciler'
 
-    def __init__(self, conf):
+    def __init__(self, conf, logger=None, swift=None):
         self.conf = conf
         # This option defines how long an un-processable misplaced object
         # marker will be retried before it is abandoned.  It is not coupled
         # with the tombstone reclaim age in the consistency engine.
         self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7))
-        self.interval = int(conf.get('interval', 30))
+        self.interval = float(conf.get('interval', 30))
         conf_path = conf.get('__file__') or \
             '/etc/swift/container-reconciler.conf'
-        self.logger = get_logger(conf, log_route='container-reconciler')
+        self.logger = logger or get_logger(
+            conf, log_route=self.log_route)
         request_tries = int(conf.get('request_tries') or 3)
-        self.swift = InternalClient(conf_path,
-                                    'Swift Container Reconciler',
-                                    request_tries)
+        self.swift = swift or InternalClient(
+            conf_path,
+            'Swift Container Reconciler',
+            request_tries,
+            use_replication_network=True,
+            global_conf={'log_name': '%s-ic' % conf.get(
+                'log_name', self.log_route)})
+        self.swift_dir = conf.get('swift_dir', '/etc/swift')
         self.stats = defaultdict(int)
         self.last_stat_time = time.time()
+        self.ring_check_interval = float(conf.get('ring_check_interval', 15))
+        self.concurrency = int(conf.get('concurrency', 1))
+        if self.concurrency < 1:
+            raise ValueError("concurrency must be set to at least 1")
+        self.processes = int(self.conf.get('processes', 0))
+        if self.processes < 0:
+            raise ValueError(
+                'processes must be an integer greater than or equal to 0')
+        self.process = int(self.conf.get('process', 0))
+        if self.process < 0:
+            raise ValueError(
+                'process must be an integer greater than or equal to 0')
+        if self.processes and self.process >= self.processes:
+            raise ValueError(
+                'process must be less than processes')
 
     def stats_log(self, metric, msg, *args, **kwargs):
         """
@@ -400,7 +430,7 @@ class ContainerReconciler(Daemon):
         an object was manually re-enqued.
         """
         q_path = '/%s/%s/%s' % (MISPLACED_OBJECTS_ACCOUNT, container, obj)
-        x_timestamp = slightly_later_timestamp(max(q_record, q_ts))
+        x_timestamp = slightly_later_timestamp(max(q_record, q_ts), offset=2)
         self.stats_log('pop_queue', 'remove %r (%f) from the queue (%s)',
                        q_path, q_ts, x_timestamp)
         headers = {'X-Timestamp': x_timestamp}
@@ -408,6 +438,13 @@ class ContainerReconciler(Daemon):
             self.swift.container_ring, MISPLACED_OBJECTS_ACCOUNT,
             container, obj, headers=headers)
 
+    def can_reconcile_policy(self, policy_index):
+        pol = POLICIES.get_by_index(policy_index)
+        if pol:
+            pol.load_ring(self.swift_dir, reload_time=self.ring_check_interval)
+            return pol.object_ring.next_part_power is None
+        return False
+
     def throw_tombstones(self, account, container, obj, timestamp,
                          policy_index, path):
         """
@@ -474,15 +511,33 @@ class ContainerReconciler(Daemon):
                            container_policy_index, q_policy_index)
             return True
 
+        # don't reconcile if the source or container policy_index is in the
+        # middle of a PPI
+        if not self.can_reconcile_policy(q_policy_index):
+            self.stats_log('ppi_skip', 'Source policy (%r) in the middle of '
+                           'a part power increase (PPI)', q_policy_index)
+            return False
+        if not self.can_reconcile_policy(container_policy_index):
+            self.stats_log('ppi_skip', 'Container policy (%r) in the middle '
+                           'of a part power increase (PPI)',
+                           container_policy_index)
+            return False
+
         # check if object exists in the destination already
         self.logger.debug('checking for %r (%f) in destination '
                           'policy_index %s', path, q_ts,
                           container_policy_index)
         headers = {
             'X-Backend-Storage-Policy-Index': container_policy_index}
-        dest_obj = self.swift.get_object_metadata(account, container, obj,
-                                                  headers=headers,
-                                                  acceptable_statuses=(2, 4))
+        try:
+            dest_obj = self.swift.get_object_metadata(
+                account, container, obj, headers=headers,
+                acceptable_statuses=(2, 4))
+        except UnexpectedResponse:
+            self.stats_log('unavailable_destination', '%r (%f) unable to '
+                           'determine the destination timestamp, if any',
+                           path, q_ts)
+            return False
         dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
         if dest_ts >= q_ts:
             self.stats_log('found_object', '%r (%f) in policy_index %s '
@@ -563,7 +618,7 @@ class ContainerReconciler(Daemon):
         # optimistically move any source with a timestamp >= q_ts
         ts = max(Timestamp(source_ts), q_ts)
         # move the object
-        put_timestamp = slightly_later_timestamp(ts, offset=2)
+        put_timestamp = slightly_later_timestamp(ts, offset=3)
         self.stats_log('copy_attempt', '%r (%f) in policy_index %s will be '
                        'moved to policy_index %s (%s)', path, source_ts,
                        q_policy_index, container_policy_index, put_timestamp)
@@ -603,7 +658,7 @@ class ContainerReconciler(Daemon):
         Issue a DELETE request against the destination to match the
         misplaced DELETE against the source.
         """
-        delete_timestamp = slightly_later_timestamp(q_ts, offset=2)
+        delete_timestamp = slightly_later_timestamp(q_ts, offset=3)
         self.stats_log('delete_attempt', '%r (%f) in policy_index %s '
                        'will be deleted from policy_index %s (%s)', path,
                        source_ts, q_policy_index, container_policy_index,
@@ -679,9 +734,9 @@ class ContainerReconciler(Daemon):
         # hit most recent container first instead of waiting on the updaters
         current_container = get_reconciler_container_name(time.time())
         yield current_container
-        container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT)
         self.logger.debug('looking for containers in %s',
                           MISPLACED_OBJECTS_ACCOUNT)
+        container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT)
         while True:
             one_page = None
             try:
@@ -698,9 +753,6 @@ class ContainerReconciler(Daemon):
             # reversed order since we expect older containers to be empty
             for c in reversed(one_page):
                 container = c['name']
-                if six.PY2:
-                    # encoding here is defensive
-                    container = container.encode('utf8')
                 if container == current_container:
                     continue  # we've already hit this one this pass
                 yield container
@@ -732,29 +784,57 @@ class ContainerReconciler(Daemon):
                 MISPLACED_OBJECTS_ACCOUNT, container,
                 acceptable_statuses=(2, 404, 409, 412))
 
+    def should_process(self, queue_item):
+        """
+        Check if a given entry should be handled by this process.
+
+        :param container: the queue container
+        :param queue_item: an entry from the queue
+        """
+        if not self.processes:
+            return True
+        hexdigest = hash_path(
+            queue_item['account'], queue_item['container'], queue_item['obj'])
+        return int(hexdigest, 16) % self.processes == self.process
+
+    def process_queue_item(self, q_container, q_entry, queue_item):
+        """
+        Process an entry and remove from queue on success.
+
+        :param q_container: the queue container
+        :param q_entry: the raw_obj name from the q_container
+        :param queue_item: a parsed entry from the queue
+        """
+        finished = self.reconcile_object(queue_item)
+        if finished:
+            self.pop_queue(q_container, q_entry,
+                           queue_item['q_ts'],
+                           queue_item['q_record'])
+
     def reconcile(self):
         """
-        Main entry point for processing misplaced objects.
+        Main entry point for concurrent processing of misplaced objects.
 
-        Iterate over all queue entries and delegate to reconcile_object.
+        Iterate over all queue entries and delegate processing to spawned
+        workers in the pool.
         """
         self.logger.debug('pulling items from the queue')
+        pool = GreenPool(self.concurrency)
         for container in self._iter_containers():
+            self.logger.debug('checking container %s', container)
             for raw_obj in self._iter_objects(container):
                 try:
-                    obj_info = parse_raw_obj(raw_obj)
+                    queue_item = parse_raw_obj(raw_obj)
                 except Exception:
                     self.stats_log('invalid_record',
                                    'invalid queue record: %r', raw_obj,
                                    level=logging.ERROR, exc_info=True)
                     continue
-                finished = self.reconcile_object(obj_info)
-                if finished:
-                    self.pop_queue(container, raw_obj['name'],
-                                   obj_info['q_ts'],
-                                   obj_info['q_record'])
+                if self.should_process(queue_item):
+                    pool.spawn_n(self.process_queue_item,
+                                 container, raw_obj['name'], queue_item)
             self.log_stats()
-            self.logger.debug('finished container %s', container)
+        pool.waitall()
 
     def run_once(self, *args, **kwargs):
         """
@@ -772,3 +852,12 @@ class ContainerReconciler(Daemon):
             self.stats = defaultdict(int)
             self.logger.info('sleeping between intervals (%ss)', self.interval)
             time.sleep(self.interval)
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerReconciler, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/container/replicator.py

@@ -14,22 +14,54 @@
 # limitations under the License.
 
 import os
-import itertools
 import json
 from collections import defaultdict
 from eventlet import Timeout
+import optparse
+from random import choice
 
 from swift.container.sync_store import ContainerSyncStore
-from swift.container.backend import ContainerBroker, DATADIR, SHARDED
+from swift.container.backend import ContainerBroker, DATADIR, SHARDED, \
+    merge_shards
 from swift.container.reconciler import (
     MISPLACED_OBJECTS_ACCOUNT, incorrect_policy_index,
     get_reconciler_container_name, get_row_to_q_entry_translator)
 from swift.common import db_replicator
+from swift.common.daemon import run_daemon
 from swift.common.storage_policy import POLICIES
 from swift.common.swob import HTTPOk, HTTPAccepted
-from swift.common.exceptions import DeviceUnavailable
 from swift.common.http import is_success
-from swift.common.utils import Timestamp, majority_size, get_db_files
+from swift.common.utils import Timestamp, majority_size, get_db_files, \
+    parse_options
+
+
+def check_merge_own_shard_range(shards, broker, logger, source):
+    """
+    If broker has own_shard_range *with an epoch* then filter out an
+    own_shard_range *without an epoch*, and log a warning about it.
+
+    :param shards: a list of candidate ShardRanges to merge
+    :param broker: a ContainerBroker
+    :param logger: a logger
+    :param source: string to log as source of shards
+    :return: a list of ShardRanges to actually merge
+    """
+    # work-around for https://bugs.launchpad.net/swift/+bug/1980451
+    own_sr = broker.get_own_shard_range()
+    if own_sr.epoch is None:
+        return shards
+    to_merge = []
+    for shard in shards:
+        if shard['name'] == own_sr.name and not shard['epoch']:
+            shard_copy = dict(shard)
+            new_content = merge_shards(shard_copy, dict(own_sr))
+            if new_content and shard_copy['epoch'] is None:
+                logger.warning(
+                    'Ignoring remote osr w/o epoch, own_sr: %r, remote_sr: %r,'
+                    ' source: %s', dict(own_sr), shard, source)
+            continue
+        to_merge.append(shard)
+    return to_merge
 
 
 class ContainerReplicator(db_replicator.Replicator):
@@ -139,23 +171,44 @@ class ContainerReplicator(db_replicator.Replicator):
         with Timeout(self.node_timeout):
             response = http.replicate('get_shard_ranges')
         if response and is_success(response.status):
-            broker.merge_shard_ranges(json.loads(
-                response.data.decode('ascii')))
+            shards = json.loads(response.data.decode('ascii'))
+            shards = check_merge_own_shard_range(
+                shards, broker, self.logger, '%s%s' % (http.host, http.path))
+            broker.merge_shard_ranges(shards)
 
     def find_local_handoff_for_part(self, part):
         """
-        Look through devices in the ring for the first handoff device that was
-        identified during job creation as available on this node.
+        Find a device in the ring that is on this node on which to place a
+        partition. Preference is given to a device that is a primary location
+        for the partition. If no such device is found then a local device with
+        weight is chosen, and failing that any local device.
 
+        :param part: a partition
         :returns: a node entry from the ring
         """
-        nodes = self.ring.get_part_nodes(part)
-        more_nodes = self.ring.get_more_nodes(part)
+        if not self._local_device_ids:
+            raise RuntimeError('Cannot find local handoff; no local devices')
 
-        for node in itertools.chain(nodes, more_nodes):
+        for node in self.ring.get_part_nodes(part):
             if node['id'] in self._local_device_ids:
                 return node
-        return None
+
+        # don't attempt to minimize handoff depth: just choose any local
+        # device, but start by only picking a device with a weight, just in
+        # case some devices are being drained...
+        local_devs_with_weight = [
+            dev for dev in self._local_device_ids.values()
+            if dev.get('weight', 0)]
+        if local_devs_with_weight:
+            return choice(local_devs_with_weight)
+
+        # we have to return something, so choose any local device..
+        node = choice(list(self._local_device_ids.values()))
+        self.logger.warning(
+            "Could not find a non-zero weight device for handoff partition "
+            "%d, falling back device %s" %
+            (part, node['device']))
+        return node
 
     def get_reconciler_broker(self, timestamp):
         """
@@ -173,14 +226,12 @@ class ContainerReplicator(db_replicator.Replicator):
         account = MISPLACED_OBJECTS_ACCOUNT
         part = self.ring.get_part(account, container)
         node = self.find_local_handoff_for_part(part)
-        if not node:
-            raise DeviceUnavailable(
-                'No mounted devices found suitable to Handoff reconciler '
-                'container %s in partition %s' % (container, part))
-        broker = ContainerBroker.create_broker(
+        broker, initialized = ContainerBroker.create_broker(
             os.path.join(self.root, node['device']), part, account, container,
             logger=self.logger, put_timestamp=timestamp,
             storage_policy_index=0)
+        self.logger.increment('reconciler_db_created' if initialized
+                              else 'reconciler_db_exists')
         if self.reconciler_containers is not None:
             self.reconciler_containers[container] = part, broker, node['id']
         return broker
@@ -198,8 +249,9 @@ class ContainerReplicator(db_replicator.Replicator):
 
         try:
             reconciler = self.get_reconciler_broker(container)
-        except DeviceUnavailable as e:
-            self.logger.warning('DeviceUnavailable: %s', e)
+        except Exception:
+            self.logger.exception('Failed to get reconciler broker for '
+                                  'container %s', container)
             return False
         self.logger.debug('Adding %d objects to the reconciler at %s',
                           len(item_list), reconciler.db_file)
@@ -377,13 +429,35 @@ class ContainerReplicatorRpc(db_replicator.ReplicatorRpc):
     def _post_rsync_then_merge_hook(self, existing_broker, new_broker):
         # Note the following hook will need to change to using a pointer and
         # limit in the future.
-        new_broker.merge_shard_ranges(
-            existing_broker.get_all_shard_range_data())
+        shards = existing_broker.get_all_shard_range_data()
+        shards = check_merge_own_shard_range(
+            shards, new_broker, self.logger, 'rsync')
+        new_broker.merge_shard_ranges(shards)
 
     def merge_shard_ranges(self, broker, args):
-        broker.merge_shard_ranges(args[0])
+        shards = check_merge_own_shard_range(
+            args[0], broker, self.logger, 'repl_req')
+        broker.merge_shard_ranges(shards)
         return HTTPAccepted()
 
     def get_shard_ranges(self, broker, args):
         return HTTPOk(headers={'Content-Type': 'application/json'},
                       body=json.dumps(broker.get_all_shard_range_data()))
+
+
+def main():
+    parser = optparse.OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help=('Replicate only given devices. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    parser.add_option('-p', '--partitions',
+                      help=('Replicate only given partitions. '
+                            'Comma-separated list. '
+                            'Only has effect if --once is used.'))
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ContainerReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()