swift 2.33.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/obj/expirer.py

@@ -14,9 +14,11 @@
 # limitations under the License.
 
 import six
+from six.moves import urllib
 
 from random import random
 from time import time
+from optparse import OptionParser
 from os.path import join
 from collections import defaultdict, deque
 
@@ -24,11 +26,12 @@ from eventlet import sleep, Timeout
 from eventlet.greenpool import GreenPool
 
 from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.internal_client import InternalClient, UnexpectedResponse
 from swift.common.utils import get_logger, dump_recon_cache, split_path, \
     Timestamp, config_true_value, normalize_delete_at_timestamp, \
-    RateLimitedIterator, md5
+    RateLimitedIterator, md5, non_negative_float, non_negative_int, \
+    parse_content_type, parse_options
 from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \
     HTTP_PRECONDITION_FAILED
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
@@ -36,6 +39,7 @@ from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.container.reconciler import direct_delete_container_entry
 
 MAX_OBJECTS_TO_CACHE = 100000
+X_DELETE_TYPE = 'text/plain'
 ASYNC_DELETE_TYPE = 'application/async-deleted'
 
 
@@ -66,6 +70,80 @@ def parse_task_obj(task_obj):
     return timestamp, target_account, target_container, target_obj
 
 
+def extract_expirer_bytes_from_ctype(content_type):
+    """
+    Parse a content-type and return the number of bytes.
+
+    :param content_type: a content-type string
+    :return: int or None
+    """
+    content_type, params = parse_content_type(content_type)
+    bytes_size = None
+    for k, v in params:
+        if k == 'swift_expirer_bytes':
+            bytes_size = int(v)
+    return bytes_size
+
+
+def embed_expirer_bytes_in_ctype(content_type, metadata):
+    """
+    Embed number of bytes into content-type.  The bytes should come from
+    content-length on regular objects, but future extensions to "bytes in
+    expirer queue" monitoring may want to more closely consider expiration of
+    large multipart object manifests.
+
+    :param content_type: a content-type string
+    :param metadata: a dict, from Diskfile metadata
+    :return: str
+    """
+    # as best I can tell this key is required by df.open
+    report_bytes = metadata['Content-Length']
+    return "%s;swift_expirer_bytes=%d" % (content_type, int(report_bytes))
+
+
+def read_conf_for_delay_reaping_times(conf):
+    delay_reaping_times = {}
+    for conf_key in conf:
+        delay_reaping_prefix = "delay_reaping_"
+        if not conf_key.startswith(delay_reaping_prefix):
+            continue
+        delay_reaping_key = urllib.parse.unquote(
+            conf_key[len(delay_reaping_prefix):])
+        if delay_reaping_key.strip('/') != delay_reaping_key:
+            raise ValueError(
+                '%s '
+                'should be in the form delay_reaping_<account> '
+                'or delay_reaping_<account>/<container> '
+                '(leading or trailing "/" is not allowed)' % conf_key)
+        try:
+            # If split_path fails, have multiple '/' or
+            # account name is invalid
+            account, container = split_path(
+                '/' + delay_reaping_key, 1, 2
+            )
+        except ValueError:
+            raise ValueError(
+                '%s '
+                'should be in the form delay_reaping_<account> '
+                'or delay_reaping_<account>/<container> '
+                '(at most one "/" is allowed)' % conf_key)
+        try:
+            delay_reaping_times[(account, container)] = non_negative_float(
+                conf.get(conf_key)
+            )
+        except ValueError:
+            raise ValueError(
+                '%s must be a float '
+                'greater than or equal to 0' % conf_key)
+    return delay_reaping_times
+
+
+def get_delay_reaping(delay_reaping_times, target_account, target_container):
+    return delay_reaping_times.get(
+        (target_account, target_container),
+        delay_reaping_times.get((target_account, None), 0.0))
+
+
 class ObjectExpirer(Daemon):
     """
     Daemon that queries the internal hidden task accounts to discover objects
@@ -89,16 +167,8 @@ class ObjectExpirer(Daemon):
         self.dequeue_from_legacy = \
             True if is_legacy_conf else \
             config_true_value(conf.get('dequeue_from_legacy', 'false'))
-
-        if is_legacy_conf:
-            self.ic_conf_path = self.conf_path
-        else:
-            self.ic_conf_path = \
-                self.conf.get('internal_client_conf_path') or \
-                '/etc/swift/internal-client.conf'
-
-        self.read_conf_for_queue_access(swift)
-
+        self.swift = swift or self._make_internal_client(is_legacy_conf)
+        self.read_conf_for_queue_access()
         self.report_interval = float(conf.get('report_interval') or 300)
         self.report_first_time = self.report_last_time = time()
         self.report_objects = 0
@@ -113,23 +183,32 @@ class ObjectExpirer(Daemon):
         # with the tombstone reclaim age in the consistency engine.
         self.reclaim_age = int(conf.get('reclaim_age', 604800))
 
-    def read_conf_for_queue_access(self, swift):
-        self.expiring_objects_account = AUTO_CREATE_ACCOUNT_PREFIX + \
-            (self.conf.get('expiring_objects_account_name') or
-             'expiring_objects')
-
-        # This is for common parameter with general task queue in future
-        self.task_container_prefix = ''
+        self.delay_reaping_times = read_conf_for_delay_reaping_times(conf)
 
+    def _make_internal_client(self, is_legacy_conf):
+        if is_legacy_conf:
+            ic_conf_path = self.conf_path
+        else:
+            ic_conf_path = \
+                self.conf.get('internal_client_conf_path') or \
+                '/etc/swift/internal-client.conf'
         request_tries = int(self.conf.get('request_tries') or 3)
-        self.swift = swift or InternalClient(
-            self.ic_conf_path, 'Swift Object Expirer', request_tries,
+        return InternalClient(
+            ic_conf_path, 'Swift Object Expirer', request_tries,
             use_replication_network=True,
             global_conf={'log_name': '%s-ic' % self.conf.get(
                 'log_name', self.log_route)})
 
-        self.processes = int(self.conf.get('processes', 0))
-        self.process = int(self.conf.get('process', 0))
+    def read_conf_for_queue_access(self):
+        self.expiring_objects_account = AUTO_CREATE_ACCOUNT_PREFIX + \
+            (self.conf.get('expiring_objects_account_name') or
+             'expiring_objects')
+
+        # This is for common parameter with general task queue in future
+        self.task_container_prefix = ''
+        self.processes = non_negative_int(self.conf.get('processes', 0))
+        self.process = non_negative_int(self.conf.get('process', 0))
+        self._validate_processes_config()
 
     def report(self, final=False):
         """
@@ -246,6 +325,10 @@ class ObjectExpirer(Daemon):
                 break
             yield task_container
 
+    def get_delay_reaping(self, target_account, target_container):
+        return get_delay_reaping(self.delay_reaping_times, target_account,
+                                 target_container)
+
     def iter_task_to_expire(self, task_account_container_list,
                             my_index, divisor):
         """
@@ -266,18 +349,30 @@ class ObjectExpirer(Daemon):
                 except ValueError:
                     self.logger.exception('Unexcepted error handling task %r' %
                                           task_object)
+                    self.logger.increment('tasks.parse_errors')
                     continue
+                is_async = o.get('content_type') == ASYNC_DELETE_TYPE
+                delay_reaping = self.get_delay_reaping(target_account,
+                                                       target_container)
+
                 if delete_timestamp > Timestamp.now():
-                    # we shouldn't yield the object that doesn't reach
+                    # we shouldn't yield ANY more objects that can't reach
                     # the expiration date yet.
                     break
 
-                # Only one expirer daemon assigned for one task
+                # Only one expirer daemon assigned for each task
                 if self.hash_mod('%s/%s' % (task_container, task_object),
                                  divisor) != my_index:
+                    self.logger.increment('tasks.skipped')
                     continue
 
-                is_async = o.get('content_type') == ASYNC_DELETE_TYPE
+                if delete_timestamp > Timestamp(time() - delay_reaping) \
+                        and not is_async:
+                    # we shouldn't yield the object during the delay
+                    self.logger.increment('tasks.delayed')
+                    continue
+
+                self.logger.increment('tasks.assigned')
                 yield {'task_account': task_account,
                        'task_container': task_container,
                        'task_object': task_object,
@@ -308,6 +403,9 @@ class ObjectExpirer(Daemon):
                        These will override the values from the config file if
                        provided.
         """
+        # these config values are available to override at the command line,
+        # blow-up now if they're wrong
+        self.override_proceses_config_from_command_line(**kwargs)
         # This if-clause will be removed when general task queue feature is
         # implemented.
         if not self.dequeue_from_legacy:
@@ -318,7 +416,6 @@ class ObjectExpirer(Daemon):
                              'with dequeue_from_legacy == true.')
             return
 
-        self.get_process_values(kwargs)
         pool = GreenPool(self.concurrency)
         self.report_first_time = self.report_last_time = time()
         self.report_objects = 0
@@ -373,6 +470,9 @@ class ObjectExpirer(Daemon):
         :param kwargs: Extra keyword args to fulfill the Daemon interface; this
                        daemon has no additional keyword args.
         """
+        # these config values are available to override at the command line
+        # blow-up now if they're wrong
+        self.override_proceses_config_from_command_line(**kwargs)
         sleep(random() * self.interval)
         while True:
             begin = time()
@@ -384,7 +484,7 @@ class ObjectExpirer(Daemon):
             if elapsed < self.interval:
                 sleep(random() * (self.interval - elapsed))
 
-    def get_process_values(self, kwargs):
+    def override_proceses_config_from_command_line(self, **kwargs):
         """
         Sets self.processes and self.process from the kwargs if those
         values exist, otherwise, leaves those values as they were set in
@@ -395,19 +495,20 @@ class ObjectExpirer(Daemon):
                        line when the daemon is run.
         """
         if kwargs.get('processes') is not None:
-            self.processes = int(kwargs['processes'])
+            self.processes = non_negative_int(kwargs['processes'])
 
         if kwargs.get('process') is not None:
-            self.process = int(kwargs['process'])
+            self.process = non_negative_int(kwargs['process'])
 
-        if self.process < 0:
-            raise ValueError(
-                'process must be an integer greater than or equal to 0')
+        self._validate_processes_config()
 
-        if self.processes < 0:
-            raise ValueError(
-                'processes must be an integer greater than or equal to 0')
+    def _validate_processes_config(self):
+        """
+        Used in constructor and in override_proceses_config_from_command_line
+        to validate the processes configuration requirements.
 
+        :raiess: ValueError if processes config is invalid
+        """
         if self.processes and self.process >= self.processes:
             raise ValueError(
                 'process must be less than processes')
@@ -484,3 +585,21 @@ class ObjectExpirer(Daemon):
         self.swift.delete_object(*split_path('/' + actual_obj, 3, 3, True),
                                  headers=headers,
                                  acceptable_statuses=acceptable_statuses)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('--processes', dest='processes',
+                      help="Number of processes to use to do the work, don't "
+                      "use this option to do all the work in one process")
+    parser.add_option('--process', dest='process',
+                      help="Process number for this process, don't use "
+                      "this option to do all the work in one process, this "
+                      "is used to determine which part of the work this "
+                      "process should do")
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectExpirer, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/reconstructor.py

@@ -15,6 +15,7 @@
 import itertools
 import json
 import errno
+from optparse import OptionParser
 import os
 from os.path import join
 import random
@@ -33,10 +34,10 @@ from swift.common.utils import (
     GreenAsyncPile, Timestamp, remove_file, node_to_string,
     load_recon_cache, parse_override_options, distribute_evenly,
     PrefixLoggerAdapter, remove_directory, config_request_node_count_value,
-    non_negative_int)
+    non_negative_int, parse_options)
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.bufferedhttp import http_connect
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.common.ring.utils import is_local_device
 from swift.obj.ssync_sender import Sender as ssync_sender
@@ -911,12 +912,14 @@ class ObjectReconstructor(Daemon):
             except StopIteration:
                 break
             attempts_remaining -= 1
+            conn = None
             try:
                 with Timeout(self.http_timeout):
-                    resp = http_connect(
+                    conn = http_connect(
                         node['replication_ip'], node['replication_port'],
                         node['device'], job['partition'], 'REPLICATE',
-                        '', headers=headers).getresponse()
+                        '', headers=headers)
+                    resp = conn.getresponse()
                 if resp.status == HTTP_INSUFFICIENT_STORAGE:
                     self.logger.error(
                         '%s responded as unmounted',
@@ -939,6 +942,9 @@ class ObjectReconstructor(Daemon):
                                       'from %r' % _full_path(
                                           node, job['partition'], '',
                                           job['policy']))
+            finally:
+                if conn:
+                    conn.close()
         if remote_suffixes is None:
             raise SuffixSyncError('Unable to get remote suffix hashes')
 
@@ -1556,3 +1562,21 @@ class ObjectReconstructor(Daemon):
             self.logger.debug('reconstruction sleeping for %s seconds.',
                               self.interval)
             sleep(self.interval)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help='Reconstruct only given devices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-p', '--partitions',
+                      help='Reconstruct only given partitions. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectReconstructor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/replicator.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from collections import defaultdict
+from optparse import OptionParser
 import os
 import errno
 from os.path import isdir, isfile, join, dirname
@@ -35,9 +36,9 @@ from swift.common.utils import whataremyips, unlink_older_than, \
     rsync_module_interpolation, mkdirs, config_true_value, \
     config_auto_int_value, storage_directory, \
     load_recon_cache, PrefixLoggerAdapter, parse_override_options, \
-    distribute_evenly, listdir, node_to_string
+    distribute_evenly, listdir, node_to_string, parse_options
 from swift.common.bufferedhttp import http_connect
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj import ssync_sender
@@ -195,6 +196,12 @@ class ObjectReplicator(Daemon):
                                 'operation, please disable handoffs_first and '
                                 'handoff_delete before the next '
                                 'normal rebalance')
+        if all(self.load_object_ring(p).replica_count <= self.handoff_delete
+               for p in self.policies):
+            self.logger.warning('No storage policies found for which '
+                                'handoff_delete=%d would have an effect. '
+                                'Disabling.', self.handoff_delete)
+            self.handoff_delete = 0
         self.is_multiprocess_worker = None
         self._df_router = DiskFileRouter(conf, self.logger)
         self._child_process_reaper_queue = queue.LightQueue()
@@ -473,7 +480,10 @@ class ObjectReplicator(Daemon):
                     node['replication_ip'], node['replication_port'],
                     node['device'], job['partition'], 'REPLICATE',
                     '/' + '-'.join(suffixes), headers=headers)
-                conn.getresponse().read()
+                try:
+                    conn.getresponse().read()
+                finally:
+                    conn.close()
         return success, {}
 
     def ssync(self, node, job, suffixes, remote_check_objs=None):
@@ -554,7 +564,8 @@ class ObjectReplicator(Daemon):
                 if self.handoff_delete:
                     # delete handoff if we have had handoff_delete successes
                     successes_count = len([resp for resp in responses if resp])
-                    delete_handoff = successes_count >= self.handoff_delete
+                    delete_handoff = successes_count >= min(
+                        self.handoff_delete, len(job['nodes']))
                 else:
                     # delete handoff if all syncs were successful
                     delete_handoff = len(responses) == len(job['nodes']) and \
@@ -608,7 +619,8 @@ class ObjectReplicator(Daemon):
         try:
             tpool.execute(shutil.rmtree, path)
         except OSError as e:
-            if e.errno not in (errno.ENOENT, errno.ENOTEMPTY, errno.ENODATA):
+            if e.errno not in (errno.ENOENT, errno.ENOTEMPTY, errno.ENODATA,
+                               errno.EUCLEAN):
                 # Don't worry if there was a race to create or delete,
                 # or some disk corruption that happened after the sync
                 raise
@@ -675,25 +687,30 @@ class ObjectReplicator(Daemon):
                     continue
                 try:
                     with Timeout(self.http_timeout):
-                        resp = http_connect(
+                        conn = http_connect(
                             node['replication_ip'], node['replication_port'],
                             node['device'], job['partition'], 'REPLICATE',
-                            '', headers=headers).getresponse()
-                        if resp.status == HTTP_INSUFFICIENT_STORAGE:
-                            self.logger.error('%s responded as unmounted',
-                                              node_str)
-                            attempts_left += 1
-                            failure_devs_info.add((node['replication_ip'],
-                                                   node['device']))
-                            continue
-                        if resp.status != HTTP_OK:
-                            self.logger.error(
-                                "Invalid response %(resp)s from %(remote)s",
-                                {'resp': resp.status, 'remote': node_str})
-                            failure_devs_info.add((node['replication_ip'],
-                                                   node['device']))
-                            continue
-                        remote_hash = pickle.loads(resp.read())
+                            '', headers=headers)
+                        try:
+                            resp = conn.getresponse()
+                            if resp.status == HTTP_INSUFFICIENT_STORAGE:
+                                self.logger.error('%s responded as unmounted',
+                                                  node_str)
+                                attempts_left += 1
+                                failure_devs_info.add((node['replication_ip'],
+                                                       node['device']))
+                                continue
+                            if resp.status != HTTP_OK:
+                                self.logger.error(
+                                    "Invalid response %(resp)s "
+                                    "from %(remote)s",
+                                    {'resp': resp.status, 'remote': node_str})
+                                failure_devs_info.add((node['replication_ip'],
+                                                       node['device']))
+                                continue
+                            remote_hash = pickle.loads(resp.read())
+                        finally:
+                            conn.close()
                         del resp
                     suffixes = [suffix for suffix in local_hash if
                                 local_hash[suffix] !=
@@ -1140,3 +1157,25 @@ class ObjectReplicator(Daemon):
         # This method is called after run_once using multiple workers.
         update = self.aggregate_recon_update()
         dump_recon_cache(update, self.rcache, self.logger)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help='Replicate only given devices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-p', '--partitions',
+                      help='Replicate only given partitions. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-i', '--policies',
+                      help='Replicate only given policy indices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()