swift 2.32.0__py2.py3-none-any.whl → 2.34.0__py2.py3-none-any.whl

swift/obj/reconstructor.py

@@ -15,6 +15,7 @@
 import itertools
 import json
 import errno
+from optparse import OptionParser
 import os
 from os.path import join
 import random
@@ -33,10 +34,10 @@ from swift.common.utils import (
     GreenAsyncPile, Timestamp, remove_file, node_to_string,
     load_recon_cache, parse_override_options, distribute_evenly,
     PrefixLoggerAdapter, remove_directory, config_request_node_count_value,
-    non_negative_int)
+    non_negative_int, parse_options)
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.bufferedhttp import http_connect
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.common.ring.utils import is_local_device
 from swift.obj.ssync_sender import Sender as ssync_sender
@@ -911,12 +912,14 @@ class ObjectReconstructor(Daemon):
             except StopIteration:
                 break
             attempts_remaining -= 1
+            conn = None
             try:
                 with Timeout(self.http_timeout):
-                    resp = http_connect(
+                    conn = http_connect(
                         node['replication_ip'], node['replication_port'],
                         node['device'], job['partition'], 'REPLICATE',
-                        '', headers=headers).getresponse()
+                        '', headers=headers)
+                    resp = conn.getresponse()
                 if resp.status == HTTP_INSUFFICIENT_STORAGE:
                     self.logger.error(
                         '%s responded as unmounted',
@@ -939,6 +942,9 @@ class ObjectReconstructor(Daemon):
                                       'from %r' % _full_path(
                                           node, job['partition'], '',
                                           job['policy']))
+            finally:
+                if conn:
+                    conn.close()
         if remote_suffixes is None:
             raise SuffixSyncError('Unable to get remote suffix hashes')
 
@@ -1556,3 +1562,21 @@ class ObjectReconstructor(Daemon):
             self.logger.debug('reconstruction sleeping for %s seconds.',
                               self.interval)
             sleep(self.interval)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help='Reconstruct only given devices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-p', '--partitions',
+                      help='Reconstruct only given partitions. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectReconstructor, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/replicator.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from collections import defaultdict
+from optparse import OptionParser
 import os
 import errno
 from os.path import isdir, isfile, join, dirname
@@ -35,9 +36,9 @@ from swift.common.utils import whataremyips, unlink_older_than, \
     rsync_module_interpolation, mkdirs, config_true_value, \
     config_auto_int_value, storage_directory, \
     load_recon_cache, PrefixLoggerAdapter, parse_override_options, \
-    distribute_evenly, listdir, node_to_string
+    distribute_evenly, listdir, node_to_string, parse_options
 from swift.common.bufferedhttp import http_connect
-from swift.common.daemon import Daemon
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj import ssync_sender
@@ -195,6 +196,12 @@ class ObjectReplicator(Daemon):
                                 'operation, please disable handoffs_first and '
                                 'handoff_delete before the next '
                                 'normal rebalance')
+        if all(self.load_object_ring(p).replica_count <= self.handoff_delete
+               for p in self.policies):
+            self.logger.warning('No storage policies found for which '
+                                'handoff_delete=%d would have an effect. '
+                                'Disabling.', self.handoff_delete)
+            self.handoff_delete = 0
         self.is_multiprocess_worker = None
         self._df_router = DiskFileRouter(conf, self.logger)
         self._child_process_reaper_queue = queue.LightQueue()
@@ -473,7 +480,10 @@ class ObjectReplicator(Daemon):
                     node['replication_ip'], node['replication_port'],
                     node['device'], job['partition'], 'REPLICATE',
                     '/' + '-'.join(suffixes), headers=headers)
-                conn.getresponse().read()
+                try:
+                    conn.getresponse().read()
+                finally:
+                    conn.close()
         return success, {}
 
     def ssync(self, node, job, suffixes, remote_check_objs=None):
@@ -493,7 +503,7 @@ class ObjectReplicator(Daemon):
                 return False
         return True
 
-    def update_deleted(self, job):
+    def revert(self, job):
         """
         High-level method that replicates a single partition that doesn't
         belong on this node.
@@ -554,7 +564,8 @@ class ObjectReplicator(Daemon):
                 if self.handoff_delete:
                     # delete handoff if we have had handoff_delete successes
                     successes_count = len([resp for resp in responses if resp])
-                    delete_handoff = successes_count >= self.handoff_delete
+                    delete_handoff = successes_count >= min(
+                        self.handoff_delete, len(job['nodes']))
                 else:
                     # delete handoff if all syncs were successful
                     delete_handoff = len(responses) == len(job['nodes']) and \
@@ -608,7 +619,8 @@ class ObjectReplicator(Daemon):
         try:
             tpool.execute(shutil.rmtree, path)
         except OSError as e:
-            if e.errno not in (errno.ENOENT, errno.ENOTEMPTY, errno.ENODATA):
+            if e.errno not in (errno.ENOENT, errno.ENOTEMPTY, errno.ENODATA,
+                               errno.EUCLEAN):
                 # Don't worry if there was a race to create or delete,
                 # or some disk corruption that happened after the sync
                 raise
@@ -675,25 +687,30 @@ class ObjectReplicator(Daemon):
                     continue
                 try:
                     with Timeout(self.http_timeout):
-                        resp = http_connect(
+                        conn = http_connect(
                             node['replication_ip'], node['replication_port'],
                             node['device'], job['partition'], 'REPLICATE',
-                            '', headers=headers).getresponse()
-                        if resp.status == HTTP_INSUFFICIENT_STORAGE:
-                            self.logger.error('%s responded as unmounted',
-                                              node_str)
-                            attempts_left += 1
-                            failure_devs_info.add((node['replication_ip'],
-                                                   node['device']))
-                            continue
-                        if resp.status != HTTP_OK:
-                            self.logger.error(
-                                "Invalid response %(resp)s from %(remote)s",
-                                {'resp': resp.status, 'remote': node_str})
-                            failure_devs_info.add((node['replication_ip'],
-                                                   node['device']))
-                            continue
-                        remote_hash = pickle.loads(resp.read())
+                            '', headers=headers)
+                        try:
+                            resp = conn.getresponse()
+                            if resp.status == HTTP_INSUFFICIENT_STORAGE:
+                                self.logger.error('%s responded as unmounted',
+                                                  node_str)
+                                attempts_left += 1
+                                failure_devs_info.add((node['replication_ip'],
+                                                       node['device']))
+                                continue
+                            if resp.status != HTTP_OK:
+                                self.logger.error(
+                                    "Invalid response %(resp)s "
+                                    "from %(remote)s",
+                                    {'resp': resp.status, 'remote': node_str})
+                                failure_devs_info.add((node['replication_ip'],
+                                                       node['device']))
+                                continue
+                            remote_hash = pickle.loads(resp.read())
+                        finally:
+                            conn.close()
                         del resp
                     suffixes = [suffix for suffix in local_hash if
                                 local_hash[suffix] !=
@@ -993,7 +1010,7 @@ class ObjectReplicator(Daemon):
                 except OSError:
                     continue
                 if job['delete']:
-                    self.run_pool.spawn(self.update_deleted, job)
+                    self.run_pool.spawn(self.revert, job)
                 else:
                     self.run_pool.spawn(self.update, job)
             current_nodes = None
@@ -1140,3 +1157,25 @@ class ObjectReplicator(Daemon):
         # This method is called after run_once using multiple workers.
         update = self.aggregate_recon_update()
         dump_recon_cache(update, self.rcache, self.logger)
+
+
+def main():
+    parser = OptionParser("%prog CONFIG [options]")
+    parser.add_option('-d', '--devices',
+                      help='Replicate only given devices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-p', '--partitions',
+                      help='Replicate only given partitions. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    parser.add_option('-i', '--policies',
+                      help='Replicate only given policy indices. '
+                           'Comma-separated list. '
+                           'Only has effect if --once is used.')
+    conf_file, options = parse_options(parser=parser, once=True)
+    run_daemon(ObjectReplicator, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()

swift/obj/server.py

@@ -21,6 +21,7 @@ from six.moves.urllib.parse import unquote
 import json
 import os
 import multiprocessing
+import sys
 import time
 import traceback
 import socket
@@ -34,7 +35,7 @@ from swift.common.utils import public, get_logger, \
     get_expirer_container, parse_mime_headers, \
     iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \
     config_auto_int_value, split_path, get_redirect_data, \
-    normalize_timestamp, md5
+    normalize_timestamp, md5, parse_options
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, \
     valid_timestamp, check_utf8, AUTO_CREATE_ACCOUNT_PREFIX
@@ -42,7 +43,7 @@ from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
     DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
     DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \
     ChunkReadError, DiskFileXattrNotSupported
-from swift.common.request_helpers import \
+from swift.common.request_helpers import resolve_ignore_range_header, \
     OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX
 from swift.obj import ssync_receiver
 from swift.common.http import is_success, HTTP_MOVED_PERMANENTLY
@@ -50,15 +51,18 @@ from swift.common.base_storage_server import BaseStorageServer
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.request_helpers import get_name_and_placement, \
     is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \
-    resolve_etag_is_at_header, is_sys_meta, validate_internal_obj
+    resolve_etag_is_at_header, is_sys_meta, validate_internal_obj, \
+    is_backend_open_expired
 from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
     HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
     HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
     HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
     HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag
+from swift.common.wsgi import run_wsgi
 from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
-from swift.obj.expirer import build_task_obj
+from swift.obj.expirer import build_task_obj, embed_expirer_bytes_in_ctype, \
+    X_DELETE_TYPE
 
 
 def iter_mime_headers_and_bodies(wsgi_input, mime_boundary, read_chunk_size):
@@ -152,6 +156,7 @@ class ObjectController(BaseStorageServer):
             config_true_value(conf.get('keep_cache_private', 'false'))
         self.keep_cache_slo_manifest = \
             config_true_value(conf.get('keep_cache_slo_manifest', 'false'))
+        self.cooperative_period = int(conf.get("cooperative_period", 0))
 
         default_allowed_headers = '''
             content-disposition,
@@ -173,18 +178,8 @@ class ObjectController(BaseStorageServer):
         for header in extra_allowed_headers:
             if header not in RESERVED_DATAFILE_META:
                 self.allowed_headers.add(header)
-        if conf.get('auto_create_account_prefix'):
-            self.logger.warning('Option auto_create_account_prefix is '
-                                'deprecated. Configure '
-                                'auto_create_account_prefix under the '
-                                'swift-constraints section of '
-                                'swift.conf. This option will '
-                                'be ignored in a future release.')
-            self.auto_create_account_prefix = \
-                conf['auto_create_account_prefix']
-        else:
-            self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
 
+        self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
         self.expiring_objects_account = self.auto_create_account_prefix + \
             (conf.get('expiring_objects_account_name') or 'expiring_objects')
         self.expiring_objects_container_divisor = \
@@ -277,7 +272,8 @@ class ObjectController(BaseStorageServer):
 
     def async_update(self, op, account, container, obj, host, partition,
                      contdevice, headers_out, objdevice, policy,
-                     logger_thread_locals=None, container_path=None):
+                     logger_thread_locals=None, container_path=None,
+                     db_state=None):
         """
         Sends or saves an async update.
 
@@ -299,6 +295,8 @@ class ObjectController(BaseStorageServer):
             to which the update should be sent. If given this path will be used
             instead of constructing a path from the ``account`` and
             ``container`` params.
+        :param db_state: The current database state of the container as
+            supplied to us by the proxy.
         """
         if logger_thread_locals:
             self.logger.thread_locals = logger_thread_locals
@@ -342,7 +340,7 @@ class ObjectController(BaseStorageServer):
                     '%(ip)s:%(port)s/%(dev)s (saving for async update later)',
                     {'ip': ip, 'port': port, 'dev': contdevice})
         data = {'op': op, 'account': account, 'container': container,
-                'obj': obj, 'headers': headers_out}
+                'obj': obj, 'headers': headers_out, 'db_state': db_state}
         if redirect_data:
             self.logger.debug(
                 'Update to %(path)s redirected to %(redirect)s',
@@ -376,6 +374,7 @@ class ObjectController(BaseStorageServer):
         contdevices = [d.strip() for d in
                        headers_in.get('X-Container-Device', '').split(',')]
         contpartition = headers_in.get('X-Container-Partition', '')
+        contdbstate = headers_in.get('X-Container-Root-Db-State')
 
         if len(conthosts) != len(contdevices):
             # This shouldn't happen unless there's a bug in the proxy,
@@ -426,7 +425,7 @@ class ObjectController(BaseStorageServer):
                        conthost, contpartition, contdevice, headers_out,
                        objdevice, policy,
                        logger_thread_locals=self.logger.thread_locals,
-                       container_path=contpath)
+                       container_path=contpath, db_state=contdbstate)
             update_greenthreads.append(gt)
         # Wait a little bit to see if the container updates are successful.
         # If we immediately return after firing off the greenthread above, then
@@ -445,7 +444,7 @@ class ObjectController(BaseStorageServer):
                 self.container_update_timeout, updates)
 
     def delete_at_update(self, op, delete_at, account, container, obj,
-                         request, objdevice, policy):
+                         request, objdevice, policy, extra_headers=None):
         """
         Update the expiring objects container when objects are updated.
 
@@ -457,6 +456,7 @@ class ObjectController(BaseStorageServer):
         :param request: the original request driving the update
         :param objdevice: device name that the object is in
         :param policy: the BaseStoragePolicy instance (used for tmp dir)
+        :param extra_headers: dict of additional headers for the update
         """
         if config_true_value(
                 request.headers.get('x-backend-replication', 'f')):
@@ -502,8 +502,10 @@ class ObjectController(BaseStorageServer):
             if not updates:
                 updates = [(None, None)]
             headers_out['x-size'] = '0'
-            headers_out['x-content-type'] = 'text/plain'
+            headers_out['x-content-type'] = X_DELETE_TYPE
             headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
+            if extra_headers:
+                headers_out.update(extra_headers)
         else:
             if not config_true_value(
                 request.headers.get(
@@ -628,6 +630,24 @@ class ObjectController(BaseStorageServer):
                     override = key.lower().replace(override_prefix, 'x-')
                     update_headers[override] = val
 
+    def _conditional_delete_at_update(self, request, device, account,
+                                      container, obj, policy, metadata,
+                                      orig_delete_at, new_delete_at):
+        if new_delete_at:
+            extra_headers = {
+                'x-content-type': embed_expirer_bytes_in_ctype(
+                    X_DELETE_TYPE, metadata),
+                'x-content-type-timestamp':
+                metadata.get('X-Timestamp'),
+            }
+            self.delete_at_update(
+                'PUT', new_delete_at, account, container, obj, request,
+                device, policy, extra_headers)
+        if orig_delete_at and orig_delete_at != new_delete_at:
+            self.delete_at_update(
+                'DELETE', orig_delete_at, account, container, obj,
+                request, device, policy)
+
     @public
     @timing_stats()
     def POST(self, request):
@@ -644,8 +664,7 @@ class ObjectController(BaseStorageServer):
         try:
             disk_file = self.get_diskfile(
                 device, partition, account, container, obj,
-                policy=policy, open_expired=config_true_value(
-                    request.headers.get('x-backend-replication', 'false')),
+                policy=policy, open_expired=is_backend_open_expired(request),
                 next_part_power=next_part_power)
         except DiskFileDeviceUnavailable:
             return HTTPInsufficientStorage(drive=device, request=request)
@@ -684,15 +703,11 @@ class ObjectController(BaseStorageServer):
                         wsgi_to_bytes(header_key).title())
                     metadata[header_caps] = request.headers[header_key]
             orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
-            if orig_delete_at != new_delete_at:
-                if new_delete_at:
-                    self.delete_at_update(
-                        'PUT', new_delete_at, account, container, obj, request,
-                        device, policy)
-                if orig_delete_at:
-                    self.delete_at_update('DELETE', orig_delete_at, account,
-                                          container, obj, request, device,
-                                          policy)
+            disk_file_metadata = disk_file.get_datafile_metadata()
+            self._conditional_delete_at_update(
+                request, device, account, container, obj, policy,
+                disk_file_metadata, orig_delete_at, new_delete_at
+            )
         else:
             # preserve existing metadata, only content-type may be updated
             metadata = dict(disk_file.get_metafile_metadata())
@@ -1002,15 +1017,10 @@ class ObjectController(BaseStorageServer):
                              orig_metadata, footers_metadata, metadata):
         orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
         new_delete_at = int(request.headers.get('X-Delete-At') or 0)
-        if orig_delete_at != new_delete_at:
-            if new_delete_at:
-                self.delete_at_update(
-                    'PUT', new_delete_at, account, container, obj, request,
-                    device, policy)
-            if orig_delete_at:
-                self.delete_at_update(
-                    'DELETE', orig_delete_at, account, container, obj,
-                    request, device, policy)
+
+        self._conditional_delete_at_update(request, device, account, container,
+                                           obj, policy, metadata,
+                                           orig_delete_at, new_delete_at)
 
         update_headers = HeaderKeyDict({
             'x-size': metadata['Content-Length'],
@@ -1083,21 +1093,13 @@ class ObjectController(BaseStorageServer):
             disk_file = self.get_diskfile(
                 device, partition, account, container, obj,
                 policy=policy, frag_prefs=frag_prefs,
-                open_expired=config_true_value(
-                    request.headers.get('x-backend-replication', 'false')))
+                open_expired=is_backend_open_expired(request))
         except DiskFileDeviceUnavailable:
             return HTTPInsufficientStorage(drive=device, request=request)
         try:
             with disk_file.open(current_time=req_timestamp):
                 metadata = disk_file.get_metadata()
-                ignore_range_headers = set(
-                    h.strip().lower()
-                    for h in request.headers.get(
-                        'X-Backend-Ignore-Range-If-Metadata-Present',
-                        '').split(','))
-                if ignore_range_headers.intersection(
-                        h.lower() for h in metadata):
-                    request.headers.pop('Range', None)
+                resolve_ignore_range_header(request, metadata)
                 obj_size = int(metadata['Content-Length'])
                 file_x_ts = Timestamp(metadata['X-Timestamp'])
                 keep_cache = (
@@ -1114,10 +1116,15 @@ class ObjectController(BaseStorageServer):
                     )
                 )
                 conditional_etag = resolve_etag_is_at_header(request, metadata)
+                app_iter = disk_file.reader(
+                    keep_cache=keep_cache,
+                    cooperative_period=self.cooperative_period,
+                )
                 response = Response(
-                    app_iter=disk_file.reader(keep_cache=keep_cache),
-                    request=request, conditional_response=True,
-                    conditional_etag=conditional_etag)
+                    app_iter=app_iter, request=request,
+                    conditional_response=True,
+                    conditional_etag=conditional_etag,
+                )
                 response.headers['Content-Type'] = metadata.get(
                     'Content-Type', 'application/octet-stream')
                 for key, value in metadata.items():
@@ -1168,8 +1175,7 @@ class ObjectController(BaseStorageServer):
             disk_file = self.get_diskfile(
                 device, partition, account, container, obj,
                 policy=policy, frag_prefs=frag_prefs,
-                open_expired=config_true_value(
-                    request.headers.get('x-backend-replication', 'false')))
+                open_expired=is_backend_open_expired(request))
         except DiskFileDeviceUnavailable:
             return HTTPInsufficientStorage(drive=device, request=request)
         try:
@@ -1275,10 +1281,10 @@ class ObjectController(BaseStorageServer):
             else:
                 # differentiate success from no object at all
                 response_class = HTTPNoContent
-        if orig_delete_at:
-            self.delete_at_update('DELETE', orig_delete_at, account,
-                                  container, obj, request, device,
-                                  policy)
+        self._conditional_delete_at_update(
+            request, device, account, container, obj, policy, {},
+            orig_delete_at, 0
+        )
         if orig_timestamp < req_timestamp:
             try:
                 disk_file.delete(req_timestamp)
@@ -1454,3 +1460,14 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return ObjectController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'object-server',
+                      global_conf_callback=global_conf_callback,
+                      **options))
+
+
+if __name__ == '__main__':
+    main()

swift/obj/updater.py

@@ -34,8 +34,8 @@ from swift.common.utils import get_logger, renamer, write_pickle, \
     dump_recon_cache, config_true_value, RateLimitedIterator, split_path, \
     eventlet_monkey_patch, get_redirect_data, ContextPool, hash_path, \
     non_negative_float, config_positive_int_value, non_negative_int, \
-    EventletRateLimiter, node_to_string
-from swift.common.daemon import Daemon
+    EventletRateLimiter, node_to_string, parse_options
+from swift.common.daemon import Daemon, run_daemon
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.storage_policy import split_policy_string, PolicyError
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
@@ -381,6 +381,7 @@ class ObjectUpdater(Daemon):
                     pids.append(pid)
                 else:
                     signal.signal(signal.SIGTERM, signal.SIG_DFL)
+                    os.environ.pop('NOTIFY_SOCKET', None)
                     eventlet_monkey_patch()
                     self.stats.reset()
                     forkbegin = time.time()
@@ -757,3 +758,12 @@ class ObjectUpdater(Daemon):
             self.logger.timing('updater.timing.status.%s' % status,
                                elapsed * 1000)
         return HTTP_INTERNAL_SERVER_ERROR, node['id'], redirect
+
+
+def main():
+    conf_file, options = parse_options(once=True)
+    run_daemon(ObjectUpdater, conf_file, **options)
+
+
+if __name__ == '__main__':
+    main()