swift 2.23.2__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/obj/server.py

@@ -15,51 +15,53 @@
 
 """ Object Server for Swift """
 
-import six
-import six.moves.cPickle as pickle
+import pickle  # nosec: B403
+from urllib.parse import unquote
 import json
 import os
 import multiprocessing
+import sys
 import time
 import traceback
 import socket
-import math
-from swift import gettext_ as _
-from hashlib import md5
 
 from eventlet import sleep, wsgi, Timeout, tpool
 from eventlet.greenthread import spawn
 
 from swift.common.utils import public, get_logger, \
-    config_true_value, timing_stats, replication, \
+    config_true_value, config_percent_value, timing_stats, replication, \
     normalize_delete_at_timestamp, get_log_line, Timestamp, \
-    get_expirer_container, parse_mime_headers, \
+    parse_mime_headers, \
     iter_multipart_mime_documents, extract_swift_bytes, safe_json_loads, \
-    config_auto_int_value, split_path, get_redirect_data, normalize_timestamp
+    config_auto_int_value, split_path, get_redirect_data, \
+    normalize_timestamp, md5, parse_options, CooperativeIterator
 from swift.common.bufferedhttp import http_connect
 from swift.common.constraints import check_object_creation, \
-    valid_timestamp, check_utf8
+    valid_timestamp, check_utf8, AUTO_CREATE_ACCOUNT_PREFIX
 from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \
     DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \
     DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \
     ChunkReadError, DiskFileXattrNotSupported
-from swift.common.request_helpers import \
+from swift.common.request_helpers import resolve_ignore_range_header, \
     OBJECT_SYSMETA_CONTAINER_UPDATE_OVERRIDE_PREFIX
-from swift.obj import ssync_receiver
+from swift.obj import ssync_receiver, expirer
 from swift.common.http import is_success, HTTP_MOVED_PERMANENTLY
 from swift.common.base_storage_server import BaseStorageServer
 from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.request_helpers import get_name_and_placement, \
     is_user_meta, is_sys_or_user_meta, is_object_transient_sysmeta, \
-    resolve_etag_is_at_header, is_sys_meta
+    resolve_etag_is_at_header, is_sys_meta, validate_internal_obj, \
+    is_backend_open_expired
 from swift.common.swob import HTTPAccepted, HTTPBadRequest, HTTPCreated, \
     HTTPInternalServerError, HTTPNoContent, HTTPNotFound, \
     HTTPPreconditionFailed, HTTPRequestTimeout, HTTPUnprocessableEntity, \
     HTTPClientDisconnect, HTTPMethodNotAllowed, Request, Response, \
     HTTPInsufficientStorage, HTTPForbidden, HTTPException, HTTPConflict, \
-    HTTPServerError, wsgi_to_bytes, wsgi_to_str
+    HTTPServerError, bytes_to_wsgi, wsgi_to_bytes, wsgi_to_str, normalize_etag
+from swift.common.wsgi import run_wsgi
 from swift.obj.diskfile import RESERVED_DATAFILE_META, DiskFileRouter
-from swift.obj.expirer import build_task_obj
+from swift.obj.expirer import build_task_obj, embed_expirer_bytes_in_ctype, \
+    X_DELETE_TYPE
 
 
 def iter_mime_headers_and_bodies(wsgi_input, mime_boundary, read_chunk_size):
@@ -89,6 +91,20 @@ def drain(file_like, read_size, timeout):
                 break
 
 
+def get_obj_name_and_placement(request):
+    """
+    Split and validate path for an object.
+
+    :param request: a swob request
+
+    :returns: a tuple of path parts and storage policy
+    """
+    device, partition, account, container, obj, policy = \
+        get_name_and_placement(request, 5, 5, True)
+    validate_internal_obj(account, container, obj)
+    return device, partition, account, container, obj, policy
+
+
 def _make_backend_fragments_header(fragments):
     if fragments:
         result = {}
@@ -129,7 +145,7 @@ class ObjectController(BaseStorageServer):
         self.container_update_timeout = float(
             conf.get('container_update_timeout', 1))
         self.conn_timeout = float(conf.get('conn_timeout', 0.5))
-        self.client_timeout = int(conf.get('client_timeout', 60))
+        self.client_timeout = float(conf.get('client_timeout', 60))
         self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
         self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
         self.log_requests = config_true_value(conf.get('log_requests', 'true'))
@@ -137,6 +153,11 @@ class ObjectController(BaseStorageServer):
         self.slow = int(conf.get('slow', 0))
         self.keep_cache_private = \
             config_true_value(conf.get('keep_cache_private', 'false'))
+        self.keep_cache_slo_manifest = \
+            config_true_value(conf.get('keep_cache_slo_manifest', 'false'))
+        self.cooperative_period = int(conf.get("cooperative_period", 0))
+        self.etag_validate_frac = config_percent_value(
+            conf.get("etag_validate_pct", 100))
 
         default_allowed_headers = '''
             content-disposition,
@@ -158,12 +179,9 @@ class ObjectController(BaseStorageServer):
         for header in extra_allowed_headers:
             if header not in RESERVED_DATAFILE_META:
                 self.allowed_headers.add(header)
-        self.auto_create_account_prefix = \
-            conf.get('auto_create_account_prefix') or '.'
-        self.expiring_objects_account = self.auto_create_account_prefix + \
-            (conf.get('expiring_objects_account_name') or 'expiring_objects')
-        self.expiring_objects_container_divisor = \
-            int(conf.get('expiring_objects_container_divisor') or 86400)
+
+        self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX
+        self.expirer_config = expirer.ExpirerConfig(conf, logger=self.logger)
         # Initialization was successful, so now apply the network chunk size
         # parameter as the default read / write buffer size for the network
         # sockets.
@@ -178,8 +196,8 @@ class ObjectController(BaseStorageServer):
         # disk_chunk_size parameter. However, it affects all created sockets
         # using this class so we have chosen to tie it to the
         # network_chunk_size parameter value instead.
-        if six.PY2:
-            socket._fileobject.default_bufsize = self.network_chunk_size
+        # if six.PY2:
+        #    socket._fileobject.default_bufsize = self.network_chunk_size
         # TODO: find a way to enable similar functionality in py3
 
         # Provide further setup specific to an object server implementation.
@@ -252,7 +270,8 @@ class ObjectController(BaseStorageServer):
 
     def async_update(self, op, account, container, obj, host, partition,
                      contdevice, headers_out, objdevice, policy,
-                     logger_thread_locals=None, container_path=None):
+                     logger_thread_locals=None, container_path=None,
+                     db_state=None):
         """
         Sends or saves an async update.
 
@@ -274,6 +293,8 @@ class ObjectController(BaseStorageServer):
             to which the update should be sent. If given this path will be used
             instead of constructing a path from the ``account`` and
             ``container`` params.
+        :param db_state: The current database state of the container as
+            supplied to us by the proxy.
         """
         if logger_thread_locals:
             self.logger.thread_locals = logger_thread_locals
@@ -305,19 +326,19 @@ class ObjectController(BaseStorageServer):
                             'Container update failed for %r; problem with '
                             'redirect location: %s' % (obj, err))
                 else:
-                    self.logger.error(_(
+                    self.logger.error(
                         'ERROR Container update failed '
                         '(saving for async update later): %(status)d '
-                        'response from %(ip)s:%(port)s/%(dev)s'),
+                        'response from %(ip)s:%(port)s/%(dev)s',
                         {'status': response.status, 'ip': ip, 'port': port,
                          'dev': contdevice})
             except (Exception, Timeout):
-                self.logger.exception(_(
+                self.logger.exception(
                     'ERROR container update failed with '
-                    '%(ip)s:%(port)s/%(dev)s (saving for async update later)'),
+                    '%(ip)s:%(port)s/%(dev)s (saving for async update later)',
                     {'ip': ip, 'port': port, 'dev': contdevice})
         data = {'op': op, 'account': account, 'container': container,
-                'obj': obj, 'headers': headers_out}
+                'obj': obj, 'headers': headers_out, 'db_state': db_state}
         if redirect_data:
             self.logger.debug(
                 'Update to %(path)s redirected to %(redirect)s',
@@ -351,19 +372,25 @@ class ObjectController(BaseStorageServer):
         contdevices = [d.strip() for d in
                        headers_in.get('X-Container-Device', '').split(',')]
         contpartition = headers_in.get('X-Container-Partition', '')
-        contpath = headers_in.get('X-Backend-Container-Path')
+        contdbstate = headers_in.get('X-Container-Root-Db-State')
 
         if len(conthosts) != len(contdevices):
             # This shouldn't happen unless there's a bug in the proxy,
             # but if there is, we want to know about it.
-            self.logger.error(_(
+            self.logger.error(
                 'ERROR Container update failed: different '
                 'numbers of hosts and devices in request: '
-                '"%(hosts)s" vs "%(devices)s"') % {
+                '"%(hosts)s" vs "%(devices)s"', {
                     'hosts': headers_in.get('X-Container-Host', ''),
                     'devices': headers_in.get('X-Container-Device', '')})
             return
 
+        contpath = headers_in.get('X-Backend-Quoted-Container-Path')
+        if contpath:
+            contpath = unquote(contpath)
+        else:
+            contpath = headers_in.get('X-Backend-Container-Path')
+
         if contpath:
             try:
                 # TODO: this is very late in request handling to be validating
@@ -396,7 +423,7 @@ class ObjectController(BaseStorageServer):
                        conthost, contpartition, contdevice, headers_out,
                        objdevice, policy,
                        logger_thread_locals=self.logger.thread_locals,
-                       container_path=contpath)
+                       container_path=contpath, db_state=contdbstate)
             update_greenthreads.append(gt)
         # Wait a little bit to see if the container updates are successful.
         # If we immediately return after firing off the greenthread above, then
@@ -415,7 +442,7 @@ class ObjectController(BaseStorageServer):
                 self.container_update_timeout, updates)
 
     def delete_at_update(self, op, delete_at, account, container, obj,
-                         request, objdevice, policy):
+                         request, objdevice, policy, extra_headers=None):
         """
         Update the expiring objects container when objects are updated.
 
@@ -427,15 +454,14 @@ class ObjectController(BaseStorageServer):
         :param request: the original request driving the update
         :param objdevice: device name that the object is in
         :param policy: the BaseStoragePolicy instance (used for tmp dir)
+        :param extra_headers: dict of additional headers for the update
         """
         if config_true_value(
                 request.headers.get('x-backend-replication', 'f')):
             return
+
         delete_at = normalize_delete_at_timestamp(delete_at)
-        updates = [(None, None)]
 
-        partition = None
-        hosts = contdevices = [None]
         headers_in = request.headers
         headers_out = HeaderKeyDict({
             # system accounts are always Policy-0
@@ -443,26 +469,42 @@ class ObjectController(BaseStorageServer):
             'x-timestamp': request.timestamp.internal,
             'x-trans-id': headers_in.get('x-trans-id', '-'),
             'referer': request.as_referer()})
+
+        expiring_objects_account_name, delete_at_container = \
+            self.expirer_config.get_expirer_account_and_container(
+                delete_at, account, container, obj)
         if op != 'DELETE':
             hosts = headers_in.get('X-Delete-At-Host', None)
             if hosts is None:
                 # If header is missing, no update needed as sufficient other
                 # object servers should perform the required update.
                 return
-            delete_at_container = headers_in.get('X-Delete-At-Container', None)
-            if not delete_at_container:
-                # older proxy servers did not send X-Delete-At-Container so for
-                # backwards compatibility calculate the value here, but also
-                # log a warning because this is prone to inconsistent
-                # expiring_objects_container_divisor configurations.
-                # See https://bugs.launchpad.net/swift/+bug/1187200
-                self.logger.warning(
-                    'X-Delete-At-Container header must be specified for '
-                    'expiring objects background %s to work properly. Making '
-                    'best guess as to the container name for now.' % op)
-                delete_at_container = get_expirer_container(
-                    delete_at, self.expiring_objects_container_divisor,
-                    account, container, obj)
+
+            proxy_delete_at_container = headers_in.get(
+                'X-Delete-At-Container', None)
+            if delete_at_container != proxy_delete_at_container:
+                if not proxy_delete_at_container:
+                    # We carry this warning around for pre-2013 proxies
+                    self.logger.warning(
+                        'X-Delete-At-Container header must be specified for '
+                        'expiring objects background %s to work properly. '
+                        'Making best guess as to the container name '
+                        'for now.', op)
+                    proxy_delete_at_container = delete_at_container
+                else:
+                    # Inconsistent configuration may lead to orphaned expirer
+                    # task queue objects when X-Delete-At is updated, which can
+                    # stick around for a whole reclaim age.
+                    self.logger.debug(
+                        'Proxy X-Delete-At-Container %r does not match '
+                        'expected %r for current expirer_config.',
+                        proxy_delete_at_container, delete_at_container)
+                # it's not possible to say which is "more correct", this will
+                # at least match the host/part/device
+                delete_at_container = normalize_delete_at_timestamp(
+                    proxy_delete_at_container)
+
+            # new updates need to enqueue new x-delete-at
             partition = headers_in.get('X-Delete-At-Partition', None)
             contdevices = headers_in.get('X-Delete-At-Device', '')
             updates = [upd for upd in
@@ -472,30 +514,22 @@ class ObjectController(BaseStorageServer):
             if not updates:
                 updates = [(None, None)]
             headers_out['x-size'] = '0'
-            headers_out['x-content-type'] = 'text/plain'
+            headers_out['x-content-type'] = X_DELETE_TYPE
             headers_out['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'
+            if extra_headers:
+                headers_out.update(extra_headers)
         else:
             if not config_true_value(
                 request.headers.get(
                     'X-Backend-Clean-Expiring-Object-Queue', 't')):
                 return
-
-            # DELETEs of old expiration data have no way of knowing what the
-            # old X-Delete-At-Container was at the time of the initial setting
-            # of the data, so a best guess is made here.
-            # Worst case is a DELETE is issued now for something that doesn't
-            # exist there and the original data is left where it is, where
-            # it will be ignored when the expirer eventually tries to issue the
-            # object DELETE later since the X-Delete-At value won't match up.
-            delete_at_container = get_expirer_container(
-                delete_at, self.expiring_objects_container_divisor,
-                account, container, obj)
-        delete_at_container = normalize_delete_at_timestamp(
-            delete_at_container)
+            # DELETE op always go directly to async_pending
+            partition = None
+            updates = [(None, None)]
 
         for host, contdevice in updates:
             self.async_update(
-                op, self.expiring_objects_account, delete_at_container,
+                op, expiring_objects_account_name, delete_at_container,
                 build_task_obj(delete_at, account, container, obj),
                 host, partition, contdevice, headers_out, objdevice,
                 policy)
@@ -552,7 +586,7 @@ class ObjectController(BaseStorageServer):
         footer_md5 = footer_hdrs.get('Content-MD5')
         if not footer_md5:
             raise HTTPBadRequest(body="no Content-MD5 in footer")
-        if footer_md5 != md5(footer_body).hexdigest():
+        if footer_md5 != md5(footer_body, usedforsecurity=False).hexdigest():
             raise HTTPUnprocessableEntity(body="footer MD5 mismatch")
 
         try:
@@ -598,12 +632,31 @@ class ObjectController(BaseStorageServer):
                     override = key.lower().replace(override_prefix, 'x-')
                     update_headers[override] = val
 
+    def _conditional_delete_at_update(self, request, device, account,
+                                      container, obj, policy, metadata,
+                                      orig_delete_at, new_delete_at):
+        if new_delete_at:
+            extra_headers = {
+                'x-content-type': embed_expirer_bytes_in_ctype(
+                    X_DELETE_TYPE, metadata),
+                'x-content-type-timestamp':
+                metadata.get('X-Timestamp'),
+            }
+            self.delete_at_update(
+                'PUT', new_delete_at, account, container, obj, request,
+                device, policy, extra_headers)
+        if orig_delete_at and orig_delete_at != new_delete_at:
+            self.delete_at_update(
+                'DELETE', orig_delete_at, account, container, obj,
+                request, device, policy)
+
     @public
     @timing_stats()
     def POST(self, request):
         """Handle HTTP POST requests for the Swift Object Server."""
         device, partition, account, container, obj, policy = \
-            get_name_and_placement(request, 5, 5, True)
+            get_obj_name_and_placement(request)
+
         req_timestamp = valid_timestamp(request)
         new_delete_at = int(request.headers.get('X-Delete-At') or 0)
         if new_delete_at and new_delete_at < req_timestamp:
@@ -613,8 +666,7 @@ class ObjectController(BaseStorageServer):
         try:
             disk_file = self.get_diskfile(
                 device, partition, account, container, obj,
-                policy=policy, open_expired=config_true_value(
-                    request.headers.get('x-backend-replication', 'false')),
+                policy=policy, open_expired=is_backend_open_expired(request),
                 next_part_power=next_part_power)
         except DiskFileDeviceUnavailable:
             return HTTPInsufficientStorage(drive=device, request=request)
@@ -649,18 +701,15 @@ class ObjectController(BaseStorageServer):
                 list(self.allowed_headers))
             for header_key in headers_to_copy:
                 if header_key in request.headers:
-                    header_caps = header_key.title()
+                    header_caps = bytes_to_wsgi(
+                        wsgi_to_bytes(header_key).title())
                     metadata[header_caps] = request.headers[header_key]
             orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
-            if orig_delete_at != new_delete_at:
-                if new_delete_at:
-                    self.delete_at_update(
-                        'PUT', new_delete_at, account, container, obj, request,
-                        device, policy)
-                if orig_delete_at:
-                    self.delete_at_update('DELETE', orig_delete_at, account,
-                                          container, obj, request, device,
-                                          policy)
+            disk_file_metadata = disk_file.get_datafile_metadata()
+            self._conditional_delete_at_update(
+                request, device, account, container, obj, policy,
+                disk_file_metadata, orig_delete_at, new_delete_at
+            )
         else:
             # preserve existing metadata, only content-type may be updated
             metadata = dict(disk_file.get_metafile_metadata())
@@ -732,8 +781,9 @@ class ObjectController(BaseStorageServer):
             'PUT', account, container, obj, request, update_headers,
             device, policy)
 
-        # Add sysmeta to response
-        resp_headers = {}
+        # Add current content-type and sysmeta to response
+        resp_headers = {
+            'X-Backend-Content-Type': content_type_headers['Content-Type']}
         for key, value in orig_metadata.items():
             if is_sys_meta('object', key):
                 resp_headers[key] = value
@@ -860,13 +910,20 @@ class ObjectController(BaseStorageServer):
         elapsed_time = 0
         upload_expiration = time.time() + self.max_upload_time
         timeout_reader = self._make_timeout_reader(obj_input)
-        for chunk in iter(timeout_reader, b''):
+
+        # Wrap the chunks in CooperativeIterator with specified period
+        cooperative_reader = CooperativeIterator(
+            iter(timeout_reader, b''), period=self.cooperative_period
+        )
+
+        for chunk in cooperative_reader:
             start_time = time.time()
             if start_time > upload_expiration:
                 self.logger.increment('PUT.timeouts')
                 raise HTTPRequestTimeout(request=request)
             writer.write(chunk)
             elapsed_time += time.time() - start_time
+
         upload_size, etag = writer.chunks_finished()
         if fsize is not None and fsize != upload_size:
             raise HTTPClientDisconnect(request=request)
@@ -897,7 +954,8 @@ class ObjectController(BaseStorageServer):
             list(self.allowed_headers))
         for header_key in headers_to_copy:
             if header_key in request.headers:
-                header_caps = header_key.title()
+                header_caps = bytes_to_wsgi(
+                    wsgi_to_bytes(header_key).title())
                 metadata[header_caps] = request.headers[header_key]
         return metadata
 
@@ -926,8 +984,8 @@ class ObjectController(BaseStorageServer):
                         if (is_sys_or_user_meta('object', val[0]) or
                             is_object_transient_sysmeta(val[0])))
         # N.B. footers_metadata is a HeaderKeyDict
-        received_etag = footers_metadata.get('etag', request.headers.get(
-            'etag', '')).strip('"')
+        received_etag = normalize_etag(footers_metadata.get(
+            'etag', request.headers.get('etag', '')))
         if received_etag and received_etag != metadata['ETag']:
             raise HTTPUnprocessableEntity(request=request)
 
@@ -968,15 +1026,10 @@ class ObjectController(BaseStorageServer):
                              orig_metadata, footers_metadata, metadata):
         orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
         new_delete_at = int(request.headers.get('X-Delete-At') or 0)
-        if orig_delete_at != new_delete_at:
-            if new_delete_at:
-                self.delete_at_update(
-                    'PUT', new_delete_at, account, container, obj, request,
-                    device, policy)
-            if orig_delete_at:
-                self.delete_at_update(
-                    'DELETE', orig_delete_at, account, container, obj,
-                    request, device, policy)
+
+        self._conditional_delete_at_update(request, device, account, container,
+                                           obj, policy, metadata,
+                                           orig_delete_at, new_delete_at)
 
         update_headers = HeaderKeyDict({
             'x-size': metadata['Content-Length'],
@@ -995,7 +1048,7 @@ class ObjectController(BaseStorageServer):
     def PUT(self, request):
         """Handle HTTP PUT requests for the Swift Object Server."""
         device, partition, account, container, obj, policy = \
-            get_name_and_placement(request, 5, 5, True)
+            get_obj_name_and_placement(request)
         disk_file, fsize, orig_metadata = self._pre_create_checks(
             request, device, partition, account, container, obj, policy)
         writer = disk_file.writer(size=fsize)
@@ -1016,7 +1069,9 @@ class ObjectController(BaseStorageServer):
             if multi_stage_mime_state:
                 self._send_multi_stage_continue_headers(
                     request, **multi_stage_mime_state)
-            writer.commit(request.timestamp)
+            if not config_true_value(
+                    request.headers.get('X-Backend-No-Commit', False)):
+                writer.commit(request.timestamp)
             if multi_stage_mime_state:
                 self._drain_mime_request(**multi_stage_mime_state)
         except (DiskFileXattrNotSupported, DiskFileNoSpace):
@@ -1037,7 +1092,7 @@ class ObjectController(BaseStorageServer):
     def GET(self, request):
         """Handle HTTP GET requests for the Swift Object Server."""
         device, partition, account, container, obj, policy = \
-            get_name_and_placement(request, 5, 5, True)
+            get_obj_name_and_placement(request)
         request.headers.setdefault('X-Timestamp',
                                    normalize_timestamp(time.time()))
         req_timestamp = valid_timestamp(request)
@@ -1047,23 +1102,39 @@ class ObjectController(BaseStorageServer):
             disk_file = self.get_diskfile(
                 device, partition, account, container, obj,
                 policy=policy, frag_prefs=frag_prefs,
-                open_expired=config_true_value(
-                    request.headers.get('x-backend-replication', 'false')))
+                open_expired=is_backend_open_expired(request))
         except DiskFileDeviceUnavailable:
             return HTTPInsufficientStorage(drive=device, request=request)
         try:
             with disk_file.open(current_time=req_timestamp):
                 metadata = disk_file.get_metadata()
+                resolve_ignore_range_header(request, metadata)
                 obj_size = int(metadata['Content-Length'])
                 file_x_ts = Timestamp(metadata['X-Timestamp'])
-                keep_cache = (self.keep_cache_private or
-                              ('X-Auth-Token' not in request.headers and
-                               'X-Storage-Token' not in request.headers))
+                keep_cache = (
+                    self.keep_cache_private
+                    or (
+                        "X-Auth-Token" not in request.headers
+                        and "X-Storage-Token" not in request.headers
+                    )
+                    or (
+                        self.keep_cache_slo_manifest
+                        and config_true_value(
+                            metadata.get("X-Static-Large-Object")
+                        )
+                    )
+                )
                 conditional_etag = resolve_etag_is_at_header(request, metadata)
+                app_iter = disk_file.reader(
+                    keep_cache=keep_cache,
+                    cooperative_period=self.cooperative_period,
+                    etag_validate_frac=self.etag_validate_frac,
+                )
                 response = Response(
-                    app_iter=disk_file.reader(keep_cache=keep_cache),
-                    request=request, conditional_response=True,
-                    conditional_etag=conditional_etag)
+                    app_iter=app_iter, request=request,
+                    conditional_response=True,
+                    conditional_etag=conditional_etag,
+                )
                 response.headers['Content-Type'] = metadata.get(
                     'Content-Type', 'application/octet-stream')
                 for key, value in metadata.items():
@@ -1072,7 +1143,7 @@ class ObjectController(BaseStorageServer):
                             key.lower() in self.allowed_headers):
                         response.headers[key] = value
                 response.etag = metadata['ETag']
-                response.last_modified = math.ceil(float(file_x_ts))
+                response.last_modified = file_x_ts.ceil()
                 response.content_length = obj_size
                 try:
                     response.content_encoding = metadata[
@@ -1104,7 +1175,7 @@ class ObjectController(BaseStorageServer):
     def HEAD(self, request):
         """Handle HTTP HEAD requests for the Swift Object Server."""
         device, partition, account, container, obj, policy = \
-            get_name_and_placement(request, 5, 5, True)
+            get_obj_name_and_placement(request)
         request.headers.setdefault('X-Timestamp',
                                    normalize_timestamp(time.time()))
         req_timestamp = valid_timestamp(request)
@@ -1114,8 +1185,7 @@ class ObjectController(BaseStorageServer):
             disk_file = self.get_diskfile(
                 device, partition, account, container, obj,
                 policy=policy, frag_prefs=frag_prefs,
-                open_expired=config_true_value(
-                    request.headers.get('x-backend-replication', 'false')))
+                open_expired=is_backend_open_expired(request))
         except DiskFileDeviceUnavailable:
             return HTTPInsufficientStorage(drive=device, request=request)
         try:
@@ -1140,7 +1210,7 @@ class ObjectController(BaseStorageServer):
                 response.headers[key] = value
         response.etag = metadata['ETag']
         ts = Timestamp(metadata['X-Timestamp'])
-        response.last_modified = math.ceil(float(ts))
+        response.last_modified = ts.ceil()
         # Needed for container sync feature
         response.headers['X-Timestamp'] = ts.normal
         response.headers['X-Backend-Timestamp'] = ts.internal
@@ -1163,7 +1233,7 @@ class ObjectController(BaseStorageServer):
     def DELETE(self, request):
         """Handle HTTP DELETE requests for the Swift Object Server."""
         device, partition, account, container, obj, policy = \
-            get_name_and_placement(request, 5, 5, True)
+            get_obj_name_and_placement(request)
         req_timestamp = valid_timestamp(request)
         next_part_power = request.headers.get('X-Backend-Next-Part-Power')
         try:
@@ -1221,10 +1291,10 @@ class ObjectController(BaseStorageServer):
             else:
                 # differentiate success from no object at all
                 response_class = HTTPNoContent
-        if orig_delete_at:
-            self.delete_at_update('DELETE', orig_delete_at, account,
-                                  container, obj, request, device,
-                                  policy)
+        self._conditional_delete_at_update(
+            request, device, account, container, obj, policy, {},
+            orig_delete_at, 0
+        )
         if orig_timestamp < req_timestamp:
             try:
                 disk_file.delete(req_timestamp)
@@ -1236,7 +1306,9 @@ class ObjectController(BaseStorageServer):
                 device, policy)
         return response_class(
             request=request,
-            headers={'X-Backend-Timestamp': response_timestamp.internal})
+            headers={'X-Backend-Timestamp': response_timestamp.internal,
+                     'X-Backend-Content-Type': orig_metadata.get(
+                         'Content-Type', '')})
 
     @public
     @replication
@@ -1255,7 +1327,8 @@ class ObjectController(BaseStorageServer):
         suffixes = suffix_parts.split('-') if suffix_parts else []
         try:
             hashes = self._diskfile_router[policy].get_hashes(
-                device, partition, suffixes, policy)
+                device, partition, suffixes, policy,
+                skip_rehash=bool(suffixes))
         except DiskFileDeviceUnavailable:
             resp = HTTPInsufficientStorage(drive=device, request=request)
         else:
@@ -1267,7 +1340,14 @@ class ObjectController(BaseStorageServer):
     @replication
     @timing_stats(sample_rate=0.1)
     def SSYNC(self, request):
-        return Response(app_iter=ssync_receiver.Receiver(self, request)())
+        # the ssync sender may want to send PUT subrequests for non-durable
+        # data that should not be committed; legacy behaviour has been to
+        # commit all PUTs (subject to EC footer metadata), so we need to
+        # indicate to the sender that this object server has been upgraded to
+        # understand the X-Backend-No-Commit header.
+        headers = {'X-Backend-Accept-No-Commit': True}
+        return Response(app_iter=ssync_receiver.Receiver(self, request)(),
+                        headers=headers)
 
     def __call__(self, env, start_response):
         """WSGI Application entry point for the Swift Object Server."""
@@ -1275,7 +1355,7 @@ class ObjectController(BaseStorageServer):
         req = Request(env)
         self.logger.txn_id = req.headers.get('x-trans-id', None)
 
-        if not check_utf8(wsgi_to_str(req.path_info)):
+        if not check_utf8(wsgi_to_str(req.path_info), internal=True):
             res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL')
         else:
             try:
@@ -1289,9 +1369,9 @@ class ObjectController(BaseStorageServer):
             except HTTPException as error_response:
                 res = error_response
             except (Exception, Timeout):
-                self.logger.exception(_(
+                self.logger.exception(
                     'ERROR __call__ error with %(method)s'
-                    ' %(path)s '), {'method': req.method, 'path': req.path})
+                    ' %(path)s ', {'method': req.method, 'path': req.path})
                 res = HTTPInternalServerError(body=traceback.format_exc())
         trans_time = time.time() - start_time
         res.fix_conditional_response()
@@ -1390,3 +1470,14 @@ def app_factory(global_conf, **local_conf):
     conf = global_conf.copy()
     conf.update(local_conf)
     return ObjectController(conf)
+
+
+def main():
+    conf_file, options = parse_options(test_config=True)
+    sys.exit(run_wsgi(conf_file, 'object-server',
+                      global_conf_callback=global_conf_callback,
+                      **options))
+
+
+if __name__ == '__main__':
+    main()