swift 2.23.3__py3-none-any.whl → 2.35.0__py3-none-any.whl

swift/obj/diskfile.py

@@ -30,7 +30,7 @@ The remaining methods in this module are considered implementation specific and
 are also not considered part of the backend API.
 """
 
-import six.moves.cPickle as pickle
+import pickle  # nosec: B403
 import binascii
 import copy
 import errno
@@ -40,12 +40,11 @@ import os
 import re
 import time
 import uuid
-from hashlib import md5
 import logging
 import traceback
 import xattr
 from os.path import basename, dirname, exists, join, splitext
-from random import shuffle
+import random
 from tempfile import mkstemp
 from contextlib import contextmanager
 from collections import defaultdict
@@ -53,11 +52,9 @@ from datetime import timedelta
 
 from eventlet import Timeout, tpool
 from eventlet.hubs import trampoline
-import six
 from pyeclib.ec_iface import ECDriverError, ECInvalidFragmentMetadata, \
     ECBadFragmentChecksum, ECInvalidParameter
 
-from swift import gettext_ as _
 from swift.common.constraints import check_drive
 from swift.common.request_helpers import is_sys_meta
 from swift.common.utils import mkdirs, Timestamp, \
@@ -66,13 +63,15 @@ from swift.common.utils import mkdirs, Timestamp, \
     config_true_value, listdir, split_path, remove_file, \
     get_md5_socket, F_SETPIPE_SZ, decode_timestamps, encode_timestamps, \
     MD5_OF_EMPTY_STRING, link_fd_to_path, \
-    O_TMPFILE, makedirs_count, replace_partition_in_path, remove_directory
+    O_TMPFILE, makedirs_count, replace_partition_in_path, remove_directory, \
+    md5, is_file_older, non_negative_float, config_fallocate_value, \
+    fs_has_free_space, CooperativeIterator, EUCLEAN
 from swift.common.splice import splice, tee
 from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
     DiskFileCollision, DiskFileNoSpace, DiskFileDeviceUnavailable, \
     DiskFileDeleted, DiskFileError, DiskFileNotOpen, PathNotDir, \
     ReplicationLockTimeout, DiskFileExpired, DiskFileXattrNotSupported, \
-    DiskFileBadMetadataChecksum
+    DiskFileBadMetadataChecksum, PartitionLockTimeout
 from swift.common.swob import multi_range_iterator
 from swift.common.storage_policy import (
     get_policy_string, split_policy_string, PolicyError, POLICIES,
@@ -81,6 +80,7 @@ from swift.common.storage_policy import (
 
 PICKLE_PROTOCOL = 2
 DEFAULT_RECLAIM_AGE = timedelta(weeks=1).total_seconds()
+DEFAULT_COMMIT_WINDOW = 60.0
 HASH_FILE = 'hashes.pkl'
 HASH_INVALIDATIONS_FILE = 'hashes.invalid'
 METADATA_KEY = b'user.swift.metadata'
@@ -153,38 +153,33 @@ def _encode_metadata(metadata):
 
     :param metadata: a dict
     """
-    if six.PY2:
-        def encode_str(item):
-            if isinstance(item, six.text_type):
-                return item.encode('utf8')
-            return item
-    else:
-        def encode_str(item):
-            if isinstance(item, six.text_type):
-                return item.encode('utf8', 'surrogateescape')
-            return item
+    def encode_str(item):
+        if isinstance(item, str):
+            return item.encode('utf8', 'surrogateescape')
+        return item
 
     return dict(((encode_str(k), encode_str(v)) for k, v in metadata.items()))
 
 
-def _decode_metadata(metadata):
+def _decode_metadata(metadata, metadata_written_by_py3):
     """
     Given a metadata dict from disk, convert keys and values to native strings.
 
     :param metadata: a dict
+    :param metadata_written_by_py3:
     """
-    if six.PY2:
-        def to_str(item):
-            if isinstance(item, six.text_type):
-                return item.encode('utf8')
-            return item
-    else:
-        def to_str(item):
-            if isinstance(item, six.binary_type):
-                return item.decode('utf8', 'surrogateescape')
-            return item
+    def to_str(item, is_name=False):
+        # For years, py2 and py3 handled non-ascii metadata differently;
+        # see https://bugs.launchpad.net/swift/+bug/2012531
+        if not metadata_written_by_py3 and isinstance(item, bytes) \
+                and not is_name:
+            # do our best to read old py2 data
+            item = item.decode('latin1')
+        if isinstance(item, bytes):
+            return item.decode('utf8', 'surrogateescape')
+        return item
 
-    return dict(((to_str(k), to_str(v)) for k, v in metadata.items()))
+    return {to_str(k): to_str(v, k == b'name') for k, v in metadata.items()}
 
 
 def read_metadata(fd, add_missing_checksum=False):
@@ -222,28 +217,28 @@ def read_metadata(fd, add_missing_checksum=False):
         # exist. This is fine; it just means that this object predates the
         # introduction of metadata checksums.
         if add_missing_checksum:
-            new_checksum = md5(metadata).hexdigest().encode('ascii')
+            new_checksum = (md5(metadata, usedforsecurity=False)
+                            .hexdigest().encode('ascii'))
             try:
                 xattr.setxattr(fd, METADATA_CHECKSUM_KEY, new_checksum)
             except (IOError, OSError) as e:
                 logging.error("Error adding metadata: %s" % e)
 
     if metadata_checksum:
-        computed_checksum = md5(metadata).hexdigest().encode('ascii')
+        computed_checksum = (md5(metadata, usedforsecurity=False)
+                             .hexdigest().encode('ascii'))
         if metadata_checksum != computed_checksum:
             raise DiskFileBadMetadataChecksum(
                 "Metadata checksum mismatch for %s: "
                 "stored checksum='%s', computed='%s'" % (
                     fd, metadata_checksum, computed_checksum))
 
+    metadata_written_by_py3 = (b'_codecs\nencode' in metadata[:32])
     # strings are utf-8 encoded when written, but have not always been
     # (see https://bugs.launchpad.net/swift/+bug/1678018) so encode them again
     # when read
-    if six.PY2:
-        metadata = pickle.loads(metadata)
-    else:
-        metadata = pickle.loads(metadata, encoding='bytes')
-    return _decode_metadata(metadata)
+    metadata = pickle.loads(metadata, encoding='bytes')  # nosec: B301
+    return _decode_metadata(metadata, metadata_written_by_py3)
 
 
 def write_metadata(fd, metadata, xattr_size=65536):
@@ -254,7 +249,8 @@ def write_metadata(fd, metadata, xattr_size=65536):
     :param metadata: metadata to write
     """
     metastr = pickle.dumps(_encode_metadata(metadata), PICKLE_PROTOCOL)
-    metastr_md5 = md5(metastr).hexdigest().encode('ascii')
+    metastr_md5 = (
+        md5(metastr, usedforsecurity=False).hexdigest().encode('ascii'))
     key = 0
     try:
         while metastr:
@@ -327,7 +323,11 @@ def quarantine_renamer(device_path, corrupted_file_path):
     to_dir = join(device_path, 'quarantined',
                   get_data_dir(policy),
                   basename(from_dir))
-    invalidate_hash(dirname(from_dir))
+    if len(basename(from_dir)) == 3:
+        # quarantining whole suffix
+        invalidate_hash(from_dir)
+    else:
+        invalidate_hash(dirname(from_dir))
     try:
         renamer(from_dir, to_dir, fsync=False)
     except OSError as e:
@@ -338,6 +338,12 @@ def quarantine_renamer(device_path, corrupted_file_path):
     return to_dir
 
 
+def valid_suffix(value):
+    if not isinstance(value, str) or len(value) != 3:
+        return False
+    return all(c in '0123456789abcdef' for c in value)
+
+
 def read_hashes(partition_dir):
     """
     Read the existing hashes.pkl
@@ -354,7 +360,7 @@ def read_hashes(partition_dir):
         pass
     else:
         try:
-            hashes = pickle.loads(pickled_hashes)
+            hashes = pickle.loads(pickled_hashes)  # nosec: B301
         except Exception:
             # pickle.loads() can raise a wide variety of exceptions when
             # given invalid input depending on the way in which the
@@ -362,9 +368,9 @@ def read_hashes(partition_dir):
             pass
 
     # Check for corrupted data that could break os.listdir()
-    for suffix in hashes.keys():
-        if not suffix.isalnum():
-            return {'valid': False}
+    if not all(valid_suffix(key) or key in ('valid', 'updated')
+               for key in hashes):
+        return {'valid': False}
 
     # hashes.pkl w/o valid updated key is "valid" but "forever old"
     hashes.setdefault('valid', True)
@@ -402,19 +408,23 @@ def consolidate_hashes(partition_dir):
     with lock_path(partition_dir):
         hashes = read_hashes(partition_dir)
 
-        found_invalidation_entry = False
+        found_invalidation_entry = hashes_updated = False
         try:
             with open(invalidations_file, 'r') as inv_fh:
                 for line in inv_fh:
                     found_invalidation_entry = True
                     suffix = line.strip()
+                    if not valid_suffix(suffix):
+                        continue
+                    hashes_updated = True
                     hashes[suffix] = None
         except (IOError, OSError) as e:
             if e.errno != errno.ENOENT:
                 raise
 
-        if found_invalidation_entry:
+        if hashes_updated:
             write_hashes(partition_dir, hashes)
+        if found_invalidation_entry:
             # Now that all the invalidations are reflected in hashes.pkl, it's
             # safe to clear out the invalidations file.
             with open(invalidations_file, 'wb') as inv_fh:
@@ -440,35 +450,58 @@ def invalidate_hash(suffix_dir):
         inv_fh.write(suffix + b"\n")
 
 
-def relink_paths(target_path, new_target_path, check_existing=False):
+def relink_paths(target_path, new_target_path, ignore_missing=True):
     """
-    Hard-links a file located in target_path using the second path
-    new_target_path. Creates intermediate directories if required.
+    Hard-links a file located in ``target_path`` using the second path
+    ``new_target_path``. Creates intermediate directories if required.
 
     :param target_path: current absolute filename
     :param new_target_path: new absolute filename for the hardlink
-    :param check_existing: if True, check whether the link is already present
-                           before attempting to create a new one
+    :param ignore_missing: if True then no exception is raised if the link
+        could not be made because ``target_path`` did not exist, otherwise an
+        OSError will be raised.
+    :raises: OSError if the hard link could not be created, unless the intended
+        hard link already exists or the ``target_path`` does not exist and
+        ``must_exist`` if False.
+    :returns: True if the link was created by the call to this method, False
+        otherwise.
     """
-
+    link_created = False
     if target_path != new_target_path:
-        logging.debug('Relinking %s to %s due to next_part_power set',
-                      target_path, new_target_path)
         new_target_dir = os.path.dirname(new_target_path)
-        if not os.path.isdir(new_target_dir):
+        try:
             os.makedirs(new_target_dir)
+        except OSError as err:
+            if err.errno != errno.EEXIST:
+                raise
 
-        link_exists = False
-        if check_existing:
-            try:
-                new_stat = os.stat(new_target_path)
-                orig_stat = os.stat(target_path)
-                link_exists = (new_stat.st_ino == orig_stat.st_ino)
-            except OSError:
-                pass  # if anything goes wrong, try anyway
-
-        if not link_exists:
+        try:
             os.link(target_path, new_target_path)
+            link_created = True
+        except OSError as err:
+            # there are some circumstances in which it may be ok that the
+            # attempted link failed
+            ok = False
+            if err.errno == errno.ENOENT:
+                # this is ok if the *target* path doesn't exist anymore
+                ok = not os.path.exists(target_path) and ignore_missing
+            if err.errno == errno.EEXIST:
+                # this is ok *if* the intended link has already been made
+                try:
+                    orig_stat = os.stat(target_path)
+                except OSError as sub_err:
+                    # this is ok: the *target* path doesn't exist anymore
+                    ok = sub_err.errno == errno.ENOENT and ignore_missing
+                else:
+                    try:
+                        new_stat = os.stat(new_target_path)
+                        ok = new_stat.st_ino == orig_stat.st_ino
+                    except OSError:
+                        # squash this exception; the original will be raised
+                        pass
+            if not ok:
+                raise err
+    return link_created
 
 
 def get_part_path(dev_path, policy, partition):
@@ -522,7 +555,7 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
         device_dirs = list(
             set(listdir(devices)).intersection(set(device_dirs)))
     # randomize devices in case of process restart before sweep completed
-    shuffle(device_dirs)
+    random.shuffle(device_dirs)
 
     base, policy = split_policy_string(datadir)
     for device in device_dirs:
@@ -546,7 +579,8 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
             try:
                 suffixes = listdir(part_path)
             except OSError as e:
-                if e.errno != errno.ENOTDIR:
+                if e.errno not in (errno.ENOTDIR, errno.ENODATA,
+                                   EUCLEAN):
                     raise
                 continue
             for asuffix in suffixes:
@@ -554,7 +588,8 @@ def object_audit_location_generator(devices, datadir, mount_check=True,
                 try:
                     hashes = listdir(suff_path)
                 except OSError as e:
-                    if e.errno != errno.ENOTDIR:
+                    if e.errno not in (errno.ENOTDIR, errno.ENODATA,
+                                       EUCLEAN):
                         raise
                     continue
                 for hsh in hashes:
@@ -570,31 +605,25 @@ def get_auditor_status(datadir_path, logger, auditor_type):
         datadir_path, "auditor_status_%s.json" % auditor_type)
     status = {}
     try:
-        if six.PY3:
-            statusfile = open(auditor_status, encoding='utf8')
-        else:
-            statusfile = open(auditor_status, 'rb')
-        with statusfile:
+        with open(auditor_status, encoding='utf8') as statusfile:
             status = statusfile.read()
     except (OSError, IOError) as e:
         if e.errno != errno.ENOENT and logger:
-            logger.warning(_('Cannot read %(auditor_status)s (%(err)s)') %
+            logger.warning('Cannot read %(auditor_status)s (%(err)s)',
                            {'auditor_status': auditor_status, 'err': e})
         return listdir(datadir_path)
     try:
         status = json.loads(status)
     except ValueError as e:
-        logger.warning(_('Loading JSON from %(auditor_status)s failed'
-                         ' (%(err)s)') %
+        logger.warning('Loading JSON from %(auditor_status)s failed'
+                       ' (%(err)s)',
                        {'auditor_status': auditor_status, 'err': e})
         return listdir(datadir_path)
     return status['partitions']
 
 
 def update_auditor_status(datadir_path, logger, partitions, auditor_type):
-    status = json.dumps({'partitions': partitions})
-    if six.PY3:
-        status = status.encode('utf8')
+    status = json.dumps({'partitions': partitions}).encode('utf8')
     auditor_status = os.path.join(
         datadir_path, "auditor_status_%s.json" % auditor_type)
     try:
@@ -612,7 +641,7 @@ def update_auditor_status(datadir_path, logger, partitions, auditor_type):
             statusfile.write(status)
     except (OSError, IOError) as e:
         if logger:
-            logger.warning(_('Cannot write %(auditor_status)s (%(err)s)') %
+            logger.warning('Cannot write %(auditor_status)s (%(err)s)',
                            {'auditor_status': auditor_status, 'err': e})
 
 
@@ -625,15 +654,6 @@ def clear_auditor_status(devices, datadir, auditor_type="ALL"):
         remove_file(auditor_status)
 
 
-def strip_self(f):
-    """
-    Wrapper to attach module level functions to base class.
-    """
-    def wrapper(self, *args, **kwargs):
-        return f(*args, **kwargs)
-    return wrapper
-
-
 class DiskFileRouter(object):
 
     def __init__(self, *args, **kwargs):
@@ -673,9 +693,9 @@ class BaseDiskFileManager(object):
     diskfile_cls = None  # must be set by subclasses
     policy = None  # must be set by subclasses
 
-    invalidate_hash = strip_self(invalidate_hash)
-    consolidate_hashes = strip_self(consolidate_hashes)
-    quarantine_renamer = strip_self(quarantine_renamer)
+    invalidate_hash = staticmethod(invalidate_hash)
+    consolidate_hashes = staticmethod(consolidate_hashes)
+    quarantine_renamer = staticmethod(quarantine_renamer)
 
     def __init__(self, conf, logger):
         self.logger = logger
@@ -685,6 +705,8 @@ class BaseDiskFileManager(object):
         self.bytes_per_sync = int(conf.get('mb_per_sync', 512)) * 1024 * 1024
         self.mount_check = config_true_value(conf.get('mount_check', 'true'))
         self.reclaim_age = int(conf.get('reclaim_age', DEFAULT_RECLAIM_AGE))
+        self.commit_window = non_negative_float(conf.get(
+            'commit_window', DEFAULT_COMMIT_WINDOW))
         replication_concurrency_per_device = conf.get(
             'replication_concurrency_per_device')
         replication_one_per_device = conf.get('replication_one_per_device')
@@ -709,6 +731,8 @@ class BaseDiskFileManager(object):
                 replication_concurrency_per_device)
         self.replication_lock_timeout = int(conf.get(
             'replication_lock_timeout', 15))
+        self.fallocate_reserve, self.fallocate_is_percent = \
+            config_fallocate_value(conf.get('fallocate_reserve', '1%'))
 
         self.use_splice = False
         self.pipe_size = None
@@ -913,7 +937,9 @@ class BaseDiskFileManager(object):
         valid fileset, or None.
 
         :param files: a list of file names.
-        :param datadir: directory name files are from.
+        :param datadir: directory name files are from; this is used to
+                        construct file paths in the results, but the datadir is
+                        not modified by this method.
         :param verify: if True verify that the ondisk file contract has not
                        been violated, otherwise do not verify.
         :param policy: storage policy used to store the files. Used to
@@ -1034,11 +1060,10 @@ class BaseDiskFileManager(object):
             key = info_key[:-5] + '_file'
             results[key] = join(datadir, info['filename']) if info else None
 
-        if verify:
-            assert self._verify_ondisk_files(
-                results, **kwargs), \
-                "On-disk file search algorithm contract is broken: %s" \
-                % str(results)
+        if verify and not self._verify_ondisk_files(results, **kwargs):
+            raise RuntimeError(
+                "On-disk file search algorithm contract is broken: %s"
+                % str(results))
 
         return results
 
@@ -1077,8 +1102,14 @@ class BaseDiskFileManager(object):
             remove_file(join(hsh_path, results['ts_info']['filename']))
             files.remove(results.pop('ts_info')['filename'])
         for file_info in results.get('possible_reclaim', []):
-            # stray files are not deleted until reclaim-age
-            if is_reclaimable(file_info['timestamp']):
+            # stray files are not deleted until reclaim-age; non-durable data
+            # files are not deleted unless they were written before
+            # commit_window
+            filepath = join(hsh_path, file_info['filename'])
+            if (is_reclaimable(file_info['timestamp']) and
+                    (file_info.get('durable', True) or
+                     self.commit_window <= 0 or
+                     is_file_older(filepath, self.commit_window))):
                 results.setdefault('obsolete', []).append(file_info)
         for file_info in results.get('obsolete', []):
             remove_file(join(hsh_path, file_info['filename']))
@@ -1112,22 +1143,19 @@ class BaseDiskFileManager(object):
         :param path: full path to directory
         :param policy: storage policy used
         """
-        if six.PY2:
-            hashes = defaultdict(md5)
-        else:
-            class shim(object):
-                def __init__(self):
-                    self.md5 = md5()
+        class shim(object):
+            def __init__(self):
+                self.md5 = md5(usedforsecurity=False)
 
-                def update(self, s):
-                    if isinstance(s, str):
-                        self.md5.update(s.encode('utf-8'))
-                    else:
-                        self.md5.update(s)
+            def update(self, s):
+                if isinstance(s, str):
+                    self.md5.update(s.encode('utf-8'))
+                else:
+                    self.md5.update(s)
 
-                def hexdigest(self):
-                    return self.md5.hexdigest()
-            hashes = defaultdict(shim)
+            def hexdigest(self):
+                return self.md5.hexdigest()
+        hashes = defaultdict(shim)
         try:
             path_contents = sorted(os.listdir(path))
         except OSError as err:
@@ -1140,10 +1168,10 @@ class BaseDiskFileManager(object):
                 ondisk_info = self.cleanup_ondisk_files(
                     hsh_path, policy=policy)
             except OSError as err:
+                partition_path = dirname(path)
+                objects_path = dirname(partition_path)
+                device_path = dirname(objects_path)
                 if err.errno == errno.ENOTDIR:
-                    partition_path = dirname(path)
-                    objects_path = dirname(partition_path)
-                    device_path = dirname(objects_path)
                     # The made-up filename is so that the eventual dirpath()
                     # will result in this object directory that we care about.
                     # Some failures will result in an object directory
@@ -1153,9 +1181,27 @@ class BaseDiskFileManager(object):
                                                    join(hsh_path,
                                                         "made-up-filename"))
                     logging.exception(
-                        _('Quarantined %(hsh_path)s to %(quar_path)s because '
-                          'it is not a directory'), {'hsh_path': hsh_path,
-                                                     'quar_path': quar_path})
+                        'Quarantined %(hsh_path)s to %(quar_path)s because '
+                        'it is not a directory', {'hsh_path': hsh_path,
+                                                  'quar_path': quar_path})
+                    continue
+                elif err.errno in (errno.ENODATA, EUCLEAN):
+                    try:
+                        # We've seen cases where bad sectors lead to ENODATA
+                        # here; use a similar hack as above
+                        quar_path = quarantine_renamer(
+                            device_path,
+                            join(hsh_path, "made-up-filename"))
+                        orig_path = hsh_path
+                    except (OSError, IOError):
+                        # We've *also* seen the bad sectors lead to us needing
+                        # to quarantine the whole suffix
+                        quar_path = quarantine_renamer(device_path, hsh_path)
+                        orig_path = path
+                    logging.exception(
+                        'Quarantined %(orig_path)s to %(quar_path)s because '
+                        'it could not be listed', {'orig_path': orig_path,
+                                                   'quar_path': quar_path})
                     continue
                 raise
             if not ondisk_info['files']:
@@ -1277,6 +1323,8 @@ class BaseDiskFileManager(object):
             self.logger.debug('Run listdir on %s', partition_path)
         hashes.update((suffix, None) for suffix in recalculate)
         for suffix, hash_ in list(hashes.items()):
+            if suffix in ('valid', 'updated'):
+                continue
             if not hash_:
                 suffix_dir = join(partition_path, suffix)
                 try:
@@ -1286,7 +1334,7 @@ class BaseDiskFileManager(object):
                 except PathNotDir:
                     del hashes[suffix]
                 except OSError:
-                    logging.exception(_('Error hashing suffix'))
+                    logging.exception('Error hashing suffix')
                 modified = True
         if modified:
             with lock_path(partition_path):
@@ -1333,8 +1381,8 @@ class BaseDiskFileManager(object):
     @contextmanager
     def replication_lock(self, device, policy, partition):
         """
-        A context manager that will lock on the device given, if
-        configured to do so.
+        A context manager that will lock on the partition and, if configured
+        to do so, on the device given.
 
         :param device: name of target device
         :param policy: policy targeted by the replication request
@@ -1342,24 +1390,36 @@ class BaseDiskFileManager(object):
         :raises ReplicationLockTimeout: If the lock on the device
             cannot be granted within the configured timeout.
         """
-        if self.replication_concurrency_per_device:
-            dev_path = self.get_dev_path(device)
-            part_path = os.path.join(dev_path, get_data_dir(policy),
-                                     str(partition))
-            limit_time = time.time() + self.replication_lock_timeout
-            with lock_path(
-                    dev_path,
-                    timeout=self.replication_lock_timeout,
-                    timeout_class=ReplicationLockTimeout,
-                    limit=self.replication_concurrency_per_device):
-                with lock_path(
-                        part_path,
-                        timeout=limit_time - time.time(),
-                        timeout_class=ReplicationLockTimeout,
-                        limit=1,
-                        name='replication'):
+        limit_time = time.time() + self.replication_lock_timeout
+        with self.partition_lock(device, policy, partition, name='replication',
+                                 timeout=self.replication_lock_timeout):
+            if self.replication_concurrency_per_device:
+                with lock_path(self.get_dev_path(device),
+                               timeout=limit_time - time.time(),
+                               timeout_class=ReplicationLockTimeout,
+                               limit=self.replication_concurrency_per_device):
                     yield True
-        else:
+            else:
+                yield True
+
+    @contextmanager
+    def partition_lock(self, device, policy, partition, name=None,
+                       timeout=None):
+        """
+        A context manager that will lock on the partition given.
+
+        :param device: device targeted by the lock request
+        :param policy: policy targeted by the lock request
+        :param partition: partition targeted by the lock request
+        :raises PartitionLockTimeout: If the lock on the partition
+            cannot be granted within the configured timeout.
+        """
+        if timeout is None:
+            timeout = self.replication_lock_timeout
+        part_path = os.path.join(self.get_dev_path(device),
+                                 get_data_dir(policy), str(partition))
+        with lock_path(part_path, timeout=timeout,
+                       timeout_class=PartitionLockTimeout, limit=1, name=name):
             yield True
 
     def pickle_async_update(self, device, account, container, obj, data,
@@ -1440,21 +1500,22 @@ class BaseDiskFileManager(object):
             self, audit_location.path, dev_path,
             audit_location.partition, policy=audit_location.policy)
 
-    def get_diskfile_from_hash(self, device, partition, object_hash,
-                               policy, **kwargs):
+    def get_diskfile_and_filenames_from_hash(self, device, partition,
+                                             object_hash, policy, **kwargs):
         """
-        Returns a DiskFile instance for an object at the given
-        object_hash. Just in case someone thinks of refactoring, be
-        sure DiskFileDeleted is *not* raised, but the DiskFile
-        instance representing the tombstoned object is returned
-        instead.
+        Returns a tuple of (a DiskFile instance for an object at the given
+        object_hash, the basenames of the files in the object's hash dir).
+        Just in case someone thinks of refactoring, be sure DiskFileDeleted is
+        *not* raised, but the DiskFile instance representing the tombstoned
+        object is returned instead.
 
         :param device: name of target device
         :param partition: partition on the device in which the object lives
         :param object_hash: the hash of an object path
         :param policy: the StoragePolicy instance
         :raises DiskFileNotExist: if the object does not exist
-        :returns: an instance of BaseDiskFile
+        :returns: a tuple comprising (an instance of BaseDiskFile, a list of
+            file basenames)
         """
         dev_path = self.get_dev_path(device)
         if not dev_path:
@@ -1475,9 +1536,27 @@ class BaseDiskFileManager(object):
                                                     join(object_path,
                                                          "made-up-filename"))
                 logging.exception(
-                    _('Quarantined %(object_path)s to %(quar_path)s because '
-                      'it is not a directory'), {'object_path': object_path,
-                                                 'quar_path': quar_path})
+                    'Quarantined %(object_path)s to %(quar_path)s because '
+                    'it is not a directory', {'object_path': object_path,
+                                              'quar_path': quar_path})
+                raise DiskFileNotExist()
+            elif err.errno in (errno.ENODATA, EUCLEAN):
+                try:
+                    # We've seen cases where bad sectors lead to ENODATA here;
+                    # use a similar hack as above
+                    quar_path = self.quarantine_renamer(
+                        dev_path,
+                        join(object_path, "made-up-filename"))
+                    orig_path = object_path
+                except (OSError, IOError):
+                    # We've *also* seen the bad sectors lead to us needing to
+                    # quarantine the whole suffix, not just the hash dir
+                    quar_path = self.quarantine_renamer(dev_path, object_path)
+                    orig_path = os.path.dirname(object_path)
+                logging.exception(
+                    'Quarantined %(orig_path)s to %(quar_path)s because '
+                    'it could not be listed', {'orig_path': orig_path,
+                                               'quar_path': quar_path})
                 raise DiskFileNotExist()
             if err.errno != errno.ENOENT:
                 raise
@@ -1493,27 +1572,55 @@ class BaseDiskFileManager(object):
                 metadata.get('name', ''), 3, 3, True)
         except ValueError:
             raise DiskFileNotExist()
-        return self.diskfile_cls(self, dev_path,
-                                 partition, account, container, obj,
-                                 policy=policy, **kwargs)
+        df = self.diskfile_cls(self, dev_path, partition, account, container,
+                               obj, policy=policy, **kwargs)
+        return df, filenames
 
-    def get_hashes(self, device, partition, suffixes, policy):
+    def get_diskfile_from_hash(self, device, partition, object_hash, policy,
+                               **kwargs):
+        """
+        Returns a DiskFile instance for an object at the given object_hash.
+        Just in case someone thinks of refactoring, be sure DiskFileDeleted is
+        *not* raised, but the DiskFile instance representing the tombstoned
+        object is returned instead.
+
+        :param device: name of target device
+        :param partition: partition on the device in which the object lives
+        :param object_hash: the hash of an object path
+        :param policy: the StoragePolicy instance
+        :raises DiskFileNotExist: if the object does not exist
+        :returns: an instance of BaseDiskFile
+        """
+        return self.get_diskfile_and_filenames_from_hash(
+            device, partition, object_hash, policy, **kwargs)[0]
+
+    def get_hashes(self, device, partition, suffixes, policy,
+                   skip_rehash=False):
         """
 
         :param device: name of target device
         :param partition: partition name
         :param suffixes: a list of suffix directories to be recalculated
         :param policy: the StoragePolicy instance
+        :param skip_rehash: just mark the suffixes dirty; return None
         :returns: a dictionary that maps suffix directories
         """
         dev_path = self.get_dev_path(device)
         if not dev_path:
             raise DiskFileDeviceUnavailable()
         partition_path = get_part_path(dev_path, policy, partition)
-        if not os.path.exists(partition_path):
-            mkdirs(partition_path)
-        _junk, hashes = tpool.execute(
-            self._get_hashes, device, partition, policy, recalculate=suffixes)
+        suffixes = [suf for suf in suffixes or [] if valid_suffix(suf)]
+
+        if skip_rehash:
+            for suffix in suffixes:
+                self.invalidate_hash(os.path.join(partition_path, suffix))
+            hashes = None
+        elif not os.path.exists(partition_path):
+            hashes = {}
+        else:
+            _junk, hashes = tpool.execute(
+                self._get_hashes, device, partition, policy,
+                recalculate=suffixes)
         return hashes
 
     def _listdir(self, path):
@@ -1567,6 +1674,7 @@ class BaseDiskFileManager(object):
         - ts_meta -> timestamp of meta file, if one exists
         - ts_ctype -> timestamp of meta file containing most recent
                       content-type value, if one exists
+        - durable -> True if data file at ts_data is durable, False otherwise
 
         where timestamps are instances of
         :class:`~swift.common.utils.Timestamp`
@@ -1588,11 +1696,15 @@ class BaseDiskFileManager(object):
                 (os.path.join(partition_path, suffix), suffix)
                 for suffix in suffixes)
 
-        key_preference = (
+        # define keys that we need to extract the result from the on disk info
+        # data:
+        #   (x, y, z) -> result[x] should take the value of y[z]
+        key_map = (
             ('ts_meta', 'meta_info', 'timestamp'),
             ('ts_data', 'data_info', 'timestamp'),
             ('ts_data', 'ts_info', 'timestamp'),
             ('ts_ctype', 'ctype_info', 'ctype_timestamp'),
+            ('durable', 'data_info', 'durable'),
         )
 
         # cleanup_ondisk_files() will remove empty hash dirs, and we'll
@@ -1603,21 +1715,24 @@ class BaseDiskFileManager(object):
             for object_hash in self._listdir(suffix_path):
                 object_path = os.path.join(suffix_path, object_hash)
                 try:
-                    results = self.cleanup_ondisk_files(
+                    diskfile_info = self.cleanup_ondisk_files(
                         object_path, **kwargs)
-                    if results['files']:
+                    if diskfile_info['files']:
                         found_files = True
-                    timestamps = {}
-                    for ts_key, info_key, info_ts_key in key_preference:
-                        if info_key not in results:
+                    result = {}
+                    for result_key, diskfile_info_key, info_key in key_map:
+                        if diskfile_info_key not in diskfile_info:
                             continue
-                        timestamps[ts_key] = results[info_key][info_ts_key]
-                    if 'ts_data' not in timestamps:
+                        info = diskfile_info[diskfile_info_key]
+                        if info_key in info:
+                            # durable key not returned from replicated Diskfile
+                            result[result_key] = info[info_key]
+                    if 'ts_data' not in result:
                         # file sets that do not include a .data or .ts
                         # file cannot be opened and therefore cannot
                         # be ssync'd
                         continue
-                    yield (object_hash, timestamps)
+                    yield object_hash, result
                 except AssertionError as err:
                     self.logger.debug('Invalid file set in %s (%s)' % (
                         object_path, err))
@@ -1656,25 +1771,27 @@ class BaseDiskFileWriter(object):
     :param bytes_per_sync: number bytes written between sync calls
     :param diskfile: the diskfile creating this DiskFileWriter instance
     :param next_part_power: the next partition power to be used
+    :param extension: the file extension to be used; may be used internally
+                      to distinguish between PUT/POST/DELETE operations
     """
 
     def __init__(self, name, datadir, size, bytes_per_sync, diskfile,
-                 next_part_power):
+                 next_part_power, extension='.data'):
         # Parameter tracking
         self._name = name
         self._datadir = datadir
         self._fd = None
         self._tmppath = None
         self._size = size
-        self._chunks_etag = md5()
+        self._chunks_etag = md5(usedforsecurity=False)
         self._bytes_per_sync = bytes_per_sync
         self._diskfile = diskfile
         self.next_part_power = next_part_power
+        self._extension = extension
 
         # Internal attributes
         self._upload_size = 0
         self._last_sync = 0
-        self._extension = '.data'
         self._put_succeeded = False
 
     @property
@@ -1696,7 +1813,7 @@ class BaseDiskFileWriter(object):
                     msg = 'open(%s, O_TMPFILE | O_WRONLY) failed: %s \
                            Falling back to using mkstemp()' \
                            % (self._datadir, os.strerror(err.errno))
-                    self.logger.warning(msg)
+                    self.logger.debug(msg)
                     self.manager.use_linkat = False
                 else:
                     raise
@@ -1719,13 +1836,26 @@ class BaseDiskFileWriter(object):
                 # No more inodes in filesystem
                 raise DiskFileNoSpace()
             raise
-        if self._size is not None and self._size > 0:
+        if self._extension == '.ts':
+            # DELETEs always bypass any free-space reserve checks
+            pass
+        elif self._size:
             try:
                 fallocate(self._fd, self._size)
             except OSError as err:
                 if err.errno in (errno.ENOSPC, errno.EDQUOT):
                     raise DiskFileNoSpace()
                 raise
+        else:
+            # If we don't know the size (i.e. self._size is None) or the size
+            # is known to be zero, we still want to block writes once we're
+            # past the reserve threshold.
+            if not fs_has_free_space(
+                    self._fd,
+                    self.manager.fallocate_reserve,
+                    self.manager.fallocate_is_percent
+            ):
+                raise DiskFileNoSpace()
         return self
 
     def close(self):
@@ -1780,7 +1910,10 @@ class BaseDiskFileWriter(object):
         """
         return self._upload_size, self._chunks_etag.hexdigest()
 
-    def _finalize_put(self, metadata, target_path, cleanup):
+    def _finalize_put(self, metadata, target_path, cleanup,
+                      logger_thread_locals):
+        if logger_thread_locals is not None:
+            self.logger.thread_locals = logger_thread_locals
         # Write the metadata before calling fsync() so that both data and
         # metadata are flushed to disk.
         write_metadata(self._fd, metadata)
@@ -1807,10 +1940,13 @@ class BaseDiskFileWriter(object):
         new_target_path = None
         if self.next_part_power:
             new_target_path = replace_partition_in_path(
-                target_path, self.next_part_power)
+                self.manager.devices, target_path, self.next_part_power)
             if target_path != new_target_path:
                 try:
                     fsync_dir(os.path.dirname(target_path))
+                    self.manager.logger.debug(
+                        'Relinking %s to %s due to next_part_power set',
+                        target_path, new_target_path)
                     relink_paths(target_path, new_target_path)
                 except OSError as exc:
                     self.manager.logger.exception(
@@ -1825,7 +1961,7 @@ class BaseDiskFileWriter(object):
             try:
                 self.manager.cleanup_ondisk_files(self._datadir)
             except OSError:
-                logging.exception(_('Problem cleaning up %s'), self._datadir)
+                logging.exception('Problem cleaning up %s', self._datadir)
 
             self._part_power_cleanup(target_path, new_target_path)
 
@@ -1854,7 +1990,9 @@ class BaseDiskFileWriter(object):
         metadata['name'] = self._name
         target_path = join(self._datadir, filename)
 
-        tpool.execute(self._finalize_put, metadata, target_path, cleanup)
+        tpool.execute(
+            self._finalize_put, metadata, target_path, cleanup,
+            logger_thread_locals=getattr(self.logger, 'thread_locals', None))
 
     def put(self, metadata):
         """
@@ -1898,19 +2036,19 @@ class BaseDiskFileWriter(object):
                 self.manager.cleanup_ondisk_files(new_target_dir)
             except OSError:
                 logging.exception(
-                    _('Problem cleaning up %s'), new_target_dir)
+                    'Problem cleaning up %s', new_target_dir)
 
         # Partition power has been increased, cleanup not yet finished
         else:
             prev_part_power = int(self.next_part_power) - 1
             old_target_path = replace_partition_in_path(
-                cur_path, prev_part_power)
+                self.manager.devices, cur_path, prev_part_power)
             old_target_dir = os.path.dirname(old_target_path)
             try:
                 self.manager.cleanup_ondisk_files(old_target_dir)
             except OSError:
                 logging.exception(
-                    _('Problem cleaning up %s'), old_target_dir)
+                    'Problem cleaning up %s', old_target_dir)
 
 
 class BaseDiskFileReader(object):
@@ -1943,11 +2081,16 @@ class BaseDiskFileReader(object):
     :param pipe_size: size of pipe buffer used in zero-copy operations
     :param diskfile: the diskfile creating this DiskFileReader instance
     :param keep_cache: should resulting reads be kept in the buffer cache
+    :param cooperative_period: the period parameter when does cooperative
+                               yielding during file read
+    :param etag_validate_frac: the probability that we should perform etag
+                               validation during a complete file read
     """
     def __init__(self, fp, data_file, obj_size, etag,
                  disk_chunk_size, keep_cache_size, device_path, logger,
                  quarantine_hook, use_splice, pipe_size, diskfile,
-                 keep_cache=False):
+                 keep_cache=False, cooperative_period=0,
+                 etag_validate_frac=1):
         # Parameter tracking
         self._fp = fp
         self._data_file = data_file
@@ -1966,6 +2109,8 @@ class BaseDiskFileReader(object):
             self._keep_cache = obj_size < keep_cache_size
         else:
             self._keep_cache = False
+        self._cooperative_period = cooperative_period
+        self._etag_validate_frac = etag_validate_frac
 
         # Internal Attributes
         self._iter_etag = None
@@ -1983,13 +2128,18 @@ class BaseDiskFileReader(object):
     def _init_checks(self):
         if self._fp.tell() == 0:
             self._started_at_0 = True
-            self._iter_etag = md5()
+            if random.random() < self._etag_validate_frac:
+                self._iter_etag = md5(usedforsecurity=False)
 
     def _update_checks(self, chunk):
         if self._iter_etag:
             self._iter_etag.update(chunk)
 
     def __iter__(self):
+        return CooperativeIterator(
+            self._inner_iter(), period=self._cooperative_period)
+
+    def _inner_iter(self):
         """Returns an iterator over the data file."""
         try:
             dropped_cache = 0
@@ -1998,7 +2148,15 @@ class BaseDiskFileReader(object):
             self._read_to_eof = False
             self._init_checks()
             while True:
-                chunk = self._fp.read(self._disk_chunk_size)
+                try:
+                    chunk = self._fp.read(self._disk_chunk_size)
+                except IOError as e:
+                    if e.errno == errno.EIO:
+                        # Note that if there's no quarantine hook set up,
+                        # this won't raise any exception
+                        self._quarantine(str(e))
+                    # ... so it's significant that this is not in an else
+                    raise
                 if chunk:
                     self._update_checks(chunk)
                     self._bytes_read += len(chunk)
@@ -2214,10 +2372,10 @@ class BaseDiskFileReader(object):
             except DiskFileQuarantined:
                 raise
             except (Exception, Timeout) as e:
-                self._logger.error(_(
+                self._logger.error(
                     'ERROR DiskFile %(data_file)s'
-                    ' close failure: %(exc)s : %(stack)s'),
-                    {'exc': e, 'stack': ''.join(traceback.format_stack()),
+                    ' close failure: %(exc)s : %(stack)s',
+                    {'exc': e, 'stack': ''.join(traceback.format_exc()),
                      'data_file': self._data_file})
             finally:
                 fp, self._fp = self._fp, None
@@ -2311,6 +2469,9 @@ class BaseDiskFile(object):
                 device_path, storage_directory(get_data_dir(policy),
                                                partition, name_hash))
 
+    def __repr__(self):
+        return '<%s datadir=%r>' % (self.__class__.__name__, self._datadir)
+
     @property
     def manager(self):
         return self._manager
@@ -2425,6 +2586,20 @@ class BaseDiskFile(object):
                     # want this one file and not its parent.
                     os.path.join(self._datadir, "made-up-filename"),
                     "Expected directory, found file at %s" % self._datadir)
+            elif err.errno in (errno.ENODATA, EUCLEAN):
+                try:
+                    # We've seen cases where bad sectors lead to ENODATA here
+                    raise self._quarantine(
+                        # similar hack to above
+                        os.path.join(self._datadir, "made-up-filename"),
+                        "Failed to list directory at %s" % self._datadir)
+                except (OSError, IOError):
+                    # We've *also* seen the bad sectors lead to us needing to
+                    # quarantine the whole suffix, not just the hash dir
+                    raise self._quarantine(
+                        # skip the above hack to rename the suffix
+                        self._datadir,
+                        "Failed to list directory at %s" % self._datadir)
             elif err.errno != errno.ENOENT:
                 raise DiskFileError(
                     "Error listing directory %s: %s" % (self._datadir, err))
@@ -2437,8 +2612,14 @@ class BaseDiskFile(object):
         self._data_file = file_info.get('data_file')
         if not self._data_file:
             raise self._construct_exception_from_ts_file(**file_info)
-        self._fp = self._construct_from_data_file(
-            current_time=current_time, modernize=modernize, **file_info)
+        try:
+            self._fp = self._construct_from_data_file(
+                current_time=current_time, modernize=modernize, **file_info)
+        except IOError as e:
+            if e.errno in (errno.ENODATA, EUCLEAN):
+                raise self._quarantine(
+                    file_info['data_file'],
+                    "Failed to open %s: %s" % (file_info['data_file'], e))
         # This method must populate the internal _metadata attribute.
         self._metadata = self._metadata or {}
         return self
@@ -2526,6 +2707,9 @@ class BaseDiskFile(object):
                 exc = DiskFileDeleted(metadata=metadata)
         return exc
 
+    def validate_metadata(self):
+        return ('Content-Length' in self._datafile_metadata)
+
     def _verify_name_matches_hash(self, data_file):
         """
 
@@ -2562,8 +2746,8 @@ class BaseDiskFile(object):
         else:
             if mname != self._name:
                 self._logger.error(
-                    _('Client path %(client)s does not match '
-                      'path stored in object metadata %(meta)s'),
+                    'Client path %(client)s does not match '
+                    'path stored in object metadata %(meta)s',
                     {'client': self._name, 'meta': mname})
                 raise DiskFileCollision('Client path does not match path '
                                         'stored in object metadata')
@@ -2774,7 +2958,8 @@ class BaseDiskFile(object):
         with self.open(current_time=current_time):
             return self.get_metadata()
 
-    def reader(self, keep_cache=False,
+    def reader(self, keep_cache=False, cooperative_period=0,
+               etag_validate_frac=1,
                _quarantine_hook=lambda m: None):
         """
         Return a :class:`swift.common.swob.Response` class compatible
@@ -2786,6 +2971,10 @@ class BaseDiskFile(object):
 
         :param keep_cache: caller's preference for keeping data read in the
                            OS buffer cache
+        :param cooperative_period: the period parameter for cooperative
+                                   yielding during file read
+        :param etag_validate_frac: the probability that we should perform etag
+                                   validation during a complete file read
         :param _quarantine_hook: 1-arg callable called when obj quarantined;
                                  the arg is the reason for quarantine.
                                  Default is to ignore it.
@@ -2797,19 +2986,24 @@ class BaseDiskFile(object):
             self._metadata['ETag'], self._disk_chunk_size,
             self._manager.keep_cache_size, self._device_path, self._logger,
             use_splice=self._use_splice, quarantine_hook=_quarantine_hook,
-            pipe_size=self._pipe_size, diskfile=self, keep_cache=keep_cache)
+            pipe_size=self._pipe_size, diskfile=self, keep_cache=keep_cache,
+            cooperative_period=cooperative_period,
+            etag_validate_frac=etag_validate_frac)
         # At this point the reader object is now responsible for closing
         # the file pointer.
         self._fp = None
         return dr
 
-    def writer(self, size=None):
+    def _writer(self, size, extension):
         return self.writer_cls(self._name, self._datadir, size,
                                self._bytes_per_sync, self,
-                               self.next_part_power)
+                               self.next_part_power, extension=extension)
+
+    def writer(self, size=None):
+        return self._writer(size, '.data')
 
     @contextmanager
-    def create(self, size=None):
+    def create(self, size=None, extension='.data'):
         """
         Context manager to create a file. We create a temporary file first, and
         then return a DiskFileWriter object to encapsulate the state.
@@ -2822,9 +3016,11 @@ class BaseDiskFile(object):
 
         :param size: optional initial size of file to explicitly allocate on
                      disk
+        :param extension: file extension to use for the newly-created file;
+                          defaults to ``.data`` for the sake of tests
         :raises DiskFileNoSpace: if a size is specified and allocation fails
         """
-        dfw = self.writer(size)
+        dfw = self._writer(size, extension)
         try:
             yield dfw.open()
         finally:
@@ -2840,8 +3036,7 @@ class BaseDiskFile(object):
         :raises DiskFileError: this implementation will raise the same
                             errors as the `create()` method.
         """
-        with self.create() as writer:
-            writer._extension = '.meta'
+        with self.create(extension='.meta') as writer:
             writer.put(metadata)
 
     def delete(self, timestamp):
@@ -2863,8 +3058,7 @@ class BaseDiskFile(object):
         """
         # this is dumb, only tests send in strings
         timestamp = Timestamp(timestamp)
-        with self.create() as deleter:
-            deleter._extension = '.ts'
+        with self.create(extension='.ts') as deleter:
             deleter.put({'X-Timestamp': timestamp.internal})
 
 
@@ -2957,11 +3151,13 @@ class ECDiskFileReader(BaseDiskFileReader):
     def __init__(self, fp, data_file, obj_size, etag,
                  disk_chunk_size, keep_cache_size, device_path, logger,
                  quarantine_hook, use_splice, pipe_size, diskfile,
-                 keep_cache=False):
+                 keep_cache=False, cooperative_period=0,
+                 etag_validate_frac=1):
         super(ECDiskFileReader, self).__init__(
             fp, data_file, obj_size, etag,
             disk_chunk_size, keep_cache_size, device_path, logger,
-            quarantine_hook, use_splice, pipe_size, diskfile, keep_cache)
+            quarantine_hook, use_splice, pipe_size, diskfile, keep_cache,
+            cooperative_period, etag_validate_frac)
         self.frag_buf = None
         self.frag_offset = 0
         self.frag_size = self._diskfile.policy.fragment_size
@@ -2980,13 +3176,13 @@ class ECDiskFileReader(BaseDiskFileReader):
     def _check_frag(self, frag):
         if not frag:
             return
-        if not isinstance(frag, six.binary_type):
+        if not isinstance(frag, bytes):
             # ECInvalidParameter can be returned if the frag violates the input
             # format so for safety, check the input chunk if it's binary to
             # avoid quarantining a valid fragment archive.
             self._diskfile._logger.warn(
-                _('Unexpected fragment data type (not quarantined) '
-                  '%(datadir)s: %(type)s at offset 0x%(offset)x'),
+                'Unexpected fragment data type (not quarantined) '
+                '%(datadir)s: %(type)s at offset 0x%(offset)x',
                 {'datadir': self._diskfile._datadir,
                  'type': type(frag),
                  'offset': self.frag_offset})
@@ -3009,7 +3205,7 @@ class ECDiskFileReader(BaseDiskFileReader):
             raise DiskFileQuarantined(msg)
         except ECDriverError as err:
             self._diskfile._logger.warn(
-                _('Problem checking EC fragment %(datadir)s: %(err)s'),
+                'Problem checking EC fragment %(datadir)s: %(err)s',
                 {'datadir': self._diskfile._datadir, 'err': err})
 
     def _update_checks(self, chunk):
@@ -3031,14 +3227,16 @@ class ECDiskFileReader(BaseDiskFileReader):
 
 class ECDiskFileWriter(BaseDiskFileWriter):
 
-    def _finalize_durable(self, data_file_path, durable_data_file_path):
+    def _finalize_durable(self, data_file_path, durable_data_file_path,
+                          timestamp):
         exc = None
         new_data_file_path = new_durable_data_file_path = None
         if self.next_part_power:
             new_data_file_path = replace_partition_in_path(
-                data_file_path, self.next_part_power)
+                self.manager.devices, data_file_path, self.next_part_power)
             new_durable_data_file_path = replace_partition_in_path(
-                durable_data_file_path, self.next_part_power)
+                self.manager.devices, durable_data_file_path,
+                self.next_part_power)
         try:
             try:
                 os.rename(data_file_path, durable_data_file_path)
@@ -3055,12 +3253,27 @@ class ECDiskFileWriter(BaseDiskFileWriter):
                             exc)
 
             except (OSError, IOError) as err:
+                if err.errno == errno.ENOENT:
+                    files = os.listdir(self._datadir)
+                    results = self.manager.get_ondisk_files(
+                        files, self._datadir,
+                        frag_index=self._diskfile._frag_index,
+                        policy=self._diskfile.policy)
+                    # We "succeeded" if another writer cleaned up our data
+                    ts_info = results.get('ts_info')
+                    durables = results.get('durable_frag_set', [])
+                    if ts_info and ts_info['timestamp'] > timestamp:
+                        return
+                    elif any(frag['timestamp'] >= timestamp
+                             for frag in durables):
+                        return
+
                 if err.errno not in (errno.ENOSPC, errno.EDQUOT):
                     # re-raise to catch all handler
                     raise
                 params = {'file': durable_data_file_path, 'err': err}
                 self.manager.logger.exception(
-                    _('No space left on device for %(file)s (%(err)s)'),
+                    'No space left on device for %(file)s (%(err)s)',
                     params)
                 exc = DiskFileNoSpace(
                     'No space left on device for %(file)s (%(err)s)' % params)
@@ -3069,7 +3282,7 @@ class ECDiskFileWriter(BaseDiskFileWriter):
                     self.manager.cleanup_ondisk_files(self._datadir)
                 except OSError as os_err:
                     self.manager.logger.exception(
-                        _('Problem cleaning up %(datadir)s (%(err)s)'),
+                        'Problem cleaning up %(datadir)s (%(err)s)',
                         {'datadir': self._datadir, 'err': os_err})
                 self._part_power_cleanup(
                     durable_data_file_path, new_durable_data_file_path)
@@ -3077,7 +3290,7 @@ class ECDiskFileWriter(BaseDiskFileWriter):
         except Exception as err:
             params = {'file': durable_data_file_path, 'err': err}
             self.manager.logger.exception(
-                _('Problem making data file durable %(file)s (%(err)s)'),
+                'Problem making data file durable %(file)s (%(err)s)',
                 params)
             exc = DiskFileError(
                 'Problem making data file durable %(file)s (%(err)s)' % params)
@@ -3102,7 +3315,8 @@ class ECDiskFileWriter(BaseDiskFileWriter):
             self._datadir, self.manager.make_on_disk_filename(
                 timestamp, '.data', self._diskfile._frag_index, durable=True))
         tpool.execute(
-            self._finalize_durable, data_file_path, durable_data_file_path)
+            self._finalize_durable, data_file_path, durable_data_file_path,
+            timestamp)
 
     def put(self, metadata):
         """
@@ -3176,6 +3390,17 @@ class ECDiskFile(BaseDiskFile):
             raise DiskFileError(
                 'Bad frag_prefs: %r: %s' % (frag_prefs, e))
 
+    def validate_metadata(self):
+        required_metadata = [
+            'Content-Length',
+            'X-Object-Sysmeta-Ec-Frag-Index',
+            'X-Object-Sysmeta-Ec-Etag',
+        ]
+        for header in required_metadata:
+            if not self._datafile_metadata.get(header):
+                return False
+        return True
+
     @property
     def durable_timestamp(self):
         """
@@ -3222,7 +3447,8 @@ class ECDiskFile(BaseDiskFile):
             frag_prefs=self._frag_prefs, policy=policy)
         return self._ondisk_info
 
-    def purge(self, timestamp, frag_index):
+    def purge(self, timestamp, frag_index, nondurable_purge_delay=0,
+              meta_timestamp=None):
         """
         Remove a tombstone file matching the specified timestamp or
         datafile matching the specified timestamp and fragment index
@@ -3239,19 +3465,36 @@ class ECDiskFile(BaseDiskFile):
                           :class:`~swift.common.utils.Timestamp`
         :param frag_index: fragment archive index, must be
                            a whole number or None.
+        :param nondurable_purge_delay: only remove a non-durable data file if
+            it's been on disk longer than this many seconds.
+        :param meta_timestamp: if not None then remove any meta file with this
+            timestamp
         """
         purge_file = self.manager.make_on_disk_filename(
             timestamp, ext='.ts')
-        remove_file(os.path.join(self._datadir, purge_file))
+        purge_path = os.path.join(self._datadir, purge_file)
+        remove_file(purge_path)
+
+        if meta_timestamp is not None:
+            purge_file = self.manager.make_on_disk_filename(
+                meta_timestamp, ext='.meta')
+            purge_path = os.path.join(self._datadir, purge_file)
+            remove_file(purge_path)
+
         if frag_index is not None:
             # data file may or may not be durable so try removing both filename
             # possibilities
             purge_file = self.manager.make_on_disk_filename(
                 timestamp, ext='.data', frag_index=frag_index)
-            remove_file(os.path.join(self._datadir, purge_file))
+            purge_path = os.path.join(self._datadir, purge_file)
+            if is_file_older(purge_path, nondurable_purge_delay):
+                remove_file(purge_path)
+
             purge_file = self.manager.make_on_disk_filename(
                 timestamp, ext='.data', frag_index=frag_index, durable=True)
-            remove_file(os.path.join(self._datadir, purge_file))
+            purge_path = os.path.join(self._datadir, purge_file)
+            remove_file(purge_path)
+
             remove_directory(self._datadir)
         self.manager.invalidate_hash(dirname(self._datadir))
 
@@ -3317,7 +3560,7 @@ class ECDiskFileManager(BaseDiskFileManager):
         """
         Returns timestamp(s) and other info extracted from a policy specific
         file name. For EC policy the data file name includes a fragment index
-        and possibly a durable marker, both of which which must be stripped off
+        and possibly a durable marker, both of which must be stripped off
         to retrieve the timestamp.
 
         :param filename: the file name including extension
@@ -3446,6 +3689,11 @@ class ECDiskFileManager(BaseDiskFileManager):
                     break
             if durable_info and durable_info['timestamp'] == timestamp:
                 durable_frag_set = frag_set
+                # a data frag filename may not have the #d part if durability
+                # is defined by a legacy .durable, so always mark all data
+                # frags as durable here
+                for frag in frag_set:
+                    frag['durable'] = True
                 break  # ignore frags that are older than durable timestamp
 
         # Choose which frag set to use
@@ -3523,7 +3771,8 @@ class ECDiskFileManager(BaseDiskFileManager):
             results.setdefault('obsolete', []).extend(exts['.durable'])
             exts.pop('.durable')
 
-        # Fragments *may* be ready for reclaim, unless they are durable
+        # Fragments *may* be ready for reclaim, unless they are most recent
+        # durable
         for frag_set in frag_sets.values():
             if frag_set in (durable_frag_set, chosen_frag_set):
                 continue