rocksdb-native 2.2.0 → 2.3.0

package/vendor/librocksdb/vendor/rocksdb/utilities/fault_injection_fs.cc

@@ -17,12 +17,16 @@
 #include "utilities/fault_injection_fs.h"
 
 #include <algorithm>
+#include <cstdio>
 #include <functional>
 #include <utility>
 
 #include "env/composite_env_wrapper.h"
 #include "port/lang.h"
 #include "port/stack_trace.h"
+#include "rocksdb/env.h"
+#include "rocksdb/io_status.h"
+#include "rocksdb/types.h"
 #include "test_util/sync_point.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
@@ -92,20 +96,15 @@ IOStatus TestFSDirectory::Fsync(const IOOptions& options, IODebugContext* dbg) {
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  {
-    IOStatus in_s = fs_->InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!s.ok()) {
+    return s;
   }
+
   fs_->SyncDir(dirname_);
-  IOStatus s = dir_->Fsync(options, dbg);
-  {
-    IOStatus in_s = fs_->InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
-  }
+  s = dir_->Fsync(options, dbg);
   return s;
 }
 
@@ -113,7 +112,14 @@ IOStatus TestFSDirectory::Close(const IOOptions& options, IODebugContext* dbg) {
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  IOStatus s = dir_->Close(options, dbg);
+
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!s.ok()) {
+    return s;
+  }
+
+  s = dir_->Close(options, dbg);
   return s;
 }
 
@@ -123,20 +129,14 @@ IOStatus TestFSDirectory::FsyncWithDirOptions(
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  {
-    IOStatus in_s = fs_->InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!s.ok()) {
+    return s;
   }
+
   fs_->SyncDir(dirname_);
-  IOStatus s = dir_->FsyncWithDirOptions(options, dbg, dir_fsync_options);
-  {
-    IOStatus in_s = fs_->InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
-  }
+  s = dir_->FsyncWithDirOptions(options, dbg, dir_fsync_options);
   return s;
 }
 
@@ -148,9 +148,11 @@ TestFSWritableFile::TestFSWritableFile(const std::string& fname,
       file_opts_(file_opts),
       target_(std::move(f)),
       writable_file_opened_(true),
-      fs_(fs) {
+      fs_(fs),
+      unsync_data_loss_(fs_->InjectUnsyncedDataLoss()) {
   assert(target_ != nullptr);
-  state_.pos_ = 0;
+  assert(state_.pos_at_last_append_ == 0);
+  assert(state_.pos_at_last_sync_ == 0);
 }
 
 TestFSWritableFile::~TestFSWritableFile() {
@@ -165,15 +167,28 @@ IOStatus TestFSWritableFile::Append(const Slice& data, const IOOptions& options,
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  if (target_->use_direct_io()) {
-    target_->Append(data, options, dbg).PermitUncheckedError();
+
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kWrite, options, state_.filename_,
+      FaultInjectionTestFS::ErrorOperation::kAppend);
+  if (!s.ok()) {
+    return s;
+  }
+
+  if (target_->use_direct_io() || !unsync_data_loss_) {
+    // TODO(hx235): buffer data for direct IO write to simulate data loss like
+    // non-direct IO write
+    s = target_->Append(data, options, dbg);
   } else {
     state_.buffer_.append(data.data(), data.size());
-    state_.pos_ += data.size();
+  }
+
+  if (s.ok()) {
+    state_.pos_at_last_append_ += data.size();
     fs_->WritableFileAppended(state_);
   }
-  IOStatus io_s = fs_->InjectWriteError(state_.filename_);
-  return io_s;
+
+  return s;
 }
 
 // By setting the IngestDataCorruptionBeforeWrite(), the data corruption is
@@ -189,6 +204,13 @@ IOStatus TestFSWritableFile::Append(
     return IOStatus::Corruption("Data is corrupted!");
   }
 
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kWrite, options, state_.filename_,
+      FaultInjectionTestFS::ErrorOperation::kAppend);
+  if (!s.ok()) {
+    return s;
+  }
+
   // Calculate the checksum
   std::string checksum;
   CalculateTypedChecksum(fs_->GetChecksumHandoffFuncType(), data.data(),
@@ -201,15 +223,66 @@ IOStatus TestFSWritableFile::Append(
         "current data checksum: " + Slice(checksum).ToString(true);
     return IOStatus::Corruption(msg);
   }
-  if (target_->use_direct_io()) {
-    target_->Append(data, options, dbg).PermitUncheckedError();
+
+  if (target_->use_direct_io() || !unsync_data_loss_) {
+    // TODO(hx235): buffer data for direct IO write to simulate data loss like
+    // non-direct IO write
+    s = target_->Append(data, options, dbg);
   } else {
     state_.buffer_.append(data.data(), data.size());
-    state_.pos_ += data.size();
+  }
+  if (s.ok()) {
+    state_.pos_at_last_append_ += data.size();
     fs_->WritableFileAppended(state_);
   }
-  IOStatus io_s = fs_->InjectWriteError(state_.filename_);
-  return io_s;
+  return s;
+}
+
+IOStatus TestFSWritableFile::Truncate(uint64_t size, const IOOptions& options,
+                                      IODebugContext* dbg) {
+  MutexLock l(&mutex_);
+  if (!fs_->IsFilesystemActive()) {
+    return fs_->GetError();
+  }
+  IOStatus s = fs_->MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite,
+                                                options, state_.filename_);
+  if (!s.ok()) {
+    return s;
+  }
+
+  s = target_->Truncate(size, options, dbg);
+  if (s.ok()) {
+    state_.pos_at_last_append_ = size;
+  }
+  return s;
+}
+
+IOStatus TestFSWritableFile::PositionedAppend(const Slice& data,
+                                              uint64_t offset,
+                                              const IOOptions& options,
+                                              IODebugContext* dbg) {
+  MutexLock l(&mutex_);
+  if (!fs_->IsFilesystemActive()) {
+    return fs_->GetError();
+  }
+  if (fs_->ShouldDataCorruptionBeforeWrite()) {
+    return IOStatus::Corruption("Data is corrupted!");
+  }
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kWrite, options, state_.filename_,
+      FaultInjectionTestFS::ErrorOperation::kPositionedAppend);
+  if (!s.ok()) {
+    return s;
+  }
+
+  // TODO(hx235): buffer data for direct IO write to simulate data loss like
+  // non-direct IO write
+  s = target_->PositionedAppend(data, offset, options, dbg);
+  if (s.ok()) {
+    state_.pos_at_last_append_ = offset + data.size();
+    fs_->WritableFileAppended(state_);
+  }
+  return s;
 }
 
 IOStatus TestFSWritableFile::PositionedAppend(
@@ -222,6 +295,12 @@ IOStatus TestFSWritableFile::PositionedAppend(
   if (fs_->ShouldDataCorruptionBeforeWrite()) {
     return IOStatus::Corruption("Data is corrupted!");
   }
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kWrite, options, state_.filename_,
+      FaultInjectionTestFS::ErrorOperation::kPositionedAppend);
+  if (!s.ok()) {
+    return s;
+  }
 
   // Calculate the checksum
   std::string checksum;
@@ -235,9 +314,14 @@ IOStatus TestFSWritableFile::PositionedAppend(
         "current data checksum: " + Slice(checksum).ToString(true);
     return IOStatus::Corruption(msg);
   }
-  target_->PositionedAppend(data, offset, options, dbg);
-  IOStatus io_s = fs_->InjectWriteError(state_.filename_);
-  return io_s;
+  // TODO(hx235): buffer data for direct IO write to simulate data loss like
+  // non-direct IO write
+  s = target_->PositionedAppend(data, offset, options, dbg);
+  if (s.ok()) {
+    state_.pos_at_last_append_ = offset + data.size();
+    fs_->WritableFileAppended(state_);
+  }
+  return s;
 }
 
 IOStatus TestFSWritableFile::Close(const IOOptions& options,
@@ -247,29 +331,17 @@ IOStatus TestFSWritableFile::Close(const IOOptions& options,
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  {
-    IOStatus in_s = fs_->InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus io_s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!io_s.ok()) {
+    return io_s;
   }
   writable_file_opened_ = false;
-  IOStatus io_s;
-  if (!target_->use_direct_io()) {
-    io_s = target_->Append(state_.buffer_, options, dbg);
-  }
-  if (io_s.ok()) {
-    state_.buffer_.resize(0);
-    // Ignore sync errors
-    target_->Sync(options, dbg).PermitUncheckedError();
-    io_s = target_->Close(options, dbg);
-  }
-  if (io_s.ok()) {
-    IOStatus in_s = fs_->InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
-  }
+
+  // Drop buffered data that was never synced because close is not a syncing
+  // mechanism in POSIX file semantics.
+  state_.buffer_.resize(0);
+  io_s = target_->Close(options, dbg);
   return io_s;
 }
 
@@ -278,9 +350,6 @@ IOStatus TestFSWritableFile::Flush(const IOOptions&, IODebugContext*) {
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  if (fs_->IsFilesystemActive()) {
-    state_.pos_at_last_flush_ = state_.pos_;
-  }
   return IOStatus::OK();
 }
 
@@ -299,7 +368,7 @@ IOStatus TestFSWritableFile::Sync(const IOOptions& options,
   state_.buffer_.resize(0);
   // Ignore sync errors
   target_->Sync(options, dbg).PermitUncheckedError();
-  state_.pos_at_last_sync_ = state_.pos_;
+  state_.pos_at_last_sync_ = state_.pos_at_last_append_;
   fs_->WritableFileSynced(state_);
   return io_s;
 }
@@ -313,15 +382,13 @@ IOStatus TestFSWritableFile::RangeSync(uint64_t offset, uint64_t nbytes,
   }
   // Assumes caller passes consecutive byte ranges.
   uint64_t sync_limit = offset + nbytes;
-  uint64_t buf_begin =
-      state_.pos_at_last_sync_ < 0 ? 0 : state_.pos_at_last_sync_;
 
   IOStatus io_s;
-  if (sync_limit < buf_begin) {
+  if (sync_limit < state_.pos_at_last_sync_) {
     return io_s;
   }
   uint64_t num_to_sync = std::min(static_cast<uint64_t>(state_.buffer_.size()),
-                                  sync_limit - buf_begin);
+                                  sync_limit - state_.pos_at_last_sync_);
   Slice buf_to_sync(state_.buffer_.data(), num_to_sync);
   io_s = target_->Append(buf_to_sync, options, dbg);
   state_.buffer_ = state_.buffer_.substr(num_to_sync);
@@ -360,6 +427,7 @@ IOStatus TestFSRandomRWFile::Read(uint64_t offset, size_t n,
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
+  // TODO (low priority): fs_->ReadUnsyncedData()
   return target_->Read(offset, n, options, result, scratch, dbg);
 }
 
@@ -403,15 +471,17 @@ IOStatus TestFSRandomAccessFile::Read(uint64_t offset, size_t n,
   if (!fs_->IsFilesystemActive()) {
     return fs_->GetError();
   }
-  IOStatus s = target_->Read(offset, n, options, result, scratch, dbg);
-  if (s.ok()) {
-    s = fs_->InjectThreadSpecificReadError(
-        FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(),
-        scratch, /*need_count_increase=*/true, /*fault_injected=*/nullptr);
-  }
-  if (s.ok() && fs_->ShouldInjectRandomReadError()) {
-    return IOStatus::IOError("injected read error");
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kRead, options, "",
+      FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(),
+      scratch, /*need_count_increase=*/true,
+      /*fault_injected=*/nullptr);
+  if (!s.ok()) {
+    return s;
   }
+
+  s = target_->Read(offset, n, options, result, scratch, dbg);
+  // TODO (low priority): fs_->ReadUnsyncedData()
   return s;
 }
 
@@ -419,28 +489,35 @@ IOStatus TestFSRandomAccessFile::ReadAsync(
     FSReadRequest& req, const IOOptions& opts,
     std::function<void(FSReadRequest&, void*)> cb, void* cb_arg,
     void** io_handle, IOHandleDeleter* del_fn, IODebugContext* /*dbg*/) {
-  IOStatus ret;
-  IOStatus s;
+  IOStatus res_status;
   FSReadRequest res;
+  IOStatus s;
   if (!fs_->IsFilesystemActive()) {
-    ret = fs_->GetError();
-  } else {
-    ret = fs_->InjectThreadSpecificReadError(
+    res_status = fs_->GetError();
+  }
+  if (res_status.ok()) {
+    res_status = fs_->MaybeInjectThreadLocalError(
+        FaultInjectionIOType::kRead, opts, "",
         FaultInjectionTestFS::ErrorOperation::kRead, &res.result,
         use_direct_io(), req.scratch, /*need_count_increase=*/true,
         /*fault_injected=*/nullptr);
   }
-  if (ret.ok()) {
-    if (fs_->ShouldInjectRandomReadError()) {
-      ret = IOStatus::IOError("injected read error");
-    } else {
-      s = target_->ReadAsync(req, opts, cb, cb_arg, io_handle, del_fn, nullptr);
-    }
-  }
-  if (!ret.ok()) {
-    res.status = ret;
+  if (res_status.ok()) {
+    s = target_->ReadAsync(req, opts, cb, cb_arg, io_handle, del_fn, nullptr);
+    // TODO (low priority): fs_->ReadUnsyncedData()
+  } else {
+    // If there’s no injected error, then cb will be called asynchronously when
+    // target_ actually finishes the read. But if there’s an injected error, it
+    // needs to immediately call cb(res, cb_arg) s since target_->ReadAsync()
+    // isn’t invoked at all.
+    res.status = res_status;
     cb(res, cb_arg);
   }
+  // We return ReadAsync()'s status intead of injected error status here since
+  // the return status is not supposed to be the status of the actual IO (i.e,
+  // the actual async read). The actual status of the IO will be passed to cb()
+  // callback upon the actual read finishes or like above when injected error
+  // happens.
   return s;
 }
 
@@ -451,6 +528,7 @@ IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs,
     return fs_->GetError();
   }
   IOStatus s = target_->MultiRead(reqs, num_reqs, options, dbg);
+  // TODO (low priority): fs_->ReadUnsyncedData()
   bool injected_error = false;
   for (size_t i = 0; i < num_reqs; i++) {
     if (!reqs[i].status.ok()) {
@@ -458,7 +536,8 @@ IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs,
       break;
     }
     bool this_injected_error;
-    reqs[i].status = fs_->InjectThreadSpecificReadError(
+    reqs[i].status = fs_->MaybeInjectThreadLocalError(
+        FaultInjectionIOType::kRead, options, "",
         FaultInjectionTestFS::ErrorOperation::kRead, &(reqs[i].result),
         use_direct_io(), reqs[i].scratch,
         /*need_count_increase=*/true,
@@ -466,14 +545,12 @@ IOStatus TestFSRandomAccessFile::MultiRead(FSReadRequest* reqs, size_t num_reqs,
     injected_error |= this_injected_error;
   }
   if (s.ok()) {
-    s = fs_->InjectThreadSpecificReadError(
+    s = fs_->MaybeInjectThreadLocalError(
+        FaultInjectionIOType::kRead, options, "",
         FaultInjectionTestFS::ErrorOperation::kMultiRead, nullptr,
         use_direct_io(), nullptr, /*need_count_increase=*/!injected_error,
         /*fault_injected=*/nullptr);
   }
-  if (s.ok() && fs_->ShouldInjectRandomReadError()) {
-    return IOStatus::IOError("injected read error");
-  }
   return s;
 }
 
@@ -484,13 +561,196 @@ size_t TestFSRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
     return target_->GetUniqueId(id, max_size);
   }
 }
+
+namespace {
+// Modifies `result` to start at the beginning of `scratch` if not already,
+// copying data there if needed.
+void MoveToScratchIfNeeded(Slice* result, char* scratch) {
+  if (result->data() != scratch) {
+    // NOTE: might overlap, where result is later in scratch
+    std::copy(result->data(), result->data() + result->size(), scratch);
+    *result = Slice(scratch, result->size());
+  }
+}
+}  // namespace
+
+void FaultInjectionTestFS::ReadUnsynced(const std::string& fname,
+                                        uint64_t offset, size_t n,
+                                        Slice* result, char* scratch,
+                                        int64_t* pos_at_last_sync) {
+  *result = Slice(scratch, 0);  // default empty result
+  assert(*pos_at_last_sync == -1);  // default "unknown"
+
+  MutexLock l(&mutex_);
+  auto it = db_file_state_.find(fname);
+  if (it != db_file_state_.end()) {
+    auto& st = it->second;
+    *pos_at_last_sync = static_cast<int64_t>(st.pos_at_last_sync_);
+    // Find overlap between [offset, offset + n) and
+    // [*pos_at_last_sync, *pos_at_last_sync + st.buffer_.size())
+    int64_t begin = std::max(static_cast<int64_t>(offset), *pos_at_last_sync);
+    int64_t end =
+        std::min(static_cast<int64_t>(offset + n),
+                 *pos_at_last_sync + static_cast<int64_t>(st.buffer_.size()));
+
+    // Copy and return overlap if there is any
+    if (begin < end) {
+      size_t offset_in_buffer = static_cast<size_t>(begin - *pos_at_last_sync);
+      size_t offset_in_scratch = static_cast<size_t>(begin - offset);
+      std::copy_n(st.buffer_.data() + offset_in_buffer, end - begin,
+                  scratch + offset_in_scratch);
+      *result = Slice(scratch + offset_in_scratch, end - begin);
+    }
+  }
+}
+
 IOStatus TestFSSequentialFile::Read(size_t n, const IOOptions& options,
                                     Slice* result, char* scratch,
                                     IODebugContext* dbg) {
-  IOStatus s = target()->Read(n, options, result, scratch, dbg);
-  if (s.ok() && fs_->ShouldInjectRandomReadError()) {
-    return IOStatus::IOError("injected seq read error");
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kRead, options, "",
+      FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(),
+      scratch, true /*need_count_increase=*/, nullptr /* fault_injected*/);
+  if (!s.ok()) {
+    return s;
+  }
+
+  // Some complex logic is needed to deal with concurrent write to the same
+  // file, while keeping good performance (e.g. not holding FS mutex during
+  // I/O op), especially in common cases.
+
+  if (read_pos_ == target_read_pos_) {
+    // Normal case: start by reading from underlying file
+    s = target()->Read(n, options, result, scratch, dbg);
+    if (!s.ok()) {
+      return s;
+    }
+    target_read_pos_ += result->size();
+  } else {
+    // We must have previously read buffered data (unsynced) not written to
+    // target. Deal with this case (and more) below.
+    *result = {};
+  }
+
+  if (fs_->ReadUnsyncedData() && result->size() < n) {
+    // We need to check if there's unsynced data to fill out the rest of the
+    // read.
+
+    // First, ensure target read data is in scratch for easy handling.
+    MoveToScratchIfNeeded(result, scratch);
+    assert(result->data() == scratch);
+
+    // If we just did a target Read, we only want unsynced data after it
+    // (target_read_pos_). Otherwise (e.g. if target is behind because of
+    // unsynced data) we want unsynced data starting at the current read pos
+    // (read_pos_, not yet updated).
+    const uint64_t unsynced_read_pos = std::max(target_read_pos_, read_pos_);
+    const size_t offset_from_read_pos =
+        static_cast<size_t>(unsynced_read_pos - read_pos_);
+    Slice unsynced_result;
+    int64_t pos_at_last_sync = -1;
+    fs_->ReadUnsynced(fname_, unsynced_read_pos, n - offset_from_read_pos,
+                      &unsynced_result, scratch + offset_from_read_pos,
+                      &pos_at_last_sync);
+    assert(unsynced_result.data() >= scratch + offset_from_read_pos);
+    assert(unsynced_result.data() < scratch + n);
+    // Now, there are several cases to consider (some grouped together):
+    if (pos_at_last_sync <= static_cast<int64_t>(unsynced_read_pos)) {
+      // 1. We didn't get any unsynced data because nothing has been written
+      // to the file beyond unsynced_read_pos (including untracked
+      // pos_at_last_sync == -1)
+      // 2. We got some unsynced data starting at unsynced_read_pos (possibly
+      // on top of some synced data from target). We don't need to try reading
+      // any more from target because we established a "point in time" for
+      // completing this Read in which we read as much tail data (unsynced) as
+      // we could.
+
+      // We got pos_at_last_sync info if we got any unsynced data.
+      assert(pos_at_last_sync >= 0 || unsynced_result.size() == 0);
+
+      // Combined data is already lined up in scratch.
+      assert(result->data() + result->size() == unsynced_result.data());
+      assert(result->size() + unsynced_result.size() <= n);
+      // Combine results
+      *result = Slice(result->data(), result->size() + unsynced_result.size());
+    } else {
+      // 3. Any unsynced data we got was after unsynced_read_pos because the
+      // file was synced some time since our last target Read (either from this
+      // Read or a prior Read). We need to read more data from target to ensure
+      // this Read is filled out, even though we might have already read some
+      // (but not all due to a race). This code handles:
+      //
+      // * Catching up target after prior read(s) of unsynced data
+      // * Racing Sync in another thread since we called target Read above
+      //
+      // And merging potentially three results together for this Read:
+      // * The original target Read above
+      // * The following (non-throw-away) target Read
+      // * The ReadUnsynced above, which is always last if it returned data,
+      // so that we have a "point in time" for completing this Read in which we
+      // read as much tail data (unsynced) as we could.
+      //
+      // Deeper note about the race: we cannot just treat the original target
+      // Read as a "point in time" view of available data in the file, because
+      // there might have been unsynced data at that time, which became synced
+      // data by the time we read unsynced data. That is the race we are
+      // resolving with this "double check"-style code.
+      const size_t supplemental_read_pos = unsynced_read_pos;
+
+      // First, if there's any data from target that we know we would need to
+      // throw away to catch up, try to do it.
+      if (target_read_pos_ < supplemental_read_pos) {
+        Slice throw_away_result;
+        size_t throw_away_n = supplemental_read_pos - target_read_pos_;
+        std::unique_ptr<char[]> throw_away_scratch{new char[throw_away_n]};
+        s = target()->Read(throw_away_n, options, &throw_away_result,
+                           throw_away_scratch.get(), dbg);
+        if (!s.ok()) {
+          read_pos_ += result->size();
+          return s;
+        }
+        target_read_pos_ += throw_away_result.size();
+        if (target_read_pos_ < supplemental_read_pos) {
+          // Because of pos_at_last_sync > supplemental_read_pos, we should
+          // have been able to catch up
+          read_pos_ += result->size();
+          return IOStatus::IOError(
+              "Unexpected truncation or short read of file " + fname_);
+        }
+      }
+      // Now we can do a productive supplemental Read from target
+      assert(target_read_pos_ == supplemental_read_pos);
+      Slice supplemental_result;
+      size_t supplemental_n =
+          unsynced_result.size() == 0
+              ? n - offset_from_read_pos
+              : unsynced_result.data() - (scratch + offset_from_read_pos);
+      s = target()->Read(supplemental_n, options, &supplemental_result,
+                         scratch + offset_from_read_pos, dbg);
+      if (!s.ok()) {
+        read_pos_ += result->size();
+        return s;
+      }
+      target_read_pos_ += supplemental_result.size();
+      MoveToScratchIfNeeded(&supplemental_result,
+                            scratch + offset_from_read_pos);
+
+      // Combined data is already lined up in scratch.
+      assert(result->data() + result->size() == supplemental_result.data());
+      assert(unsynced_result.size() == 0 ||
+             supplemental_result.data() + supplemental_result.size() ==
+                 unsynced_result.data());
+      assert(result->size() + supplemental_result.size() +
+                 unsynced_result.size() <=
+             n);
+      // Combine results
+      *result =
+          Slice(result->data(), result->size() + supplemental_result.size() +
+                                    unsynced_result.size());
+    }
   }
+  read_pos_ += result->size();
+
   return s;
 }
 
@@ -498,11 +758,16 @@ IOStatus TestFSSequentialFile::PositionedRead(uint64_t offset, size_t n,
                                               const IOOptions& options,
                                               Slice* result, char* scratch,
                                               IODebugContext* dbg) {
-  IOStatus s =
-      target()->PositionedRead(offset, n, options, result, scratch, dbg);
-  if (s.ok() && fs_->ShouldInjectRandomReadError()) {
-    return IOStatus::IOError("injected seq positioned read error");
+  IOStatus s = fs_->MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kRead, options, "",
+      FaultInjectionTestFS::ErrorOperation::kRead, result, use_direct_io(),
+      scratch, true /*need_count_increase=*/, nullptr /* fault_injected */);
+  if (!s.ok()) {
+    return s;
   }
+
+  s = target()->PositionedRead(offset, n, options, result, scratch, dbg);
+  // TODO (low priority): fs_->ReadUnsyncedData()
   return s;
 }
 
@@ -519,24 +784,77 @@ IOStatus FaultInjectionTestFS::NewDirectory(
   return IOStatus::OK();
 }
 
+IOStatus FaultInjectionTestFS::FileExists(const std::string& fname,
+                                          const IOOptions& options,
+                                          IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->FileExists(fname, options, dbg);
+  return io_s;
+}
+
+IOStatus FaultInjectionTestFS::GetChildren(const std::string& dir,
+                                           const IOOptions& options,
+                                           std::vector<std::string>* result,
+                                           IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->GetChildren(dir, options, result, dbg);
+  return io_s;
+}
+
+IOStatus FaultInjectionTestFS::GetChildrenFileAttributes(
+    const std::string& dir, const IOOptions& options,
+    std::vector<FileAttributes>* result, IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->GetChildrenFileAttributes(dir, options, result, dbg);
+  return io_s;
+}
+
 IOStatus FaultInjectionTestFS::NewWritableFile(
     const std::string& fname, const FileOptions& file_opts,
     std::unique_ptr<FSWritableFile>* result, IODebugContext* dbg) {
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  {
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
-  }
 
-  if (ShouldUseDiretWritable(fname)) {
+  if (IsFilesystemDirectWritable()) {
     return target()->NewWritableFile(fname, file_opts, result, dbg);
   }
 
-  IOStatus io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
+  IOStatus io_s = MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kWrite, file_opts.io_options, fname,
+      FaultInjectionTestFS::ErrorOperation::kOpen);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->NewWritableFile(fname, file_opts, result, dbg);
   if (io_s.ok()) {
     result->reset(
         new TestFSWritableFile(fname, file_opts, std::move(*result), this));
@@ -553,12 +871,6 @@ IOStatus FaultInjectionTestFS::NewWritableFile(
       // dropping unsynced files.
       list[dir_and_name.second] = kNewFileNoOverwrite;
     }
-    {
-      IOStatus in_s = InjectMetadataWriteError();
-      if (!in_s.ok()) {
-        return in_s;
-      }
-    }
   }
   return io_s;
 }
@@ -569,19 +881,18 @@ IOStatus FaultInjectionTestFS::ReopenWritableFile(
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  if (ShouldUseDiretWritable(fname)) {
+  if (IsFilesystemDirectWritable()) {
     return target()->ReopenWritableFile(fname, file_opts, result, dbg);
   }
-  {
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus io_s = MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite,
+                                              file_opts.io_options, fname);
+  if (!io_s.ok()) {
+    return io_s;
   }
 
   bool exists;
-  IOStatus io_s,
-      exists_s = target()->FileExists(fname, IOOptions(), nullptr /* dbg */);
+  IOStatus exists_s =
+      target()->FileExists(fname, IOOptions(), nullptr /* dbg */);
   if (exists_s.IsNotFound()) {
     exists = false;
   } else if (exists_s.ok()) {
@@ -591,10 +902,12 @@ IOStatus FaultInjectionTestFS::ReopenWritableFile(
     exists = false;
   }
 
-  if (io_s.ok()) {
-    io_s = target()->ReopenWritableFile(fname, file_opts, result, dbg);
+  if (!io_s.ok()) {
+    return io_s;
   }
 
+  io_s = target()->ReopenWritableFile(fname, file_opts, result, dbg);
+
   // Only track files we created. Files created outside of this
   // `FaultInjectionTestFS` are not eligible for tracking/data dropping
   // (for example, they may contain data a previous db_stress run expects to
@@ -623,32 +936,38 @@ IOStatus FaultInjectionTestFS::ReopenWritableFile(
       result->reset(
           new TestFSWritableFile(fname, file_opts, std::move(*result), this));
     }
-    {
-      IOStatus in_s = InjectMetadataWriteError();
-      if (!in_s.ok()) {
-        return in_s;
-      }
-    }
   }
   return io_s;
 }
 
+IOStatus FaultInjectionTestFS::ReuseWritableFile(
+    const std::string& fname, const std::string& old_fname,
+    const FileOptions& file_opts, std::unique_ptr<FSWritableFile>* result,
+    IODebugContext* dbg) {
+  IOStatus s = RenameFile(old_fname, fname, file_opts.io_options, dbg);
+  if (!s.ok()) {
+    return s;
+  }
+  return NewWritableFile(fname, file_opts, result, dbg);
+}
+
 IOStatus FaultInjectionTestFS::NewRandomRWFile(
     const std::string& fname, const FileOptions& file_opts,
     std::unique_ptr<FSRandomRWFile>* result, IODebugContext* dbg) {
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  if (ShouldUseDiretWritable(fname)) {
+  if (IsFilesystemDirectWritable()) {
     return target()->NewRandomRWFile(fname, file_opts, result, dbg);
   }
-  {
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus io_s = MaybeInjectThreadLocalError(FaultInjectionIOType::kWrite,
+                                              file_opts.io_options, fname);
+  if (!io_s.ok()) {
+    return io_s;
   }
-  IOStatus io_s = target()->NewRandomRWFile(fname, file_opts, result, dbg);
+
+  io_s = target()->NewRandomRWFile(fname, file_opts, result, dbg);
+
   if (io_s.ok()) {
     result->reset(new TestFSRandomRWFile(fname, std::move(*result), this));
     // WritableFileWriter* file is opened
@@ -663,12 +982,6 @@ IOStatus FaultInjectionTestFS::NewRandomRWFile(
       // implementation by ignoring it.
       list[dir_and_name.second] = kNewFileNoOverwrite;
     }
-    {
-      IOStatus in_s = InjectMetadataWriteError();
-      if (!in_s.ok()) {
-        return in_s;
-      }
-    }
   }
   return io_s;
 }
@@ -679,16 +992,17 @@ IOStatus FaultInjectionTestFS::NewRandomAccessFile(
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  if (ShouldInjectRandomReadError()) {
-    return IOStatus::IOError("injected error when open random access file");
-  }
-  IOStatus io_s = InjectThreadSpecificReadError(ErrorOperation::kOpen, nullptr,
-                                                false, nullptr,
-                                                /*need_count_increase=*/true,
-                                                /*fault_injected=*/nullptr);
-  if (io_s.ok()) {
-    io_s = target()->NewRandomAccessFile(fname, file_opts, result, dbg);
+  IOStatus io_s = MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kRead, file_opts.io_options, fname,
+      ErrorOperation::kOpen, nullptr /* result */, false /* direct_io */,
+      nullptr /* scratch */, true /*need_count_increase*/,
+      nullptr /*fault_injected*/);
+  if (!io_s.ok()) {
+    return io_s;
   }
+
+  io_s = target()->NewRandomAccessFile(fname, file_opts, result, dbg);
+
   if (io_s.ok()) {
     result->reset(new TestFSRandomAccessFile(fname, std::move(*result), this));
   }
@@ -701,13 +1015,19 @@ IOStatus FaultInjectionTestFS::NewSequentialFile(
   if (!IsFilesystemActive()) {
     return GetError();
   }
-
-  if (ShouldInjectRandomReadError()) {
-    return IOStatus::IOError("injected read error when creating seq file");
+  IOStatus io_s = MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kRead, file_opts.io_options, fname,
+      ErrorOperation::kOpen, nullptr /* result */, false /* direct_io */,
+      nullptr /* scratch */, true /*need_count_increase*/,
+      nullptr /*fault_injected*/);
+  if (!io_s.ok()) {
+    return io_s;
   }
-  IOStatus io_s = target()->NewSequentialFile(fname, file_opts, result, dbg);
+
+  io_s = target()->NewSequentialFile(fname, file_opts, result, dbg);
+
   if (io_s.ok()) {
-    result->reset(new TestFSSequentialFile(std::move(*result), this));
+    result->reset(new TestFSSequentialFile(std::move(*result), this, fname));
   }
   return io_s;
 }
@@ -718,25 +1038,66 @@ IOStatus FaultInjectionTestFS::DeleteFile(const std::string& f,
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  {
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus io_s = MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!io_s.ok()) {
+    return io_s;
   }
-  IOStatus io_s = FileSystemWrapper::DeleteFile(f, options, dbg);
+
+  io_s = FileSystemWrapper::DeleteFile(f, options, dbg);
+
   if (io_s.ok()) {
     UntrackFile(f);
-    {
-      IOStatus in_s = InjectMetadataWriteError();
-      if (!in_s.ok()) {
-        return in_s;
-      }
+  }
+  return io_s;
+}
+
+IOStatus FaultInjectionTestFS::GetFileSize(const std::string& f,
+                                           const IOOptions& options,
+                                           uint64_t* file_size,
+                                           IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->GetFileSize(f, options, file_size, dbg);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  if (ReadUnsyncedData()) {
+    // Need to report flushed size, not synced size
+    MutexLock l(&mutex_);
+    auto it = db_file_state_.find(f);
+    if (it != db_file_state_.end()) {
+      *file_size = it->second.pos_at_last_append_;
     }
   }
   return io_s;
 }
 
+IOStatus FaultInjectionTestFS::GetFileModificationTime(const std::string& fname,
+                                                       const IOOptions& options,
+                                                       uint64_t* file_mtime,
+                                                       IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->GetFileModificationTime(fname, options, file_mtime, dbg);
+  return io_s;
+}
+
 IOStatus FaultInjectionTestFS::RenameFile(const std::string& s,
                                           const std::string& t,
                                           const IOOptions& options,
@@ -744,11 +1105,10 @@ IOStatus FaultInjectionTestFS::RenameFile(const std::string& s,
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  {
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus io_s = MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!io_s.ok()) {
+    return io_s;
   }
 
   // We preserve contents of overwritten files up to a size threshold.
@@ -763,7 +1123,7 @@ IOStatus FaultInjectionTestFS::RenameFile(const std::string& s,
       ReadFileToString(target(), t, &previous_contents).PermitUncheckedError();
     }
   }
-  IOStatus io_s = FileSystemWrapper::RenameFile(s, t, options, dbg);
+  io_s = FileSystemWrapper::RenameFile(s, t, options, dbg);
 
   if (io_s.ok()) {
     {
@@ -777,16 +1137,10 @@ IOStatus FaultInjectionTestFS::RenameFile(const std::string& s,
       auto tdn = TestFSGetDirAndName(t);
       if (dir_to_new_files_since_last_sync_[sdn.first].erase(sdn.second) != 0) {
         auto& tlist = dir_to_new_files_since_last_sync_[tdn.first];
-        assert(tlist.find(tdn.second) == tlist.end());
         tlist[tdn.second] = previous_contents;
       }
     }
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
   }
-
   return io_s;
 }
 
@@ -797,22 +1151,27 @@ IOStatus FaultInjectionTestFS::LinkFile(const std::string& s,
   if (!IsFilesystemActive()) {
     return GetError();
   }
-  {
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
+  IOStatus io_s = MaybeInjectThreadLocalError(
+      FaultInjectionIOType::kMetadataWrite, options);
+  if (!io_s.ok()) {
+    return io_s;
   }
 
   // Using the value in `dir_to_new_files_since_last_sync_` for the source file
   // may be a more reasonable choice.
   std::string previous_contents = kNewFileNoOverwrite;
 
-  IOStatus io_s = FileSystemWrapper::LinkFile(s, t, options, dbg);
+  io_s = FileSystemWrapper::LinkFile(s, t, options, dbg);
 
   if (io_s.ok()) {
     {
       MutexLock l(&mutex_);
+      if (!allow_link_open_file_ &&
+          open_managed_files_.find(s) != open_managed_files_.end()) {
+        fprintf(stderr, "Attempt to LinkFile while open for write: %s\n",
+                s.c_str());
+        abort();
+      }
       if (db_file_state_.find(s) != db_file_state_.end()) {
         db_file_state_[t] = db_file_state_[s];
       }
@@ -826,12 +1185,74 @@ IOStatus FaultInjectionTestFS::LinkFile(const std::string& s,
         tlist[tdn.second] = previous_contents;
       }
     }
-    IOStatus in_s = InjectMetadataWriteError();
-    if (!in_s.ok()) {
-      return in_s;
-    }
   }
+  return io_s;
+}
 
+IOStatus FaultInjectionTestFS::NumFileLinks(const std::string& fname,
+                                            const IOOptions& options,
+                                            uint64_t* count,
+                                            IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->NumFileLinks(fname, options, count, dbg);
+  return io_s;
+}
+
+IOStatus FaultInjectionTestFS::AreFilesSame(const std::string& first,
+                                            const std::string& second,
+                                            const IOOptions& options, bool* res,
+                                            IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->AreFilesSame(first, second, options, res, dbg);
+  return io_s;
+}
+
+IOStatus FaultInjectionTestFS::GetAbsolutePath(const std::string& db_path,
+                                               const IOOptions& options,
+                                               std::string* output_path,
+                                               IODebugContext* dbg) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->GetAbsolutePath(db_path, options, output_path, dbg);
+  return io_s;
+}
+
+IOStatus FaultInjectionTestFS::IsDirectory(const std::string& path,
+                                           const IOOptions& options,
+                                           bool* is_dir, IODebugContext* dgb) {
+  if (!IsFilesystemActive()) {
+    return GetError();
+  }
+  IOStatus io_s =
+      MaybeInjectThreadLocalError(FaultInjectionIOType::kMetadataRead, options);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+
+  io_s = target()->IsDirectory(path, options, is_dir, dgb);
   return io_s;
 }
 
@@ -948,14 +1369,17 @@ void FaultInjectionTestFS::UntrackFile(const std::string& f) {
   open_managed_files_.erase(f);
 }
 
-IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError(
-    ErrorOperation op, Slice* result, bool direct_io, char* scratch,
-    bool need_count_increase, bool* fault_injected) {
+IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalReadError(
+    const IOOptions& io_options, ErrorOperation op, Slice* result,
+    bool direct_io, char* scratch, bool need_count_increase,
+    bool* fault_injected) {
   bool dummy_bool;
   bool& ret_fault_injected = fault_injected ? *fault_injected : dummy_bool;
   ret_fault_injected = false;
-  ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get());
-  if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in) {
+  ErrorContext* ctx =
+      static_cast<ErrorContext*>(injected_thread_local_read_error_.Get());
+  if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in ||
+      ShouldIOActivtiesExcludedFromFaultInjection(io_options.io_activity)) {
     return IOStatus::OK();
   }
 
@@ -972,9 +1396,12 @@ IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError(
     }
     ctx->callstack = port::SaveStack(&ctx->frames);
 
+    std::stringstream msg;
+    msg << FaultInjectionTestFS::kInjected << " ";
     if (op != ErrorOperation::kMultiReadSingleReq) {
       // Likely non-per read status code for MultiRead
-      ctx->message += "injected read error; ";
+      msg << "read error";
+      ctx->message = msg.str();
       ret_fault_injected = true;
       ret = IOStatus::IOError(ctx->message);
     } else if (Random::GetTLSInstance()->OneIn(8)) {
@@ -982,7 +1409,8 @@ IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError(
       // For a small chance, set the failure to status but turn the
       // result to be empty, which is supposed to be caught for a check.
       *result = Slice();
-      ctx->message += "injected empty result; ";
+      msg << "empty result";
+      ctx->message = msg.str();
       ret_fault_injected = true;
     } else if (!direct_io && Random::GetTLSInstance()->OneIn(7) &&
                scratch != nullptr && result->data() == scratch) {
@@ -999,17 +1427,19 @@ IOStatus FaultInjectionTestFS::InjectThreadSpecificReadError(
       // It would work for CRC. Not 100% sure for xxhash and will adjust
       // if it is not the case.
       const_cast<char*>(result->data())[result->size() - 1]++;
-      ctx->message += "injected corrupt last byte; ";
+      msg << "corrupt last byte";
+      ctx->message = msg.str();
       ret_fault_injected = true;
     } else {
-      ctx->message += "injected error result multiget single; ";
+      msg << "error result multiget single";
+      ctx->message = msg.str();
       ret_fault_injected = true;
       ret = IOStatus::IOError(ctx->message);
     }
   }
-  if (ctx->retryable) {
-    ret.SetRetryable(true);
-  }
+
+  ret.SetRetryable(ctx->retryable);
+  ret.SetDataLoss(ctx->has_data_loss);
   return ret;
 }
 
@@ -1020,59 +1450,58 @@ bool FaultInjectionTestFS::TryParseFileName(const std::string& file_name,
   return ParseFileName(file, number, type);
 }
 
-IOStatus FaultInjectionTestFS::InjectWriteError(const std::string& file_name) {
-  MutexLock l(&mutex_);
-  if (!enable_write_error_injection_ || !write_error_one_in_) {
+IOStatus FaultInjectionTestFS::MaybeInjectThreadLocalError(
+    FaultInjectionIOType type, const IOOptions& io_options,
+    const std::string& file_name, ErrorOperation op, Slice* result,
+    bool direct_io, char* scratch, bool need_count_increase,
+    bool* fault_injected) {
+  if (type == FaultInjectionIOType::kRead) {
+    return MaybeInjectThreadLocalReadError(io_options, op, result, direct_io,
+                                           scratch, need_count_increase,
+                                           fault_injected);
+  }
+
+  ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type);
+  if (ctx == nullptr || !ctx->enable_error_injection || !ctx->one_in ||
+      ShouldIOActivtiesExcludedFromFaultInjection(io_options.io_activity) ||
+      (type == FaultInjectionIOType::kWrite &&
+       ShouldExcludeFromWriteFaultInjection(file_name))) {
     return IOStatus::OK();
   }
-  bool allowed_type = false;
-
-  if (inject_for_all_file_types_) {
-    allowed_type = true;
-  } else {
-    uint64_t number;
-    FileType cur_type = kTempFile;
-    if (TryParseFileName(file_name, &number, &cur_type)) {
-      for (const auto& type : write_error_allowed_types_) {
-        if (cur_type == type) {
-          allowed_type = true;
-        }
-      }
-    }
-  }
 
-  if (allowed_type) {
-    if (write_error_rand_.OneIn(write_error_one_in_)) {
-      return GetError();
+  IOStatus ret;
+  if (ctx->rand.OneIn(ctx->one_in)) {
+    ctx->count++;
+    if (ctx->callstack) {
+      free(ctx->callstack);
     }
-  }
-  return IOStatus::OK();
-}
-
-IOStatus FaultInjectionTestFS::InjectMetadataWriteError() {
-  {
-    MutexLock l(&mutex_);
-    if (!enable_metadata_write_error_injection_ ||
-        !metadata_write_error_one_in_ ||
-        !write_error_rand_.OneIn(metadata_write_error_one_in_)) {
-      return IOStatus::OK();
+    ctx->callstack = port::SaveStack(&ctx->frames);
+    ctx->message = GetErrorMessage(type, file_name, op);
+    ret = IOStatus::IOError(ctx->message);
+    ret.SetRetryable(ctx->retryable);
+    ret.SetDataLoss(ctx->has_data_loss);
+    if (type == FaultInjectionIOType::kWrite) {
+      TEST_SYNC_POINT(
+          "FaultInjectionTestFS::InjectMetadataWriteError:Injected");
     }
   }
-  TEST_SYNC_POINT("FaultInjectionTestFS::InjectMetadataWriteError:Injected");
-  return IOStatus::IOError("injected metadata write error");
+  return ret;
 }
 
-void FaultInjectionTestFS::PrintFaultBacktrace() {
+void FaultInjectionTestFS::PrintInjectedThreadLocalErrorBacktrace(
+    FaultInjectionIOType type) {
 #if defined(OS_LINUX)
-  ErrorContext* ctx = static_cast<ErrorContext*>(thread_local_error_->Get());
-  if (ctx == nullptr) {
-    return;
+  ErrorContext* ctx = GetErrorContextFromFaultInjectionIOType(type);
+  if (ctx) {
+    if (type == FaultInjectionIOType::kRead) {
+      fprintf(stderr, "Injected read error type = %d\n", ctx->type);
+    }
+    fprintf(stderr, "Message: %s\n", ctx->message.c_str());
+    port::PrintAndFreeStack(ctx->callstack, ctx->frames);
+    ctx->callstack = nullptr;
   }
-  fprintf(stderr, "Injected error type = %d\n", ctx->type);
-  fprintf(stderr, "Message: %s\n", ctx->message.c_str());
-  port::PrintAndFreeStack(ctx->callstack, ctx->frames);
-  ctx->callstack = nullptr;
+#else
+  (void)type;
 #endif
 }
-
 }  // namespace ROCKSDB_NAMESPACE