rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/db/version_set_sync_and_async.h

@@ -1,169 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "util/coro_utils.h"
-
-#if defined(WITHOUT_COROUTINES) || \
-    (defined(USE_COROUTINES) && defined(WITH_COROUTINES))
-
-namespace ROCKSDB_NAMESPACE {
-
-// Lookup a batch of keys in a single SST file
-DEFINE_SYNC_AND_ASYNC(Status, Version::MultiGetFromSST)
-(const ReadOptions& read_options, MultiGetRange file_range, int hit_file_level,
- bool skip_filters, bool skip_range_deletions, FdWithKeyRange* f,
- std::unordered_map<uint64_t, BlobReadContexts>& blob_ctxs,
- TableCache::TypedHandle* table_handle, uint64_t& num_filter_read,
- uint64_t& num_index_read, uint64_t& num_sst_read) {
-  bool timer_enabled = GetPerfLevel() >= PerfLevel::kEnableTimeExceptForMutex &&
-                       get_perf_context()->per_level_perf_context_enabled;
-
-  Status s;
-  StopWatchNano timer(clock_, timer_enabled /* auto_start */);
-  s = CO_AWAIT(table_cache_->MultiGet)(
-      read_options, *internal_comparator(), *f->file_metadata, &file_range,
-      mutable_cf_options_.block_protection_bytes_per_key,
-      mutable_cf_options_.prefix_extractor,
-      cfd_->internal_stats()->GetFileReadHist(hit_file_level), skip_filters,
-      skip_range_deletions, hit_file_level, table_handle);
-  // TODO: examine the behavior for corrupted key
-  if (timer_enabled) {
-    PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(),
-                              hit_file_level);
-  }
-  if (!s.ok()) {
-    // TODO: Set status for individual keys appropriately
-    for (auto iter = file_range.begin(); iter != file_range.end(); ++iter) {
-      *iter->s = s;
-      file_range.MarkKeyDone(iter);
-    }
-    CO_RETURN s;
-  }
-  uint64_t batch_size = 0;
-  for (auto iter = file_range.begin(); s.ok() && iter != file_range.end();
-       ++iter) {
-    GetContext& get_context = *iter->get_context;
-    Status* status = iter->s;
-    // The Status in the KeyContext takes precedence over GetContext state
-    // Status may be an error if there were any IO errors in the table
-    // reader. We never expect Status to be NotFound(), as that is
-    // determined by get_context
-    assert(!status->IsNotFound());
-    if (!status->ok()) {
-      file_range.MarkKeyDone(iter);
-      continue;
-    }
-
-    if (get_context.sample()) {
-      sample_file_read_inc(f->file_metadata);
-    }
-    batch_size++;
-    num_index_read += get_context.get_context_stats_.num_index_read;
-    num_filter_read += get_context.get_context_stats_.num_filter_read;
-    num_sst_read += get_context.get_context_stats_.num_sst_read;
-    // Reset these stats since they're specific to a level
-    get_context.get_context_stats_.num_index_read = 0;
-    get_context.get_context_stats_.num_filter_read = 0;
-    get_context.get_context_stats_.num_sst_read = 0;
-
-    // report the counters before returning
-    if (get_context.State() != GetContext::kNotFound &&
-        get_context.State() != GetContext::kMerge &&
-        db_statistics_ != nullptr) {
-      get_context.ReportCounters();
-    } else {
-      if (iter->max_covering_tombstone_seq > 0) {
-        // The remaining files we look at will only contain covered keys, so
-        // we stop here for this key
-        file_range.SkipKey(iter);
-      }
-    }
-    switch (get_context.State()) {
-      case GetContext::kNotFound:
-        // Keep searching in other files
-        break;
-      case GetContext::kMerge:
-        // TODO: update per-level perfcontext user_key_return_count for kMerge
-        break;
-      case GetContext::kFound:
-        if (hit_file_level == 0) {
-          RecordTick(db_statistics_, GET_HIT_L0);
-        } else if (hit_file_level == 1) {
-          RecordTick(db_statistics_, GET_HIT_L1);
-        } else if (hit_file_level >= 2) {
-          RecordTick(db_statistics_, GET_HIT_L2_AND_UP);
-        }
-
-        PERF_COUNTER_BY_LEVEL_ADD(user_key_return_count, 1, hit_file_level);
-
-        file_range.MarkKeyDone(iter);
-
-        if (iter->is_blob_index) {
-          BlobIndex blob_index;
-          Status tmp_s;
-
-          if (iter->value) {
-            TEST_SYNC_POINT_CALLBACK("Version::MultiGet::TamperWithBlobIndex",
-                                     &(*iter));
-
-            tmp_s = blob_index.DecodeFrom(*(iter->value));
-
-          } else {
-            assert(iter->columns);
-
-            tmp_s = blob_index.DecodeFrom(
-                WideColumnsHelper::GetDefaultColumn(iter->columns->columns()));
-          }
-
-          if (tmp_s.ok()) {
-            const uint64_t blob_file_num = blob_index.file_number();
-            blob_ctxs[blob_file_num].emplace_back(blob_index, &*iter);
-          } else {
-            *(iter->s) = tmp_s;
-          }
-        } else {
-          if (iter->value) {
-            file_range.AddValueSize(iter->value->size());
-          } else {
-            assert(iter->columns);
-            file_range.AddValueSize(iter->columns->serialized_size());
-          }
-
-          if (file_range.GetValueSize() > read_options.value_size_soft_limit) {
-            s = Status::Aborted();
-            break;
-          }
-        }
-        continue;
-      case GetContext::kDeleted:
-        // Use empty error message for speed
-        *status = Status::NotFound();
-        file_range.MarkKeyDone(iter);
-        continue;
-      case GetContext::kCorrupt:
-        *status =
-            Status::Corruption("corrupted key for ", iter->lkey->user_key());
-        file_range.MarkKeyDone(iter);
-        continue;
-      case GetContext::kUnexpectedBlobIndex:
-        ROCKS_LOG_ERROR(info_log_, "Encounter unexpected blob index.");
-        *status = Status::NotSupported(
-            "Encounter unexpected blob index. Please open DB with "
-            "ROCKSDB_NAMESPACE::blob_db::BlobDB instead.");
-        file_range.MarkKeyDone(iter);
-        continue;
-      case GetContext::kMergeOperatorFailed:
-        *status = Status::Corruption(Status::SubCode::kMergeOperatorFailed);
-        file_range.MarkKeyDone(iter);
-        continue;
-    }
-  }
-
-  RecordInHistogram(db_statistics_, SST_BATCH_SIZE, batch_size);
-  CO_RETURN s;
-}
-}  // namespace ROCKSDB_NAMESPACE
-#endif

package/vendor/librocksdb/vendor/rocksdb/db/version_util.h

@@ -1,77 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include "db/version_set.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Instead of opening a `DB` to perform certain manifest updates, this
-// uses the underlying `VersionSet` API to read and modify the MANIFEST. This
-// allows us to use the user's real options, while not having to worry about
-// the DB persisting new SST files via flush/compaction or attempting to read/
-// compact files which may fail, particularly for the file we intend to remove
-// (the user may want to remove an already deleted file from MANIFEST).
-class OfflineManifestWriter {
- public:
-  OfflineManifestWriter(const DBOptions& options, const std::string& db_path)
-      : wc_(options.delayed_write_rate),
-        wb_(options.db_write_buffer_size),
-        immutable_db_options_(WithDbPath(options, db_path)),
-        tc_(NewLRUCache(1 << 20 /* capacity */,
-                        options.table_cache_numshardbits)),
-        versions_(db_path, &immutable_db_options_, sopt_, tc_.get(), &wb_, &wc_,
-                  /*block_cache_tracer=*/nullptr, /*io_tracer=*/nullptr,
-                  /*db_id=*/"", /*db_session_id=*/"",
-                  options.daily_offpeak_time_utc,
-                  /*error_handler=*/nullptr,
-                  /*read_only=*/false) {}
-
-  Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families) {
-    return versions_.Recover(column_families, /*read_only*/ false,
-                             /*db_id*/ nullptr,
-                             /*no_error_if_files_missing*/ true);
-  }
-
-  Status LogAndApply(const ReadOptions& read_options,
-                     const WriteOptions& write_options, ColumnFamilyData* cfd,
-                     VersionEdit* edit,
-                     FSDirectory* dir_contains_current_file) {
-    // Use `mutex` to imitate a locked DB mutex when calling `LogAndApply()`.
-    InstrumentedMutex mutex;
-    mutex.Lock();
-    Status s = versions_.LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                     read_options, write_options, edit, &mutex,
-                                     dir_contains_current_file,
-                                     false /* new_descriptor_log */);
-    mutex.Unlock();
-    return s;
-  }
-
-  VersionSet& Versions() { return versions_; }
-  const ImmutableDBOptions& IOptions() { return immutable_db_options_; }
-
- private:
-  WriteController wc_;
-  WriteBufferManager wb_;
-  ImmutableDBOptions immutable_db_options_;
-  std::shared_ptr<Cache> tc_;
-  EnvOptions sopt_;
-  VersionSet versions_;
-
-  static ImmutableDBOptions WithDbPath(const DBOptions& options,
-                                       const std::string& db_path) {
-    ImmutableDBOptions rv(options);
-    if (rv.db_paths.empty()) {
-      // `VersionSet` expects options that have been through
-      // `SanitizeOptions()`, which would sanitize an empty `db_paths`.
-      rv.db_paths.emplace_back(db_path, 0 /* target_size */);
-    }
-    return rv;
-  }
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/wal_edit.cc

@@ -1,211 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-// This source code is licensed under both the GPLv2 (found in the
-// COPYING file in the root directory) and Apache 2.0 License
-// (found in the LICENSE.Apache file in the root directory).
-
-#include "db/wal_edit.h"
-
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-void WalAddition::EncodeTo(std::string* dst) const {
-  PutVarint64(dst, number_);
-
-  if (metadata_.HasSyncedSize()) {
-    PutVarint32(dst, static_cast<uint32_t>(WalAdditionTag::kSyncedSize));
-    PutVarint64(dst, metadata_.GetSyncedSizeInBytes());
-  }
-
-  PutVarint32(dst, static_cast<uint32_t>(WalAdditionTag::kTerminate));
-}
-
-Status WalAddition::DecodeFrom(Slice* src) {
-  constexpr char class_name[] = "WalAddition";
-
-  if (!GetVarint64(src, &number_)) {
-    return Status::Corruption(class_name, "Error decoding WAL log number");
-  }
-
-  while (true) {
-    uint32_t tag_value = 0;
-    if (!GetVarint32(src, &tag_value)) {
-      return Status::Corruption(class_name, "Error decoding tag");
-    }
-    WalAdditionTag tag = static_cast<WalAdditionTag>(tag_value);
-    switch (tag) {
-      case WalAdditionTag::kSyncedSize: {
-        uint64_t size = 0;
-        if (!GetVarint64(src, &size)) {
-          return Status::Corruption(class_name, "Error decoding WAL file size");
-        }
-        metadata_.SetSyncedSizeInBytes(size);
-        break;
-      }
-      // TODO: process future tags such as checksum.
-      case WalAdditionTag::kTerminate:
-        return Status::OK();
-      default: {
-        std::stringstream ss;
-        ss << "Unknown tag " << tag_value;
-        return Status::Corruption(class_name, ss.str());
-      }
-    }
-  }
-}
-
-JSONWriter& operator<<(JSONWriter& jw, const WalAddition& wal) {
-  jw << "LogNumber" << wal.GetLogNumber() << "SyncedSizeInBytes"
-     << wal.GetMetadata().GetSyncedSizeInBytes();
-  return jw;
-}
-
-std::ostream& operator<<(std::ostream& os, const WalAddition& wal) {
-  os << "log_number: " << wal.GetLogNumber()
-     << " synced_size_in_bytes: " << wal.GetMetadata().GetSyncedSizeInBytes();
-  return os;
-}
-
-std::string WalAddition::DebugString() const {
-  std::ostringstream oss;
-  oss << *this;
-  return oss.str();
-}
-
-void WalDeletion::EncodeTo(std::string* dst) const {
-  PutVarint64(dst, number_);
-}
-
-Status WalDeletion::DecodeFrom(Slice* src) {
-  constexpr char class_name[] = "WalDeletion";
-
-  if (!GetVarint64(src, &number_)) {
-    return Status::Corruption(class_name, "Error decoding WAL log number");
-  }
-
-  return Status::OK();
-}
-
-JSONWriter& operator<<(JSONWriter& jw, const WalDeletion& wal) {
-  jw << "LogNumber" << wal.GetLogNumber();
-  return jw;
-}
-
-std::ostream& operator<<(std::ostream& os, const WalDeletion& wal) {
-  os << "log_number: " << wal.GetLogNumber();
-  return os;
-}
-
-std::string WalDeletion::DebugString() const {
-  std::ostringstream oss;
-  oss << *this;
-  return oss.str();
-}
-
-Status WalSet::AddWal(const WalAddition& wal) {
-  if (wal.GetLogNumber() < min_wal_number_to_keep_) {
-    // The WAL has been obsolete, ignore it.
-    return Status::OK();
-  }
-
-  auto it = wals_.lower_bound(wal.GetLogNumber());
-  bool existing = it != wals_.end() && it->first == wal.GetLogNumber();
-
-  if (!existing) {
-    wals_.insert(it, {wal.GetLogNumber(), wal.GetMetadata()});
-    return Status::OK();
-  }
-
-  assert(existing);
-  if (!wal.GetMetadata().HasSyncedSize()) {
-    std::stringstream ss;
-    ss << "WAL " << wal.GetLogNumber() << " is created more than once";
-    return Status::Corruption("WalSet::AddWal", ss.str());
-  }
-
-  assert(wal.GetMetadata().HasSyncedSize());
-  if (it->second.HasSyncedSize() && wal.GetMetadata().GetSyncedSizeInBytes() <=
-                                        it->second.GetSyncedSizeInBytes()) {
-    // This is possible because version edits with different synced WAL sizes
-    // for the same WAL can be committed out-of-order. For example, thread
-    // 1 synces the first 10 bytes of 1.log, while thread 2 synces the first 20
-    // bytes of 1.log. It's possible that thread 1 calls LogAndApply() after
-    // thread 2.
-    // In this case, just return ok.
-    return Status::OK();
-  }
-
-  // Update synced size for the given WAL.
-  it->second.SetSyncedSizeInBytes(wal.GetMetadata().GetSyncedSizeInBytes());
-  return Status::OK();
-}
-
-Status WalSet::AddWals(const WalAdditions& wals) {
-  Status s;
-  for (const WalAddition& wal : wals) {
-    s = AddWal(wal);
-    if (!s.ok()) {
-      break;
-    }
-  }
-  return s;
-}
-
-Status WalSet::DeleteWalsBefore(WalNumber wal) {
-  if (wal > min_wal_number_to_keep_) {
-    min_wal_number_to_keep_ = wal;
-    wals_.erase(wals_.begin(), wals_.lower_bound(wal));
-  }
-  return Status::OK();
-}
-
-void WalSet::Reset() {
-  wals_.clear();
-  min_wal_number_to_keep_ = 0;
-}
-
-Status WalSet::CheckWals(
-    Env* env,
-    const std::unordered_map<WalNumber, std::string>& logs_on_disk) const {
-  assert(env != nullptr);
-
-  Status s;
-  for (const auto& wal : wals_) {
-    const uint64_t log_number = wal.first;
-    const WalMetadata& wal_meta = wal.second;
-
-    if (!wal_meta.HasSyncedSize()) {
-      // The WAL and WAL directory is not even synced,
-      // so the WAL's inode may not be persisted,
-      // then the WAL might not show up when listing WAL directory.
-      continue;
-    }
-
-    if (logs_on_disk.find(log_number) == logs_on_disk.end()) {
-      std::stringstream ss;
-      ss << "Missing WAL with log number: " << log_number << ".";
-      s = Status::Corruption(ss.str());
-      break;
-    }
-
-    uint64_t log_file_size = 0;
-    s = env->GetFileSize(logs_on_disk.at(log_number), &log_file_size);
-    if (!s.ok()) {
-      break;
-    }
-    if (log_file_size < wal_meta.GetSyncedSizeInBytes()) {
-      std::stringstream ss;
-      ss << "Size mismatch: WAL (log number: " << log_number
-         << ") in MANIFEST is " << wal_meta.GetSyncedSizeInBytes()
-         << " bytes , but actually is " << log_file_size << " bytes on disk.";
-      s = Status::Corruption(ss.str());
-      break;
-    }
-  }
-
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/wal_edit.h

@@ -1,177 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-// This source code is licensed under both the GPLv2 (found in the
-// COPYING file in the root directory) and Apache 2.0 License
-// (found in the LICENSE.Apache file in the root directory).
-
-// WAL related classes used in VersionEdit and VersionSet.
-// Modifications to WalAddition and WalDeletion may need to update
-// VersionEdit and its related tests.
-
-#pragma once
-
-#include <map>
-#include <ostream>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "logging/event_logger.h"
-#include "port/port.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class JSONWriter;
-class Slice;
-class Status;
-
-using WalNumber = uint64_t;
-
-// Metadata of a WAL.
-class WalMetadata {
- public:
-  WalMetadata() = default;
-
-  explicit WalMetadata(uint64_t synced_size_bytes)
-      : synced_size_bytes_(synced_size_bytes) {}
-
-  bool HasSyncedSize() const { return synced_size_bytes_ != kUnknownWalSize; }
-
-  void SetSyncedSizeInBytes(uint64_t bytes) { synced_size_bytes_ = bytes; }
-
-  uint64_t GetSyncedSizeInBytes() const { return synced_size_bytes_; }
-
- private:
-  friend bool operator==(const WalMetadata& lhs, const WalMetadata& rhs);
-  friend bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs);
-  // The size of WAL is unknown, used when the WAL is not synced yet or is
-  // empty.
-  constexpr static uint64_t kUnknownWalSize =
-      std::numeric_limits<uint64_t>::max();
-
-  // Size of the most recently synced WAL in bytes.
-  uint64_t synced_size_bytes_ = kUnknownWalSize;
-};
-
-inline bool operator==(const WalMetadata& lhs, const WalMetadata& rhs) {
-  return lhs.synced_size_bytes_ == rhs.synced_size_bytes_;
-}
-
-inline bool operator!=(const WalMetadata& lhs, const WalMetadata& rhs) {
-  return !(lhs == rhs);
-}
-
-// These tags are persisted to MANIFEST, so it's part of the user API.
-enum class WalAdditionTag : uint32_t {
-  // Indicates that there are no more tags.
-  kTerminate = 1,
-  // Synced Size in bytes.
-  kSyncedSize = 2,
-  // Add tags in the future, such as checksum?
-};
-
-// Records the event of adding a WAL in VersionEdit.
-class WalAddition {
- public:
-  WalAddition() : number_(0), metadata_() {}
-
-  explicit WalAddition(WalNumber number) : number_(number), metadata_() {}
-
-  WalAddition(WalNumber number, WalMetadata meta)
-      : number_(number), metadata_(std::move(meta)) {}
-
-  WalNumber GetLogNumber() const { return number_; }
-
-  const WalMetadata& GetMetadata() const { return metadata_; }
-
-  void EncodeTo(std::string* dst) const;
-
-  Status DecodeFrom(Slice* src);
-
-  std::string DebugString() const;
-
- private:
-  WalNumber number_;
-  WalMetadata metadata_;
-};
-
-std::ostream& operator<<(std::ostream& os, const WalAddition& wal);
-JSONWriter& operator<<(JSONWriter& jw, const WalAddition& wal);
-
-using WalAdditions = std::vector<WalAddition>;
-
-// Records the event of deleting WALs before the specified log number.
-class WalDeletion {
- public:
-  WalDeletion() : number_(kEmpty) {}
-
-  explicit WalDeletion(WalNumber number) : number_(number) {}
-
-  WalNumber GetLogNumber() const { return number_; }
-
-  void EncodeTo(std::string* dst) const;
-
-  Status DecodeFrom(Slice* src);
-
-  std::string DebugString() const;
-
-  bool IsEmpty() const { return number_ == kEmpty; }
-
-  void Reset() { number_ = kEmpty; }
-
- private:
-  static constexpr WalNumber kEmpty = 0;
-
-  WalNumber number_;
-};
-
-std::ostream& operator<<(std::ostream& os, const WalDeletion& wal);
-JSONWriter& operator<<(JSONWriter& jw, const WalDeletion& wal);
-
-// Used in VersionSet to keep the current set of WALs.
-//
-// When a WAL is synced or becomes obsoleted,
-// a VersionEdit is logged to MANIFEST and
-// the WAL is added to or deleted from WalSet.
-//
-// Not thread safe, needs external synchronization such as holding DB mutex.
-class WalSet {
- public:
-  // Add WAL(s).
-  // If the WAL is closed,
-  // then there must be an existing unclosed WAL,
-  // otherwise, return Status::Corruption.
-  // Can happen when applying a VersionEdit or recovering from MANIFEST.
-  Status AddWal(const WalAddition& wal);
-  Status AddWals(const WalAdditions& wals);
-
-  // Delete WALs with log number smaller than the specified wal number.
-  // Can happen when applying a VersionEdit or recovering from MANIFEST.
-  Status DeleteWalsBefore(WalNumber wal);
-
-  // Resets the internal state.
-  void Reset();
-
-  // WALs with number less than MinWalNumberToKeep should not exist in WalSet.
-  WalNumber GetMinWalNumberToKeep() const { return min_wal_number_to_keep_; }
-
-  const std::map<WalNumber, WalMetadata>& GetWals() const { return wals_; }
-
-  // Checks whether there are missing or corrupted WALs.
-  // Returns Status::OK if there is no missing nor corrupted WAL,
-  // otherwise returns Status::Corruption.
-  // logs_on_disk is a map from log number to the log filename.
-  // Note that logs_on_disk may contain logs that is obsolete but
-  // haven't been deleted from disk.
-  Status CheckWals(
-      Env* env,
-      const std::unordered_map<WalNumber, std::string>& logs_on_disk) const;
-
- private:
-  std::map<WalNumber, WalMetadata> wals_;
-  // WAL number < min_wal_number_to_keep_ should not exist in wals_.
-  // It's monotonically increasing, in-memory only, not written to MANIFEST.
-  WalNumber min_wal_number_to_keep_ = 0;
-};
-
-}  // namespace ROCKSDB_NAMESPACE