rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_state.cc

@@ -1,46 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/compaction/compaction_state.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Slice CompactionState::SmallestUserKey() {
-  for (const auto& sub_compact_state : sub_compact_states) {
-    Slice smallest = sub_compact_state.SmallestUserKey();
-    if (!smallest.empty()) {
-      return smallest;
-    }
-  }
-  // If there is no finished output, return an empty slice.
-  return Slice{nullptr, 0};
-}
-
-Slice CompactionState::LargestUserKey() {
-  for (auto it = sub_compact_states.rbegin(); it < sub_compact_states.rend();
-       ++it) {
-    Slice largest = it->LargestUserKey();
-    if (!largest.empty()) {
-      return largest;
-    }
-  }
-  // If there is no finished output, return an empty slice.
-  return Slice{nullptr, 0};
-}
-
-void CompactionState::AggregateCompactionStats(
-    InternalStats::CompactionStatsFull& compaction_stats,
-    CompactionJobStats& compaction_job_stats) {
-  for (const auto& sc : sub_compact_states) {
-    sc.AggregateCompactionStats(compaction_stats);
-    compaction_job_stats.Add(sc.compaction_job_stats);
-  }
-}
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_state.h

@@ -1,42 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#include "db/compaction/compaction.h"
-#include "db/compaction/subcompaction_state.h"
-#include "db/internal_stats.h"
-
-// Data structures used for compaction_job and compaction_service_job which has
-// the list of sub_compact_states and the aggregated information for the
-// compaction.
-namespace ROCKSDB_NAMESPACE {
-
-// Maintains state for the entire compaction
-class CompactionState {
- public:
-  Compaction* const compaction;
-
-  // REQUIRED: subcompaction states are stored in order of increasing key-range
-  std::vector<SubcompactionState> sub_compact_states;
-  Status status;
-
-  void AggregateCompactionStats(
-      InternalStats::CompactionStatsFull& compaction_stats,
-      CompactionJobStats& compaction_job_stats);
-
-  explicit CompactionState(Compaction* c) : compaction(c) {}
-
-  Slice SmallestUserKey();
-
-  Slice LargestUserKey();
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/compaction/file_pri.h

@@ -1,94 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#pragma once
-#include <algorithm>
-
-#include "db/version_edit.h"
-
-namespace ROCKSDB_NAMESPACE {
-// We boost files that are closer to TTL limit. This boosting could be
-// through FileMetaData.compensated_file_size but this compensated size
-// is widely used as something similar to file size so dramatically boost
-// the value might cause unintended consequences.
-//
-// This boosting algorithm can go very fancy, but here we use a simple
-// formula which can satisify:
-// (1) Different levels are triggered slightly differently to avoid
-//     too many cascading cases
-// (2) Files in the same level get boosting more when TTL gets closer.
-//
-// Don't do any boosting before TTL has past by half. This is to make
-// sure lower write amp for most of the case. And all levels should be
-// fully boosted when total TTL compaction threshold triggers.
-// Differientiate boosting ranges of each level by 1/2. This will make
-// range for each level exponentially increasing. We could do it by
-// having them to be equal, or go even fancier. We can adjust it after
-// we observe the behavior in production.
-// The threshold starting boosting:
-// +------------------------------------------------------------------ +
-// ^                            ^   ^     ^       ^                 ^
-// Age 0                        ... |     |    second last level    thresold
-//                                  |     |
-//                                  |  third last level
-//                                  |
-//                            forth last level
-//
-// We arbitrarily set with 0 when a file is aged boost_age_start and
-// grow linearly. The ratio is arbitrarily set so that when the next level
-// starts to boost, the previous level's boosting amount is 16.
-class FileTtlBooster {
- public:
-  FileTtlBooster(uint64_t current_time, uint64_t ttl, int num_non_empty_levels,
-                 int level)
-      : current_time_(current_time) {
-    if (ttl == 0 || level == 0 || level >= num_non_empty_levels - 1) {
-      enabled_ = false;
-      boost_age_start_ = 0;
-      boost_step_ = 1;
-    } else {
-      enabled_ = true;
-      uint64_t all_boost_start_age = ttl / 2;
-      uint64_t all_boost_age_range = (ttl / 32) * 31 - all_boost_start_age;
-      // TODO(cbi): more reasonable algorithm that gives different values
-      //   when num_non_empty_levels - level - 1 > 63.
-      uint64_t boost_age_range =
-          all_boost_age_range >> std::min(63, num_non_empty_levels - level - 1);
-      boost_age_start_ = all_boost_start_age + boost_age_range;
-      const uint64_t kBoostRatio = 16;
-      // prevent 0 value to avoid divide 0 error.
-      boost_step_ = std::max(boost_age_range / kBoostRatio, uint64_t{1});
-    }
-  }
-
-  uint64_t GetBoostScore(FileMetaData* f) {
-    if (!enabled_) {
-      return 1;
-    }
-    uint64_t oldest_ancester_time = f->TryGetOldestAncesterTime();
-    if (oldest_ancester_time >= current_time_) {
-      return 1;
-    }
-    uint64_t age = current_time_ - oldest_ancester_time;
-    if (age > boost_age_start_) {
-      // Use integer just for convenience.
-      // We could make all file_to_order double if we want.
-      // Technically this can overflow if users override timing and
-      // give a very high current time. Ignore the case for simplicity.
-      // Boosting is addition to current value, so +1. This will effectively
-      // make boosting to kick in after the first boost_step_ is reached.
-      return (age - boost_age_start_) / boost_step_ + 1;
-    }
-    return 1;
-  }
-
- private:
-  bool enabled_;
-  uint64_t current_time_;
-  uint64_t boost_age_start_;
-  uint64_t boost_step_;
-};
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/compaction/sst_partitioner.cc

@@ -1,83 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#include "rocksdb/sst_partitioner.h"
-
-#include <algorithm>
-
-#include "rocksdb/utilities/customizable_util.h"
-#include "rocksdb/utilities/object_registry.h"
-#include "rocksdb/utilities/options_type.h"
-
-namespace ROCKSDB_NAMESPACE {
-static std::unordered_map<std::string, OptionTypeInfo>
-    sst_fixed_prefix_type_info = {
-        {"length",
-         {0, OptionType::kSizeT, OptionVerificationType::kNormal,
-          OptionTypeFlags::kNone}},
-};
-
-SstPartitionerFixedPrefixFactory::SstPartitionerFixedPrefixFactory(size_t len)
-    : len_(len) {
-  RegisterOptions("Length", &len_, &sst_fixed_prefix_type_info);
-}
-
-PartitionerResult SstPartitionerFixedPrefix::ShouldPartition(
-    const PartitionerRequest& request) {
-  Slice last_key_fixed(*request.prev_user_key);
-  if (last_key_fixed.size() > len_) {
-    last_key_fixed.size_ = len_;
-  }
-  Slice current_key_fixed(*request.current_user_key);
-  if (current_key_fixed.size() > len_) {
-    current_key_fixed.size_ = len_;
-  }
-  return last_key_fixed.compare(current_key_fixed) != 0 ? kRequired
-                                                        : kNotRequired;
-}
-
-bool SstPartitionerFixedPrefix::CanDoTrivialMove(
-    const Slice& smallest_user_key, const Slice& largest_user_key) {
-  return ShouldPartition(PartitionerRequest(smallest_user_key, largest_user_key,
-                                            0)) == kNotRequired;
-}
-
-std::unique_ptr<SstPartitioner>
-SstPartitionerFixedPrefixFactory::CreatePartitioner(
-    const SstPartitioner::Context& /* context */) const {
-  return std::unique_ptr<SstPartitioner>(new SstPartitionerFixedPrefix(len_));
-}
-
-std::shared_ptr<SstPartitionerFactory> NewSstPartitionerFixedPrefixFactory(
-    size_t prefix_len) {
-  return std::make_shared<SstPartitionerFixedPrefixFactory>(prefix_len);
-}
-
-namespace {
-static int RegisterSstPartitionerFactories(ObjectLibrary& library,
-                                           const std::string& /*arg*/) {
-  library.AddFactory<SstPartitionerFactory>(
-      SstPartitionerFixedPrefixFactory::kClassName(),
-      [](const std::string& /*uri*/,
-         std::unique_ptr<SstPartitionerFactory>* guard,
-         std::string* /* errmsg */) {
-        guard->reset(new SstPartitionerFixedPrefixFactory(0));
-        return guard->get();
-      });
-  return 1;
-}
-}  // namespace
-
-Status SstPartitionerFactory::CreateFromString(
-    const ConfigOptions& options, const std::string& value,
-    std::shared_ptr<SstPartitionerFactory>* result) {
-  static std::once_flag once;
-  std::call_once(once, [&]() {
-    RegisterSstPartitionerFactories(*(ObjectLibrary::Default().get()), "");
-  });
-  return LoadSharedObject<SstPartitionerFactory>(options, value, result);
-}
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/compaction/subcompaction_state.cc

@@ -1,106 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/compaction/subcompaction_state.h"
-
-#include "rocksdb/sst_partitioner.h"
-
-namespace ROCKSDB_NAMESPACE {
-void SubcompactionState::AggregateCompactionStats(
-    InternalStats::CompactionStatsFull& compaction_stats) const {
-  compaction_stats.stats.Add(compaction_outputs_.stats_);
-  if (HasPenultimateLevelOutputs()) {
-    compaction_stats.has_penultimate_level_output = true;
-    compaction_stats.penultimate_level_stats.Add(
-        penultimate_level_outputs_.stats_);
-  }
-}
-
-OutputIterator SubcompactionState::GetOutputs() const {
-  return OutputIterator(penultimate_level_outputs_.outputs_,
-                        compaction_outputs_.outputs_);
-}
-
-void SubcompactionState::Cleanup(Cache* cache) {
-  penultimate_level_outputs_.Cleanup();
-  compaction_outputs_.Cleanup();
-
-  if (!status.ok()) {
-    for (const auto& out : GetOutputs()) {
-      // If this file was inserted into the table cache then remove
-      // them here because this compaction was not committed.
-      TableCache::Evict(cache, out.meta.fd.GetNumber());
-    }
-  }
-  // TODO: sub_compact.io_status is not checked like status. Not sure if thats
-  // intentional. So ignoring the io_status as of now.
-  io_status.PermitUncheckedError();
-}
-
-Slice SubcompactionState::SmallestUserKey() const {
-  if (has_penultimate_level_outputs_) {
-    Slice a = compaction_outputs_.SmallestUserKey();
-    Slice b = penultimate_level_outputs_.SmallestUserKey();
-    if (a.empty()) {
-      return b;
-    }
-    if (b.empty()) {
-      return a;
-    }
-    const Comparator* user_cmp =
-        compaction->column_family_data()->user_comparator();
-    if (user_cmp->Compare(a, b) > 0) {
-      return b;
-    } else {
-      return a;
-    }
-  } else {
-    return compaction_outputs_.SmallestUserKey();
-  }
-}
-
-Slice SubcompactionState::LargestUserKey() const {
-  if (has_penultimate_level_outputs_) {
-    Slice a = compaction_outputs_.LargestUserKey();
-    Slice b = penultimate_level_outputs_.LargestUserKey();
-    if (a.empty()) {
-      return b;
-    }
-    if (b.empty()) {
-      return a;
-    }
-    const Comparator* user_cmp =
-        compaction->column_family_data()->user_comparator();
-    if (user_cmp->Compare(a, b) < 0) {
-      return b;
-    } else {
-      return a;
-    }
-  } else {
-    return compaction_outputs_.LargestUserKey();
-  }
-}
-
-Status SubcompactionState::AddToOutput(
-    const CompactionIterator& iter,
-    const CompactionFileOpenFunc& open_file_func,
-    const CompactionFileCloseFunc& close_file_func) {
-  // update target output first
-  is_current_penultimate_level_ = iter.output_to_penultimate_level();
-  current_outputs_ = is_current_penultimate_level_ ? &penultimate_level_outputs_
-                                                   : &compaction_outputs_;
-  if (is_current_penultimate_level_) {
-    has_penultimate_level_outputs_ = true;
-  }
-
-  return Current().AddToOutput(iter, open_file_func, close_file_func);
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/compaction/subcompaction_state.h

@@ -1,220 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#include <optional>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_garbage_meter.h"
-#include "db/compaction/compaction.h"
-#include "db/compaction/compaction_iterator.h"
-#include "db/compaction/compaction_outputs.h"
-#include "db/internal_stats.h"
-#include "db/output_validator.h"
-#include "db/range_del_aggregator.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Maintains state and outputs for each sub-compaction
-// It contains 2 `CompactionOutputs`:
-//  1. one for the normal output files
-//  2. another for the penultimate level outputs
-// a `current` pointer maintains the current output group, when calling
-// `AddToOutput()`, it checks the output of the current compaction_iterator key
-// and point `current` to the target output group. By default, it just points to
-// normal compaction_outputs, if the compaction_iterator key should be placed on
-// the penultimate level, `current` is changed to point to
-// `penultimate_level_outputs`.
-// The later operations uses `Current()` to get the target group.
-//
-// +----------+          +-----------------------------+      +---------+
-// | *current |--------> | compaction_outputs          |----->| output  |
-// +----------+          +-----------------------------+      +---------+
-//       |                                                    | output  |
-//       |                                                    +---------+
-//       |                                                    |  ...    |
-//       |
-//       |               +-----------------------------+      +---------+
-//       +-------------> | penultimate_level_outputs   |----->| output  |
-//                       +-----------------------------+      +---------+
-//                                                            |  ...    |
-
-class SubcompactionState {
- public:
-  const Compaction* compaction;
-
-  // The boundaries of the key-range this compaction is interested in. No two
-  // sub-compactions may have overlapping key-ranges.
-  // 'start' is inclusive, 'end' is exclusive, and nullptr means unbounded
-  const std::optional<Slice> start, end;
-
-  // The return status of this sub-compaction
-  Status status;
-
-  // The return IO Status of this sub-compaction
-  IOStatus io_status;
-
-  // Notify on sub-compaction completion only if listener was notified on
-  // sub-compaction begin.
-  bool notify_on_subcompaction_completion = false;
-
-  // compaction job stats for this sub-compaction
-  CompactionJobStats compaction_job_stats;
-
-  // sub-compaction job id, which is used to identify different sub-compaction
-  // within the same compaction job.
-  const uint32_t sub_job_id;
-
-  Slice SmallestUserKey() const;
-
-  Slice LargestUserKey() const;
-
-  // Get all outputs from the subcompaction. For per_key_placement compaction,
-  // it returns both the last level outputs and penultimate level outputs.
-  OutputIterator GetOutputs() const;
-
-  // Assign range dels aggregator, for each range_del, it can only be assigned
-  // to one output level, for per_key_placement, it's going to be the
-  // penultimate level.
-  // TODO: This does not work for per_key_placement + user-defined timestamp +
-  //  DeleteRange() combo. If user-defined timestamp is enabled,
-  //  it is possible for a range tombstone to belong to bottommost level (
-  //  seqno < earliest snapshot) without being dropped (garbage collection
-  //  for user-defined timestamp).
-  void AssignRangeDelAggregator(
-      std::unique_ptr<CompactionRangeDelAggregator>&& range_del_agg) {
-    if (compaction->SupportsPerKeyPlacement()) {
-      penultimate_level_outputs_.AssignRangeDelAggregator(
-          std::move(range_del_agg));
-    } else {
-      compaction_outputs_.AssignRangeDelAggregator(std::move(range_del_agg));
-    }
-  }
-
-  void RemoveLastEmptyOutput() {
-    compaction_outputs_.RemoveLastEmptyOutput();
-    penultimate_level_outputs_.RemoveLastEmptyOutput();
-  }
-
-  void BuildSubcompactionJobInfo(
-      SubcompactionJobInfo& subcompaction_job_info) const {
-    const Compaction* c = compaction;
-    const ColumnFamilyData* cfd = c->column_family_data();
-
-    subcompaction_job_info.cf_id = cfd->GetID();
-    subcompaction_job_info.cf_name = cfd->GetName();
-    subcompaction_job_info.status = status;
-    subcompaction_job_info.subcompaction_job_id = static_cast<int>(sub_job_id);
-    subcompaction_job_info.base_input_level = c->start_level();
-    subcompaction_job_info.output_level = c->output_level();
-    subcompaction_job_info.stats = compaction_job_stats;
-  }
-
-  SubcompactionState() = delete;
-  SubcompactionState(const SubcompactionState&) = delete;
-  SubcompactionState& operator=(const SubcompactionState&) = delete;
-
-  SubcompactionState(Compaction* c, const std::optional<Slice> _start,
-                     const std::optional<Slice> _end, uint32_t _sub_job_id)
-      : compaction(c),
-        start(_start),
-        end(_end),
-        sub_job_id(_sub_job_id),
-        compaction_outputs_(c, /*is_penultimate_level=*/false),
-        penultimate_level_outputs_(c, /*is_penultimate_level=*/true) {
-    assert(compaction != nullptr);
-    // Set output split key (used for RoundRobin feature) only for normal
-    // compaction_outputs, output to penultimate_level feature doesn't support
-    // RoundRobin feature (and may never going to be supported, because for
-    // RoundRobin, the data time is mostly naturally sorted, no need to have
-    // per-key placement with output_to_penultimate_level).
-    compaction_outputs_.SetOutputSlitKey(start, end);
-  }
-
-  SubcompactionState(SubcompactionState&& state) noexcept
-      : compaction(state.compaction),
-        start(state.start),
-        end(state.end),
-        status(std::move(state.status)),
-        io_status(std::move(state.io_status)),
-        notify_on_subcompaction_completion(
-            state.notify_on_subcompaction_completion),
-        compaction_job_stats(std::move(state.compaction_job_stats)),
-        sub_job_id(state.sub_job_id),
-        compaction_outputs_(std::move(state.compaction_outputs_)),
-        penultimate_level_outputs_(std::move(state.penultimate_level_outputs_)),
-        is_current_penultimate_level_(state.is_current_penultimate_level_),
-        has_penultimate_level_outputs_(state.has_penultimate_level_outputs_) {
-    current_outputs_ = is_current_penultimate_level_
-                           ? &penultimate_level_outputs_
-                           : &compaction_outputs_;
-  }
-
-  bool HasPenultimateLevelOutputs() const {
-    return has_penultimate_level_outputs_ ||
-           penultimate_level_outputs_.HasRangeDel();
-  }
-
-  bool IsCurrentPenultimateLevel() const {
-    return is_current_penultimate_level_;
-  }
-
-  // Add all the new files from this compaction to version_edit
-  void AddOutputsEdit(VersionEdit* out_edit) const {
-    for (const auto& file : penultimate_level_outputs_.outputs_) {
-      out_edit->AddFile(compaction->GetPenultimateLevel(), file.meta);
-    }
-    for (const auto& file : compaction_outputs_.outputs_) {
-      out_edit->AddFile(compaction->output_level(), file.meta);
-    }
-  }
-
-  void Cleanup(Cache* cache);
-
-  void AggregateCompactionStats(
-      InternalStats::CompactionStatsFull& compaction_stats) const;
-
-  CompactionOutputs& Current() const {
-    assert(current_outputs_);
-    return *current_outputs_;
-  }
-
-  // Add compaction_iterator key/value to the `Current` output group.
-  Status AddToOutput(const CompactionIterator& iter,
-                     const CompactionFileOpenFunc& open_file_func,
-                     const CompactionFileCloseFunc& close_file_func);
-
-  // Close all compaction output files, both output_to_penultimate_level outputs
-  // and normal outputs.
-  Status CloseCompactionFiles(const Status& curr_status,
-                              const CompactionFileOpenFunc& open_file_func,
-                              const CompactionFileCloseFunc& close_file_func) {
-    // Call FinishCompactionOutputFile() even if status is not ok: it needs to
-    // close the output file.
-    // CloseOutput() may open new compaction output files.
-    is_current_penultimate_level_ = true;
-    Status s = penultimate_level_outputs_.CloseOutput(
-        curr_status, open_file_func, close_file_func);
-    is_current_penultimate_level_ = false;
-    s = compaction_outputs_.CloseOutput(s, open_file_func, close_file_func);
-    return s;
-  }
-
- private:
-  // State kept for output being generated
-  CompactionOutputs compaction_outputs_;
-  CompactionOutputs penultimate_level_outputs_;
-  CompactionOutputs* current_outputs_ = &compaction_outputs_;
-  bool is_current_penultimate_level_ = false;
-  bool has_penultimate_level_outputs_ = false;
-};
-
-}  // namespace ROCKSDB_NAMESPACE