rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_compaction_filter.h

@@ -1,202 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <unordered_set>
-
-#include "db/blob/blob_index.h"
-#include "monitoring/statistics_impl.h"
-#include "rocksdb/compaction_filter.h"
-#include "utilities/blob_db/blob_db_gc_stats.h"
-#include "utilities/blob_db/blob_db_impl.h"
-#include "utilities/compaction_filters/layered_compaction_filter_base.h"
-
-namespace ROCKSDB_NAMESPACE {
-class SystemClock;
-namespace blob_db {
-
-struct BlobCompactionContext {
-  BlobDBImpl* blob_db_impl = nullptr;
-  uint64_t next_file_number = 0;
-  std::unordered_set<uint64_t> current_blob_files;
-  SequenceNumber fifo_eviction_seq = 0;
-  uint64_t evict_expiration_up_to = 0;
-};
-
-struct BlobCompactionContextGC {
-  uint64_t cutoff_file_number = 0;
-};
-
-// Compaction filter that deletes expired blob indexes from the base DB.
-// Comes into two varieties, one for the non-GC case and one for the GC case.
-class BlobIndexCompactionFilterBase : public LayeredCompactionFilterBase {
- public:
-  BlobIndexCompactionFilterBase(
-      BlobCompactionContext&& _context,
-      const CompactionFilter* _user_comp_filter,
-      std::unique_ptr<const CompactionFilter> _user_comp_filter_from_factory,
-      uint64_t current_time, Statistics* stats)
-      : LayeredCompactionFilterBase(_user_comp_filter,
-                                    std::move(_user_comp_filter_from_factory)),
-        context_(std::move(_context)),
-        current_time_(current_time),
-        statistics_(stats) {}
-
-  ~BlobIndexCompactionFilterBase() override;
-
-  // Filter expired blob indexes regardless of snapshots.
-  bool IgnoreSnapshots() const override { return true; }
-
-  Decision FilterV2(int level, const Slice& key, ValueType value_type,
-                    const Slice& value, std::string* new_value,
-                    std::string* skip_until) const override;
-
-  bool IsStackedBlobDbInternalCompactionFilter() const override { return true; }
-
- protected:
-  bool IsBlobFileOpened() const;
-  virtual bool OpenNewBlobFileIfNeeded() const;
-  bool ReadBlobFromOldFile(const Slice& key, const BlobIndex& blob_index,
-                           PinnableSlice* blob, bool need_decompress,
-                           CompressionType* compression_type) const;
-  bool WriteBlobToNewFile(const Slice& key, const Slice& blob,
-                          uint64_t* new_blob_file_number,
-                          uint64_t* new_blob_offset) const;
-  bool CloseAndRegisterNewBlobFileIfNeeded() const;
-  bool CloseAndRegisterNewBlobFile() const;
-
-  Statistics* statistics() const { return statistics_; }
-  const BlobCompactionContext& context() const { return context_; }
-
- private:
-  Decision HandleValueChange(const Slice& key, std::string* new_value) const;
-
- private:
-  BlobCompactionContext context_;
-  const uint64_t current_time_;
-  Statistics* statistics_;
-
-  mutable std::shared_ptr<BlobFile> blob_file_;
-  mutable std::shared_ptr<BlobLogWriter> writer_;
-
-  // It is safe to not using std::atomic since the compaction filter, created
-  // from a compaction filter factroy, will not be called from multiple threads.
-  mutable uint64_t expired_count_ = 0;
-  mutable uint64_t expired_size_ = 0;
-  mutable uint64_t evicted_count_ = 0;
-  mutable uint64_t evicted_size_ = 0;
-};
-
-class BlobIndexCompactionFilter : public BlobIndexCompactionFilterBase {
- public:
-  BlobIndexCompactionFilter(
-      BlobCompactionContext&& _context,
-      const CompactionFilter* _user_comp_filter,
-      std::unique_ptr<const CompactionFilter> _user_comp_filter_from_factory,
-      uint64_t current_time, Statistics* stats)
-      : BlobIndexCompactionFilterBase(std::move(_context), _user_comp_filter,
-                                      std::move(_user_comp_filter_from_factory),
-                                      current_time, stats) {}
-
-  const char* Name() const override { return "BlobIndexCompactionFilter"; }
-};
-
-class BlobIndexCompactionFilterGC : public BlobIndexCompactionFilterBase {
- public:
-  BlobIndexCompactionFilterGC(
-      BlobCompactionContext&& _context, BlobCompactionContextGC&& context_gc,
-      const CompactionFilter* _user_comp_filter,
-      std::unique_ptr<const CompactionFilter> _user_comp_filter_from_factory,
-      uint64_t current_time, Statistics* stats)
-      : BlobIndexCompactionFilterBase(std::move(_context), _user_comp_filter,
-                                      std::move(_user_comp_filter_from_factory),
-                                      current_time, stats),
-        context_gc_(std::move(context_gc)) {}
-
-  ~BlobIndexCompactionFilterGC() override;
-
-  const char* Name() const override { return "BlobIndexCompactionFilterGC"; }
-
-  BlobDecision PrepareBlobOutput(const Slice& key, const Slice& existing_value,
-                                 std::string* new_value) const override;
-
- private:
-  bool OpenNewBlobFileIfNeeded() const override;
-
- private:
-  BlobCompactionContextGC context_gc_;
-  mutable BlobDBGarbageCollectionStats gc_stats_;
-};
-
-// Compaction filter factory; similarly to the filters above, it comes
-// in two flavors, one that creates filters that support GC, and one
-// that creates non-GC filters.
-class BlobIndexCompactionFilterFactoryBase : public CompactionFilterFactory {
- public:
-  BlobIndexCompactionFilterFactoryBase(BlobDBImpl* _blob_db_impl,
-                                       SystemClock* _clock,
-                                       const ColumnFamilyOptions& _cf_options,
-                                       Statistics* _statistics)
-      : blob_db_impl_(_blob_db_impl),
-        clock_(_clock),
-        statistics_(_statistics),
-        user_comp_filter_(_cf_options.compaction_filter),
-        user_comp_filter_factory_(_cf_options.compaction_filter_factory) {}
-
- protected:
-  std::unique_ptr<CompactionFilter> CreateUserCompactionFilterFromFactory(
-      const CompactionFilter::Context& context) const;
-
-  BlobDBImpl* blob_db_impl() const { return blob_db_impl_; }
-  SystemClock* clock() const { return clock_; }
-  Statistics* statistics() const { return statistics_; }
-  const CompactionFilter* user_comp_filter() const { return user_comp_filter_; }
-
- private:
-  BlobDBImpl* blob_db_impl_;
-  SystemClock* clock_;
-  Statistics* statistics_;
-  const CompactionFilter* user_comp_filter_;
-  std::shared_ptr<CompactionFilterFactory> user_comp_filter_factory_;
-};
-
-class BlobIndexCompactionFilterFactory
-    : public BlobIndexCompactionFilterFactoryBase {
- public:
-  BlobIndexCompactionFilterFactory(BlobDBImpl* _blob_db_impl,
-                                   SystemClock* _clock,
-                                   const ColumnFamilyOptions& _cf_options,
-                                   Statistics* _statistics)
-      : BlobIndexCompactionFilterFactoryBase(_blob_db_impl, _clock, _cf_options,
-                                             _statistics) {}
-
-  const char* Name() const override {
-    return "BlobIndexCompactionFilterFactory";
-  }
-
-  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
-      const CompactionFilter::Context& context) override;
-};
-
-class BlobIndexCompactionFilterFactoryGC
-    : public BlobIndexCompactionFilterFactoryBase {
- public:
-  BlobIndexCompactionFilterFactoryGC(BlobDBImpl* _blob_db_impl,
-                                     SystemClock* _clock,
-                                     const ColumnFamilyOptions& _cf_options,
-                                     Statistics* _statistics)
-      : BlobIndexCompactionFilterFactoryBase(_blob_db_impl, _clock, _cf_options,
-                                             _statistics) {}
-
-  const char* Name() const override {
-    return "BlobIndexCompactionFilterFactoryGC";
-  }
-
-  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
-      const CompactionFilter::Context& context) override;
-};
-
-}  // namespace blob_db
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_db.cc

@@ -1,109 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#include "utilities/blob_db/blob_db.h"
-
-#include <cinttypes>
-
-#include "logging/logging.h"
-#include "utilities/blob_db/blob_db_impl.h"
-
-namespace ROCKSDB_NAMESPACE::blob_db {
-
-Status BlobDB::Open(const Options& options, const BlobDBOptions& bdb_options,
-                    const std::string& dbname, BlobDB** blob_db) {
-  *blob_db = nullptr;
-  DBOptions db_options(options);
-  ColumnFamilyOptions cf_options(options);
-  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.emplace_back(kDefaultColumnFamilyName, cf_options);
-  std::vector<ColumnFamilyHandle*> handles;
-  Status s = BlobDB::Open(db_options, bdb_options, dbname, column_families,
-                          &handles, blob_db);
-  if (s.ok()) {
-    assert(handles.size() == 1);
-    // i can delete the handle since DBImpl is always holding a reference to
-    // default column family
-    delete handles[0];
-  }
-  return s;
-}
-
-Status BlobDB::Open(const DBOptions& db_options,
-                    const BlobDBOptions& bdb_options, const std::string& dbname,
-                    const std::vector<ColumnFamilyDescriptor>& column_families,
-                    std::vector<ColumnFamilyHandle*>* handles,
-                    BlobDB** blob_db) {
-  assert(handles);
-
-  if (column_families.size() != 1 ||
-      column_families[0].name != kDefaultColumnFamilyName) {
-    return Status::NotSupported(
-        "Blob DB doesn't support non-default column family.");
-  }
-
-  BlobDBImpl* blob_db_impl = new BlobDBImpl(dbname, bdb_options, db_options,
-                                            column_families[0].options);
-  Status s = blob_db_impl->Open(handles);
-  if (s.ok()) {
-    *blob_db = static_cast<BlobDB*>(blob_db_impl);
-  } else {
-    if (!handles->empty()) {
-      for (ColumnFamilyHandle* cfh : *handles) {
-        blob_db_impl->DestroyColumnFamilyHandle(cfh);
-      }
-
-      handles->clear();
-    }
-
-    delete blob_db_impl;
-    *blob_db = nullptr;
-  }
-  return s;
-}
-
-BlobDB::BlobDB() : StackableDB(nullptr) {}
-
-void BlobDBOptions::Dump(Logger* log) const {
-  ROCKS_LOG_HEADER(
-      log, "                                  BlobDBOptions.blob_dir: %s",
-      blob_dir.c_str());
-  ROCKS_LOG_HEADER(
-      log, "                             BlobDBOptions.path_relative: %d",
-      path_relative);
-  ROCKS_LOG_HEADER(
-      log, "                                   BlobDBOptions.is_fifo: %d",
-      is_fifo);
-  ROCKS_LOG_HEADER(
-      log, "                               BlobDBOptions.max_db_size: %" PRIu64,
-      max_db_size);
-  ROCKS_LOG_HEADER(
-      log, "                            BlobDBOptions.ttl_range_secs: %" PRIu64,
-      ttl_range_secs);
-  ROCKS_LOG_HEADER(
-      log, "                             BlobDBOptions.min_blob_size: %" PRIu64,
-      min_blob_size);
-  ROCKS_LOG_HEADER(
-      log, "                            BlobDBOptions.bytes_per_sync: %" PRIu64,
-      bytes_per_sync);
-  ROCKS_LOG_HEADER(
-      log, "                            BlobDBOptions.blob_file_size: %" PRIu64,
-      blob_file_size);
-  ROCKS_LOG_HEADER(
-      log, "                               BlobDBOptions.compression: %d",
-      static_cast<int>(compression));
-  ROCKS_LOG_HEADER(
-      log, "                 BlobDBOptions.enable_garbage_collection: %d",
-      enable_garbage_collection);
-  ROCKS_LOG_HEADER(
-      log, "                 BlobDBOptions.garbage_collection_cutoff: %f",
-      garbage_collection_cutoff);
-  ROCKS_LOG_HEADER(
-      log, "                  BlobDBOptions.disable_background_tasks: %d",
-      disable_background_tasks);
-}
-
-}  // namespace ROCKSDB_NAMESPACE::blob_db

package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_db.h

@@ -1,231 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-
-#include <functional>
-#include <limits>
-#include <string>
-#include <vector>
-
-#include "rocksdb/db.h"
-#include "rocksdb/status.h"
-#include "rocksdb/utilities/stackable_db.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-namespace blob_db {
-
-// A wrapped database which puts values of KV pairs in a separate log
-// and store location to the log in the underlying DB.
-//
-// The factory needs to be moved to include/rocksdb/utilities to allow
-// users to use blob DB.
-
-constexpr uint64_t kNoExpiration = std::numeric_limits<uint64_t>::max();
-
-struct BlobDBOptions {
-  // Name of the directory under the base DB where blobs will be stored. Using
-  // a directory where the base DB stores its SST files is not supported.
-  // Default is "blob_dir"
-  std::string blob_dir = "blob_dir";
-
-  // whether the blob_dir path is relative or absolute.
-  bool path_relative = true;
-
-  // When max_db_size is reached, evict blob files to free up space
-  // instead of returnning NoSpace error on write. Blob files will be
-  // evicted from oldest to newest, based on file creation time.
-  bool is_fifo = false;
-
-  // Maximum size of the database (including SST files and blob files).
-  //
-  // Default: 0 (no limits)
-  uint64_t max_db_size = 0;
-
-  // a new bucket is opened, for ttl_range. So if ttl_range is 600seconds
-  // (10 minutes), and the first bucket starts at 1471542000
-  // then the blob buckets will be
-  // first bucket is 1471542000 - 1471542600
-  // second bucket is 1471542600 - 1471543200
-  // and so on
-  uint64_t ttl_range_secs = 3600;
-
-  // The smallest value to store in blob log. Values smaller than this threshold
-  // will be inlined in base DB together with the key.
-  uint64_t min_blob_size = 0;
-
-  // Allows OS to incrementally sync blob files to disk for every
-  // bytes_per_sync bytes written. Users shouldn't rely on it for
-  // persistency guarantee.
-  uint64_t bytes_per_sync = 512 * 1024;
-
-  // the target size of each blob file. File will become immutable
-  // after it exceeds that size
-  uint64_t blob_file_size = 256 * 1024 * 1024;
-
-  // what compression to use for Blob's
-  CompressionType compression = kNoCompression;
-
-  // If enabled, BlobDB cleans up stale blobs in non-TTL files during compaction
-  // by rewriting the remaining live blobs to new files.
-  bool enable_garbage_collection = false;
-
-  // The cutoff in terms of blob file age for garbage collection. Blobs in
-  // the oldest N non-TTL blob files will be rewritten when encountered during
-  // compaction, where N = garbage_collection_cutoff * number_of_non_TTL_files.
-  double garbage_collection_cutoff = 0.25;
-
-  // Disable all background job. Used for test only.
-  bool disable_background_tasks = false;
-
-  void Dump(Logger* log) const;
-};
-
-class BlobDB : public StackableDB {
- public:
-  using ROCKSDB_NAMESPACE::StackableDB::Put;
-  Status Put(const WriteOptions& options, const Slice& key,
-             const Slice& value) override = 0;
-  Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family,
-             const Slice& key, const Slice& value) override {
-    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
-      return Status::NotSupported(
-          "Blob DB doesn't support non-default column family.");
-    }
-    return Put(options, key, value);
-  }
-
-  using ROCKSDB_NAMESPACE::StackableDB::Delete;
-  Status Delete(const WriteOptions& options, ColumnFamilyHandle* column_family,
-                const Slice& key) override {
-    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
-      return Status::NotSupported(
-          "Blob DB doesn't support non-default column family.");
-    }
-    assert(db_ != nullptr);
-    return db_->Delete(options, column_family, key);
-  }
-
-  virtual Status PutWithTTL(const WriteOptions& options, const Slice& key,
-                            const Slice& value, uint64_t ttl) = 0;
-  virtual Status PutWithTTL(const WriteOptions& options,
-                            ColumnFamilyHandle* column_family, const Slice& key,
-                            const Slice& value, uint64_t ttl) {
-    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
-      return Status::NotSupported(
-          "Blob DB doesn't support non-default column family.");
-    }
-    return PutWithTTL(options, key, value, ttl);
-  }
-
-  // Put with expiration. Key with expiration time equal to
-  // std::numeric_limits<uint64_t>::max() means the key don't expire.
-  virtual Status PutUntil(const WriteOptions& options, const Slice& key,
-                          const Slice& value, uint64_t expiration) = 0;
-  virtual Status PutUntil(const WriteOptions& options,
-                          ColumnFamilyHandle* column_family, const Slice& key,
-                          const Slice& value, uint64_t expiration) {
-    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
-      return Status::NotSupported(
-          "Blob DB doesn't support non-default column family.");
-    }
-    return PutUntil(options, key, value, expiration);
-  }
-
-  using ROCKSDB_NAMESPACE::StackableDB::Get;
-  Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
-             const Slice& key, PinnableSlice* value,
-             std::string* timestamp) override = 0;
-
-  // Get value and expiration.
-  virtual Status Get(const ReadOptions& options,
-                     ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value, uint64_t* expiration) = 0;
-  virtual Status Get(const ReadOptions& options, const Slice& key,
-                     PinnableSlice* value, uint64_t* expiration) {
-    return Get(options, DefaultColumnFamily(), key, value, expiration);
-  }
-
-  using ROCKSDB_NAMESPACE::StackableDB::SingleDelete;
-  Status SingleDelete(const WriteOptions& /*wopts*/,
-                      ColumnFamilyHandle* /*column_family*/,
-                      const Slice& /*key*/) override {
-    return Status::NotSupported("Not supported operation in blob db.");
-  }
-
-  using ROCKSDB_NAMESPACE::StackableDB::Merge;
-  Status Merge(const WriteOptions& /*options*/,
-               ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
-               const Slice& /*value*/) override {
-    return Status::NotSupported("Not supported operation in blob db.");
-  }
-
-  Status Write(const WriteOptions& opts, WriteBatch* updates) override = 0;
-
-  using ROCKSDB_NAMESPACE::StackableDB::NewIterator;
-  Iterator* NewIterator(const ReadOptions& options) override = 0;
-  Iterator* NewIterator(const ReadOptions& options,
-                        ColumnFamilyHandle* column_family) override {
-    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
-      // Blob DB doesn't support non-default column family.
-      return nullptr;
-    }
-    return NewIterator(options);
-  }
-
-  Status CompactFiles(
-      const CompactionOptions& compact_options,
-      const std::vector<std::string>& input_file_names, const int output_level,
-      const int output_path_id = -1,
-      std::vector<std::string>* const output_file_names = nullptr,
-      CompactionJobInfo* compaction_job_info = nullptr) override = 0;
-  Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names, const int output_level,
-      const int output_path_id = -1,
-      std::vector<std::string>* const output_file_names = nullptr,
-      CompactionJobInfo* compaction_job_info = nullptr) override {
-    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
-      return Status::NotSupported(
-          "Blob DB doesn't support non-default column family.");
-    }
-
-    return CompactFiles(compact_options, input_file_names, output_level,
-                        output_path_id, output_file_names, compaction_job_info);
-  }
-
-  using ROCKSDB_NAMESPACE::StackableDB::Close;
-  Status Close() override = 0;
-
-  // Opening blob db.
-  static Status Open(const Options& options, const BlobDBOptions& bdb_options,
-                     const std::string& dbname, BlobDB** blob_db);
-
-  static Status Open(const DBOptions& db_options,
-                     const BlobDBOptions& bdb_options,
-                     const std::string& dbname,
-                     const std::vector<ColumnFamilyDescriptor>& column_families,
-                     std::vector<ColumnFamilyHandle*>* handles,
-                     BlobDB** blob_db);
-
-  virtual BlobDBOptions GetBlobDBOptions() const = 0;
-
-  virtual Status SyncBlobFiles(const WriteOptions& write_options) = 0;
-
-  ~BlobDB() override {}
-
- protected:
-  explicit BlobDB();
-};
-
-// Destroy the content of the database.
-Status DestroyBlobDB(const std::string& dbname, const Options& options,
-                     const BlobDBOptions& bdb_options);
-
-}  // namespace blob_db
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_db_gc_stats.h

@@ -1,54 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#pragma once
-
-#include <cstdint>
-
-#include "rocksdb/rocksdb_namespace.h"
-
-
-namespace ROCKSDB_NAMESPACE {
-
-namespace blob_db {
-
-/**
- * Statistics related to a single garbage collection pass (i.e. a single
- * (sub)compaction).
- */
-class BlobDBGarbageCollectionStats {
- public:
-  uint64_t AllBlobs() const { return all_blobs_; }
-  uint64_t AllBytes() const { return all_bytes_; }
-  uint64_t RelocatedBlobs() const { return relocated_blobs_; }
-  uint64_t RelocatedBytes() const { return relocated_bytes_; }
-  uint64_t NewFiles() const { return new_files_; }
-  bool HasError() const { return error_; }
-
-  void AddBlob(uint64_t size) {
-    ++all_blobs_;
-    all_bytes_ += size;
-  }
-
-  void AddRelocatedBlob(uint64_t size) {
-    ++relocated_blobs_;
-    relocated_bytes_ += size;
-  }
-
-  void AddNewFile() { ++new_files_; }
-
-  void SetError() { error_ = true; }
-
- private:
-  uint64_t all_blobs_ = 0;
-  uint64_t all_bytes_ = 0;
-  uint64_t relocated_blobs_ = 0;
-  uint64_t relocated_bytes_ = 0;
-  uint64_t new_files_ = 0;
-  bool error_ = false;
-};
-
-}  // namespace blob_db
-}  // namespace ROCKSDB_NAMESPACE