rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/table/plain/plain_table_bloom.h

@@ -1,132 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "port/port.h"
-#include "rocksdb/slice.h"
-#include "util/bloom_impl.h"
-#include "util/hash.h"
-#include "util/math.h"
-
-namespace ROCKSDB_NAMESPACE {
-class Slice;
-class Allocator;
-class Logger;
-
-// A legacy Bloom filter implementation used by Plain Table db format, for
-// schema backward compatibility. Not for use in new filter applications.
-class PlainTableBloomV1 {
- public:
-  // allocator: pass allocator to bloom filter, hence trace the usage of memory
-  // total_bits: fixed total bits for the bloom
-  // num_probes: number of hash probes for a single key
-  // locality:  If positive, optimize for cache line locality, 0 otherwise.
-  // hash_func:  customized hash function
-  // huge_page_tlb_size:  if >0, try to allocate bloom bytes from huge page TLB
-  //                      within this page size. Need to reserve huge pages for
-  //                      it to be allocated, like:
-  //                         sysctl -w vm.nr_hugepages=20
-  //                     See linux doc Documentation/vm/hugetlbpage.txt
-  explicit PlainTableBloomV1(uint32_t num_probes = 6);
-  void SetTotalBits(Allocator* allocator, uint32_t total_bits,
-                    uint32_t locality, size_t huge_page_tlb_size,
-                    Logger* logger);
-
-  ~PlainTableBloomV1() {}
-
-  // Assuming single threaded access to this function.
-  void AddHash(uint32_t hash);
-
-  // Multithreaded access to this function is OK
-  bool MayContainHash(uint32_t hash) const;
-
-  void Prefetch(uint32_t hash);
-
-  uint32_t GetNumBlocks() const { return kNumBlocks; }
-
-  Slice GetRawData() const { return Slice(data_, GetTotalBits() / 8); }
-
-  void SetRawData(char* raw_data, uint32_t total_bits, uint32_t num_blocks = 0);
-
-  uint32_t GetTotalBits() const { return kTotalBits; }
-
-  bool IsInitialized() const { return kNumBlocks > 0 || kTotalBits > 0; }
-
- private:
-  uint32_t kTotalBits;
-  uint32_t kNumBlocks;
-  const uint32_t kNumProbes;
-
-  char* data_;
-
-  static constexpr int LOG2_CACHE_LINE_SIZE =
-      ConstexprFloorLog2(CACHE_LINE_SIZE);
-};
-
-#if defined(_MSC_VER)
-#pragma warning(push)
-// local variable is initialized but not referenced
-#pragma warning(disable : 4189)
-#endif
-inline void PlainTableBloomV1::Prefetch(uint32_t h) {
-  if (kNumBlocks != 0) {
-    uint32_t ignored;
-    LegacyLocalityBloomImpl</*ExtraRotates*/ true>::PrepareHashMayMatch(
-        h, kNumBlocks, data_, &ignored, LOG2_CACHE_LINE_SIZE);
-  }
-}
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-
-inline bool PlainTableBloomV1::MayContainHash(uint32_t h) const {
-  assert(IsInitialized());
-  if (kNumBlocks != 0) {
-    return LegacyLocalityBloomImpl<true>::HashMayMatch(
-        h, kNumBlocks, kNumProbes, data_, LOG2_CACHE_LINE_SIZE);
-  } else {
-    return LegacyNoLocalityBloomImpl::HashMayMatch(h, kTotalBits, kNumProbes,
-                                                   data_);
-  }
-}
-
-inline void PlainTableBloomV1::AddHash(uint32_t h) {
-  assert(IsInitialized());
-  if (kNumBlocks != 0) {
-    LegacyLocalityBloomImpl<true>::AddHash(h, kNumBlocks, kNumProbes, data_,
-                                           LOG2_CACHE_LINE_SIZE);
-  } else {
-    LegacyNoLocalityBloomImpl::AddHash(h, kTotalBits, kNumProbes, data_);
-  }
-}
-
-class BloomBlockBuilder {
- public:
-  static const std::string kBloomBlock;
-
-  explicit BloomBlockBuilder(uint32_t num_probes = 6) : bloom_(num_probes) {}
-
-  void SetTotalBits(Allocator* allocator, uint32_t total_bits,
-                    uint32_t locality, size_t huge_page_tlb_size,
-                    Logger* logger) {
-    bloom_.SetTotalBits(allocator, total_bits, locality, huge_page_tlb_size,
-                        logger);
-  }
-
-  uint32_t GetNumBlocks() const { return bloom_.GetNumBlocks(); }
-
-  void AddKeysHashes(const std::vector<uint32_t>& keys_hashes);
-
-  Slice Finish();
-
- private:
-  PlainTableBloomV1 bloom_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/table/plain/plain_table_builder.cc

@@ -1,348 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "table/plain/plain_table_builder.h"
-
-#include <cassert>
-#include <limits>
-#include <map>
-#include <string>
-
-#include "db/dbformat.h"
-#include "file/writable_file_writer.h"
-#include "logging/logging.h"
-#include "rocksdb/comparator.h"
-#include "rocksdb/env.h"
-#include "rocksdb/filter_policy.h"
-#include "rocksdb/options.h"
-#include "rocksdb/table.h"
-#include "table/block_based/block_builder.h"
-#include "table/format.h"
-#include "table/meta_blocks.h"
-#include "table/plain/plain_table_bloom.h"
-#include "table/plain/plain_table_factory.h"
-#include "table/plain/plain_table_index.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-#include "util/stop_watch.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-namespace {
-
-// a utility that helps writing block content to the file
-//   @offset will advance if @block_contents was successfully written.
-//   @block_handle the block handle this particular block.
-IOStatus WriteBlock(const Slice& block_contents, WritableFileWriter* file,
-                    uint64_t* offset, BlockHandle* block_handle) {
-  block_handle->set_offset(*offset);
-  block_handle->set_size(block_contents.size());
-  IOStatus io_s = file->Append(IOOptions(), block_contents);
-
-  if (io_s.ok()) {
-    *offset += block_contents.size();
-  }
-  return io_s;
-}
-
-}  // namespace
-
-// kPlainTableMagicNumber was picked by running
-//    echo rocksdb.table.plain | sha1sum
-// and taking the leading 64 bits.
-const uint64_t kPlainTableMagicNumber = 0x8242229663bf9564ull;
-const uint64_t kLegacyPlainTableMagicNumber = 0x4f3418eb7a8f13b8ull;
-
-PlainTableBuilder::PlainTableBuilder(
-    const ImmutableOptions& ioptions, const MutableCFOptions& moptions,
-    const InternalTblPropCollFactories* internal_tbl_prop_coll_factories,
-    uint32_t column_family_id, int level_at_creation, WritableFileWriter* file,
-    uint32_t user_key_len, EncodingType encoding_type, size_t index_sparseness,
-    uint32_t bloom_bits_per_key, const std::string& column_family_name,
-    uint32_t num_probes, size_t huge_page_tlb_size, double hash_table_ratio,
-    bool store_index_in_file, const std::string& db_id,
-    const std::string& db_session_id, uint64_t file_number)
-    : ioptions_(ioptions),
-      moptions_(moptions),
-      bloom_block_(num_probes),
-      file_(file),
-      bloom_bits_per_key_(bloom_bits_per_key),
-      huge_page_tlb_size_(huge_page_tlb_size),
-      encoder_(encoding_type, user_key_len, moptions.prefix_extractor.get(),
-               index_sparseness),
-      store_index_in_file_(store_index_in_file),
-      prefix_extractor_(moptions.prefix_extractor.get()) {
-  // Build index block and save it in the file if hash_table_ratio > 0
-  if (store_index_in_file_) {
-    assert(hash_table_ratio > 0 || IsTotalOrderMode());
-    index_builder_.reset(new PlainTableIndexBuilder(
-        &arena_, ioptions, moptions.prefix_extractor.get(), index_sparseness,
-        hash_table_ratio, huge_page_tlb_size_));
-    properties_
-        .user_collected_properties[PlainTablePropertyNames::kBloomVersion] =
-        "1";  // For future use
-  }
-
-  properties_.fixed_key_len = user_key_len;
-
-  // for plain table, we put all the data in a big chuck.
-  properties_.num_data_blocks = 1;
-  // Fill it later if store_index_in_file_ == true
-  properties_.index_size = 0;
-  properties_.filter_size = 0;
-  // To support roll-back to previous version, now still use version 0 for
-  // plain encoding.
-  properties_.format_version = (encoding_type == kPlain) ? 0 : 1;
-  properties_.column_family_id = column_family_id;
-  properties_.column_family_name = column_family_name;
-  properties_.db_id = db_id;
-  properties_.db_session_id = db_session_id;
-  properties_.db_host_id = ioptions.db_host_id;
-  if (!ReifyDbHostIdProperty(ioptions_.env, &properties_.db_host_id).ok()) {
-    ROCKS_LOG_INFO(ioptions_.logger, "db_host_id property will not be set");
-  }
-  properties_.orig_file_number = file_number;
-  properties_.prefix_extractor_name =
-      moptions_.prefix_extractor != nullptr
-          ? moptions_.prefix_extractor->AsString()
-          : "nullptr";
-
-  std::string val;
-  PutFixed32(&val, static_cast<uint32_t>(encoder_.GetEncodingType()));
-  properties_
-      .user_collected_properties[PlainTablePropertyNames::kEncodingType] = val;
-
-  assert(internal_tbl_prop_coll_factories);
-  for (auto& factory : *internal_tbl_prop_coll_factories) {
-    assert(factory);
-
-    std::unique_ptr<InternalTblPropColl> collector{
-        factory->CreateInternalTblPropColl(column_family_id, level_at_creation,
-                                           ioptions.num_levels)};
-    if (collector) {
-      table_properties_collectors_.emplace_back(std::move(collector));
-    }
-  }
-}
-
-PlainTableBuilder::~PlainTableBuilder() {
-  // They are supposed to have been passed to users through Finish()
-  // if the file succeeds.
-  status_.PermitUncheckedError();
-  io_status_.PermitUncheckedError();
-}
-
-void PlainTableBuilder::Add(const Slice& key, const Slice& value) {
-  // temp buffer for metadata bytes between key and value.
-  char meta_bytes_buf[6];
-  size_t meta_bytes_buf_size = 0;
-  const IOOptions opts;
-
-  ParsedInternalKey internal_key;
-  if (!ParseInternalKey(key, &internal_key, false /* log_err_key */)
-           .ok()) {  // TODO
-    assert(false);
-    return;
-  }
-  if (internal_key.type == kTypeRangeDeletion) {
-    status_ = Status::NotSupported("Range deletion unsupported");
-    return;
-  }
-
-  // Store key hash
-  if (store_index_in_file_) {
-    if (moptions_.prefix_extractor == nullptr) {
-      keys_or_prefixes_hashes_.push_back(GetSliceHash(internal_key.user_key));
-    } else {
-      Slice prefix =
-          moptions_.prefix_extractor->Transform(internal_key.user_key);
-      keys_or_prefixes_hashes_.push_back(GetSliceHash(prefix));
-    }
-  }
-
-  // Write value
-  assert(offset_ <= std::numeric_limits<uint32_t>::max());
-  auto prev_offset = static_cast<uint32_t>(offset_);
-  // Write out the key
-  io_status_ = encoder_.AppendKey(key, file_, &offset_, meta_bytes_buf,
-                                  &meta_bytes_buf_size);
-  if (SaveIndexInFile()) {
-    index_builder_->AddKeyPrefix(GetPrefix(internal_key), prev_offset);
-  }
-
-  // Write value length
-  uint32_t value_size = static_cast<uint32_t>(value.size());
-  if (io_status_.ok()) {
-    char* end_ptr =
-        EncodeVarint32(meta_bytes_buf + meta_bytes_buf_size, value_size);
-    assert(end_ptr <= meta_bytes_buf + sizeof(meta_bytes_buf));
-    meta_bytes_buf_size = end_ptr - meta_bytes_buf;
-    io_status_ =
-        file_->Append(opts, Slice(meta_bytes_buf, meta_bytes_buf_size));
-  }
-
-  // Write value
-  if (io_status_.ok()) {
-    io_status_ = file_->Append(opts, value);
-    offset_ += value_size + meta_bytes_buf_size;
-  }
-
-  if (io_status_.ok()) {
-    properties_.num_entries++;
-    properties_.raw_key_size += key.size();
-    properties_.raw_value_size += value.size();
-    if (internal_key.type == kTypeDeletion ||
-        internal_key.type == kTypeSingleDeletion) {
-      properties_.num_deletions++;
-    } else if (internal_key.type == kTypeMerge) {
-      properties_.num_merge_operands++;
-    }
-  }
-
-  // notify property collectors
-  NotifyCollectTableCollectorsOnAdd(
-      key, value, offset_, table_properties_collectors_, ioptions_.logger);
-  status_ = io_status_;
-}
-
-Status PlainTableBuilder::Finish() {
-  assert(!closed_);
-  closed_ = true;
-
-  properties_.data_size = offset_;
-
-  //  Write the following blocks
-  //  1. [meta block: bloom] - optional
-  //  2. [meta block: index] - optional
-  //  3. [meta block: properties]
-  //  4. [metaindex block]
-  //  5. [footer]
-
-  MetaIndexBuilder meta_index_builer;
-
-  if (store_index_in_file_ && (properties_.num_entries > 0)) {
-    assert(properties_.num_entries <= std::numeric_limits<uint32_t>::max());
-    BlockHandle bloom_block_handle;
-    if (bloom_bits_per_key_ > 0) {
-      bloom_block_.SetTotalBits(
-          &arena_,
-          static_cast<uint32_t>(properties_.num_entries) * bloom_bits_per_key_,
-          ioptions_.bloom_locality, huge_page_tlb_size_, ioptions_.logger);
-
-      PutVarint32(&properties_.user_collected_properties
-                       [PlainTablePropertyNames::kNumBloomBlocks],
-                  bloom_block_.GetNumBlocks());
-
-      bloom_block_.AddKeysHashes(keys_or_prefixes_hashes_);
-
-      Slice bloom_finish_result = bloom_block_.Finish();
-
-      properties_.filter_size = bloom_finish_result.size();
-      io_status_ =
-          WriteBlock(bloom_finish_result, file_, &offset_, &bloom_block_handle);
-
-      if (!io_status_.ok()) {
-        status_ = io_status_;
-        return status_;
-      }
-      meta_index_builer.Add(BloomBlockBuilder::kBloomBlock, bloom_block_handle);
-    }
-    BlockHandle index_block_handle;
-    Slice index_finish_result = index_builder_->Finish();
-
-    properties_.index_size = index_finish_result.size();
-    io_status_ =
-        WriteBlock(index_finish_result, file_, &offset_, &index_block_handle);
-
-    if (!io_status_.ok()) {
-      status_ = io_status_;
-      return status_;
-    }
-
-    meta_index_builer.Add(PlainTableIndexBuilder::kPlainTableIndexBlock,
-                          index_block_handle);
-  }
-
-  // Calculate bloom block size and index block size
-  PropertyBlockBuilder property_block_builder;
-  // -- Add basic properties
-  property_block_builder.AddTableProperty(properties_);
-  // -- Add eixsting user collected properties
-  property_block_builder.Add(properties_.user_collected_properties);
-  // -- Add more user collected properties
-  UserCollectedProperties more_user_collected_properties;
-  NotifyCollectTableCollectorsOnFinish(
-      table_properties_collectors_, ioptions_.logger, &property_block_builder,
-      more_user_collected_properties, properties_.readable_properties);
-  properties_.user_collected_properties.insert(
-      more_user_collected_properties.begin(),
-      more_user_collected_properties.end());
-
-  // -- Write property block
-  BlockHandle property_block_handle;
-  io_status_ = WriteBlock(property_block_builder.Finish(), file_, &offset_,
-                          &property_block_handle);
-  if (!io_status_.ok()) {
-    status_ = io_status_;
-    return status_;
-  }
-  meta_index_builer.Add(kPropertiesBlockName, property_block_handle);
-
-  // -- write metaindex block
-  BlockHandle metaindex_block_handle;
-  io_status_ = WriteBlock(meta_index_builer.Finish(), file_, &offset_,
-                          &metaindex_block_handle);
-  if (!io_status_.ok()) {
-    status_ = io_status_;
-    return status_;
-  }
-
-  // Write Footer
-  // no need to write out new footer if we're using default checksum
-  FooterBuilder footer;
-  Status s = footer.Build(kPlainTableMagicNumber, /* format_version */ 0,
-                          offset_, kNoChecksum, metaindex_block_handle);
-  if (!s.ok()) {
-    status_ = s;
-    return status_;
-  }
-  io_status_ = file_->Append(IOOptions(), footer.GetSlice());
-  if (io_status_.ok()) {
-    offset_ += footer.GetSlice().size();
-  }
-  status_ = io_status_;
-  return status_;
-}
-
-void PlainTableBuilder::Abandon() { closed_ = true; }
-
-uint64_t PlainTableBuilder::NumEntries() const {
-  return properties_.num_entries;
-}
-
-uint64_t PlainTableBuilder::FileSize() const { return offset_; }
-
-std::string PlainTableBuilder::GetFileChecksum() const {
-  if (file_ != nullptr) {
-    return file_->GetFileChecksum();
-  } else {
-    return kUnknownFileChecksum;
-  }
-}
-
-const char* PlainTableBuilder::GetFileChecksumFuncName() const {
-  if (file_ != nullptr) {
-    return file_->GetFileChecksumFuncName();
-  } else {
-    return kUnknownFileChecksumFuncName;
-  }
-}
-void PlainTableBuilder::SetSeqnoTimeTableProperties(
-    const SeqnoToTimeMapping& relevant_mapping, uint64_t uint_64) {
-  // TODO: storing seqno to time mapping is not yet support for plain table.
-  TableBuilder::SetSeqnoTimeTableProperties(relevant_mapping, uint_64);
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/table/plain/plain_table_builder.h

@@ -1,151 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <stdint.h>
-
-#include <string>
-#include <vector>
-
-#include "db/version_edit.h"
-#include "rocksdb/options.h"
-#include "rocksdb/status.h"
-#include "rocksdb/table.h"
-#include "rocksdb/table_properties.h"
-#include "table/plain/plain_table_bloom.h"
-#include "table/plain/plain_table_index.h"
-#include "table/plain/plain_table_key_coding.h"
-#include "table/table_builder.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlockBuilder;
-class BlockHandle;
-class WritableFile;
-class TableBuilder;
-
-// The builder class of PlainTable. For description of PlainTable format
-// See comments of class PlainTableFactory, where instances of
-// PlainTableReader are created.
-class PlainTableBuilder : public TableBuilder {
- public:
-  // Create a builder that will store the contents of the table it is
-  // building in *file.  Does not close the file.  It is up to the
-  // caller to close the file after calling Finish(). The output file
-  // will be part of level specified by 'level'.  A value of -1 means
-  // that the caller does not know which level the output file will reside.
-  PlainTableBuilder(
-      const ImmutableOptions& ioptions, const MutableCFOptions& moptions,
-      const InternalTblPropCollFactories* internal_tbl_prop_coll_factories,
-      uint32_t column_family_id, int level_at_creation,
-      WritableFileWriter* file, uint32_t user_key_size,
-      EncodingType encoding_type, size_t index_sparseness,
-      uint32_t bloom_bits_per_key, const std::string& column_family_name,
-      uint32_t num_probes = 6, size_t huge_page_tlb_size = 0,
-      double hash_table_ratio = 0, bool store_index_in_file = false,
-      const std::string& db_id = "", const std::string& db_session_id = "",
-      uint64_t file_number = 0);
-  // No copying allowed
-  PlainTableBuilder(const PlainTableBuilder&) = delete;
-  void operator=(const PlainTableBuilder&) = delete;
-
-  // REQUIRES: Either Finish() or Abandon() has been called.
-  ~PlainTableBuilder();
-
-  // Add key,value to the table being constructed.
-  // REQUIRES: key is after any previously added key according to comparator.
-  // REQUIRES: Finish(), Abandon() have not been called
-  void Add(const Slice& key, const Slice& value) override;
-
-  // Return non-ok iff some error has been detected.
-  Status status() const override { return status_; }
-
-  // Return non-ok iff some error happens during IO.
-  IOStatus io_status() const override { return io_status_; }
-
-  // Finish building the table.  Stops using the file passed to the
-  // constructor after this function returns.
-  // REQUIRES: Finish(), Abandon() have not been called
-  Status Finish() override;
-
-  // Indicate that the contents of this builder should be abandoned.  Stops
-  // using the file passed to the constructor after this function returns.
-  // If the caller is not going to call Finish(), it must call Abandon()
-  // before destroying this builder.
-  // REQUIRES: Finish(), Abandon() have not been called
-  void Abandon() override;
-
-  // Number of calls to Add() so far.
-  uint64_t NumEntries() const override;
-
-  // Size of the file generated so far.  If invoked after a successful
-  // Finish() call, returns the size of the final generated file.
-  uint64_t FileSize() const override;
-
-  TableProperties GetTableProperties() const override { return properties_; }
-
-  bool SaveIndexInFile() const { return store_index_in_file_; }
-
-  // Get file checksum
-  std::string GetFileChecksum() const override;
-
-  // Get file checksum function name
-  const char* GetFileChecksumFuncName() const override;
-
-  void SetSeqnoTimeTableProperties(const SeqnoToTimeMapping& relevant_mapping,
-                                   uint64_t uint_64) override;
-
- private:
-  Arena arena_;
-  const ImmutableOptions& ioptions_;
-  const MutableCFOptions& moptions_;
-  std::vector<std::unique_ptr<InternalTblPropColl>>
-      table_properties_collectors_;
-
-  BloomBlockBuilder bloom_block_;
-  std::unique_ptr<PlainTableIndexBuilder> index_builder_;
-
-  WritableFileWriter* file_;
-  uint64_t offset_ = 0;
-  uint32_t bloom_bits_per_key_;
-  size_t huge_page_tlb_size_;
-  Status status_;
-  IOStatus io_status_;
-  TableProperties properties_;
-  PlainTableKeyEncoder encoder_;
-
-  bool store_index_in_file_;
-
-  std::vector<uint32_t> keys_or_prefixes_hashes_;
-  bool closed_ = false;  // Either Finish() or Abandon() has been called.
-
-  const SliceTransform* prefix_extractor_;
-
-  Slice GetPrefix(const Slice& target) const {
-    assert(target.size() >= 8);  // target is internal key
-    return GetPrefixFromUserKey(ExtractUserKey(target));
-  }
-
-  Slice GetPrefix(const ParsedInternalKey& target) const {
-    return GetPrefixFromUserKey(target.user_key);
-  }
-
-  Slice GetPrefixFromUserKey(const Slice& user_key) const {
-    if (!IsTotalOrderMode()) {
-      return prefix_extractor_->Transform(user_key);
-    } else {
-      // Use empty slice as prefix if prefix_extractor is not set.
-      // In that case,
-      // it falls back to pure binary search and
-      // total iterator seek is supported.
-      return Slice();
-    }
-  }
-
-  bool IsTotalOrderMode() const { return (prefix_extractor_ == nullptr); }
-};
-
-}  // namespace ROCKSDB_NAMESPACE