rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_filter_block.h

@@ -1,188 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <deque>
-#include <list>
-#include <string>
-#include <unordered_map>
-
-#include "block_cache.h"
-#include "rocksdb/options.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/slice_transform.h"
-#include "table/block_based/block.h"
-#include "table/block_based/filter_block_reader_common.h"
-#include "table/block_based/full_filter_block.h"
-#include "table/block_based/index_builder.h"
-#include "util/autovector.h"
-#include "util/hash_containers.h"
-
-namespace ROCKSDB_NAMESPACE {
-class InternalKeyComparator;
-
-class PartitionedFilterBlockBuilder : public FullFilterBlockBuilder {
- public:
-  explicit PartitionedFilterBlockBuilder(
-      const SliceTransform* prefix_extractor, bool whole_key_filtering,
-      FilterBitsBuilder* filter_bits_builder, int index_block_restart_interval,
-      const bool use_value_delta_encoding,
-      PartitionedIndexBuilder* const p_index_builder,
-      const uint32_t partition_size, size_t ts_sz,
-      const bool persist_user_defined_timestamps);
-
-  virtual ~PartitionedFilterBlockBuilder();
-
-  void AddKey(const Slice& key) override;
-  void Add(const Slice& key_without_ts) override;
-  size_t EstimateEntriesAdded() override;
-
-  Status Finish(const BlockHandle& last_partition_block_handle, Slice* filter,
-                std::unique_ptr<const char[]>* filter_owner = nullptr) override;
-
-  void ResetFilterBitsBuilder() override {
-    filters_.clear();
-    total_added_in_built_ = 0;
-    index_on_filter_block_builder_.Reset();
-    index_on_filter_block_builder_without_seq_.Reset();
-    FullFilterBlockBuilder::ResetFilterBitsBuilder();
-  }
-
-  // For PartitionFilter, optional post-verifing the filter is done
-  // as part of PartitionFilterBlockBuilder::Finish
-  // to avoid implementation complexity of doing it elsewhere.
-  // Therefore we are skipping it in here.
-  Status MaybePostVerifyFilter(const Slice& /* filter_content */) override {
-    return Status::OK();
-  }
-
- private:  // fns
-  // The policy of when cut a filter block and Finish it
-  void MaybeCutAFilterBlock(const Slice* next_key);
-
- private:  // data
-  // Currently we keep the same number of partitions for filters and indexes.
-  // This would allow for some potentioal optimizations in future. If such
-  // optimizations did not realize we can use different number of partitions and
-  // eliminate p_index_builder_
-  PartitionedIndexBuilder* const p_index_builder_;
-
-  // Filter data
-  struct FilterEntry {
-    std::string ikey;  // internal key or separator *after* this filter
-    std::unique_ptr<const char[]> filter_owner;
-    Slice filter;
-  };
-  std::deque<FilterEntry> filters_;  // list of partitioned filters and keys
-                                     // used in building the index
-  // The desired number of keys per partition
-  uint32_t keys_per_partition_;
-  // The number of keys added to the last partition so far
-  uint32_t keys_added_to_partition_ = 0;
-  // According to the bits builders, how many keys/prefixes added
-  // in all the filters we have fully built
-  uint64_t total_added_in_built_ = 0;
-
-  // Set to the first non-okay status if any of the filter
-  // partitions experiences construction error.
-  // If partitioned_filters_construction_status_ is non-okay,
-  // then the whole partitioned filters should not be used.
-  Status partitioned_filters_construction_status_;
-
-  // ===== State for Finish() =====
-
-  // top-level index builder on internal keys
-  BlockBuilder index_on_filter_block_builder_;
-  // same for user keys
-  BlockBuilder index_on_filter_block_builder_without_seq_;
-  // For delta-encoding handles
-  BlockHandle last_encoded_handle_;
-  // True if we are between two calls to Finish(), because we have returned
-  // the filter at the front of filters_ but haven't yet added it to the
-  // partition index.
-  bool finishing_front_filter_ = false;
-};
-
-class PartitionedFilterBlockReader
-    : public FilterBlockReaderCommon<Block_kFilterPartitionIndex> {
- public:
-  PartitionedFilterBlockReader(
-      const BlockBasedTable* t,
-      CachableEntry<Block_kFilterPartitionIndex>&& filter_block);
-
-  static std::unique_ptr<FilterBlockReader> Create(
-      const BlockBasedTable* table, const ReadOptions& ro,
-      FilePrefetchBuffer* prefetch_buffer, bool use_cache, bool prefetch,
-      bool pin, BlockCacheLookupContext* lookup_context);
-
-  bool KeyMayMatch(const Slice& key, const Slice* const const_ikey_ptr,
-                   GetContext* get_context,
-                   BlockCacheLookupContext* lookup_context,
-                   const ReadOptions& read_options) override;
-  void KeysMayMatch(MultiGetRange* range,
-                    BlockCacheLookupContext* lookup_context,
-                    const ReadOptions& read_options) override;
-
-  bool PrefixMayMatch(const Slice& prefix, const Slice* const const_ikey_ptr,
-                      GetContext* get_context,
-                      BlockCacheLookupContext* lookup_context,
-                      const ReadOptions& read_options) override;
-  void PrefixesMayMatch(MultiGetRange* range,
-                        const SliceTransform* prefix_extractor,
-                        BlockCacheLookupContext* lookup_context,
-                        const ReadOptions& read_options) override;
-
-  size_t ApproximateMemoryUsage() const override;
-
- private:
-  BlockHandle GetFilterPartitionHandle(
-      const CachableEntry<Block_kFilterPartitionIndex>& filter_block,
-      const Slice& entry) const;
-  Status GetFilterPartitionBlock(
-      FilePrefetchBuffer* prefetch_buffer, const BlockHandle& handle,
-      GetContext* get_context, BlockCacheLookupContext* lookup_context,
-      const ReadOptions& read_options,
-      CachableEntry<ParsedFullFilterBlock>* filter_block) const;
-
-  using FilterFunction = bool (FullFilterBlockReader::*)(
-      const Slice& slice, const Slice* const const_ikey_ptr,
-      GetContext* get_context, BlockCacheLookupContext* lookup_context,
-      const ReadOptions& read_options);
-  bool MayMatch(const Slice& slice, const Slice* const_ikey_ptr,
-                GetContext* get_context,
-                BlockCacheLookupContext* lookup_context,
-                const ReadOptions& read_options,
-                FilterFunction filter_function) const;
-  using FilterManyFunction = void (FullFilterBlockReader::*)(
-      MultiGetRange* range, const SliceTransform* prefix_extractor,
-      BlockCacheLookupContext* lookup_context, const ReadOptions& read_options);
-  void MayMatch(MultiGetRange* range, const SliceTransform* prefix_extractor,
-                BlockCacheLookupContext* lookup_context,
-                const ReadOptions& read_options,
-                FilterManyFunction filter_function) const;
-  void MayMatchPartition(MultiGetRange* range,
-                         const SliceTransform* prefix_extractor,
-                         BlockHandle filter_handle,
-                         BlockCacheLookupContext* lookup_context,
-                         const ReadOptions& read_options,
-                         FilterManyFunction filter_function) const;
-  Status CacheDependencies(const ReadOptions& ro, bool pin,
-                           FilePrefetchBuffer* tail_prefetch_buffer) override;
-  void EraseFromCacheBeforeDestruction(
-      uint32_t /*uncache_aggressiveness*/) override;
-
-  const InternalKeyComparator* internal_comparator() const;
-  bool index_key_includes_seq() const;
-  bool index_value_is_full() const;
-  bool user_defined_timestamps_persisted() const;
-
- protected:
-  // For partition blocks pinned in cache. Can be a subset of blocks
-  // in case some fail insertion on attempt to pin.
-  UnorderedMap<uint64_t, CachableEntry<ParsedFullFilterBlock>> filter_map_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_index_iterator.cc

@@ -1,164 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "table/block_based/partitioned_index_iterator.h"
-
-namespace ROCKSDB_NAMESPACE {
-void PartitionedIndexIterator::Seek(const Slice& target) { SeekImpl(&target); }
-
-void PartitionedIndexIterator::SeekToFirst() { SeekImpl(nullptr); }
-
-void PartitionedIndexIterator::SeekImpl(const Slice* target) {
-  SavePrevIndexValue();
-
-  if (target) {
-    index_iter_->Seek(*target);
-  } else {
-    index_iter_->SeekToFirst();
-  }
-
-  if (!index_iter_->Valid()) {
-    ResetPartitionedIndexIter();
-    return;
-  }
-
-  InitPartitionedIndexBlock();
-
-  if (target) {
-    block_iter_.Seek(*target);
-  } else {
-    block_iter_.SeekToFirst();
-  }
-  FindKeyForward();
-
-  // We could check upper bound here, but that would be too complicated
-  // and checking index upper bound is less useful than for data blocks.
-
-  if (target) {
-    assert(!Valid() || (table_->get_rep()->index_key_includes_seq
-                            ? (icomp_.Compare(*target, key()) <= 0)
-                            : (user_comparator_.Compare(ExtractUserKey(*target),
-                                                        key()) <= 0)));
-  }
-}
-
-void PartitionedIndexIterator::SeekToLast() {
-  SavePrevIndexValue();
-  index_iter_->SeekToLast();
-  if (!index_iter_->Valid()) {
-    ResetPartitionedIndexIter();
-    return;
-  }
-  InitPartitionedIndexBlock();
-  block_iter_.SeekToLast();
-  FindKeyBackward();
-}
-
-void PartitionedIndexIterator::Next() {
-  assert(block_iter_points_to_real_block_);
-  block_iter_.Next();
-  FindKeyForward();
-}
-
-void PartitionedIndexIterator::Prev() {
-  assert(block_iter_points_to_real_block_);
-  block_iter_.Prev();
-
-  FindKeyBackward();
-}
-
-void PartitionedIndexIterator::InitPartitionedIndexBlock() {
-  BlockHandle partitioned_index_handle = index_iter_->value().handle;
-  if (!block_iter_points_to_real_block_ ||
-      partitioned_index_handle.offset() != prev_block_offset_ ||
-      // if previous attempt of reading the block missed cache, try again
-      block_iter_.status().IsIncomplete()) {
-    if (block_iter_points_to_real_block_) {
-      ResetPartitionedIndexIter();
-    }
-    auto* rep = table_->get_rep();
-    bool is_for_compaction =
-        lookup_context_.caller == TableReaderCaller::kCompaction;
-    // Prefetch additional data for range scans (iterators).
-    // Implicit auto readahead:
-    //   Enabled after 2 sequential IOs when ReadOptions.readahead_size == 0.
-    // Explicit user requested readahead:
-    //   Enabled from the very first IO when ReadOptions.readahead_size is set.
-    block_prefetcher_.PrefetchIfNeeded(
-        rep, partitioned_index_handle, read_options_.readahead_size,
-        is_for_compaction, /*no_sequential_checking=*/false, read_options_,
-        /*readaheadsize_cb=*/nullptr, /*is_async_io_prefetch=*/false);
-    Status s;
-    table_->NewDataBlockIterator<IndexBlockIter>(
-        read_options_, partitioned_index_handle, &block_iter_,
-        BlockType::kIndex,
-        /*get_context=*/nullptr, &lookup_context_,
-        block_prefetcher_.prefetch_buffer(),
-        /*for_compaction=*/is_for_compaction, /*async_read=*/false, s,
-        /*use_block_cache_for_lookup=*/true);
-    block_iter_points_to_real_block_ = true;
-    // We could check upper bound here but it is complicated to reason about
-    // upper bound in index iterator. On the other than, in large scans, index
-    // iterators are moved much less frequently compared to data blocks. So
-    // the upper bound check is skipped for simplicity.
-  }
-}
-
-void PartitionedIndexIterator::FindKeyForward() {
-  // This method's code is kept short to make it likely to be inlined.
-
-  assert(block_iter_points_to_real_block_);
-
-  if (!block_iter_.Valid()) {
-    // This is the only call site of FindBlockForward(), but it's extracted into
-    // a separate method to keep FindKeyForward() short and likely to be
-    // inlined. When transitioning to a different block, we call
-    // FindBlockForward(), which is much longer and is probably not inlined.
-    FindBlockForward();
-  } else {
-    // This is the fast path that avoids a function call.
-  }
-}
-
-void PartitionedIndexIterator::FindBlockForward() {
-  // TODO the while loop inherits from two-level-iterator. We don't know
-  // whether a block can be empty so it can be replaced by an "if".
-  do {
-    if (!block_iter_.status().ok()) {
-      return;
-    }
-    ResetPartitionedIndexIter();
-    index_iter_->Next();
-
-    if (!index_iter_->Valid()) {
-      return;
-    }
-
-    InitPartitionedIndexBlock();
-    block_iter_.SeekToFirst();
-  } while (!block_iter_.Valid());
-}
-
-void PartitionedIndexIterator::FindKeyBackward() {
-  while (!block_iter_.Valid()) {
-    if (!block_iter_.status().ok()) {
-      return;
-    }
-
-    ResetPartitionedIndexIter();
-    index_iter_->Prev();
-
-    if (index_iter_->Valid()) {
-      InitPartitionedIndexBlock();
-      block_iter_.SeekToLast();
-    } else {
-      return;
-    }
-  }
-}
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_index_iterator.h

@@ -1,160 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-#include "table/block_based/block_based_table_reader.h"
-#include "table/block_based/block_based_table_reader_impl.h"
-#include "table/block_based/block_prefetcher.h"
-#include "table/block_based/reader_common.h"
-
-namespace ROCKSDB_NAMESPACE {
-// Iterator that iterates over partitioned index.
-// Some upper and lower bound tricks played in block based table iterators
-// could be played here, but it's too complicated to reason about index
-// keys with upper or lower bound, so we skip it for simplicity.
-class PartitionedIndexIterator : public InternalIteratorBase<IndexValue> {
-  // compaction_readahead_size: its value will only be used if for_compaction =
-  // true
- public:
-  PartitionedIndexIterator(
-      const BlockBasedTable* table, const ReadOptions& read_options,
-      const InternalKeyComparator& icomp,
-      std::unique_ptr<InternalIteratorBase<IndexValue>>&& index_iter,
-      TableReaderCaller caller, size_t compaction_readahead_size = 0)
-      : index_iter_(std::move(index_iter)),
-        table_(table),
-        read_options_(read_options),
-#ifndef NDEBUG
-        icomp_(icomp),
-#endif
-        user_comparator_(icomp.user_comparator()),
-        block_iter_points_to_real_block_(false),
-        lookup_context_(caller),
-        block_prefetcher_(
-            compaction_readahead_size,
-            table_->get_rep()->table_options.initial_auto_readahead_size) {
-  }
-
-  ~PartitionedIndexIterator() override {}
-
-  void Seek(const Slice& target) override;
-  void SeekForPrev(const Slice&) override {
-    // Shouldn't be called.
-    assert(false);
-  }
-  void SeekToFirst() override;
-  void SeekToLast() override;
-  void Next() final override;
-  bool NextAndGetResult(IterateResult*) override {
-    assert(false);
-    return false;
-  }
-  void Prev() override;
-  bool Valid() const override {
-    return block_iter_points_to_real_block_ && block_iter_.Valid();
-  }
-  Slice key() const override {
-    assert(Valid());
-    return block_iter_.key();
-  }
-  Slice user_key() const override {
-    assert(Valid());
-    return block_iter_.user_key();
-  }
-  IndexValue value() const override {
-    assert(Valid());
-    return block_iter_.value();
-  }
-  Status status() const override {
-    // Prefix index set status to NotFound when the prefix does not exist
-    if (!index_iter_->status().ok() && !index_iter_->status().IsNotFound()) {
-      return index_iter_->status();
-    } else if (block_iter_points_to_real_block_) {
-      return block_iter_.status();
-    } else {
-      return Status::OK();
-    }
-  }
-  inline IterBoundCheck UpperBoundCheckResult() override {
-    // Shouldn't be called.
-    assert(false);
-    return IterBoundCheck::kUnknown;
-  }
-  void SetPinnedItersMgr(PinnedIteratorsManager*) override {
-    // Shouldn't be called.
-    assert(false);
-  }
-  bool IsKeyPinned() const override {
-    // Shouldn't be called.
-    assert(false);
-    return false;
-  }
-  bool IsValuePinned() const override {
-    // Shouldn't be called.
-    assert(false);
-    return false;
-  }
-
-  void ResetPartitionedIndexIter() {
-    if (block_iter_points_to_real_block_) {
-      block_iter_.Invalidate(Status::OK());
-      block_iter_points_to_real_block_ = false;
-    }
-  }
-
-  void SavePrevIndexValue() {
-    if (block_iter_points_to_real_block_) {
-      // Reseek. If they end up with the same data block, we shouldn't re-fetch
-      // the same data block.
-      prev_block_offset_ = index_iter_->value().handle.offset();
-    }
-  }
-
-  void GetReadaheadState(ReadaheadFileInfo* readahead_file_info) override {
-    if (block_prefetcher_.prefetch_buffer() != nullptr &&
-        read_options_.adaptive_readahead) {
-      block_prefetcher_.prefetch_buffer()->GetReadaheadState(
-          &(readahead_file_info->index_block_readahead_info));
-    }
-  }
-
-  void SetReadaheadState(ReadaheadFileInfo* readahead_file_info) override {
-    if (read_options_.adaptive_readahead) {
-      block_prefetcher_.SetReadaheadState(
-          &(readahead_file_info->index_block_readahead_info));
-    }
-  }
-
-  std::unique_ptr<InternalIteratorBase<IndexValue>> index_iter_;
-
- private:
-  friend class BlockBasedTableReaderTestVerifyChecksum_ChecksumMismatch_Test;
-  const BlockBasedTable* table_;
-  const ReadOptions read_options_;
-#ifndef NDEBUG
-  const InternalKeyComparator& icomp_;
-#endif
-  UserComparatorWrapper user_comparator_;
-  IndexBlockIter block_iter_;
-
-  // True if block_iter_ is initialized and points to the same block
-  // as index iterator.
-  bool block_iter_points_to_real_block_;
-  uint64_t prev_block_offset_ = std::numeric_limits<uint64_t>::max();
-  BlockCacheLookupContext lookup_context_;
-  BlockPrefetcher block_prefetcher_;
-
-  // If `target` is null, seek to first.
-  void SeekImpl(const Slice* target);
-
-  void InitPartitionedIndexBlock();
-  void FindKeyForward();
-  void FindBlockForward();
-  void FindKeyBackward();
-};
-}  // namespace ROCKSDB_NAMESPACE