rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_file_reader.h

@@ -1,111 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cinttypes>
-#include <memory>
-
-#include "db/blob/blob_read_request.h"
-#include "file/random_access_file_reader.h"
-#include "rocksdb/compression_type.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "util/autovector.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Status;
-struct ImmutableOptions;
-struct FileOptions;
-class HistogramImpl;
-struct ReadOptions;
-class Slice;
-class FilePrefetchBuffer;
-class BlobContents;
-class Statistics;
-
-class BlobFileReader {
- public:
-  static Status Create(const ImmutableOptions& immutable_options,
-                       const ReadOptions& read_options,
-                       const FileOptions& file_options,
-                       uint32_t column_family_id,
-                       HistogramImpl* blob_file_read_hist,
-                       uint64_t blob_file_number,
-                       const std::shared_ptr<IOTracer>& io_tracer,
-                       std::unique_ptr<BlobFileReader>* reader);
-
-  BlobFileReader(const BlobFileReader&) = delete;
-  BlobFileReader& operator=(const BlobFileReader&) = delete;
-
-  ~BlobFileReader();
-
-  Status GetBlob(const ReadOptions& read_options, const Slice& user_key,
-                 uint64_t offset, uint64_t value_size,
-                 CompressionType compression_type,
-                 FilePrefetchBuffer* prefetch_buffer,
-                 MemoryAllocator* allocator,
-                 std::unique_ptr<BlobContents>* result,
-                 uint64_t* bytes_read) const;
-
-  // offsets must be sorted in ascending order by caller.
-  void MultiGetBlob(
-      const ReadOptions& read_options, MemoryAllocator* allocator,
-      autovector<std::pair<BlobReadRequest*, std::unique_ptr<BlobContents>>>&
-          blob_reqs,
-      uint64_t* bytes_read) const;
-
-  CompressionType GetCompressionType() const { return compression_type_; }
-
-  uint64_t GetFileSize() const { return file_size_; }
-
- private:
-  BlobFileReader(std::unique_ptr<RandomAccessFileReader>&& file_reader,
-                 uint64_t file_size, CompressionType compression_type,
-                 SystemClock* clock, Statistics* statistics);
-
-  static Status OpenFile(const ImmutableOptions& immutable_options,
-                         const FileOptions& file_opts,
-                         HistogramImpl* blob_file_read_hist,
-                         uint64_t blob_file_number,
-                         const std::shared_ptr<IOTracer>& io_tracer,
-                         uint64_t* file_size,
-                         std::unique_ptr<RandomAccessFileReader>* file_reader);
-
-  static Status ReadHeader(const RandomAccessFileReader* file_reader,
-                           const ReadOptions& read_options,
-                           uint32_t column_family_id, Statistics* statistics,
-                           CompressionType* compression_type);
-
-  static Status ReadFooter(const RandomAccessFileReader* file_reader,
-                           const ReadOptions& read_options, uint64_t file_size,
-                           Statistics* statistics);
-
-  using Buffer = std::unique_ptr<char[]>;
-
-  static Status ReadFromFile(const RandomAccessFileReader* file_reader,
-                             const ReadOptions& read_options,
-                             uint64_t read_offset, size_t read_size,
-                             Statistics* statistics, Slice* slice, Buffer* buf,
-                             AlignedBuf* aligned_buf);
-
-  static Status VerifyBlob(const Slice& record_slice, const Slice& user_key,
-                           uint64_t value_size);
-
-  static Status UncompressBlobIfNeeded(const Slice& value_slice,
-                                       CompressionType compression_type,
-                                       MemoryAllocator* allocator,
-                                       SystemClock* clock,
-                                       Statistics* statistics,
-                                       std::unique_ptr<BlobContents>* result);
-
-  std::unique_ptr<RandomAccessFileReader> file_reader_;
-  uint64_t file_size_;
-  CompressionType compression_type_;
-  SystemClock* clock_;
-  Statistics* statistics_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_garbage_meter.cc

@@ -1,100 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_garbage_meter.h"
-
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/dbformat.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status BlobGarbageMeter::ProcessInFlow(const Slice& key, const Slice& value) {
-  uint64_t blob_file_number = kInvalidBlobFileNumber;
-  uint64_t bytes = 0;
-
-  const Status s = Parse(key, value, &blob_file_number, &bytes);
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (blob_file_number == kInvalidBlobFileNumber) {
-    return Status::OK();
-  }
-
-  flows_[blob_file_number].AddInFlow(bytes);
-
-  return Status::OK();
-}
-
-Status BlobGarbageMeter::ProcessOutFlow(const Slice& key, const Slice& value) {
-  uint64_t blob_file_number = kInvalidBlobFileNumber;
-  uint64_t bytes = 0;
-
-  const Status s = Parse(key, value, &blob_file_number, &bytes);
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (blob_file_number == kInvalidBlobFileNumber) {
-    return Status::OK();
-  }
-
-  // Note: in order to measure the amount of additional garbage, we only need to
-  // track the outflow for preexisting files, i.e. those that also had inflow.
-  // (Newly written files would only have outflow.)
-  auto it = flows_.find(blob_file_number);
-  if (it == flows_.end()) {
-    return Status::OK();
-  }
-
-  it->second.AddOutFlow(bytes);
-
-  return Status::OK();
-}
-
-Status BlobGarbageMeter::Parse(const Slice& key, const Slice& value,
-                               uint64_t* blob_file_number, uint64_t* bytes) {
-  assert(blob_file_number);
-  assert(*blob_file_number == kInvalidBlobFileNumber);
-  assert(bytes);
-  assert(*bytes == 0);
-
-  ParsedInternalKey ikey;
-
-  {
-    constexpr bool log_err_key = false;
-    const Status s = ParseInternalKey(key, &ikey, log_err_key);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (ikey.type != kTypeBlobIndex) {
-    return Status::OK();
-  }
-
-  BlobIndex blob_index;
-
-  {
-    const Status s = blob_index.DecodeFrom(value);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (blob_index.IsInlined() || blob_index.HasTTL()) {
-    return Status::Corruption("Unexpected TTL/inlined blob index");
-  }
-
-  *blob_file_number = blob_index.file_number();
-  *bytes =
-      blob_index.size() +
-      BlobLogRecord::CalculateAdjustmentForRecordHeader(ikey.user_key.size());
-
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_garbage_meter.h

@@ -1,102 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-#include <cstdint>
-#include <unordered_map>
-
-#include "db/blob/blob_constants.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Slice;
-
-// A class that can be used to compute the amount of additional garbage
-// generated by a compaction. It parses the keys and blob references in the
-// input and output of a compaction, and aggregates the "inflow" and "outflow"
-// on a per-blob file basis. The amount of additional garbage for any given blob
-// file can then be computed by subtracting the outflow from the inflow.
-class BlobGarbageMeter {
- public:
-  // A class to store the number and total size of blobs on a per-blob file
-  // basis.
-  class BlobStats {
-   public:
-    void Add(uint64_t bytes) {
-      ++count_;
-      bytes_ += bytes;
-    }
-    void Add(uint64_t count, uint64_t bytes) {
-      count_ += count;
-      bytes_ += bytes;
-    }
-
-    uint64_t GetCount() const { return count_; }
-    uint64_t GetBytes() const { return bytes_; }
-
-   private:
-    uint64_t count_ = 0;
-    uint64_t bytes_ = 0;
-  };
-
-  // A class to keep track of the "inflow" and the "outflow" and to compute the
-  // amount of additional garbage for a given blob file.
-  class BlobInOutFlow {
-   public:
-    void AddInFlow(uint64_t bytes) {
-      in_flow_.Add(bytes);
-      assert(IsValid());
-    }
-    void AddOutFlow(uint64_t bytes) {
-      out_flow_.Add(bytes);
-      assert(IsValid());
-    }
-
-    const BlobStats& GetInFlow() const { return in_flow_; }
-    const BlobStats& GetOutFlow() const { return out_flow_; }
-
-    bool IsValid() const {
-      return in_flow_.GetCount() >= out_flow_.GetCount() &&
-             in_flow_.GetBytes() >= out_flow_.GetBytes();
-    }
-    bool HasGarbage() const {
-      assert(IsValid());
-      return in_flow_.GetCount() > out_flow_.GetCount();
-    }
-    uint64_t GetGarbageCount() const {
-      assert(IsValid());
-      assert(HasGarbage());
-      return in_flow_.GetCount() - out_flow_.GetCount();
-    }
-    uint64_t GetGarbageBytes() const {
-      assert(IsValid());
-      assert(HasGarbage());
-      return in_flow_.GetBytes() - out_flow_.GetBytes();
-    }
-
-   private:
-    BlobStats in_flow_;
-    BlobStats out_flow_;
-  };
-
-  Status ProcessInFlow(const Slice& key, const Slice& value);
-  Status ProcessOutFlow(const Slice& key, const Slice& value);
-
-  const std::unordered_map<uint64_t, BlobInOutFlow>& flows() const {
-    return flows_;
-  }
-
- private:
-  static Status Parse(const Slice& key, const Slice& value,
-                      uint64_t* blob_file_number, uint64_t* bytes);
-
-  std::unordered_map<uint64_t, BlobInOutFlow> flows_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_index.h

@@ -1,187 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <sstream>
-#include <string>
-
-#include "rocksdb/compression_type.h"
-#include "util/coding.h"
-#include "util/compression.h"
-#include "util/string_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// BlobIndex is a pointer to the blob and metadata of the blob. The index is
-// stored in base DB as ValueType::kTypeBlobIndex.
-// There are three types of blob index:
-//
-//    kInlinedTTL:
-//      +------+------------+---------------+
-//      | type | expiration | value         |
-//      +------+------------+---------------+
-//      | char | varint64   | variable size |
-//      +------+------------+---------------+
-//
-//    kBlob:
-//      +------+-------------+----------+----------+-------------+
-//      | type | file number | offset   | size     | compression |
-//      +------+-------------+----------+----------+-------------+
-//      | char | varint64    | varint64 | varint64 | char        |
-//      +------+-------------+----------+----------+-------------+
-//
-//    kBlobTTL:
-//      +------+------------+-------------+----------+----------+-------------+
-//      | type | expiration | file number | offset   | size     | compression |
-//      +------+------------+-------------+----------+----------+-------------+
-//      | char | varint64   | varint64    | varint64 | varint64 | char        |
-//      +------+------------+-------------+----------+----------+-------------+
-//
-// There isn't a kInlined (without TTL) type since we can store it as a plain
-// value (i.e. ValueType::kTypeValue).
-class BlobIndex {
- public:
-  enum class Type : unsigned char {
-    kInlinedTTL = 0,
-    kBlob = 1,
-    kBlobTTL = 2,
-    kUnknown = 3,
-  };
-
-  BlobIndex() : type_(Type::kUnknown) {}
-
-  BlobIndex(const BlobIndex&) = default;
-  BlobIndex& operator=(const BlobIndex&) = default;
-
-  bool IsInlined() const { return type_ == Type::kInlinedTTL; }
-
-  bool HasTTL() const {
-    return type_ == Type::kInlinedTTL || type_ == Type::kBlobTTL;
-  }
-
-  uint64_t expiration() const {
-    assert(HasTTL());
-    return expiration_;
-  }
-
-  const Slice& value() const {
-    assert(IsInlined());
-    return value_;
-  }
-
-  uint64_t file_number() const {
-    assert(!IsInlined());
-    return file_number_;
-  }
-
-  uint64_t offset() const {
-    assert(!IsInlined());
-    return offset_;
-  }
-
-  uint64_t size() const {
-    assert(!IsInlined());
-    return size_;
-  }
-
-  CompressionType compression() const {
-    assert(!IsInlined());
-    return compression_;
-  }
-
-  Status DecodeFrom(Slice slice) {
-    const char* kErrorMessage = "Error while decoding blob index";
-    assert(slice.size() > 0);
-    type_ = static_cast<Type>(*slice.data());
-    if (type_ >= Type::kUnknown) {
-      return Status::Corruption(kErrorMessage,
-                                "Unknown blob index type: " +
-                                    std::to_string(static_cast<char>(type_)));
-    }
-    slice = Slice(slice.data() + 1, slice.size() - 1);
-    if (HasTTL()) {
-      if (!GetVarint64(&slice, &expiration_)) {
-        return Status::Corruption(kErrorMessage, "Corrupted expiration");
-      }
-    }
-    if (IsInlined()) {
-      value_ = slice;
-    } else {
-      if (GetVarint64(&slice, &file_number_) && GetVarint64(&slice, &offset_) &&
-          GetVarint64(&slice, &size_) && slice.size() == 1) {
-        compression_ = static_cast<CompressionType>(*slice.data());
-      } else {
-        return Status::Corruption(kErrorMessage, "Corrupted blob offset");
-      }
-    }
-    return Status::OK();
-  }
-
-  std::string DebugString(bool output_hex) const {
-    std::ostringstream oss;
-
-    if (IsInlined()) {
-      oss << "[inlined blob] value:" << value_.ToString(output_hex);
-    } else {
-      oss << "[blob ref] file:" << file_number_ << " offset:" << offset_
-          << " size:" << size_
-          << " compression: " << CompressionTypeToString(compression_);
-    }
-
-    if (HasTTL()) {
-      oss << " exp:" << expiration_;
-    }
-
-    return oss.str();
-  }
-
-  static void EncodeInlinedTTL(std::string* dst, uint64_t expiration,
-                               const Slice& value) {
-    assert(dst != nullptr);
-    dst->clear();
-    dst->reserve(1 + kMaxVarint64Length + value.size());
-    dst->push_back(static_cast<char>(Type::kInlinedTTL));
-    PutVarint64(dst, expiration);
-    dst->append(value.data(), value.size());
-  }
-
-  static void EncodeBlob(std::string* dst, uint64_t file_number,
-                         uint64_t offset, uint64_t size,
-                         CompressionType compression) {
-    assert(dst != nullptr);
-    dst->clear();
-    dst->reserve(kMaxVarint64Length * 3 + 2);
-    dst->push_back(static_cast<char>(Type::kBlob));
-    PutVarint64(dst, file_number);
-    PutVarint64(dst, offset);
-    PutVarint64(dst, size);
-    dst->push_back(static_cast<char>(compression));
-  }
-
-  static void EncodeBlobTTL(std::string* dst, uint64_t expiration,
-                            uint64_t file_number, uint64_t offset,
-                            uint64_t size, CompressionType compression) {
-    assert(dst != nullptr);
-    dst->clear();
-    dst->reserve(kMaxVarint64Length * 4 + 2);
-    dst->push_back(static_cast<char>(Type::kBlobTTL));
-    PutVarint64(dst, expiration);
-    PutVarint64(dst, file_number);
-    PutVarint64(dst, offset);
-    PutVarint64(dst, size);
-    dst->push_back(static_cast<char>(compression));
-  }
-
- private:
-  Type type_ = Type::kUnknown;
-  uint64_t expiration_ = 0;
-  Slice value_;
-  uint64_t file_number_ = 0;
-  uint64_t offset_ = 0;
-  uint64_t size_ = 0;
-  CompressionType compression_ = kNoCompression;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_log_format.cc

@@ -1,143 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#include "db/blob/blob_log_format.h"
-
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-void BlobLogHeader::EncodeTo(std::string* dst) {
-  assert(dst != nullptr);
-  dst->clear();
-  dst->reserve(BlobLogHeader::kSize);
-  PutFixed32(dst, kMagicNumber);
-  PutFixed32(dst, version);
-  PutFixed32(dst, column_family_id);
-  unsigned char flags = (has_ttl ? 1 : 0);
-  dst->push_back(flags);
-  dst->push_back(compression);
-  PutFixed64(dst, expiration_range.first);
-  PutFixed64(dst, expiration_range.second);
-}
-
-Status BlobLogHeader::DecodeFrom(Slice src) {
-  const char* kErrorMessage = "Error while decoding blob log header";
-  if (src.size() != BlobLogHeader::kSize) {
-    return Status::Corruption(kErrorMessage,
-                              "Unexpected blob file header size");
-  }
-  uint32_t magic_number;
-  unsigned char flags;
-  if (!GetFixed32(&src, &magic_number) || !GetFixed32(&src, &version) ||
-      !GetFixed32(&src, &column_family_id)) {
-    return Status::Corruption(
-        kErrorMessage,
-        "Error decoding magic number, version and column family id");
-  }
-  if (magic_number != kMagicNumber) {
-    return Status::Corruption(kErrorMessage, "Magic number mismatch");
-  }
-  if (version != kVersion1) {
-    return Status::Corruption(kErrorMessage, "Unknown header version");
-  }
-  flags = src.data()[0];
-  compression = static_cast<CompressionType>(src.data()[1]);
-  has_ttl = (flags & 1) == 1;
-  src.remove_prefix(2);
-  if (!GetFixed64(&src, &expiration_range.first) ||
-      !GetFixed64(&src, &expiration_range.second)) {
-    return Status::Corruption(kErrorMessage, "Error decoding expiration range");
-  }
-  return Status::OK();
-}
-
-void BlobLogFooter::EncodeTo(std::string* dst) {
-  assert(dst != nullptr);
-  dst->clear();
-  dst->reserve(BlobLogFooter::kSize);
-  PutFixed32(dst, kMagicNumber);
-  PutFixed64(dst, blob_count);
-  PutFixed64(dst, expiration_range.first);
-  PutFixed64(dst, expiration_range.second);
-  crc = crc32c::Value(dst->c_str(), dst->size());
-  crc = crc32c::Mask(crc);
-  PutFixed32(dst, crc);
-}
-
-Status BlobLogFooter::DecodeFrom(Slice src) {
-  const char* kErrorMessage = "Error while decoding blob log footer";
-  if (src.size() != BlobLogFooter::kSize) {
-    return Status::Corruption(kErrorMessage,
-                              "Unexpected blob file footer size");
-  }
-  uint32_t src_crc = 0;
-  src_crc = crc32c::Value(src.data(), BlobLogFooter::kSize - sizeof(uint32_t));
-  src_crc = crc32c::Mask(src_crc);
-  uint32_t magic_number = 0;
-  if (!GetFixed32(&src, &magic_number) || !GetFixed64(&src, &blob_count) ||
-      !GetFixed64(&src, &expiration_range.first) ||
-      !GetFixed64(&src, &expiration_range.second) || !GetFixed32(&src, &crc)) {
-    return Status::Corruption(kErrorMessage, "Error decoding content");
-  }
-  if (magic_number != kMagicNumber) {
-    return Status::Corruption(kErrorMessage, "Magic number mismatch");
-  }
-  if (src_crc != crc) {
-    return Status::Corruption(kErrorMessage, "CRC mismatch");
-  }
-  return Status::OK();
-}
-
-void BlobLogRecord::EncodeHeaderTo(std::string* dst) {
-  assert(dst != nullptr);
-  dst->clear();
-  dst->reserve(BlobLogRecord::kHeaderSize + key.size() + value.size());
-  PutFixed64(dst, key.size());
-  PutFixed64(dst, value.size());
-  PutFixed64(dst, expiration);
-  header_crc = crc32c::Value(dst->c_str(), dst->size());
-  header_crc = crc32c::Mask(header_crc);
-  PutFixed32(dst, header_crc);
-  blob_crc = crc32c::Value(key.data(), key.size());
-  blob_crc = crc32c::Extend(blob_crc, value.data(), value.size());
-  blob_crc = crc32c::Mask(blob_crc);
-  PutFixed32(dst, blob_crc);
-}
-
-Status BlobLogRecord::DecodeHeaderFrom(Slice src) {
-  const char* kErrorMessage = "Error while decoding blob record";
-  if (src.size() != BlobLogRecord::kHeaderSize) {
-    return Status::Corruption(kErrorMessage,
-                              "Unexpected blob record header size");
-  }
-  uint32_t src_crc = 0;
-  src_crc = crc32c::Value(src.data(), BlobLogRecord::kHeaderSize - 8);
-  src_crc = crc32c::Mask(src_crc);
-  if (!GetFixed64(&src, &key_size) || !GetFixed64(&src, &value_size) ||
-      !GetFixed64(&src, &expiration) || !GetFixed32(&src, &header_crc) ||
-      !GetFixed32(&src, &blob_crc)) {
-    return Status::Corruption(kErrorMessage, "Error decoding content");
-  }
-  if (src_crc != header_crc) {
-    return Status::Corruption(kErrorMessage, "Header CRC mismatch");
-  }
-  return Status::OK();
-}
-
-Status BlobLogRecord::CheckBlobCRC() const {
-  uint32_t expected_crc = 0;
-  expected_crc = crc32c::Value(key.data(), key.size());
-  expected_crc = crc32c::Extend(expected_crc, value.data(), value.size());
-  expected_crc = crc32c::Mask(expected_crc);
-  if (expected_crc != blob_crc) {
-    return Status::Corruption("Blob CRC mismatch");
-  }
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE