rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_log_format.h

@@ -1,164 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Log format information shared by reader and writer.
-
-#pragma once
-
-#include <memory>
-#include <utility>
-
-#include "rocksdb/options.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "rocksdb/types.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-constexpr uint32_t kMagicNumber = 2395959;  // 0x00248f37
-constexpr uint32_t kVersion1 = 1;
-
-using ExpirationRange = std::pair<uint64_t, uint64_t>;
-
-// clang-format off
-
-// Format of blob log file header (30 bytes):
-//
-//    +--------------+---------+---------+-------+-------------+-------------------+
-//    | magic number | version |  cf id  | flags | compression | expiration range  |
-//    +--------------+---------+---------+-------+-------------+-------------------+
-//    |   Fixed32    | Fixed32 | Fixed32 | char  |    char     | Fixed64   Fixed64 |
-//    +--------------+---------+---------+-------+-------------+-------------------+
-//
-// List of flags:
-//   has_ttl: Whether the file contain TTL data.
-//
-// Expiration range in the header is a rough range based on
-// blob_db_options.ttl_range_secs.
-
-// clang-format on
-
-struct BlobLogHeader {
-  static constexpr size_t kSize = 30;
-
-  BlobLogHeader() = default;
-  BlobLogHeader(uint32_t _column_family_id, CompressionType _compression,
-                bool _has_ttl, const ExpirationRange& _expiration_range)
-      : column_family_id(_column_family_id),
-        compression(_compression),
-        has_ttl(_has_ttl),
-        expiration_range(_expiration_range) {}
-
-  uint32_t version = kVersion1;
-  uint32_t column_family_id = 0;
-  CompressionType compression = kNoCompression;
-  bool has_ttl = false;
-  ExpirationRange expiration_range;
-
-  void EncodeTo(std::string* dst);
-
-  Status DecodeFrom(Slice slice);
-};
-
-// clang-format off
-
-// Format of blob log file footer (32 bytes):
-//
-//    +--------------+------------+-------------------+------------+
-//    | magic number | blob count | expiration range  | footer CRC |
-//    +--------------+------------+-------------------+------------+
-//    |   Fixed32    |  Fixed64   | Fixed64 + Fixed64 |   Fixed32  |
-//    +--------------+------------+-------------------+------------+
-//
-// The footer will be presented only when the blob file is properly closed.
-//
-// Unlike the same field in file header, expiration range in the footer is the
-// range of smallest and largest expiration of the data in this file.
-
-// clang-format on
-
-struct BlobLogFooter {
-  static constexpr size_t kSize = 32;
-
-  uint64_t blob_count = 0;
-  ExpirationRange expiration_range = std::make_pair(0, 0);
-  uint32_t crc = 0;
-
-  void EncodeTo(std::string* dst);
-
-  Status DecodeFrom(Slice slice);
-};
-
-// clang-format off
-
-// Blob record format (32 bytes header + key + value):
-//
-//    +------------+--------------+------------+------------+----------+---------+-----------+
-//    | key length | value length | expiration | header CRC | blob CRC |   key   |   value   |
-//    +------------+--------------+------------+------------+----------+---------+-----------+
-//    |   Fixed64  |   Fixed64    |  Fixed64   |  Fixed32   | Fixed32  | key len | value len |
-//    +------------+--------------+------------+------------+----------+---------+-----------+
-//
-// If file has has_ttl = false, expiration field is always 0, and the blob
-// doesn't has expiration.
-//
-// Also note that if compression is used, value is compressed value and value
-// length is compressed value length.
-//
-// Header CRC is the checksum of (key_len + val_len + expiration), while
-// blob CRC is the checksum of (key + value).
-//
-// We could use variable length encoding (Varint64) to save more space, but it
-// make reader more complicated.
-
-// clang-format on
-
-struct BlobLogRecord {
-  // header include fields up to blob CRC
-  static constexpr size_t kHeaderSize = 32;
-
-  // Note that the offset field of BlobIndex actually points to the blob value
-  // as opposed to the start of the blob record. The following method can
-  // be used to calculate the adjustment needed to read the blob record header.
-  static constexpr uint64_t CalculateAdjustmentForRecordHeader(
-      uint64_t key_size) {
-    return key_size + kHeaderSize;
-  }
-
-  uint64_t key_size = 0;
-  uint64_t value_size = 0;
-  uint64_t expiration = 0;
-  uint32_t header_crc = 0;
-  uint32_t blob_crc = 0;
-  Slice key;
-  Slice value;
-  std::unique_ptr<char[]> key_buf;
-  std::unique_ptr<char[]> value_buf;
-
-  uint64_t record_size() const { return kHeaderSize + key_size + value_size; }
-
-  void EncodeHeaderTo(std::string* dst);
-
-  Status DecodeHeaderFrom(Slice src);
-
-  Status CheckBlobCRC() const;
-};
-
-// Checks whether a blob offset is potentially valid or not.
-inline bool IsValidBlobOffset(uint64_t value_offset, uint64_t key_size,
-                              uint64_t value_size, uint64_t file_size) {
-  if (value_offset <
-      BlobLogHeader::kSize + BlobLogRecord::kHeaderSize + key_size) {
-    return false;
-  }
-
-  if (value_offset + value_size + BlobLogFooter::kSize > file_size) {
-    return false;
-  }
-
-  return true;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_log_sequential_reader.cc

@@ -1,133 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#include "db/blob/blob_log_sequential_reader.h"
-
-#include "file/random_access_file_reader.h"
-#include "monitoring/statistics_impl.h"
-#include "util/stop_watch.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobLogSequentialReader::BlobLogSequentialReader(
-    std::unique_ptr<RandomAccessFileReader>&& file_reader, SystemClock* clock,
-    Statistics* statistics)
-    : file_(std::move(file_reader)),
-      clock_(clock),
-      statistics_(statistics),
-      next_byte_(0) {}
-
-BlobLogSequentialReader::~BlobLogSequentialReader() = default;
-
-Status BlobLogSequentialReader::ReadSlice(uint64_t size, Slice* slice,
-                                          char* buf) {
-  assert(slice);
-  assert(file_);
-
-  StopWatch read_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_READ_MICROS);
-  // TODO: rate limit `BlobLogSequentialReader` reads (it appears unused?)
-  Status s = file_->Read(IOOptions(), next_byte_, static_cast<size_t>(size),
-                         slice, buf, nullptr);
-  next_byte_ += size;
-  if (!s.ok()) {
-    return s;
-  }
-  RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, slice->size());
-  if (slice->size() != size) {
-    return Status::Corruption("EOF reached while reading record");
-  }
-  return s;
-}
-
-Status BlobLogSequentialReader::ReadHeader(BlobLogHeader* header) {
-  assert(header);
-  assert(next_byte_ == 0);
-
-  static_assert(BlobLogHeader::kSize <= sizeof(header_buf_),
-                "Buffer is smaller than BlobLogHeader::kSize");
-
-  Status s = ReadSlice(BlobLogHeader::kSize, &buffer_, header_buf_);
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (buffer_.size() != BlobLogHeader::kSize) {
-    return Status::Corruption("EOF reached before file header");
-  }
-
-  return header->DecodeFrom(buffer_);
-}
-
-Status BlobLogSequentialReader::ReadRecord(BlobLogRecord* record,
-                                           ReadLevel level,
-                                           uint64_t* blob_offset) {
-  assert(record);
-  static_assert(BlobLogRecord::kHeaderSize <= sizeof(header_buf_),
-                "Buffer is smaller than BlobLogRecord::kHeaderSize");
-
-  Status s = ReadSlice(BlobLogRecord::kHeaderSize, &buffer_, header_buf_);
-  if (!s.ok()) {
-    return s;
-  }
-  if (buffer_.size() != BlobLogRecord::kHeaderSize) {
-    return Status::Corruption("EOF reached before record header");
-  }
-
-  s = record->DecodeHeaderFrom(buffer_);
-  if (!s.ok()) {
-    return s;
-  }
-
-  uint64_t kb_size = record->key_size + record->value_size;
-  if (blob_offset != nullptr) {
-    *blob_offset = next_byte_ + record->key_size;
-  }
-
-  switch (level) {
-    case kReadHeader:
-      next_byte_ += kb_size;
-      break;
-
-    case kReadHeaderKey:
-      record->key_buf.reset(new char[record->key_size]);
-      s = ReadSlice(record->key_size, &record->key, record->key_buf.get());
-      next_byte_ += record->value_size;
-      break;
-
-    case kReadHeaderKeyBlob:
-      record->key_buf.reset(new char[record->key_size]);
-      s = ReadSlice(record->key_size, &record->key, record->key_buf.get());
-      if (s.ok()) {
-        record->value_buf.reset(new char[record->value_size]);
-        s = ReadSlice(record->value_size, &record->value,
-                      record->value_buf.get());
-      }
-      if (s.ok()) {
-        s = record->CheckBlobCRC();
-      }
-      break;
-  }
-  return s;
-}
-
-Status BlobLogSequentialReader::ReadFooter(BlobLogFooter* footer) {
-  assert(footer);
-  static_assert(BlobLogFooter::kSize <= sizeof(header_buf_),
-                "Buffer is smaller than BlobLogFooter::kSize");
-
-  Status s = ReadSlice(BlobLogFooter::kSize, &buffer_, header_buf_);
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (buffer_.size() != BlobLogFooter::kSize) {
-    return Status::Corruption("EOF reached before file footer");
-  }
-
-  return footer->DecodeFrom(buffer_);
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_log_sequential_reader.h

@@ -1,83 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#pragma once
-
-#include <memory>
-
-#include "db/blob/blob_log_format.h"
-#include "rocksdb/slice.h"
-
-#define MAX_HEADER_SIZE(a, b, c) (a > b ? (a > c ? a : c) : (b > c ? b : c))
-
-namespace ROCKSDB_NAMESPACE {
-
-class RandomAccessFileReader;
-class Env;
-class Statistics;
-class Status;
-class SystemClock;
-
-/**
- * BlobLogSequentialReader is a general purpose log stream reader
- * implementation. The actual job of reading from the device is implemented by
- * the RandomAccessFileReader interface.
- *
- * Please see BlobLogWriter for details on the file and record layout.
- */
-
-class BlobLogSequentialReader {
- public:
-  enum ReadLevel {
-    kReadHeader,
-    kReadHeaderKey,
-    kReadHeaderKeyBlob,
-  };
-
-  // Create a reader that will return log records from "*file_reader".
-  BlobLogSequentialReader(std::unique_ptr<RandomAccessFileReader>&& file_reader,
-                          SystemClock* clock, Statistics* statistics);
-
-  // No copying allowed
-  BlobLogSequentialReader(const BlobLogSequentialReader&) = delete;
-  BlobLogSequentialReader& operator=(const BlobLogSequentialReader&) = delete;
-
-  ~BlobLogSequentialReader();
-
-  Status ReadHeader(BlobLogHeader* header);
-
-  // Read the next record into *record.  Returns true if read
-  // successfully, false if we hit end of the input. The contents filled in
-  // *record will only be valid until the next mutating operation on this
-  // reader.
-  // If blob_offset is non-null, return offset of the blob through it.
-  Status ReadRecord(BlobLogRecord* record, ReadLevel level = kReadHeader,
-                    uint64_t* blob_offset = nullptr);
-
-  Status ReadFooter(BlobLogFooter* footer);
-
-  void ResetNextByte() { next_byte_ = 0; }
-
-  uint64_t GetNextByte() const { return next_byte_; }
-
- private:
-  Status ReadSlice(uint64_t size, Slice* slice, char* buf);
-
-  const std::unique_ptr<RandomAccessFileReader> file_;
-  SystemClock* clock_;
-
-  Statistics* statistics_;
-
-  Slice buffer_;
-  char header_buf_[MAX_HEADER_SIZE(BlobLogHeader::kSize, BlobLogFooter::kSize,
-                                   BlobLogRecord::kHeaderSize)];
-
-  // which byte to read next
-  uint64_t next_byte_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE
-
-#undef MAX_HEADER_SIZE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_log_writer.cc

@@ -1,207 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_log_writer.h"
-
-#include <cstdint>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "file/writable_file_writer.h"
-#include "monitoring/statistics_impl.h"
-#include "rocksdb/system_clock.h"
-#include "test_util/sync_point.h"
-#include "util/coding.h"
-#include "util/stop_watch.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobLogWriter::BlobLogWriter(std::unique_ptr<WritableFileWriter>&& dest,
-                             SystemClock* clock, Statistics* statistics,
-                             uint64_t log_number, bool use_fs, bool do_flush,
-                             uint64_t boffset)
-    : dest_(std::move(dest)),
-      clock_(clock),
-      statistics_(statistics),
-      log_number_(log_number),
-      block_offset_(boffset),
-      use_fsync_(use_fs),
-      do_flush_(do_flush),
-      last_elem_type_(kEtNone) {}
-
-BlobLogWriter::~BlobLogWriter() = default;
-
-Status BlobLogWriter::Sync(const WriteOptions& write_options) {
-  TEST_SYNC_POINT("BlobLogWriter::Sync");
-
-  StopWatch sync_sw(clock_, statistics_, BLOB_DB_BLOB_FILE_SYNC_MICROS);
-  IOOptions opts;
-  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (s.ok()) {
-    s = dest_->Sync(opts, use_fsync_);
-  }
-  if (s.ok()) {
-    RecordTick(statistics_, BLOB_DB_BLOB_FILE_SYNCED);
-  }
-  return s;
-}
-
-Status BlobLogWriter::WriteHeader(const WriteOptions& write_options,
-                                  BlobLogHeader& header) {
-  assert(block_offset_ == 0);
-  assert(last_elem_type_ == kEtNone);
-  std::string str;
-  header.EncodeTo(&str);
-
-  IOOptions opts;
-  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (s.ok()) {
-    s = dest_->Append(opts, Slice(str));
-  }
-  if (s.ok()) {
-    block_offset_ += str.size();
-    if (do_flush_) {
-      s = dest_->Flush(opts);
-    }
-  }
-  last_elem_type_ = kEtFileHdr;
-  if (s.ok()) {
-    RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_WRITTEN,
-               BlobLogHeader::kSize);
-  }
-  return s;
-}
-
-Status BlobLogWriter::AppendFooter(const WriteOptions& write_options,
-                                   BlobLogFooter& footer,
-                                   std::string* checksum_method,
-                                   std::string* checksum_value) {
-  assert(block_offset_ != 0);
-  assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord);
-
-  std::string str;
-  footer.EncodeTo(&str);
-
-  Status s;
-  if (dest_->seen_error()) {
-    s.PermitUncheckedError();
-    return Status::IOError("Seen Error. Skip closing.");
-  } else {
-    IOOptions opts;
-    s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-    if (s.ok()) {
-      s = dest_->Append(opts, Slice(str));
-    }
-    if (s.ok()) {
-      block_offset_ += str.size();
-      s = Sync(write_options);
-
-      if (s.ok()) {
-        s = dest_->Close(opts);
-
-        if (s.ok()) {
-          assert(!!checksum_method == !!checksum_value);
-
-          if (checksum_method) {
-            assert(checksum_method->empty());
-
-            std::string method = dest_->GetFileChecksumFuncName();
-            if (method != kUnknownFileChecksumFuncName) {
-              *checksum_method = std::move(method);
-            }
-          }
-          if (checksum_value) {
-            assert(checksum_value->empty());
-
-            std::string value = dest_->GetFileChecksum();
-            if (value != kUnknownFileChecksum) {
-              *checksum_value = std::move(value);
-            }
-          }
-        }
-      }
-    }
-
-    dest_.reset();
-  }
-
-  last_elem_type_ = kEtFileFooter;
-  if (s.ok()) {
-    RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_WRITTEN,
-               BlobLogFooter::kSize);
-  }
-  return s;
-}
-
-Status BlobLogWriter::AddRecord(const WriteOptions& write_options,
-                                const Slice& key, const Slice& val,
-                                uint64_t expiration, uint64_t* key_offset,
-                                uint64_t* blob_offset) {
-  assert(block_offset_ != 0);
-  assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord);
-
-  std::string buf;
-  ConstructBlobHeader(&buf, key, val, expiration);
-
-  Status s =
-      EmitPhysicalRecord(write_options, buf, key, val, key_offset, blob_offset);
-  return s;
-}
-
-Status BlobLogWriter::AddRecord(const WriteOptions& write_options,
-                                const Slice& key, const Slice& val,
-                                uint64_t* key_offset, uint64_t* blob_offset) {
-  assert(block_offset_ != 0);
-  assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtRecord);
-
-  std::string buf;
-  ConstructBlobHeader(&buf, key, val, 0);
-
-  Status s =
-      EmitPhysicalRecord(write_options, buf, key, val, key_offset, blob_offset);
-  return s;
-}
-
-void BlobLogWriter::ConstructBlobHeader(std::string* buf, const Slice& key,
-                                        const Slice& val, uint64_t expiration) {
-  BlobLogRecord record;
-  record.key = key;
-  record.value = val;
-  record.expiration = expiration;
-  record.EncodeHeaderTo(buf);
-}
-
-Status BlobLogWriter::EmitPhysicalRecord(const WriteOptions& write_options,
-                                         const std::string& headerbuf,
-                                         const Slice& key, const Slice& val,
-                                         uint64_t* key_offset,
-                                         uint64_t* blob_offset) {
-  IOOptions opts;
-  Status s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (s.ok()) {
-    s = dest_->Append(opts, Slice(headerbuf));
-  }
-  if (s.ok()) {
-    s = dest_->Append(opts, key);
-  }
-  if (s.ok()) {
-    s = dest_->Append(opts, val);
-  }
-  if (do_flush_ && s.ok()) {
-    s = dest_->Flush(opts);
-  }
-
-  *key_offset = block_offset_ + BlobLogRecord::kHeaderSize;
-  *blob_offset = *key_offset + key.size();
-  block_offset_ = *blob_offset + val.size();
-  last_elem_type_ = kEtRecord;
-  if (s.ok()) {
-    RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_WRITTEN,
-               BlobLogRecord::kHeaderSize + key.size() + val.size());
-  }
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/blob/blob_log_writer.h

@@ -1,87 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <cstdint>
-#include <memory>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/statistics.h"
-#include "rocksdb/status.h"
-#include "rocksdb/types.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class WritableFileWriter;
-class SystemClock;
-/**
- * BlobLogWriter is the blob log stream writer. It provides an append-only
- * abstraction for writing blob data.
- *
- *
- * Look at blob_db_format.h to see the details of the record formats.
- */
-
-class BlobLogWriter {
- public:
-  // Create a writer that will append data to "*dest".
-  // "*dest" must be initially empty.
-  // "*dest" must remain live while this BlobLogWriter is in use.
-  BlobLogWriter(std::unique_ptr<WritableFileWriter>&& dest, SystemClock* clock,
-                Statistics* statistics, uint64_t log_number, bool use_fsync,
-                bool do_flush, uint64_t boffset = 0);
-  // No copying allowed
-  BlobLogWriter(const BlobLogWriter&) = delete;
-  BlobLogWriter& operator=(const BlobLogWriter&) = delete;
-
-  ~BlobLogWriter();
-
-  static void ConstructBlobHeader(std::string* buf, const Slice& key,
-                                  const Slice& val, uint64_t expiration);
-
-  Status AddRecord(const WriteOptions& write_options, const Slice& key,
-                   const Slice& val, uint64_t* key_offset,
-                   uint64_t* blob_offset);
-
-  Status AddRecord(const WriteOptions& write_options, const Slice& key,
-                   const Slice& val, uint64_t expiration, uint64_t* key_offset,
-                   uint64_t* blob_offset);
-
-  Status EmitPhysicalRecord(const WriteOptions& write_options,
-                            const std::string& headerbuf, const Slice& key,
-                            const Slice& val, uint64_t* key_offset,
-                            uint64_t* blob_offset);
-
-  Status AppendFooter(const WriteOptions& write_options, BlobLogFooter& footer,
-                      std::string* checksum_method,
-                      std::string* checksum_value);
-
-  Status WriteHeader(const WriteOptions& write_options, BlobLogHeader& header);
-
-  WritableFileWriter* file() { return dest_.get(); }
-
-  const WritableFileWriter* file() const { return dest_.get(); }
-
-  uint64_t get_log_number() const { return log_number_; }
-
-  Status Sync(const WriteOptions& write_options);
-
- private:
-  std::unique_ptr<WritableFileWriter> dest_;
-  SystemClock* clock_;
-  Statistics* statistics_;
-  uint64_t log_number_;
-  uint64_t block_offset_;  // Current offset in block
-  bool use_fsync_;
-  bool do_flush_;
-
- public:
-  enum ElemType { kEtNone, kEtFileHdr, kEtRecord, kEtFileFooter };
-  ElemType last_elem_type_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE