rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/db/log_reader.h

@@ -1,245 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#include <stdint.h>
-
-#include <memory>
-#include <unordered_map>
-#include <vector>
-
-#include "db/log_format.h"
-#include "file/sequence_file_reader.h"
-#include "rocksdb/options.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "util/compression.h"
-#include "util/hash_containers.h"
-#include "util/udt_util.h"
-#include "util/xxhash.h"
-
-namespace ROCKSDB_NAMESPACE {
-class Logger;
-
-namespace log {
-
-/**
- * Reader is a general purpose log stream reader implementation. The actual job
- * of reading from the device is implemented by the SequentialFile interface.
- *
- * Please see Writer for details on the file and record layout.
- */
-class Reader {
- public:
-  // Interface for reporting errors.
-  class Reporter {
-   public:
-    virtual ~Reporter();
-
-    // Some corruption was detected.  "size" is the approximate number
-    // of bytes dropped due to the corruption.
-    virtual void Corruption(size_t bytes, const Status& status) = 0;
-
-    virtual void OldLogRecord(size_t /*bytes*/) {}
-  };
-
-  // Create a reader that will return log records from "*file".
-  // "*file" must remain live while this Reader is in use.
-  //
-  // If "reporter" is non-nullptr, it is notified whenever some data is
-  // dropped due to a detected corruption.  "*reporter" must remain
-  // live while this Reader is in use.
-  //
-  // If "checksum" is true, verify checksums if available.
-  Reader(std::shared_ptr<Logger> info_log,
-         std::unique_ptr<SequentialFileReader>&& file, Reporter* reporter,
-         bool checksum, uint64_t log_num);
-  // No copying allowed
-  Reader(const Reader&) = delete;
-  void operator=(const Reader&) = delete;
-
-  virtual ~Reader();
-
-  // Read the next record into *record.  Returns true if read
-  // successfully, false if we hit end of the input.  May use
-  // "*scratch" as temporary storage. The contents filled in *record
-  // will only be valid until the next mutating operation on this
-  // reader or the next mutation to *scratch.
-  // If record_checksum is not nullptr, then this function will calculate the
-  // checksum of the record read and set record_checksum to it. The checksum is
-  // calculated from the original buffers that contain the contents of the
-  // record.
-  virtual bool ReadRecord(Slice* record, std::string* scratch,
-                          WALRecoveryMode wal_recovery_mode =
-                              WALRecoveryMode::kTolerateCorruptedTailRecords,
-                          uint64_t* record_checksum = nullptr);
-
-  // Return the recorded user-defined timestamp size that have been read so
-  // far. This only applies to WAL logs.
-  const UnorderedMap<uint32_t, size_t>& GetRecordedTimestampSize() const {
-    return recorded_cf_to_ts_sz_;
-  }
-
-  // Returns the physical offset of the last record returned by ReadRecord.
-  //
-  // Undefined before the first call to ReadRecord.
-  uint64_t LastRecordOffset();
-
-  // Returns the first physical offset after the last record returned by
-  // ReadRecord, or zero before first call to ReadRecord. This can also be
-  // thought of as the "current" position in processing the file bytes.
-  uint64_t LastRecordEnd();
-
-  // returns true if the reader has encountered an eof condition.
-  bool IsEOF() { return eof_; }
-
-  // returns true if the reader has encountered read error.
-  bool hasReadError() const { return read_error_; }
-
-  // when we know more data has been written to the file. we can use this
-  // function to force the reader to look again in the file.
-  // Also aligns the file position indicator to the start of the next block
-  // by reading the rest of the data from the EOF position to the end of the
-  // block that was partially read.
-  virtual void UnmarkEOF();
-
-  SequentialFileReader* file() { return file_.get(); }
-
-  Reporter* GetReporter() const { return reporter_; }
-
-  uint64_t GetLogNumber() const { return log_number_; }
-
-  size_t GetReadOffset() const {
-    return static_cast<size_t>(end_of_buffer_offset_);
-  }
-
-  bool IsCompressedAndEmptyFile() {
-    return !first_record_read_ && compression_type_record_read_;
-  }
-
- protected:
-  std::shared_ptr<Logger> info_log_;
-  const std::unique_ptr<SequentialFileReader> file_;
-  Reporter* const reporter_;
-  bool const checksum_;
-  char* const backing_store_;
-
-  // Internal state variables used for reading records
-  Slice buffer_;
-  bool eof_;         // Last Read() indicated EOF by returning < kBlockSize
-  bool read_error_;  // Error occurred while reading from file
-
-  // Offset of the file position indicator within the last block when an
-  // EOF was detected.
-  size_t eof_offset_;
-
-  // Offset of the last record returned by ReadRecord.
-  uint64_t last_record_offset_;
-  // Offset of the first location past the end of buffer_.
-  uint64_t end_of_buffer_offset_;
-
-  // which log number this is
-  uint64_t const log_number_;
-
-  // Whether this is a recycled log file
-  bool recycled_;
-
-  // Whether the first record has been read or not.
-  bool first_record_read_;
-  // Type of compression used
-  CompressionType compression_type_;
-  // Track whether the compression type record has been read or not.
-  bool compression_type_record_read_;
-  StreamingUncompress* uncompress_;
-  // Reusable uncompressed output buffer
-  std::unique_ptr<char[]> uncompressed_buffer_;
-  // Reusable uncompressed record
-  std::string uncompressed_record_;
-  // Used for stream hashing fragment content in ReadRecord()
-  XXH3_state_t* hash_state_;
-  // Used for stream hashing uncompressed buffer in ReadPhysicalRecord()
-  XXH3_state_t* uncompress_hash_state_;
-
-  // The recorded user-defined timestamp sizes that have been read so far. This
-  // is only for WAL logs.
-  UnorderedMap<uint32_t, size_t> recorded_cf_to_ts_sz_;
-
-  // Extend record types with the following special values
-  enum {
-    kEof = kMaxRecordType + 1,
-    // Returned whenever we find an invalid physical record.
-    // Currently there are three situations in which this happens:
-    // * The record has an invalid CRC (ReadPhysicalRecord reports a drop)
-    // * The record is a 0-length record (No drop is reported)
-    kBadRecord = kMaxRecordType + 2,
-    // Returned when we fail to read a valid header.
-    kBadHeader = kMaxRecordType + 3,
-    // Returned when we read an old record from a previous user of the log.
-    kOldRecord = kMaxRecordType + 4,
-    // Returned when we get a bad record length
-    kBadRecordLen = kMaxRecordType + 5,
-    // Returned when we get a bad record checksum
-    kBadRecordChecksum = kMaxRecordType + 6,
-  };
-
-  // Return type, or one of the preceding special values
-  // If WAL compressioned is enabled, fragment_checksum is the checksum of the
-  // fragment computed from the orginal buffer containinng uncompressed
-  // fragment.
-  unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size,
-                                  uint64_t* fragment_checksum = nullptr);
-
-  // Read some more
-  bool ReadMore(size_t* drop_size, int* error);
-
-  void UnmarkEOFInternal();
-
-  // Reports dropped bytes to the reporter.
-  // buffer_ must be updated to remove the dropped bytes prior to invocation.
-  void ReportCorruption(size_t bytes, const char* reason);
-  void ReportDrop(size_t bytes, const Status& reason);
-  void ReportOldLogRecord(size_t bytes);
-
-  void InitCompression(const CompressionTypeRecord& compression_record);
-
-  Status UpdateRecordedTimestampSize(
-      const std::vector<std::pair<uint32_t, size_t>>& cf_to_ts_sz);
-};
-
-class FragmentBufferedReader : public Reader {
- public:
-  FragmentBufferedReader(std::shared_ptr<Logger> info_log,
-                         std::unique_ptr<SequentialFileReader>&& _file,
-                         Reporter* reporter, bool checksum, uint64_t log_num)
-      : Reader(info_log, std::move(_file), reporter, checksum, log_num),
-        fragments_(),
-        in_fragmented_record_(false) {}
-  ~FragmentBufferedReader() override {}
-  bool ReadRecord(Slice* record, std::string* scratch,
-                  WALRecoveryMode wal_recovery_mode =
-                      WALRecoveryMode::kTolerateCorruptedTailRecords,
-                  uint64_t* record_checksum = nullptr) override;
-  void UnmarkEOF() override;
-
- private:
-  std::string fragments_;
-  bool in_fragmented_record_;
-
-  bool TryReadFragment(Slice* result, size_t* drop_size,
-                       unsigned int* fragment_type_or_err);
-
-  bool TryReadMore(size_t* drop_size, int* error);
-
-  // No copy allowed
-  FragmentBufferedReader(const FragmentBufferedReader&);
-  void operator=(const FragmentBufferedReader&);
-};
-
-}  // namespace log
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/log_writer.cc

@@ -1,355 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_writer.h"
-
-#include <cstdint>
-
-#include "file/writable_file_writer.h"
-#include "rocksdb/env.h"
-#include "rocksdb/io_status.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-#include "util/udt_util.h"
-
-namespace ROCKSDB_NAMESPACE::log {
-
-Writer::Writer(std::unique_ptr<WritableFileWriter>&& dest, uint64_t log_number,
-               bool recycle_log_files, bool manual_flush,
-               CompressionType compression_type)
-    : dest_(std::move(dest)),
-      block_offset_(0),
-      log_number_(log_number),
-      recycle_log_files_(recycle_log_files),
-      // Header size varies depending on whether we are recycling or not.
-      header_size_(recycle_log_files ? kRecyclableHeaderSize : kHeaderSize),
-      manual_flush_(manual_flush),
-      compression_type_(compression_type),
-      compress_(nullptr) {
-  for (int i = 0; i <= kMaxRecordType; i++) {
-    char t = static_cast<char>(i);
-    type_crc_[i] = crc32c::Value(&t, 1);
-  }
-}
-
-Writer::~Writer() {
-  ThreadStatus::OperationType cur_op_type =
-      ThreadStatusUtil::GetThreadOperation();
-  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OperationType::OP_UNKNOWN);
-  if (dest_) {
-    WriteBuffer(WriteOptions()).PermitUncheckedError();
-  }
-  if (compress_) {
-    delete compress_;
-  }
-  ThreadStatusUtil::SetThreadOperation(cur_op_type);
-}
-
-IOStatus Writer::WriteBuffer(const WriteOptions& write_options) {
-  if (dest_->seen_error()) {
-#ifndef NDEBUG
-    if (dest_->seen_injected_error()) {
-      std::stringstream msg;
-      msg << "Seen " << FaultInjectionTestFS::kInjected
-          << " error. Skip writing buffer.";
-      return IOStatus::IOError(msg.str());
-    }
-#endif  // NDEBUG
-    return IOStatus::IOError("Seen error. Skip writing buffer.");
-  }
-  IOOptions opts;
-  IOStatus s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (!s.ok()) {
-    return s;
-  }
-  return dest_->Flush(opts);
-}
-
-IOStatus Writer::Close(const WriteOptions& write_options) {
-  IOStatus s;
-  IOOptions opts;
-  s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (s.ok() && dest_) {
-    s = dest_->Close(opts);
-    dest_.reset();
-  }
-  return s;
-}
-
-bool Writer::PublishIfClosed() {
-  if (dest_->IsClosed()) {
-    dest_.reset();
-    return true;
-  } else {
-    return false;
-  }
-}
-
-IOStatus Writer::AddRecord(const WriteOptions& write_options,
-                           const Slice& slice) {
-  if (dest_->seen_error()) {
-#ifndef NDEBUG
-    if (dest_->seen_injected_error()) {
-      std::stringstream msg;
-      msg << "Seen " << FaultInjectionTestFS::kInjected
-          << " error. Skip writing buffer.";
-      return IOStatus::IOError(msg.str());
-    }
-#endif  // NDEBUG
-    return IOStatus::IOError("Seen error. Skip writing buffer.");
-  }
-  const char* ptr = slice.data();
-  size_t left = slice.size();
-
-  // Fragment the record if necessary and emit it.  Note that if slice
-  // is empty, we still want to iterate once to emit a single
-  // zero-length record
-  bool begin = true;
-  int compress_remaining = 0;
-  bool compress_start = false;
-  if (compress_) {
-    compress_->Reset();
-    compress_start = true;
-  }
-
-  IOStatus s;
-  IOOptions opts;
-  s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (s.ok()) {
-    do {
-      const int64_t leftover = kBlockSize - block_offset_;
-      assert(leftover >= 0);
-      if (leftover < header_size_) {
-        // Switch to a new block
-        if (leftover > 0) {
-          // Fill the trailer (literal below relies on kHeaderSize and
-          // kRecyclableHeaderSize being <= 11)
-          assert(header_size_ <= 11);
-          s = dest_->Append(opts,
-                            Slice("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
-                                  static_cast<size_t>(leftover)),
-                            0 /* crc32c_checksum */);
-          if (!s.ok()) {
-            break;
-          }
-        }
-        block_offset_ = 0;
-      }
-
-      // Invariant: we never leave < header_size bytes in a block.
-      assert(static_cast<int64_t>(kBlockSize - block_offset_) >= header_size_);
-
-      const size_t avail = kBlockSize - block_offset_ - header_size_;
-
-      // Compress the record if compression is enabled.
-      // Compress() is called at least once (compress_start=true) and after the
-      // previous generated compressed chunk is written out as one or more
-      // physical records (left=0).
-      if (compress_ && (compress_start || left == 0)) {
-        compress_remaining = compress_->Compress(
-            slice.data(), slice.size(), compressed_buffer_.get(), &left);
-
-        if (compress_remaining < 0) {
-          // Set failure status
-          s = IOStatus::IOError("Unexpected WAL compression error");
-          s.SetDataLoss(true);
-          break;
-        } else if (left == 0) {
-          // Nothing left to compress
-          if (!compress_start) {
-            break;
-          }
-        }
-        compress_start = false;
-        ptr = compressed_buffer_.get();
-      }
-
-      const size_t fragment_length = (left < avail) ? left : avail;
-
-      RecordType type;
-      const bool end = (left == fragment_length && compress_remaining == 0);
-      if (begin && end) {
-        type = recycle_log_files_ ? kRecyclableFullType : kFullType;
-      } else if (begin) {
-        type = recycle_log_files_ ? kRecyclableFirstType : kFirstType;
-      } else if (end) {
-        type = recycle_log_files_ ? kRecyclableLastType : kLastType;
-      } else {
-        type = recycle_log_files_ ? kRecyclableMiddleType : kMiddleType;
-      }
-
-      s = EmitPhysicalRecord(write_options, type, ptr, fragment_length);
-      ptr += fragment_length;
-      left -= fragment_length;
-      begin = false;
-    } while (s.ok() && (left > 0 || compress_remaining > 0));
-  }
-  if (s.ok()) {
-    if (!manual_flush_) {
-      s = dest_->Flush(opts);
-    }
-  }
-
-  return s;
-}
-
-IOStatus Writer::AddCompressionTypeRecord(const WriteOptions& write_options) {
-  // Should be the first record
-  assert(block_offset_ == 0);
-
-  if (compression_type_ == kNoCompression) {
-    // No need to add a record
-    return IOStatus::OK();
-  }
-
-  if (dest_->seen_error()) {
-#ifndef NDEBUG
-    if (dest_->seen_injected_error()) {
-      std::stringstream msg;
-      msg << "Seen " << FaultInjectionTestFS::kInjected
-          << " error. Skip writing buffer.";
-      return IOStatus::IOError(msg.str());
-    }
-#endif  // NDEBUG
-    return IOStatus::IOError("Seen error. Skip writing buffer.");
-  }
-
-  CompressionTypeRecord record(compression_type_);
-  std::string encode;
-  record.EncodeTo(&encode);
-  IOStatus s = EmitPhysicalRecord(write_options, kSetCompressionType,
-                                  encode.data(), encode.size());
-  if (s.ok()) {
-    if (!manual_flush_) {
-      IOOptions io_opts;
-      s = WritableFileWriter::PrepareIOOptions(write_options, io_opts);
-      if (s.ok()) {
-        s = dest_->Flush(io_opts);
-      }
-    }
-    // Initialize fields required for compression
-    const size_t max_output_buffer_len = kBlockSize - header_size_;
-    CompressionOptions opts;
-    constexpr uint32_t compression_format_version = 2;
-    compress_ = StreamingCompress::Create(compression_type_, opts,
-                                          compression_format_version,
-                                          max_output_buffer_len);
-    assert(compress_ != nullptr);
-    compressed_buffer_ =
-        std::unique_ptr<char[]>(new char[max_output_buffer_len]);
-    assert(compressed_buffer_);
-  } else {
-    // Disable compression if the record could not be added.
-    compression_type_ = kNoCompression;
-  }
-  return s;
-}
-
-IOStatus Writer::MaybeAddUserDefinedTimestampSizeRecord(
-    const WriteOptions& write_options,
-    const UnorderedMap<uint32_t, size_t>& cf_to_ts_sz) {
-  std::vector<std::pair<uint32_t, size_t>> ts_sz_to_record;
-  for (const auto& [cf_id, ts_sz] : cf_to_ts_sz) {
-    if (recorded_cf_to_ts_sz_.count(cf_id) != 0) {
-      // A column family's user-defined timestamp size should not be
-      // updated while DB is running.
-      assert(recorded_cf_to_ts_sz_[cf_id] == ts_sz);
-    } else if (ts_sz != 0) {
-      ts_sz_to_record.emplace_back(cf_id, ts_sz);
-      recorded_cf_to_ts_sz_.insert(std::make_pair(cf_id, ts_sz));
-    }
-  }
-  if (ts_sz_to_record.empty()) {
-    return IOStatus::OK();
-  }
-
-  UserDefinedTimestampSizeRecord record(std::move(ts_sz_to_record));
-  std::string encoded;
-  record.EncodeTo(&encoded);
-  RecordType type = recycle_log_files_ ? kRecyclableUserDefinedTimestampSizeType
-                                       : kUserDefinedTimestampSizeType;
-
-  // If there's not enough space for this record, switch to a new block.
-  const int64_t leftover = kBlockSize - block_offset_;
-  if (leftover < header_size_ + (int)encoded.size()) {
-    IOOptions opts;
-    IOStatus s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-    if (!s.ok()) {
-      return s;
-    }
-
-    std::vector<char> trailer(leftover, '\x00');
-    s = dest_->Append(opts, Slice(trailer.data(), trailer.size()));
-    if (!s.ok()) {
-      return s;
-    }
-
-    block_offset_ = 0;
-  }
-
-  return EmitPhysicalRecord(write_options, type, encoded.data(),
-                            encoded.size());
-}
-
-bool Writer::BufferIsEmpty() { return dest_->BufferIsEmpty(); }
-
-IOStatus Writer::EmitPhysicalRecord(const WriteOptions& write_options,
-                                    RecordType t, const char* ptr, size_t n) {
-  assert(n <= 0xffff);  // Must fit in two bytes
-
-  size_t header_size;
-  char buf[kRecyclableHeaderSize];
-
-  // Format the header
-  buf[4] = static_cast<char>(n & 0xff);
-  buf[5] = static_cast<char>(n >> 8);
-  buf[6] = static_cast<char>(t);
-
-  uint32_t crc = type_crc_[t];
-  if (t < kRecyclableFullType || t == kSetCompressionType ||
-      t == kUserDefinedTimestampSizeType) {
-    // Legacy record format
-    assert(block_offset_ + kHeaderSize + n <= kBlockSize);
-    header_size = kHeaderSize;
-  } else {
-    // Recyclable record format
-    assert(block_offset_ + kRecyclableHeaderSize + n <= kBlockSize);
-    header_size = kRecyclableHeaderSize;
-
-    // Only encode low 32-bits of the 64-bit log number.  This means
-    // we will fail to detect an old record if we recycled a log from
-    // ~4 billion logs ago, but that is effectively impossible, and
-    // even if it were we'dbe far more likely to see a false positive
-    // on the 32-bit CRC.
-    EncodeFixed32(buf + 7, static_cast<uint32_t>(log_number_));
-    crc = crc32c::Extend(crc, buf + 7, 4);
-  }
-
-  // Compute the crc of the record type and the payload.
-  uint32_t payload_crc = crc32c::Value(ptr, n);
-  crc = crc32c::Crc32cCombine(crc, payload_crc, n);
-  crc = crc32c::Mask(crc);  // Adjust for storage
-  TEST_SYNC_POINT_CALLBACK("LogWriter::EmitPhysicalRecord:BeforeEncodeChecksum",
-                           &crc);
-  EncodeFixed32(buf, crc);
-
-  // Write the header and the payload
-  IOOptions opts;
-  IOStatus s = WritableFileWriter::PrepareIOOptions(write_options, opts);
-  if (s.ok()) {
-    s = dest_->Append(opts, Slice(buf, header_size), 0 /* crc32c_checksum */);
-  }
-  if (s.ok()) {
-    s = dest_->Append(opts, Slice(ptr, n), payload_crc);
-  }
-  block_offset_ += header_size + n;
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE::log