rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_file.cc

@@ -1,311 +0,0 @@
-
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#include "utilities/blob_db/blob_file.h"
-
-#include <algorithm>
-#include <cinttypes>
-#include <cstdio>
-#include <memory>
-
-#include "db/column_family.h"
-#include "db/db_impl/db_impl.h"
-#include "db/dbformat.h"
-#include "file/filename.h"
-#include "file/readahead_raf.h"
-#include "logging/logging.h"
-#include "utilities/blob_db/blob_db_impl.h"
-
-namespace ROCKSDB_NAMESPACE::blob_db {
-
-BlobFile::BlobFile(const BlobDBImpl* p, const std::string& bdir, uint64_t fn,
-                   Logger* info_log)
-    : parent_(p), path_to_dir_(bdir), file_number_(fn), info_log_(info_log) {}
-
-BlobFile::BlobFile(const BlobDBImpl* p, const std::string& bdir, uint64_t fn,
-                   Logger* info_log, uint32_t column_family_id,
-                   CompressionType compression, bool has_ttl,
-                   const ExpirationRange& expiration_range)
-    : parent_(p),
-      path_to_dir_(bdir),
-      file_number_(fn),
-      info_log_(info_log),
-      column_family_id_(column_family_id),
-      compression_(compression),
-      has_ttl_(has_ttl),
-      expiration_range_(expiration_range),
-      header_(column_family_id, compression, has_ttl, expiration_range),
-      header_valid_(true) {}
-
-BlobFile::~BlobFile() {
-  if (obsolete_) {
-    std::string pn(PathName());
-    Status s = Env::Default()->DeleteFile(PathName());
-    if (!s.ok()) {
-      // ROCKS_LOG_INFO(db_options_.info_log,
-      // "File could not be deleted %s", pn.c_str());
-    }
-  }
-}
-
-uint32_t BlobFile::GetColumnFamilyId() const { return column_family_id_; }
-
-std::string BlobFile::PathName() const {
-  return BlobFileName(path_to_dir_, file_number_);
-}
-
-std::string BlobFile::DumpState() const {
-  char str[1000];
-  snprintf(
-      str, sizeof(str),
-      "path: %s fn: %" PRIu64 " blob_count: %" PRIu64 " file_size: %" PRIu64
-      " closed: %d obsolete: %d expiration_range: (%" PRIu64 ", %" PRIu64
-      "), writer: %d reader: %d",
-      path_to_dir_.c_str(), file_number_, blob_count_.load(), file_size_.load(),
-      closed_.load(), obsolete_.load(), expiration_range_.first,
-      expiration_range_.second, (!!log_writer_), (!!ra_file_reader_));
-  return str;
-}
-
-void BlobFile::MarkObsolete(SequenceNumber sequence) {
-  assert(Immutable());
-  obsolete_sequence_ = sequence;
-  obsolete_.store(true);
-}
-
-Status BlobFile::WriteFooterAndCloseLocked(const WriteOptions& write_options,
-                                           SequenceNumber sequence) {
-  BlobLogFooter footer;
-  footer.blob_count = blob_count_;
-  if (HasTTL()) {
-    footer.expiration_range = expiration_range_;
-  }
-
-  // this will close the file and reset the Writable File Pointer.
-  Status s = log_writer_->AppendFooter(write_options, footer,
-                                       /* checksum_method */ nullptr,
-                                       /* checksum_value */ nullptr);
-  if (s.ok()) {
-    closed_ = true;
-    immutable_sequence_ = sequence;
-    file_size_ += BlobLogFooter::kSize;
-  }
-  // delete the sequential writer
-  log_writer_.reset();
-  return s;
-}
-
-Status BlobFile::ReadFooter(BlobLogFooter* bf) {
-  if (file_size_ < (BlobLogHeader::kSize + BlobLogFooter::kSize)) {
-    return Status::IOError("File does not have footer", PathName());
-  }
-
-  uint64_t footer_offset = file_size_ - BlobLogFooter::kSize;
-  // assume that ra_file_reader_ is valid before we enter this
-  assert(ra_file_reader_);
-
-  Slice result;
-  std::string buf;
-  AlignedBuf aligned_buf;
-  Status s;
-  // TODO: rate limit reading footers from blob files.
-  if (ra_file_reader_->use_direct_io()) {
-    s = ra_file_reader_->Read(IOOptions(), footer_offset, BlobLogFooter::kSize,
-                              &result, nullptr, &aligned_buf);
-  } else {
-    buf.reserve(BlobLogFooter::kSize + 10);
-    s = ra_file_reader_->Read(IOOptions(), footer_offset, BlobLogFooter::kSize,
-                              &result, buf.data(), nullptr);
-  }
-  if (!s.ok()) {
-    return s;
-  }
-  if (result.size() != BlobLogFooter::kSize) {
-    // should not happen
-    return Status::IOError("EOF reached before footer");
-  }
-
-  s = bf->DecodeFrom(result);
-  return s;
-}
-
-Status BlobFile::SetFromFooterLocked(const BlobLogFooter& footer) {
-  blob_count_ = footer.blob_count;
-  expiration_range_ = footer.expiration_range;
-  closed_ = true;
-  return Status::OK();
-}
-
-Status BlobFile::Fsync(const WriteOptions& write_options) {
-  Status s;
-  if (log_writer_.get()) {
-    s = log_writer_->Sync(write_options);
-  }
-  return s;
-}
-
-void BlobFile::CloseRandomAccessLocked() {
-  ra_file_reader_.reset();
-  last_access_ = -1;
-}
-
-Status BlobFile::GetReader(Env* env, const FileOptions& file_options,
-                           std::shared_ptr<RandomAccessFileReader>* reader,
-                           bool* fresh_open) {
-  assert(reader != nullptr);
-  assert(fresh_open != nullptr);
-  *fresh_open = false;
-  int64_t current_time = 0;
-  if (env->GetCurrentTime(&current_time).ok()) {
-    last_access_.store(current_time);
-  }
-  Status s;
-
-  {
-    ReadLock lockbfile_r(&mutex_);
-    if (ra_file_reader_) {
-      *reader = ra_file_reader_;
-      return s;
-    }
-  }
-
-  WriteLock lockbfile_w(&mutex_);
-  // Double check.
-  if (ra_file_reader_) {
-    *reader = ra_file_reader_;
-    return s;
-  }
-
-  std::unique_ptr<FSRandomAccessFile> rfile;
-  s = env->GetFileSystem()->NewRandomAccessFile(PathName(), file_options,
-                                                &rfile, nullptr);
-  if (!s.ok()) {
-    ROCKS_LOG_ERROR(info_log_,
-                    "Failed to open blob file for random-read: %s status: '%s'"
-                    " exists: '%s'",
-                    PathName().c_str(), s.ToString().c_str(),
-                    env->FileExists(PathName()).ToString().c_str());
-    return s;
-  }
-
-  ra_file_reader_ =
-      std::make_shared<RandomAccessFileReader>(std::move(rfile), PathName());
-  *reader = ra_file_reader_;
-  *fresh_open = true;
-  return s;
-}
-
-Status BlobFile::ReadMetadata(const std::shared_ptr<FileSystem>& fs,
-                              const FileOptions& file_options) {
-  assert(Immutable());
-  // Get file size.
-  uint64_t file_size = 0;
-  Status s =
-      fs->GetFileSize(PathName(), file_options.io_options, &file_size, nullptr);
-  if (s.ok()) {
-    file_size_ = file_size;
-  } else {
-    ROCKS_LOG_ERROR(info_log_,
-                    "Failed to get size of blob file %" PRIu64 ", status: %s",
-                    file_number_, s.ToString().c_str());
-    return s;
-  }
-  if (file_size < BlobLogHeader::kSize) {
-    ROCKS_LOG_ERROR(
-        info_log_, "Incomplete blob file blob file %" PRIu64 ", size: %" PRIu64,
-        file_number_, file_size);
-    return Status::Corruption("Incomplete blob file header.");
-  }
-
-  // Create file reader.
-  std::unique_ptr<RandomAccessFileReader> file_reader;
-  s = RandomAccessFileReader::Create(fs, PathName(), file_options, &file_reader,
-                                     nullptr);
-  if (!s.ok()) {
-    ROCKS_LOG_ERROR(info_log_,
-                    "Failed to open blob file %" PRIu64 ", status: %s",
-                    file_number_, s.ToString().c_str());
-    return s;
-  }
-
-  // Read file header.
-  std::string header_buf;
-  AlignedBuf aligned_buf;
-  Slice header_slice;
-  // TODO: rate limit reading headers from blob files.
-  if (file_reader->use_direct_io()) {
-    s = file_reader->Read(IOOptions(), 0, BlobLogHeader::kSize, &header_slice,
-                          nullptr, &aligned_buf);
-  } else {
-    header_buf.reserve(BlobLogHeader::kSize);
-    s = file_reader->Read(IOOptions(), 0, BlobLogHeader::kSize, &header_slice,
-                          header_buf.data(), nullptr);
-  }
-  if (!s.ok()) {
-    ROCKS_LOG_ERROR(
-        info_log_, "Failed to read header of blob file %" PRIu64 ", status: %s",
-        file_number_, s.ToString().c_str());
-    return s;
-  }
-  BlobLogHeader header;
-  s = header.DecodeFrom(header_slice);
-  if (!s.ok()) {
-    ROCKS_LOG_ERROR(info_log_,
-                    "Failed to decode header of blob file %" PRIu64
-                    ", status: %s",
-                    file_number_, s.ToString().c_str());
-    return s;
-  }
-  column_family_id_ = header.column_family_id;
-  compression_ = header.compression;
-  has_ttl_ = header.has_ttl;
-  if (has_ttl_) {
-    expiration_range_ = header.expiration_range;
-  }
-  header_valid_ = true;
-
-  // Read file footer.
-  if (file_size_ < BlobLogHeader::kSize + BlobLogFooter::kSize) {
-    // OK not to have footer.
-    assert(!footer_valid_);
-    return Status::OK();
-  }
-  std::string footer_buf;
-  Slice footer_slice;
-  // TODO: rate limit reading footers from blob files.
-  if (file_reader->use_direct_io()) {
-    s = file_reader->Read(IOOptions(), file_size - BlobLogFooter::kSize,
-                          BlobLogFooter::kSize, &footer_slice, nullptr,
-                          &aligned_buf);
-  } else {
-    footer_buf.reserve(BlobLogFooter::kSize);
-    s = file_reader->Read(IOOptions(), file_size - BlobLogFooter::kSize,
-                          BlobLogFooter::kSize, &footer_slice,
-                          footer_buf.data(), nullptr);
-  }
-  if (!s.ok()) {
-    ROCKS_LOG_ERROR(
-        info_log_, "Failed to read footer of blob file %" PRIu64 ", status: %s",
-        file_number_, s.ToString().c_str());
-    return s;
-  }
-  BlobLogFooter footer;
-  s = footer.DecodeFrom(footer_slice);
-  if (!s.ok()) {
-    // OK not to have footer.
-    assert(!footer_valid_);
-    return Status::OK();
-  }
-  blob_count_ = footer.blob_count;
-  if (has_ttl_) {
-    assert(header.expiration_range.first <= footer.expiration_range.first);
-    assert(header.expiration_range.second >= footer.expiration_range.second);
-    expiration_range_ = footer.expiration_range;
-  }
-  footer_valid_ = true;
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE::blob_db

package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_file.h

@@ -1,245 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <atomic>
-#include <limits>
-#include <memory>
-#include <unordered_set>
-
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "file/random_access_file_reader.h"
-#include "port/port.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_system.h"
-#include "rocksdb/options.h"
-
-namespace ROCKSDB_NAMESPACE {
-namespace blob_db {
-
-class BlobDBImpl;
-
-class BlobFile {
-  friend class BlobDBImpl;
-  friend struct BlobFileComparator;
-  friend struct BlobFileComparatorTTL;
-  friend class BlobIndexCompactionFilterBase;
-  friend class BlobIndexCompactionFilterGC;
-
- private:
-  // access to parent
-  const BlobDBImpl* parent_{nullptr};
-
-  // path to blob directory
-  std::string path_to_dir_;
-
-  // the id of the file.
-  // the above 2 are created during file creation and never changed
-  // after that
-  uint64_t file_number_{0};
-
-  // The file numbers of the SST files whose oldest blob file reference
-  // points to this blob file.
-  std::unordered_set<uint64_t> linked_sst_files_;
-
-  // Info log.
-  Logger* info_log_{nullptr};
-
-  // Column family id.
-  uint32_t column_family_id_{std::numeric_limits<uint32_t>::max()};
-
-  // Compression type of blobs in the file
-  CompressionType compression_{kNoCompression};
-
-  // If true, the keys in this file all has TTL. Otherwise all keys don't
-  // have TTL.
-  bool has_ttl_{false};
-
-  // TTL range of blobs in the file.
-  ExpirationRange expiration_range_;
-
-  // number of blobs in the file
-  std::atomic<uint64_t> blob_count_{0};
-
-  // size of the file
-  std::atomic<uint64_t> file_size_{0};
-
-  BlobLogHeader header_;
-
-  // closed_ = true implies the file is no more mutable
-  // no more blobs will be appended and the footer has been written out
-  std::atomic<bool> closed_{false};
-
-  // The latest sequence number when the file was closed/made immutable.
-  SequenceNumber immutable_sequence_{0};
-
-  // Whether the file was marked obsolete (due to either TTL or GC).
-  // obsolete_ still needs to do iterator/snapshot checks
-  std::atomic<bool> obsolete_{false};
-
-  // The last sequence number by the time the file marked as obsolete.
-  // Data in this file is visible to a snapshot taken before the sequence.
-  SequenceNumber obsolete_sequence_{0};
-
-  // Sequential/Append writer for blobs
-  std::shared_ptr<BlobLogWriter> log_writer_;
-
-  // random access file reader for GET calls
-  std::shared_ptr<RandomAccessFileReader> ra_file_reader_;
-
-  // This Read-Write mutex is per file specific and protects
-  // all the datastructures
-  mutable port::RWMutex mutex_;
-
-  // time when the random access reader was last created.
-  std::atomic<std::int64_t> last_access_{-1};
-
-  bool header_valid_{false};
-
-  bool footer_valid_{false};
-
- public:
-  BlobFile() = default;
-
-  BlobFile(const BlobDBImpl* parent, const std::string& bdir, uint64_t fnum,
-           Logger* info_log);
-
-  BlobFile(const BlobDBImpl* parent, const std::string& bdir, uint64_t fnum,
-           Logger* info_log, uint32_t column_family_id,
-           CompressionType compression, bool has_ttl,
-           const ExpirationRange& expiration_range);
-
-  ~BlobFile();
-
-  uint32_t GetColumnFamilyId() const;
-
-  // Returns log file's absolute pathname.
-  std::string PathName() const;
-
-  // Primary identifier for blob file.
-  // once the file is created, this never changes
-  uint64_t BlobFileNumber() const { return file_number_; }
-
-  // Get the set of SST files whose oldest blob file reference points to
-  // this file.
-  const std::unordered_set<uint64_t>& GetLinkedSstFiles() const {
-    return linked_sst_files_;
-  }
-
-  // Link an SST file whose oldest blob file reference points to this file.
-  void LinkSstFile(uint64_t sst_file_number) {
-    assert(linked_sst_files_.find(sst_file_number) == linked_sst_files_.end());
-    linked_sst_files_.insert(sst_file_number);
-  }
-
-  // Unlink an SST file whose oldest blob file reference points to this file.
-  void UnlinkSstFile(uint64_t sst_file_number) {
-    auto it = linked_sst_files_.find(sst_file_number);
-    assert(it != linked_sst_files_.end());
-    linked_sst_files_.erase(it);
-  }
-
-  // the following functions are atomic, and don't need
-  // read lock
-  uint64_t BlobCount() const {
-    return blob_count_.load(std::memory_order_acquire);
-  }
-
-  std::string DumpState() const;
-
-  // if the file is not taking any more appends.
-  bool Immutable() const { return closed_.load(); }
-
-  // Mark the file as immutable.
-  // REQUIRES: write lock held, or access from single thread (on DB open).
-  void MarkImmutable(SequenceNumber sequence) {
-    closed_ = true;
-    immutable_sequence_ = sequence;
-  }
-
-  SequenceNumber GetImmutableSequence() const {
-    assert(Immutable());
-    return immutable_sequence_;
-  }
-
-  // Whether the file was marked obsolete (due to either TTL or GC).
-  bool Obsolete() const {
-    assert(Immutable() || !obsolete_.load());
-    return obsolete_.load();
-  }
-
-  // Mark file as obsolete (due to either TTL or GC). The file is not visible to
-  // snapshots with sequence greater or equal to the given sequence.
-  void MarkObsolete(SequenceNumber sequence);
-
-  SequenceNumber GetObsoleteSequence() const {
-    assert(Obsolete());
-    return obsolete_sequence_;
-  }
-
-  Status Fsync(const WriteOptions& write_options);
-
-  uint64_t GetFileSize() const {
-    return file_size_.load(std::memory_order_acquire);
-  }
-
-  // All Get functions which are not atomic, will need ReadLock on the mutex
-
-  const ExpirationRange& GetExpirationRange() const {
-    return expiration_range_;
-  }
-
-  void ExtendExpirationRange(uint64_t expiration) {
-    expiration_range_.first = std::min(expiration_range_.first, expiration);
-    expiration_range_.second = std::max(expiration_range_.second, expiration);
-  }
-
-  bool HasTTL() const { return has_ttl_; }
-
-  void SetHasTTL(bool has_ttl) { has_ttl_ = has_ttl; }
-
-  CompressionType GetCompressionType() const { return compression_; }
-
-  std::shared_ptr<BlobLogWriter> GetWriter() const { return log_writer_; }
-
-  // Read blob file header and footer. Return corruption if file header is
-  // malform or incomplete. If footer is malform or incomplete, set
-  // footer_valid_ to false and return Status::OK.
-  Status ReadMetadata(const std::shared_ptr<FileSystem>& fs,
-                      const FileOptions& file_options);
-
-  Status GetReader(Env* env, const FileOptions& file_options,
-                   std::shared_ptr<RandomAccessFileReader>* reader,
-                   bool* fresh_open);
-
- private:
-  Status ReadFooter(BlobLogFooter* footer);
-
-  Status WriteFooterAndCloseLocked(const WriteOptions& write_options,
-                                   SequenceNumber sequence);
-
-  void CloseRandomAccessLocked();
-
-  // this is used, when you are reading only the footer of a
-  // previously closed file
-  Status SetFromFooterLocked(const BlobLogFooter& footer);
-
-  void set_expiration_range(const ExpirationRange& expiration_range) {
-    expiration_range_ = expiration_range;
-  }
-
-  // The following functions are atomic, and don't need locks
-  void SetFileSize(uint64_t fs) { file_size_ = fs; }
-
-  void SetBlobCount(uint64_t bc) { blob_count_ = bc; }
-
-  void BlobRecordAdded(uint64_t record_size) {
-    ++blob_count_;
-    file_size_ += record_size;
-  }
-};
-}  // namespace blob_db
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/utilities/cache_dump_load.cc

@@ -1,67 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-
-#include "rocksdb/utilities/cache_dump_load.h"
-
-#include "file/writable_file_writer.h"
-#include "port/lang.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_system.h"
-#include "table/format.h"
-#include "util/crc32c.h"
-#include "utilities/cache_dump_load_impl.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-IOStatus NewToFileCacheDumpWriter(const std::shared_ptr<FileSystem>& fs,
-                                  const FileOptions& file_opts,
-                                  const std::string& file_name,
-                                  std::unique_ptr<CacheDumpWriter>* writer) {
-  std::unique_ptr<WritableFileWriter> file_writer;
-  IOStatus io_s = WritableFileWriter::Create(fs, file_name, file_opts,
-                                             &file_writer, nullptr);
-  if (!io_s.ok()) {
-    return io_s;
-  }
-  writer->reset(new ToFileCacheDumpWriter(std::move(file_writer)));
-  return io_s;
-}
-
-IOStatus NewFromFileCacheDumpReader(const std::shared_ptr<FileSystem>& fs,
-                                    const FileOptions& file_opts,
-                                    const std::string& file_name,
-                                    std::unique_ptr<CacheDumpReader>* reader) {
-  std::unique_ptr<RandomAccessFileReader> file_reader;
-  IOStatus io_s = RandomAccessFileReader::Create(fs, file_name, file_opts,
-                                                 &file_reader, nullptr);
-  if (!io_s.ok()) {
-    return io_s;
-  }
-  reader->reset(new FromFileCacheDumpReader(std::move(file_reader)));
-  return io_s;
-}
-
-Status NewDefaultCacheDumper(const CacheDumpOptions& dump_options,
-                             const std::shared_ptr<Cache>& cache,
-                             std::unique_ptr<CacheDumpWriter>&& writer,
-                             std::unique_ptr<CacheDumper>* cache_dumper) {
-  cache_dumper->reset(
-      new CacheDumperImpl(dump_options, cache, std::move(writer)));
-  return Status::OK();
-}
-
-Status NewDefaultCacheDumpedLoader(
-    const CacheDumpOptions& dump_options,
-    const BlockBasedTableOptions& toptions,
-    const std::shared_ptr<SecondaryCache>& secondary_cache,
-    std::unique_ptr<CacheDumpReader>&& reader,
-    std::unique_ptr<CacheDumpedLoader>* cache_dump_loader) {
-  cache_dump_loader->reset(new CacheDumpedLoaderImpl(
-      dump_options, toptions, secondary_cache, std::move(reader)));
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE