rocksdb-native 2.6.2 → 2.6.4

package/vendor/librocksdb/vendor/rocksdb/trace_replay/block_cache_tracer.h

@@ -1,239 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <atomic>
-#include <fstream>
-
-#include "monitoring/instrumented_mutex.h"
-#include "rocksdb/block_cache_trace_writer.h"
-#include "rocksdb/options.h"
-#include "rocksdb/table_reader_caller.h"
-#include "rocksdb/trace_reader_writer.h"
-#include "trace_replay/trace_replay.h"
-
-namespace ROCKSDB_NAMESPACE {
-class Env;
-class SystemClock;
-
-extern const uint64_t kMicrosInSecond;
-extern const uint64_t kSecondInMinute;
-extern const uint64_t kSecondInHour;
-
-struct BlockCacheTraceRecord;
-
-class BlockCacheTraceHelper {
- public:
-  static bool IsGetOrMultiGetOnDataBlock(TraceType block_type,
-                                         TableReaderCaller caller);
-  static bool IsGetOrMultiGet(TableReaderCaller caller);
-  static bool IsUserAccess(TableReaderCaller caller);
-  // Row key is a concatenation of the access's fd_number and the referenced
-  // user key.
-  static std::string ComputeRowKey(const BlockCacheTraceRecord& access);
-  // The first four bytes of the referenced key in a Get request is the table
-  // id.
-  static uint64_t GetTableId(const BlockCacheTraceRecord& access);
-  // The sequence number of a get request is the last part of the referenced
-  // key.
-  static uint64_t GetSequenceNumber(const BlockCacheTraceRecord& access);
-  // Block offset in a file is the last varint64 in the block key.
-  static uint64_t GetBlockOffsetInFile(const BlockCacheTraceRecord& access);
-
-  static const std::string kUnknownColumnFamilyName;
-  static const uint64_t kReservedGetId;
-};
-
-// Lookup context for tracing block cache accesses.
-// We trace block accesses at five places:
-// 1. BlockBasedTable::GetFilter
-// 2. BlockBasedTable::GetUncompressedDict.
-// 3. BlockBasedTable::MaybeReadAndLoadToCache. (To trace access on data, index,
-// and range deletion block.)
-// 4. BlockBasedTable::Get. (To trace the referenced key and whether the
-// referenced key exists in a fetched data block.)
-// 5. BlockBasedTable::MultiGet. (To trace the referenced key and whether the
-// referenced key exists in a fetched data block.)
-// The context is created at:
-// 1. BlockBasedTable::Get. (kUserGet)
-// 2. BlockBasedTable::MultiGet. (kUserMGet)
-// 3. BlockBasedTable::NewIterator. (either kUserIterator, kCompaction, or
-// external SST ingestion calls this function.)
-// 4. BlockBasedTable::Open. (kPrefetch)
-// 5. Index/Filter::CacheDependencies. (kPrefetch)
-// 6. BlockBasedTable::ApproximateOffsetOf. (kCompaction or
-// kUserApproximateSize).
-struct BlockCacheLookupContext {
-  BlockCacheLookupContext(const TableReaderCaller& _caller) : caller(_caller) {}
-  BlockCacheLookupContext(const TableReaderCaller& _caller, uint64_t _get_id,
-                          bool _get_from_user_specified_snapshot)
-      : caller(_caller),
-        get_id(_get_id),
-        get_from_user_specified_snapshot(_get_from_user_specified_snapshot) {}
-  const TableReaderCaller caller;
-  // These are populated when we perform lookup/insert on block cache. The block
-  // cache tracer uses these inforation when logging the block access at
-  // BlockBasedTable::GET and BlockBasedTable::MultiGet.
-  bool is_cache_hit = false;
-  bool no_insert = false;
-  TraceType block_type = TraceType::kTraceMax;
-  uint64_t block_size = 0;
-  std::string block_key;
-  uint64_t num_keys_in_block = 0;
-  // The unique id associated with Get and MultiGet. This enables us to track
-  // how many blocks a Get/MultiGet request accesses. We can also measure the
-  // impact of row cache vs block cache.
-  uint64_t get_id = 0;
-  std::string referenced_key;
-  bool get_from_user_specified_snapshot = false;
-
-  void FillLookupContext(bool _is_cache_hit, bool _no_insert,
-                         TraceType _block_type, uint64_t _block_size,
-                         const std::string& _block_key,
-                         uint64_t _num_keys_in_block) {
-    is_cache_hit = _is_cache_hit;
-    no_insert = _no_insert;
-    block_type = _block_type;
-    block_size = _block_size;
-    block_key = _block_key;
-    num_keys_in_block = _num_keys_in_block;
-  }
-};
-
-struct BlockCacheTraceHeader {
-  uint64_t start_time;
-  uint32_t rocksdb_major_version;
-  uint32_t rocksdb_minor_version;
-};
-
-// BlockCacheTraceWriter captures all RocksDB block cache accesses using a
-// user-provided TraceWriter. Every RocksDB operation is written as a single
-// trace. Each trace will have a timestamp and type, followed by the trace
-// payload.
-class BlockCacheTraceWriterImpl : public BlockCacheTraceWriter {
- public:
-  BlockCacheTraceWriterImpl(SystemClock* clock,
-                            const BlockCacheTraceWriterOptions& trace_options,
-                            std::unique_ptr<TraceWriter>&& trace_writer);
-  ~BlockCacheTraceWriterImpl() = default;
-  // No copy and move.
-  BlockCacheTraceWriterImpl(const BlockCacheTraceWriterImpl&) = delete;
-  BlockCacheTraceWriterImpl& operator=(const BlockCacheTraceWriterImpl&) =
-      delete;
-  BlockCacheTraceWriterImpl(BlockCacheTraceWriterImpl&&) = delete;
-  BlockCacheTraceWriterImpl& operator=(BlockCacheTraceWriterImpl&&) = delete;
-
-  // Pass Slice references to avoid copy.
-  Status WriteBlockAccess(const BlockCacheTraceRecord& record,
-                          const Slice& block_key, const Slice& cf_name,
-                          const Slice& referenced_key) override;
-
-  // Write a trace header at the beginning, typically on initiating a trace,
-  // with some metadata like a magic number and RocksDB version.
-  Status WriteHeader() override;
-
- private:
-  SystemClock* clock_;
-  BlockCacheTraceWriterOptions trace_options_;
-  std::unique_ptr<TraceWriter> trace_writer_;
-};
-
-// Write a trace record in human readable format, see
-// https://github.com/facebook/rocksdb/wiki/Block-cache-analysis-and-simulation-tools#trace-format
-// for details.
-class BlockCacheHumanReadableTraceWriter {
- public:
-  ~BlockCacheHumanReadableTraceWriter();
-
-  Status NewWritableFile(const std::string& human_readable_trace_file_path,
-                         ROCKSDB_NAMESPACE::Env* env);
-
-  Status WriteHumanReadableTraceRecord(const BlockCacheTraceRecord& access,
-                                       uint64_t block_id, uint64_t get_key_id);
-
- private:
-  char trace_record_buffer_[1024 * 1024];
-  std::unique_ptr<ROCKSDB_NAMESPACE::WritableFile>
-      human_readable_trace_file_writer_;
-};
-
-// BlockCacheTraceReader helps read the trace file generated by
-// BlockCacheTraceWriter using a user provided TraceReader.
-class BlockCacheTraceReader {
- public:
-  BlockCacheTraceReader(std::unique_ptr<TraceReader>&& reader);
-  virtual ~BlockCacheTraceReader() = default;
-  // No copy and move.
-  BlockCacheTraceReader(const BlockCacheTraceReader&) = delete;
-  BlockCacheTraceReader& operator=(const BlockCacheTraceReader&) = delete;
-  BlockCacheTraceReader(BlockCacheTraceReader&&) = delete;
-  BlockCacheTraceReader& operator=(BlockCacheTraceReader&&) = delete;
-
-  Status ReadHeader(BlockCacheTraceHeader* header);
-
-  Status ReadAccess(BlockCacheTraceRecord* record);
-
- private:
-  std::unique_ptr<TraceReader> trace_reader_;
-};
-
-// Read a trace record in human readable format, see
-// https://github.com/facebook/rocksdb/wiki/Block-cache-analysis-and-simulation-tools#trace-format
-// for detailed.
-class BlockCacheHumanReadableTraceReader : public BlockCacheTraceReader {
- public:
-  BlockCacheHumanReadableTraceReader(const std::string& trace_file_path);
-
-  ~BlockCacheHumanReadableTraceReader();
-
-  Status ReadHeader(BlockCacheTraceHeader* header);
-
-  Status ReadAccess(BlockCacheTraceRecord* record);
-
- private:
-  std::ifstream human_readable_trace_reader_;
-};
-
-// A block cache tracer. It downsamples the accesses according to
-// trace_options and uses BlockCacheTraceWriter to write the access record to
-// the trace file.
-class BlockCacheTracer {
- public:
-  BlockCacheTracer();
-  ~BlockCacheTracer();
-  // No copy and move.
-  BlockCacheTracer(const BlockCacheTracer&) = delete;
-  BlockCacheTracer& operator=(const BlockCacheTracer&) = delete;
-  BlockCacheTracer(BlockCacheTracer&&) = delete;
-  BlockCacheTracer& operator=(BlockCacheTracer&&) = delete;
-
-  // Start writing block cache accesses to the trace_writer.
-  Status StartTrace(const BlockCacheTraceOptions& trace_options,
-                    std::unique_ptr<BlockCacheTraceWriter>&& trace_writer);
-
-  // Stop writing block cache accesses to the trace_writer.
-  void EndTrace();
-
-  bool is_tracing_enabled() const {
-    return writer_.load(std::memory_order_relaxed);
-  }
-
-  Status WriteBlockAccess(const BlockCacheTraceRecord& record,
-                          const Slice& block_key, const Slice& cf_name,
-                          const Slice& referenced_key);
-
-  // GetId cycles from 1 to std::numeric_limits<uint64_t>::max().
-  uint64_t NextGetId();
-
- private:
-  BlockCacheTraceOptions trace_options_;
-  // A mutex protects the writer_.
-  InstrumentedMutex trace_writer_mutex_;
-  std::atomic<BlockCacheTraceWriter*> writer_;
-  std::atomic<uint64_t> get_id_counter_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/trace_replay/io_tracer.cc

@@ -1,303 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "trace_replay/io_tracer.h"
-
-#include <cinttypes>
-#include <cstdio>
-#include <cstdlib>
-
-#include "db/db_impl/db_impl.h"
-#include "db/dbformat.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/system_clock.h"
-#include "rocksdb/trace_reader_writer.h"
-#include "util/coding.h"
-#include "util/hash.h"
-#include "util/string_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-IOTraceWriter::IOTraceWriter(SystemClock* clock,
-                             const TraceOptions& trace_options,
-                             std::unique_ptr<TraceWriter>&& trace_writer)
-    : clock_(clock),
-      trace_options_(trace_options),
-      trace_writer_(std::move(trace_writer)) {}
-
-Status IOTraceWriter::WriteIOOp(const IOTraceRecord& record,
-                                IODebugContext* dbg) {
-  uint64_t trace_file_size = trace_writer_->GetFileSize();
-  if (trace_file_size > trace_options_.max_trace_file_size) {
-    return Status::OK();
-  }
-  Trace trace;
-  trace.ts = record.access_timestamp;
-  trace.type = record.trace_type;
-  PutFixed64(&trace.payload, record.io_op_data);
-  Slice file_operation(record.file_operation);
-  PutLengthPrefixedSlice(&trace.payload, file_operation);
-  PutFixed64(&trace.payload, record.latency);
-  Slice io_status(record.io_status);
-  PutLengthPrefixedSlice(&trace.payload, io_status);
-  Slice file_name(record.file_name);
-  PutLengthPrefixedSlice(&trace.payload, file_name);
-
-  // Each bit in io_op_data stores which corresponding info from IOTraceOp will
-  // be added in the trace. Foreg, if bit at position 1 is set then
-  // IOTraceOp::kIOLen (length) will be logged in the record (Since
-  // IOTraceOp::kIOLen = 1 in the enum). So find all the set positions in
-  // io_op_data one by one and, update corresponsing info in the trace record,
-  // unset that bit to find other set bits until io_op_data = 0.
-  /* Write remaining options based on io_op_data set by file operation */
-  int64_t io_op_data = static_cast<int64_t>(record.io_op_data);
-  while (io_op_data) {
-    // Find the rightmost set bit.
-    uint32_t set_pos = static_cast<uint32_t>(log2(io_op_data & -io_op_data));
-    switch (set_pos) {
-      case IOTraceOp::kIOFileSize:
-        PutFixed64(&trace.payload, record.file_size);
-        break;
-      case IOTraceOp::kIOLen:
-        PutFixed64(&trace.payload, record.len);
-        break;
-      case IOTraceOp::kIOOffset:
-        PutFixed64(&trace.payload, record.offset);
-        break;
-      default:
-        assert(false);
-    }
-    // unset the rightmost bit.
-    io_op_data &= (io_op_data - 1);
-  }
-
-  int64_t trace_data = 0;
-  if (dbg) {
-    trace_data = static_cast<int64_t>(dbg->trace_data);
-  }
-  PutFixed64(&trace.payload, trace_data);
-  while (trace_data) {
-    // Find the rightmost set bit.
-    uint32_t set_pos = static_cast<uint32_t>(log2(trace_data & -trace_data));
-    switch (set_pos) {
-      case IODebugContext::TraceData::kRequestID: {
-        Slice request_id(dbg->request_id);
-        PutLengthPrefixedSlice(&trace.payload, request_id);
-      } break;
-      default:
-        assert(false);
-    }
-    // unset the rightmost bit.
-    trace_data &= (trace_data - 1);
-  }
-
-  std::string encoded_trace;
-  TracerHelper::EncodeTrace(trace, &encoded_trace);
-  return trace_writer_->Write(encoded_trace);
-}
-
-Status IOTraceWriter::WriteHeader() {
-  Trace trace;
-  trace.ts = clock_->NowMicros();
-  trace.type = TraceType::kTraceBegin;
-  PutLengthPrefixedSlice(&trace.payload, kTraceMagic);
-  PutFixed32(&trace.payload, kMajorVersion);
-  PutFixed32(&trace.payload, kMinorVersion);
-  std::string encoded_trace;
-  TracerHelper::EncodeTrace(trace, &encoded_trace);
-  return trace_writer_->Write(encoded_trace);
-}
-
-IOTraceReader::IOTraceReader(std::unique_ptr<TraceReader>&& reader)
-    : trace_reader_(std::move(reader)) {}
-
-Status IOTraceReader::ReadHeader(IOTraceHeader* header) {
-  assert(header != nullptr);
-  std::string encoded_trace;
-  Status s = trace_reader_->Read(&encoded_trace);
-  if (!s.ok()) {
-    return s;
-  }
-  Trace trace;
-  s = TracerHelper::DecodeTrace(encoded_trace, &trace);
-  if (!s.ok()) {
-    return s;
-  }
-  header->start_time = trace.ts;
-  Slice enc_slice = Slice(trace.payload);
-  Slice magic_number;
-  if (!GetLengthPrefixedSlice(&enc_slice, &magic_number)) {
-    return Status::Corruption(
-        "Corrupted header in the trace file: Failed to read the magic number.");
-  }
-  if (magic_number.ToString() != kTraceMagic) {
-    return Status::Corruption(
-        "Corrupted header in the trace file: Magic number does not match.");
-  }
-  if (!GetFixed32(&enc_slice, &header->rocksdb_major_version)) {
-    return Status::Corruption(
-        "Corrupted header in the trace file: Failed to read rocksdb major "
-        "version number.");
-  }
-  if (!GetFixed32(&enc_slice, &header->rocksdb_minor_version)) {
-    return Status::Corruption(
-        "Corrupted header in the trace file: Failed to read rocksdb minor "
-        "version number.");
-  }
-  // We should have retrieved all information in the header.
-  if (!enc_slice.empty()) {
-    return Status::Corruption(
-        "Corrupted header in the trace file: The length of header is too "
-        "long.");
-  }
-  return Status::OK();
-}
-
-Status IOTraceReader::ReadIOOp(IOTraceRecord* record) {
-  assert(record);
-  std::string encoded_trace;
-  Status s = trace_reader_->Read(&encoded_trace);
-  if (!s.ok()) {
-    return s;
-  }
-  Trace trace;
-  s = TracerHelper::DecodeTrace(encoded_trace, &trace);
-  if (!s.ok()) {
-    return s;
-  }
-  record->access_timestamp = trace.ts;
-  record->trace_type = trace.type;
-  Slice enc_slice = Slice(trace.payload);
-
-  if (!GetFixed64(&enc_slice, &record->io_op_data)) {
-    return Status::Incomplete(
-        "Incomplete access record: Failed to read trace data.");
-  }
-  Slice file_operation;
-  if (!GetLengthPrefixedSlice(&enc_slice, &file_operation)) {
-    return Status::Incomplete(
-        "Incomplete access record: Failed to read file operation.");
-  }
-  record->file_operation = file_operation.ToString();
-  if (!GetFixed64(&enc_slice, &record->latency)) {
-    return Status::Incomplete(
-        "Incomplete access record: Failed to read latency.");
-  }
-  Slice io_status;
-  if (!GetLengthPrefixedSlice(&enc_slice, &io_status)) {
-    return Status::Incomplete(
-        "Incomplete access record: Failed to read IO status.");
-  }
-  record->io_status = io_status.ToString();
-  Slice file_name;
-  if (!GetLengthPrefixedSlice(&enc_slice, &file_name)) {
-    return Status::Incomplete(
-        "Incomplete access record: Failed to read file name.");
-  }
-  record->file_name = file_name.ToString();
-
-  // Each bit in io_op_data stores which corresponding info from IOTraceOp will
-  // be added in the trace. Foreg, if bit at position 1 is set then
-  // IOTraceOp::kIOLen (length) will be logged in the record (Since
-  // IOTraceOp::kIOLen = 1 in the enum). So find all the set positions in
-  // io_op_data one by one and, update corresponsing info in the trace record,
-  // unset that bit to find other set bits until io_op_data = 0.
-  /* Read remaining options based on io_op_data set by file operation */
-  // Assuming 63 bits will be used at max.
-  int64_t io_op_data = static_cast<int64_t>(record->io_op_data);
-  while (io_op_data) {
-    // Find the rightmost set bit.
-    uint32_t set_pos = static_cast<uint32_t>(log2(io_op_data & -io_op_data));
-    switch (set_pos) {
-      case IOTraceOp::kIOFileSize:
-        if (!GetFixed64(&enc_slice, &record->file_size)) {
-          return Status::Incomplete(
-              "Incomplete access record: Failed to read file size.");
-        }
-        break;
-      case IOTraceOp::kIOLen:
-        if (!GetFixed64(&enc_slice, &record->len)) {
-          return Status::Incomplete(
-              "Incomplete access record: Failed to read length.");
-        }
-        break;
-      case IOTraceOp::kIOOffset:
-        if (!GetFixed64(&enc_slice, &record->offset)) {
-          return Status::Incomplete(
-              "Incomplete access record: Failed to read offset.");
-        }
-        break;
-      default:
-        assert(false);
-    }
-    // unset the rightmost bit.
-    io_op_data &= (io_op_data - 1);
-  }
-
-  if (!GetFixed64(&enc_slice, &record->trace_data)) {
-    return Status::Incomplete(
-        "Incomplete access record: Failed to read trace op.");
-  }
-  int64_t trace_data = static_cast<int64_t>(record->trace_data);
-  while (trace_data) {
-    // Find the rightmost set bit.
-    uint32_t set_pos = static_cast<uint32_t>(log2(trace_data & -trace_data));
-    switch (set_pos) {
-      case IODebugContext::TraceData::kRequestID: {
-        Slice request_id;
-        if (!GetLengthPrefixedSlice(&enc_slice, &request_id)) {
-          return Status::Incomplete(
-              "Incomplete access record: Failed to request id.");
-        }
-        record->request_id = request_id.ToString();
-      } break;
-      default:
-        assert(false);
-    }
-    // unset the rightmost bit.
-    trace_data &= (trace_data - 1);
-  }
-
-  return Status::OK();
-}
-
-IOTracer::IOTracer() : tracing_enabled(false) { writer_.store(nullptr); }
-
-IOTracer::~IOTracer() { EndIOTrace(); }
-
-Status IOTracer::StartIOTrace(SystemClock* clock,
-                              const TraceOptions& trace_options,
-                              std::unique_ptr<TraceWriter>&& trace_writer) {
-  InstrumentedMutexLock lock_guard(&trace_writer_mutex_);
-  if (writer_.load()) {
-    return Status::Busy();
-  }
-  trace_options_ = trace_options;
-  writer_.store(
-      new IOTraceWriter(clock, trace_options, std::move(trace_writer)));
-  tracing_enabled = true;
-  return writer_.load()->WriteHeader();
-}
-
-void IOTracer::EndIOTrace() {
-  InstrumentedMutexLock lock_guard(&trace_writer_mutex_);
-  if (!writer_.load()) {
-    return;
-  }
-  delete writer_.load();
-  writer_.store(nullptr);
-  tracing_enabled = false;
-}
-
-void IOTracer::WriteIOOp(const IOTraceRecord& record, IODebugContext* dbg) {
-  if (!writer_.load()) {
-    return;
-  }
-  InstrumentedMutexLock lock_guard(&trace_writer_mutex_);
-  if (!writer_.load()) {
-    return;
-  }
-  writer_.load()->WriteIOOp(record, dbg).PermitUncheckedError();
-}
-}  // namespace ROCKSDB_NAMESPACE