rocksdb-native 2.6.3 → 2.6.5

package/vendor/librocksdb/vendor/rocksdb/db/error_handler.h

@@ -1,149 +0,0 @@
-//  Copyright (c) 2018-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <sstream>
-
-#include "monitoring/instrumented_mutex.h"
-#include "options/db_options.h"
-#include "rocksdb/io_status.h"
-#include "rocksdb/listener.h"
-#include "rocksdb/status.h"
-#include "util/autovector.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class DBImpl;
-
-// This structure is used to store the DB recovery context. The context is
-// the information that related to the recover actions. For example, it contains
-// FlushReason, which tells the flush job why this flush is called.
-struct DBRecoverContext {
-  FlushReason flush_reason;
-  bool flush_after_recovery;
-
-  DBRecoverContext()
-      : flush_reason(FlushReason::kErrorRecovery),
-        flush_after_recovery(false) {}
-  DBRecoverContext(FlushReason reason)
-      : flush_reason(reason), flush_after_recovery(false) {}
-};
-
-class ErrorHandler {
- public:
-  ErrorHandler(DBImpl* db, const ImmutableDBOptions& db_options,
-               InstrumentedMutex* db_mutex)
-      : db_(db),
-        db_options_(db_options),
-        cv_(db_mutex),
-        end_recovery_(false),
-        recovery_thread_(nullptr),
-        db_mutex_(db_mutex),
-        auto_recovery_(false),
-        recovery_in_prog_(false),
-        soft_error_no_bg_work_(false),
-        is_db_stopped_(false),
-        bg_error_stats_(db_options.statistics) {
-    // Clear the checked flag for uninitialized errors
-    bg_error_.PermitUncheckedError();
-    recovery_error_.PermitUncheckedError();
-  }
-
-  void EnableAutoRecovery() { auto_recovery_ = true; }
-
-  Status::Severity GetErrorSeverity(BackgroundErrorReason reason,
-                                    Status::Code code, Status::SubCode subcode);
-
-  void SetBGError(const Status& bg_err, BackgroundErrorReason reason);
-
-  Status GetBGError() const { return bg_error_; }
-
-  Status GetRecoveryError() const { return recovery_error_; }
-
-  Status ClearBGError();
-
-  bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
-
-  bool IsBGWorkStopped() {
-    assert(db_mutex_);
-    db_mutex_->AssertHeld();
-    return !bg_error_.ok() &&
-           (bg_error_.severity() >= Status::Severity::kHardError ||
-            !auto_recovery_ || soft_error_no_bg_work_);
-  }
-
-  bool IsSoftErrorNoBGWork() { return soft_error_no_bg_work_; }
-
-  bool IsRecoveryInProgress() { return recovery_in_prog_; }
-
-  Status RecoverFromBGError(bool is_manual = false);
-  void CancelErrorRecovery();
-
-  void EndAutoRecovery();
-
-  void AddFilesToQuarantine(
-      autovector<const autovector<uint64_t>*> files_to_quarantine);
-
-  const autovector<uint64_t>& GetFilesToQuarantine() const {
-    db_mutex_->AssertHeld();
-    return files_to_quarantine_;
-  }
-
-  void ClearFilesToQuarantine();
-
- private:
-  void RecordStats(
-      const std::vector<Tickers>& ticker_types,
-      const std::vector<std::tuple<Histograms, uint64_t>>& int_histograms);
-
-  DBImpl* db_;
-  const ImmutableDBOptions& db_options_;
-  Status bg_error_;
-  // A separate Status variable used to record any errors during the
-  // recovery process from hard errors
-  IOStatus recovery_error_;
-  // The condition variable used with db_mutex during auto resume for time
-  // wait.
-  InstrumentedCondVar cv_;
-  bool end_recovery_;
-  std::unique_ptr<port::Thread> recovery_thread_;
-
-  InstrumentedMutex* db_mutex_;
-  // A flag indicating whether automatic recovery from errors is enabled. Auto
-  // recovery applies for delegating to SstFileManager to handle no space type
-  // of errors. This flag doesn't control the auto resume behavior to recover
-  // from retryable IO errors.
-  bool auto_recovery_;
-  bool recovery_in_prog_;
-  // A flag to indicate that for the soft error, we should not allow any
-  // background work except the work is from recovery.
-  bool soft_error_no_bg_work_;
-
-  // Used to store the context for recover, such as flush reason.
-  DBRecoverContext recover_context_;
-  std::atomic<bool> is_db_stopped_;
-
-  // The pointer of DB statistics.
-  std::shared_ptr<Statistics> bg_error_stats_;
-
-  // During recovery from manifest IO errors, files whose VersionEdits entries
-  // could be in an ambiguous state are quarantined and file deletion refrain
-  // from deleting them. Successful recovery will clear this vector. Files are
-  // added to this vector while DB mutex was locked, this data structure is
-  // unsorted.
-  autovector<uint64_t> files_to_quarantine_;
-
-  void HandleKnownErrors(const Status& bg_err, BackgroundErrorReason reason);
-  Status OverrideNoSpaceError(const Status& bg_error, bool* auto_recovery);
-  void RecoverFromNoSpace();
-  void StartRecoverFromRetryableBGIOError(const IOStatus& io_error);
-  void RecoverFromRetryableBGIOError();
-  // First, if it is in recovery and the recovery_error is ok. Set the
-  // recovery_error_ to bg_err. Second, if the severity is higher than the
-  // current bg_error_, overwrite it.
-  void CheckAndSetRecoveryAndBGError(const Status& bg_err);
-};
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/event_helpers.cc

@@ -1,332 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/event_helpers.h"
-
-#include "rocksdb/convenience.h"
-#include "rocksdb/listener.h"
-#include "rocksdb/utilities/customizable_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-Status EventListener::CreateFromString(const ConfigOptions& config_options,
-                                       const std::string& id,
-                                       std::shared_ptr<EventListener>* result) {
-  return LoadSharedObject<EventListener>(config_options, id, result);
-}
-
-namespace {
-template <class T>
-inline T SafeDivide(T a, T b) {
-  return b == 0 ? 0 : a / b;
-}
-}  // anonymous namespace
-
-void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
-  *jwriter << "time_micros"
-           << std::chrono::duration_cast<std::chrono::microseconds>(
-                  std::chrono::system_clock::now().time_since_epoch())
-                  .count();
-}
-
-void EventHelpers::NotifyTableFileCreationStarted(
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    const std::string& db_name, const std::string& cf_name,
-    const std::string& file_path, int job_id, TableFileCreationReason reason) {
-  if (listeners.empty()) {
-    return;
-  }
-  TableFileCreationBriefInfo info;
-  info.db_name = db_name;
-  info.cf_name = cf_name;
-  info.file_path = file_path;
-  info.job_id = job_id;
-  info.reason = reason;
-  for (auto& listener : listeners) {
-    listener->OnTableFileCreationStarted(info);
-  }
-}
-
-void EventHelpers::NotifyOnBackgroundError(
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    BackgroundErrorReason reason, Status* bg_error, InstrumentedMutex* db_mutex,
-    bool* auto_recovery) {
-  if (listeners.empty()) {
-    return;
-  }
-  db_mutex->AssertHeld();
-  // release lock while notifying events
-  db_mutex->Unlock();
-  for (auto& listener : listeners) {
-    listener->OnBackgroundError(reason, bg_error);
-    bg_error->PermitUncheckedError();
-    if (*auto_recovery) {
-      listener->OnErrorRecoveryBegin(reason, *bg_error, auto_recovery);
-    }
-  }
-  db_mutex->Lock();
-}
-
-void EventHelpers::LogAndNotifyTableFileCreationFinished(
-    EventLogger* event_logger,
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    const std::string& db_name, const std::string& cf_name,
-    const std::string& file_path, int job_id, const FileDescriptor& fd,
-    uint64_t oldest_blob_file_number, const TableProperties& table_properties,
-    TableFileCreationReason reason, const Status& s,
-    const std::string& file_checksum,
-    const std::string& file_checksum_func_name) {
-  if (s.ok() && event_logger) {
-    JSONWriter jwriter;
-    AppendCurrentTime(&jwriter);
-    jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "table_file_creation"
-            << "file_number" << fd.GetNumber() << "file_size"
-            << fd.GetFileSize() << "file_checksum"
-            << Slice(file_checksum).ToString(true) << "file_checksum_func_name"
-            << file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno
-            << "largest_seqno" << fd.largest_seqno;
-
-    // table_properties
-    {
-      jwriter << "table_properties";
-      jwriter.StartObject();
-
-      // basic properties:
-      jwriter << "data_size" << table_properties.data_size << "index_size"
-              << table_properties.index_size << "index_partitions"
-              << table_properties.index_partitions << "top_level_index_size"
-              << table_properties.top_level_index_size
-              << "index_key_is_user_key"
-              << table_properties.index_key_is_user_key
-              << "index_value_is_delta_encoded"
-              << table_properties.index_value_is_delta_encoded << "filter_size"
-              << table_properties.filter_size << "raw_key_size"
-              << table_properties.raw_key_size << "raw_average_key_size"
-              << SafeDivide(table_properties.raw_key_size,
-                            table_properties.num_entries)
-              << "raw_value_size" << table_properties.raw_value_size
-              << "raw_average_value_size"
-              << SafeDivide(table_properties.raw_value_size,
-                            table_properties.num_entries)
-              << "num_data_blocks" << table_properties.num_data_blocks
-              << "num_entries" << table_properties.num_entries
-              << "num_filter_entries" << table_properties.num_filter_entries
-              << "num_deletions" << table_properties.num_deletions
-              << "num_merge_operands" << table_properties.num_merge_operands
-              << "num_range_deletions" << table_properties.num_range_deletions
-              << "format_version" << table_properties.format_version
-              << "fixed_key_len" << table_properties.fixed_key_len
-              << "filter_policy" << table_properties.filter_policy_name
-              << "column_family_name" << table_properties.column_family_name
-              << "column_family_id" << table_properties.column_family_id
-              << "comparator" << table_properties.comparator_name
-              << "user_defined_timestamps_persisted"
-              << table_properties.user_defined_timestamps_persisted
-              << "merge_operator" << table_properties.merge_operator_name
-              << "prefix_extractor_name"
-              << table_properties.prefix_extractor_name << "property_collectors"
-              << table_properties.property_collectors_names << "compression"
-              << table_properties.compression_name << "compression_options"
-              << table_properties.compression_options << "creation_time"
-              << table_properties.creation_time << "oldest_key_time"
-              << table_properties.oldest_key_time << "file_creation_time"
-              << table_properties.file_creation_time
-              << "slow_compression_estimated_data_size"
-              << table_properties.slow_compression_estimated_data_size
-              << "fast_compression_estimated_data_size"
-              << table_properties.fast_compression_estimated_data_size
-              << "db_id" << table_properties.db_id << "db_session_id"
-              << table_properties.db_session_id << "orig_file_number"
-              << table_properties.orig_file_number << "seqno_to_time_mapping";
-
-      if (table_properties.seqno_to_time_mapping.empty()) {
-        jwriter << "N/A";
-      } else {
-        SeqnoToTimeMapping tmp;
-        Status status = tmp.DecodeFrom(table_properties.seqno_to_time_mapping);
-        if (status.ok()) {
-          jwriter << tmp.ToHumanString();
-        } else {
-          jwriter << "Invalid";
-        }
-      }
-
-      // user collected properties
-      for (const auto& prop : table_properties.readable_properties) {
-        jwriter << prop.first << prop.second;
-      }
-      jwriter.EndObject();
-    }
-
-    if (oldest_blob_file_number != kInvalidBlobFileNumber) {
-      jwriter << "oldest_blob_file_number" << oldest_blob_file_number;
-    }
-
-    jwriter.EndObject();
-
-    event_logger->Log(jwriter);
-  }
-
-  if (listeners.empty()) {
-    return;
-  }
-  TableFileCreationInfo info;
-  info.db_name = db_name;
-  info.cf_name = cf_name;
-  info.file_path = file_path;
-  info.file_size = fd.file_size;
-  info.job_id = job_id;
-  info.table_properties = table_properties;
-  info.reason = reason;
-  info.status = s;
-  info.file_checksum = file_checksum;
-  info.file_checksum_func_name = file_checksum_func_name;
-  for (auto& listener : listeners) {
-    listener->OnTableFileCreated(info);
-  }
-  info.status.PermitUncheckedError();
-}
-
-void EventHelpers::LogAndNotifyTableFileDeletion(
-    EventLogger* event_logger, int job_id, uint64_t file_number,
-    const std::string& file_path, const Status& status,
-    const std::string& dbname,
-    const std::vector<std::shared_ptr<EventListener>>& listeners) {
-  JSONWriter jwriter;
-  AppendCurrentTime(&jwriter);
-
-  jwriter << "job" << job_id << "event"
-          << "table_file_deletion"
-          << "file_number" << file_number;
-  if (!status.ok()) {
-    jwriter << "status" << status.ToString();
-  }
-
-  jwriter.EndObject();
-
-  event_logger->Log(jwriter);
-
-  if (listeners.empty()) {
-    return;
-  }
-  TableFileDeletionInfo info;
-  info.db_name = dbname;
-  info.job_id = job_id;
-  info.file_path = file_path;
-  info.status = status;
-  for (auto& listener : listeners) {
-    listener->OnTableFileDeleted(info);
-  }
-  info.status.PermitUncheckedError();
-}
-
-void EventHelpers::NotifyOnErrorRecoveryEnd(
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    const Status& old_bg_error, const Status& new_bg_error,
-    InstrumentedMutex* db_mutex) {
-  if (!listeners.empty()) {
-    db_mutex->AssertHeld();
-    // Make copies before releasing mutex to avoid race.
-    Status old_bg_error_cp = old_bg_error;
-    Status new_bg_error_cp = new_bg_error;
-    // release lock while notifying events
-    db_mutex->Unlock();
-    TEST_SYNC_POINT("NotifyOnErrorRecoveryEnd:MutexUnlocked:1");
-    TEST_SYNC_POINT("NotifyOnErrorRecoveryEnd:MutexUnlocked:2");
-    for (auto& listener : listeners) {
-      BackgroundErrorRecoveryInfo info;
-      info.old_bg_error = old_bg_error_cp;
-      info.new_bg_error = new_bg_error_cp;
-      listener->OnErrorRecoveryCompleted(old_bg_error_cp);
-      listener->OnErrorRecoveryEnd(info);
-      info.old_bg_error.PermitUncheckedError();
-      info.new_bg_error.PermitUncheckedError();
-    }
-    db_mutex->Lock();
-  } else {
-    old_bg_error.PermitUncheckedError();
-  }
-}
-
-void EventHelpers::NotifyBlobFileCreationStarted(
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    const std::string& db_name, const std::string& cf_name,
-    const std::string& file_path, int job_id,
-    BlobFileCreationReason creation_reason) {
-  if (listeners.empty()) {
-    return;
-  }
-  BlobFileCreationBriefInfo info(db_name, cf_name, file_path, job_id,
-                                 creation_reason);
-  for (const auto& listener : listeners) {
-    listener->OnBlobFileCreationStarted(info);
-  }
-}
-
-void EventHelpers::LogAndNotifyBlobFileCreationFinished(
-    EventLogger* event_logger,
-    const std::vector<std::shared_ptr<EventListener>>& listeners,
-    const std::string& db_name, const std::string& cf_name,
-    const std::string& file_path, int job_id, uint64_t file_number,
-    BlobFileCreationReason creation_reason, const Status& s,
-    const std::string& file_checksum,
-    const std::string& file_checksum_func_name, uint64_t total_blob_count,
-    uint64_t total_blob_bytes) {
-  if (s.ok() && event_logger) {
-    JSONWriter jwriter;
-    AppendCurrentTime(&jwriter);
-    jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "blob_file_creation"
-            << "file_number" << file_number << "total_blob_count"
-            << total_blob_count << "total_blob_bytes" << total_blob_bytes
-            << "file_checksum" << file_checksum << "file_checksum_func_name"
-            << file_checksum_func_name << "status" << s.ToString();
-
-    jwriter.EndObject();
-    event_logger->Log(jwriter);
-  }
-
-  if (listeners.empty()) {
-    return;
-  }
-  BlobFileCreationInfo info(db_name, cf_name, file_path, job_id,
-                            creation_reason, total_blob_count, total_blob_bytes,
-                            s, file_checksum, file_checksum_func_name);
-  for (const auto& listener : listeners) {
-    listener->OnBlobFileCreated(info);
-  }
-  info.status.PermitUncheckedError();
-}
-
-void EventHelpers::LogAndNotifyBlobFileDeletion(
-    EventLogger* event_logger,
-    const std::vector<std::shared_ptr<EventListener>>& listeners, int job_id,
-    uint64_t file_number, const std::string& file_path, const Status& status,
-    const std::string& dbname) {
-  if (event_logger) {
-    JSONWriter jwriter;
-    AppendCurrentTime(&jwriter);
-
-    jwriter << "job" << job_id << "event"
-            << "blob_file_deletion"
-            << "file_number" << file_number;
-    if (!status.ok()) {
-      jwriter << "status" << status.ToString();
-    }
-
-    jwriter.EndObject();
-    event_logger->Log(jwriter);
-  }
-  if (listeners.empty()) {
-    return;
-  }
-  BlobFileDeletionInfo info(dbname, file_path, job_id, status);
-  for (const auto& listener : listeners) {
-    listener->OnBlobFileDeleted(info);
-  }
-  info.status.PermitUncheckedError();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

package/vendor/librocksdb/vendor/rocksdb/db/event_helpers.h

@@ -1,78 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "db/column_family.h"
-#include "db/version_edit.h"
-#include "logging/event_logger.h"
-#include "rocksdb/listener.h"
-#include "rocksdb/table_properties.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class EventHelpers {
- public:
-  static void AppendCurrentTime(JSONWriter* json_writer);
-  static void NotifyTableFileCreationStarted(
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      const std::string& db_name, const std::string& cf_name,
-      const std::string& file_path, int job_id, TableFileCreationReason reason);
-  static void NotifyOnBackgroundError(
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      BackgroundErrorReason reason, Status* bg_error,
-      InstrumentedMutex* db_mutex, bool* auto_recovery);
-  static void LogAndNotifyTableFileCreationFinished(
-      EventLogger* event_logger,
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      const std::string& db_name, const std::string& cf_name,
-      const std::string& file_path, int job_id, const FileDescriptor& fd,
-      uint64_t oldest_blob_file_number, const TableProperties& table_properties,
-      TableFileCreationReason reason, const Status& s,
-      const std::string& file_checksum,
-      const std::string& file_checksum_func_name);
-  static void LogAndNotifyTableFileDeletion(
-      EventLogger* event_logger, int job_id, uint64_t file_number,
-      const std::string& file_path, const Status& status,
-      const std::string& db_name,
-      const std::vector<std::shared_ptr<EventListener>>& listeners);
-  static void NotifyOnErrorRecoveryEnd(
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      const Status& old_bg_error, const Status& new_bg_error,
-      InstrumentedMutex* db_mutex);
-
-  static void NotifyBlobFileCreationStarted(
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      const std::string& db_name, const std::string& cf_name,
-      const std::string& file_path, int job_id,
-      BlobFileCreationReason creation_reason);
-
-  static void LogAndNotifyBlobFileCreationFinished(
-      EventLogger* event_logger,
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      const std::string& db_name, const std::string& cf_name,
-      const std::string& file_path, int job_id, uint64_t file_number,
-      BlobFileCreationReason creation_reason, const Status& s,
-      const std::string& file_checksum,
-      const std::string& file_checksum_func_name, uint64_t total_blob_count,
-      uint64_t total_blob_bytes);
-
-  static void LogAndNotifyBlobFileDeletion(
-      EventLogger* event_logger,
-      const std::vector<std::shared_ptr<EventListener>>& listeners, int job_id,
-      uint64_t file_number, const std::string& file_path, const Status& status,
-      const std::string& db_name);
-
- private:
-  static void LogAndNotifyTableFileCreation(
-      EventLogger* event_logger,
-      const std::vector<std::shared_ptr<EventListener>>& listeners,
-      const FileDescriptor& fd, const TableFileCreationInfo& info);
-};
-
-}  // namespace ROCKSDB_NAMESPACE