rocksdb-native 2.6.3 → 2.6.5

package/vendor/librocksdb/vendor/rocksdb/utilities/persistent_cache/hash_table.h

@@ -1,237 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#pragma once
-
-
-#include <assert.h>
-
-#include <list>
-#include <vector>
-
-#ifdef OS_LINUX
-#include <sys/mman.h>
-#endif
-
-#include "rocksdb/env.h"
-#include "util/mutexlock.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// HashTable<T, Hash, Equal>
-//
-// Traditional implementation of hash table with synchronization built on top
-// don't perform very well in multi-core scenarios. This is an implementation
-// designed for multi-core scenarios with high lock contention.
-//
-//                         |<-------- alpha ------------->|
-//               Buckets   Collision list
-//          ---- +----+    +---+---+--- ...... ---+---+---+
-//         /     |    |--->|   |   |              |   |   |
-//        /      +----+    +---+---+--- ...... ---+---+---+
-//       /       |    |
-// Locks/        +----+
-// +--+/         .    .
-// |  |          .    .
-// +--+          .    .
-// |  |          .    .
-// +--+          .    .
-// |  |          .    .
-// +--+          .    .
-//     \         +----+
-//      \        |    |
-//       \       +----+
-//        \      |    |
-//         \---- +----+
-//
-// The lock contention is spread over an array of locks. This helps improve
-// concurrent access. The spine is designed for a certain capacity and load
-// factor. When the capacity planning is done correctly we can expect
-// O(load_factor = 1) insert, access and remove time.
-//
-// Micro benchmark on debug build gives about .5 Million/sec rate of insert,
-// erase and lookup in parallel (total of about 1.5 Million ops/sec). If the
-// blocks were of 4K, the hash table can support  a virtual throughput of
-// 6 GB/s.
-//
-// T      Object type (contains both key and value)
-// Hash   Function that returns an hash from type T
-// Equal  Returns if two objects are equal
-//        (We need explicit equal for pointer type)
-//
-template <class T, class Hash, class Equal>
-class HashTable {
- public:
-  explicit HashTable(const size_t capacity = 1024 * 1024,
-                     const float load_factor = 2.0, const uint32_t nlocks = 256)
-      : nbuckets_(
-            static_cast<uint32_t>(load_factor ? capacity / load_factor : 0)),
-        nlocks_(nlocks) {
-    // pre-conditions
-    assert(capacity);
-    assert(load_factor);
-    assert(nbuckets_);
-    assert(nlocks_);
-
-    buckets_.reset(new Bucket[nbuckets_]);
-#ifdef OS_LINUX
-    mlock(buckets_.get(), nbuckets_ * sizeof(Bucket));
-#endif
-
-    // initialize locks
-    locks_.reset(new port::RWMutex[nlocks_]);
-#ifdef OS_LINUX
-    mlock(locks_.get(), nlocks_ * sizeof(port::RWMutex));
-#endif
-
-    // post-conditions
-    assert(buckets_);
-    assert(locks_);
-  }
-
-  virtual ~HashTable() { AssertEmptyBuckets(); }
-
-  //
-  // Insert given record to hash table
-  //
-  bool Insert(const T& t) {
-    const uint64_t h = Hash()(t);
-    const uint32_t bucket_idx = h % nbuckets_;
-    const uint32_t lock_idx = bucket_idx % nlocks_;
-
-    WriteLock _(&locks_[lock_idx]);
-    auto& bucket = buckets_[bucket_idx];
-    return Insert(&bucket, t);
-  }
-
-  // Lookup hash table
-  //
-  // Please note that read lock should be held by the caller. This is because
-  // the caller owns the data, and should hold the read lock as long as he
-  // operates on the data.
-  bool Find(const T& t, T* ret, port::RWMutex** ret_lock) {
-    const uint64_t h = Hash()(t);
-    const uint32_t bucket_idx = h % nbuckets_;
-    const uint32_t lock_idx = bucket_idx % nlocks_;
-
-    port::RWMutex& lock = locks_[lock_idx];
-    lock.ReadLock();
-
-    auto& bucket = buckets_[bucket_idx];
-    if (Find(&bucket, t, ret)) {
-      *ret_lock = &lock;
-      return true;
-    }
-
-    lock.ReadUnlock();
-    return false;
-  }
-
-  //
-  // Erase a given key from the hash table
-  //
-  bool Erase(const T& t, T* ret) {
-    const uint64_t h = Hash()(t);
-    const uint32_t bucket_idx = h % nbuckets_;
-    const uint32_t lock_idx = bucket_idx % nlocks_;
-
-    WriteLock _(&locks_[lock_idx]);
-
-    auto& bucket = buckets_[bucket_idx];
-    return Erase(&bucket, t, ret);
-  }
-
-  // Fetch the mutex associated with a key
-  // This call is used to hold the lock for a given data for extended period of
-  // time.
-  port::RWMutex* GetMutex(const T& t) {
-    const uint64_t h = Hash()(t);
-    const uint32_t bucket_idx = h % nbuckets_;
-    const uint32_t lock_idx = bucket_idx % nlocks_;
-
-    return &locks_[lock_idx];
-  }
-
-  void Clear(void (*fn)(T)) {
-    for (uint32_t i = 0; i < nbuckets_; ++i) {
-      const uint32_t lock_idx = i % nlocks_;
-      WriteLock _(&locks_[lock_idx]);
-      for (auto& t : buckets_[i].list_) {
-        (*fn)(t);
-      }
-      buckets_[i].list_.clear();
-    }
-  }
-
- protected:
-  // Models bucket of keys that hash to the same bucket number
-  struct Bucket {
-    std::list<T> list_;
-  };
-
-  // Substitute for std::find with custom comparator operator
-  typename std::list<T>::iterator Find(std::list<T>* list, const T& t) {
-    for (auto it = list->begin(); it != list->end(); ++it) {
-      if (Equal()(*it, t)) {
-        return it;
-      }
-    }
-    return list->end();
-  }
-
-  bool Insert(Bucket* bucket, const T& t) {
-    // Check if the key already exists
-    auto it = Find(&bucket->list_, t);
-    if (it != bucket->list_.end()) {
-      return false;
-    }
-
-    // insert to bucket
-    bucket->list_.push_back(t);
-    return true;
-  }
-
-  bool Find(Bucket* bucket, const T& t, T* ret) {
-    auto it = Find(&bucket->list_, t);
-    if (it != bucket->list_.end()) {
-      if (ret) {
-        *ret = *it;
-      }
-      return true;
-    }
-    return false;
-  }
-
-  bool Erase(Bucket* bucket, const T& t, T* ret) {
-    auto it = Find(&bucket->list_, t);
-    if (it != bucket->list_.end()) {
-      if (ret) {
-        *ret = *it;
-      }
-
-      bucket->list_.erase(it);
-      return true;
-    }
-    return false;
-  }
-
-  // assert that all buckets are empty
-  void AssertEmptyBuckets() {
-#ifndef NDEBUG
-    for (size_t i = 0; i < nbuckets_; ++i) {
-      WriteLock _(&locks_[i % nlocks_]);
-      assert(buckets_[i].list_.empty());
-    }
-#endif
-  }
-
-  const uint32_t nbuckets_;                 // No. of buckets in the spine
-  std::unique_ptr<Bucket[]> buckets_;       // Spine of the hash buckets
-  const uint32_t nlocks_;                   // No. of locks
-  std::unique_ptr<port::RWMutex[]> locks_;  // Granular locks
-};
-
-}  // namespace ROCKSDB_NAMESPACE
-

package/vendor/librocksdb/vendor/rocksdb/utilities/persistent_cache/hash_table_bench.cc

@@ -1,310 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-
-#if !defined(OS_WIN)
-
-#ifndef GFLAGS
-#include <cstdio>
-int main() { fprintf(stderr, "Please install gflags to run tools\n"); }
-#else
-
-#include <sys/time.h>
-#include <unistd.h>
-
-#include <atomic>
-#include <functional>
-#include <string>
-#include <unordered_map>
-
-#include "port/port_posix.h"
-#include "port/sys_time.h"
-#include "rocksdb/env.h"
-#include "util/gflags_compat.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-#include "utilities/persistent_cache/hash_table.h"
-
-using std::string;
-
-DEFINE_int32(nsec, 10, "nsec");
-DEFINE_int32(nthread_write, 1, "insert %");
-DEFINE_int32(nthread_read, 1, "lookup %");
-DEFINE_int32(nthread_erase, 1, "erase %");
-
-namespace ROCKSDB_NAMESPACE {
-
-//
-// HashTableImpl interface
-//
-// Abstraction of a hash table implementation
-template <class Key, class Value>
-class HashTableImpl {
- public:
-  virtual ~HashTableImpl() {}
-
-  virtual bool Insert(const Key& key, const Value& val) = 0;
-  virtual bool Erase(const Key& key) = 0;
-  virtual bool Lookup(const Key& key, Value* val) = 0;
-};
-
-// HashTableBenchmark
-//
-// Abstraction to test a given hash table implementation. The test mostly
-// focus on insert, lookup and erase. The test can operate in test mode and
-// benchmark mode.
-class HashTableBenchmark {
- public:
-  explicit HashTableBenchmark(HashTableImpl<size_t, std::string>* impl,
-                              const size_t sec = 10,
-                              const size_t nthread_write = 1,
-                              const size_t nthread_read = 1,
-                              const size_t nthread_erase = 1)
-      : impl_(impl),
-        sec_(sec),
-        ninserts_(0),
-        nreads_(0),
-        nerases_(0),
-        nerases_failed_(0),
-        quit_(false) {
-    Prepop();
-
-    StartThreads(nthread_write, WriteMain);
-    StartThreads(nthread_read, ReadMain);
-    StartThreads(nthread_erase, EraseMain);
-
-    uint64_t start = NowInMillSec();
-    while (!quit_) {
-      quit_ = NowInMillSec() - start > sec_ * 1000;
-      /* sleep override */ sleep(1);
-    }
-
-    Env* env = Env::Default();
-    env->WaitForJoin();
-
-    if (sec_) {
-      printf("Result \n");
-      printf("====== \n");
-      printf("insert/sec = %f \n", ninserts_ / static_cast<double>(sec_));
-      printf("read/sec = %f \n", nreads_ / static_cast<double>(sec_));
-      printf("erases/sec = %f \n", nerases_ / static_cast<double>(sec_));
-      const uint64_t ops = ninserts_ + nreads_ + nerases_;
-      printf("ops/sec = %f \n", ops / static_cast<double>(sec_));
-      printf("erase fail = %d (%f%%)\n", static_cast<int>(nerases_failed_),
-             static_cast<float>(nerases_failed_ / nerases_ * 100));
-      printf("====== \n");
-    }
-  }
-
-  void RunWrite() {
-    while (!quit_) {
-      size_t k = insert_key_++;
-      std::string tmp(1000, k % 255);
-      bool status = impl_->Insert(k, tmp);
-      assert(status);
-      ninserts_++;
-    }
-  }
-
-  void RunRead() {
-    Random64 rgen(time(nullptr));
-    while (!quit_) {
-      std::string s;
-      size_t k = rgen.Next() % max_prepop_key;
-      bool status = impl_->Lookup(k, &s);
-      assert(status);
-      assert(s == std::string(1000, k % 255));
-      nreads_++;
-    }
-  }
-
-  void RunErase() {
-    while (!quit_) {
-      size_t k = erase_key_++;
-      bool status = impl_->Erase(k);
-      nerases_failed_ += !status;
-      nerases_++;
-    }
-  }
-
- private:
-  // Start threads for a given function
-  void StartThreads(const size_t n, void (*fn)(void*)) {
-    Env* env = Env::Default();
-    for (size_t i = 0; i < n; ++i) {
-      env->StartThread(fn, this);
-    }
-  }
-
-  // Prepop the hash table with 1M keys
-  void Prepop() {
-    for (size_t i = 0; i < max_prepop_key; ++i) {
-      bool status = impl_->Insert(i, std::string(1000, i % 255));
-      assert(status);
-    }
-
-    erase_key_ = insert_key_ = max_prepop_key;
-
-    for (size_t i = 0; i < 10 * max_prepop_key; ++i) {
-      bool status = impl_->Insert(insert_key_++, std::string(1000, 'x'));
-      assert(status);
-    }
-  }
-
-  static uint64_t NowInMillSec() {
-    port::TimeVal tv;
-    port::GetTimeOfDay(&tv, /*tz=*/nullptr);
-    return tv.tv_sec * 1000 + tv.tv_usec / 1000;
-  }
-
-  //
-  //  Wrapper functions for thread entry
-  //
-  static void WriteMain(void* args) {
-    static_cast<HashTableBenchmark*>(args)->RunWrite();
-  }
-
-  static void ReadMain(void* args) {
-    static_cast<HashTableBenchmark*>(args)->RunRead();
-  }
-
-  static void EraseMain(void* args) {
-    static_cast<HashTableBenchmark*>(args)->RunErase();
-  }
-
-  HashTableImpl<size_t, std::string>* impl_;         // Implementation to test
-  const size_t sec_;                                 // Test time
-  const size_t max_prepop_key = 1ULL * 1024 * 1024;  // Max prepop key
-  std::atomic<size_t> insert_key_;                   // Last inserted key
-  std::atomic<size_t> erase_key_;                    // Erase key
-  std::atomic<size_t> ninserts_;                     // Number of inserts
-  std::atomic<size_t> nreads_;                       // Number of reads
-  std::atomic<size_t> nerases_;                      // Number of erases
-  std::atomic<size_t> nerases_failed_;               // Number of erases failed
-  bool quit_;  // Should the threads quit ?
-};
-
-//
-// SimpleImpl
-// Lock safe unordered_map implementation
-class SimpleImpl : public HashTableImpl<size_t, string> {
- public:
-  bool Insert(const size_t& key, const string& val) override {
-    WriteLock _(&rwlock_);
-    map_.insert(make_pair(key, val));
-    return true;
-  }
-
-  bool Erase(const size_t& key) override {
-    WriteLock _(&rwlock_);
-    auto it = map_.find(key);
-    if (it == map_.end()) {
-      return false;
-    }
-    map_.erase(it);
-    return true;
-  }
-
-  bool Lookup(const size_t& key, string* val) override {
-    ReadLock _(&rwlock_);
-    auto it = map_.find(key);
-    if (it != map_.end()) {
-      *val = it->second;
-    }
-    return it != map_.end();
-  }
-
- private:
-  port::RWMutex rwlock_;
-  std::unordered_map<size_t, string> map_;
-};
-
-//
-// GranularLockImpl
-// Thread safe custom RocksDB implementation of hash table with granular
-// locking
-class GranularLockImpl : public HashTableImpl<size_t, string> {
- public:
-  bool Insert(const size_t& key, const string& val) override {
-    Node n(key, val);
-    return impl_.Insert(n);
-  }
-
-  bool Erase(const size_t& key) override {
-    Node n(key, string());
-    return impl_.Erase(n, nullptr);
-  }
-
-  bool Lookup(const size_t& key, string* val) override {
-    Node n(key, string());
-    port::RWMutex* rlock;
-    bool status = impl_.Find(n, &n, &rlock);
-    if (status) {
-      ReadUnlock _(rlock);
-      *val = n.val_;
-    }
-    return status;
-  }
-
- private:
-  struct Node {
-    explicit Node(const size_t key, const string& val) : key_(key), val_(val) {}
-
-    size_t key_ = 0;
-    string val_;
-  };
-
-  struct Hash {
-    uint64_t operator()(const Node& node) {
-      return std::hash<uint64_t>()(node.key_);
-    }
-  };
-
-  struct Equal {
-    bool operator()(const Node& lhs, const Node& rhs) {
-      return lhs.key_ == rhs.key_;
-    }
-  };
-
-  HashTable<Node, Hash, Equal> impl_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE
-
-//
-// main
-//
-int main(int argc, char** argv) {
-  GFLAGS_NAMESPACE::SetUsageMessage(std::string("\nUSAGE:\n") +
-                                    std::string(argv[0]) + " [OPTIONS]...");
-  GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, false);
-
-  //
-  // Micro benchmark unordered_map
-  //
-  printf("Micro benchmarking std::unordered_map \n");
-  {
-    ROCKSDB_NAMESPACE::SimpleImpl impl;
-    ROCKSDB_NAMESPACE::HashTableBenchmark _(
-        &impl, FLAGS_nsec, FLAGS_nthread_write, FLAGS_nthread_read,
-        FLAGS_nthread_erase);
-  }
-  //
-  // Micro benchmark scalable hash table
-  //
-  printf("Micro benchmarking scalable hash map \n");
-  {
-    ROCKSDB_NAMESPACE::GranularLockImpl impl;
-    ROCKSDB_NAMESPACE::HashTableBenchmark _(
-        &impl, FLAGS_nsec, FLAGS_nthread_write, FLAGS_nthread_read,
-        FLAGS_nthread_erase);
-  }
-
-  return 0;
-}
-#endif  // #ifndef GFLAGS
-#else
-int main(int /*argc*/, char** /*argv*/) { return 0; }
-#endif

package/vendor/librocksdb/vendor/rocksdb/utilities/persistent_cache/hash_table_evictable.h

@@ -1,166 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#pragma once
-
-
-#include <functional>
-
-#include "util/random.h"
-#include "utilities/persistent_cache/hash_table.h"
-#include "utilities/persistent_cache/lrulist.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Evictable Hash Table
-//
-// Hash table index where least accessed (or one of the least accessed) elements
-// can be evicted.
-//
-// Please note EvictableHashTable can only be created for pointer type objects
-template <class T, class Hash, class Equal>
-class EvictableHashTable : private HashTable<T*, Hash, Equal> {
- public:
-  using hash_table = HashTable<T*, Hash, Equal>;
-
-  explicit EvictableHashTable(const size_t capacity = 1024 * 1024,
-                              const float load_factor = 2.0,
-                              const uint32_t nlocks = 256)
-      : HashTable<T*, Hash, Equal>(capacity, load_factor, nlocks),
-        lru_lists_(new LRUList<T>[hash_table::nlocks_]) {
-    assert(lru_lists_);
-  }
-
-  virtual ~EvictableHashTable() { AssertEmptyLRU(); }
-
-  //
-  // Insert given record to hash table (and LRU list)
-  //
-  bool Insert(T* t) {
-    const uint64_t h = Hash()(t);
-    typename hash_table::Bucket& bucket = GetBucket(h);
-    LRUListType& lru = GetLRUList(h);
-    port::RWMutex& lock = GetMutex(h);
-
-    WriteLock _(&lock);
-    if (hash_table::Insert(&bucket, t)) {
-      lru.Push(t);
-      return true;
-    }
-    return false;
-  }
-
-  //
-  // Lookup hash table
-  //
-  // Please note that read lock should be held by the caller. This is because
-  // the caller owns the data, and should hold the read lock as long as he
-  // operates on the data.
-  bool Find(T* t, T** ret) {
-    const uint64_t h = Hash()(t);
-    typename hash_table::Bucket& bucket = GetBucket(h);
-    LRUListType& lru = GetLRUList(h);
-    port::RWMutex& lock = GetMutex(h);
-
-    ReadLock _(&lock);
-    if (hash_table::Find(&bucket, t, ret)) {
-      ++(*ret)->refs_;
-      lru.Touch(*ret);
-      return true;
-    }
-    return false;
-  }
-
-  //
-  // Evict one of the least recently used object
-  //
-  T* Evict(const std::function<void(T*)>& fn = nullptr) {
-    uint32_t random = Random::GetTLSInstance()->Next();
-    const size_t start_idx = random % hash_table::nlocks_;
-    T* t = nullptr;
-
-    // iterate from start_idx .. 0 .. start_idx
-    for (size_t i = 0; !t && i < hash_table::nlocks_; ++i) {
-      const size_t idx = (start_idx + i) % hash_table::nlocks_;
-
-      WriteLock _(&hash_table::locks_[idx]);
-      LRUListType& lru = lru_lists_[idx];
-      if (!lru.IsEmpty() && (t = lru.Pop()) != nullptr) {
-        assert(!t->refs_);
-        // We got an item to evict, erase from the bucket
-        const uint64_t h = Hash()(t);
-        typename hash_table::Bucket& bucket = GetBucket(h);
-        T* tmp = nullptr;
-        bool status = hash_table::Erase(&bucket, t, &tmp);
-        assert(t == tmp);
-        (void)status;
-        assert(status);
-        if (fn) {
-          fn(t);
-        }
-        break;
-      }
-      assert(!t);
-    }
-    return t;
-  }
-
-  void Clear(void (*fn)(T*)) {
-    for (uint32_t i = 0; i < hash_table::nbuckets_; ++i) {
-      const uint32_t lock_idx = i % hash_table::nlocks_;
-      WriteLock _(&hash_table::locks_[lock_idx]);
-      auto& lru_list = lru_lists_[lock_idx];
-      auto& bucket = hash_table::buckets_[i];
-      for (auto* t : bucket.list_) {
-        lru_list.Unlink(t);
-        (*fn)(t);
-      }
-      bucket.list_.clear();
-    }
-    // make sure that all LRU lists are emptied
-    AssertEmptyLRU();
-  }
-
-  void AssertEmptyLRU() {
-#ifndef NDEBUG
-    for (uint32_t i = 0; i < hash_table::nlocks_; ++i) {
-      WriteLock _(&hash_table::locks_[i]);
-      auto& lru_list = lru_lists_[i];
-      assert(lru_list.IsEmpty());
-    }
-#endif
-  }
-
-  //
-  // Fetch the mutex associated with a key
-  // This call is used to hold the lock for a given data for extended period of
-  // time.
-  port::RWMutex* GetMutex(T* t) { return hash_table::GetMutex(t); }
-
- private:
-  using LRUListType = LRUList<T>;
-
-  typename hash_table::Bucket& GetBucket(const uint64_t h) {
-    const uint32_t bucket_idx = h % hash_table::nbuckets_;
-    return hash_table::buckets_[bucket_idx];
-  }
-
-  LRUListType& GetLRUList(const uint64_t h) {
-    const uint32_t bucket_idx = h % hash_table::nbuckets_;
-    const uint32_t lock_idx = bucket_idx % hash_table::nlocks_;
-    return lru_lists_[lock_idx];
-  }
-
-  port::RWMutex& GetMutex(const uint64_t h) {
-    const uint32_t bucket_idx = h % hash_table::nbuckets_;
-    const uint32_t lock_idx = bucket_idx % hash_table::nlocks_;
-    return hash_table::locks_[lock_idx];
-  }
-
-  std::unique_ptr<LRUListType[]> lru_lists_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE
-