npm - rocksdb-native - Versions diffs - 0.0.0 - Mend

rocksdb-native 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1951) hide show

package/vendor/rocksdb/docs/_includes/powered_by.html ADDED Viewed

@@ -0,0 +1,28 @@
+{% if site.data.powered_by.first.items or site.data.powered_by_highlight.first.items %}
+<div class="poweredByContainer">
+  <div class="wrapper mainWrapper poweredByWrapper">
+    {% if site.data.powered_by_highlight.first.title %}
+    <h2>{{ site.data.powered_by_highlight.first.title }}</h2>
+    {% else %}
+    <h2>{{ site.data.powered_by.first.title }}</h2>
+    {% endif %}
+    {% if site.data.powered_by_highlight.first.items %}
+    <div class="poweredByItems">
+      {% for item in site.data.powered_by_highlight.first.items %}
+      <div class="poweredByItem itemLarge">
+        <a href="{{ item.url }}" target="_blank"><img src="{{ item.img }}" alt="{{ item.name }}" /></a>
+      </div>
+      {% endfor %}
+    </div>
+    {% endif %}
+    <div class="poweredByItems">
+      {% for item in site.data.powered_by.first.items %}
+      <div class="poweredByItem itemSmall">
+        <a href="{{ item.url }}" target="_blank">{{ item.name }}</a>
+      </div>
+      {% endfor %}
+    </div>
+    <div class="poweredByMessage">Does your app use {{ site.title }}? Add it to this list with <a href="https://github.com/{{ site.ghrepo }}/edit/gh-pages/_data/powered_by.yml" target="_blank">a pull request!</a></div>
+  </div>
+</div>
+{% endif %}

package/vendor/rocksdb/docs/_includes/social_plugins.html ADDED Viewed

@@ -0,0 +1,31 @@
+<a
+  href="https://twitter.com/share"
+  class="twitter-share-button"
+  data-url="http://facebook.github.io/fresco{{ page.url }}"
+  data-text="Fresco | {{ page.title }}"
+  data-hashtags="fresco">Tweet</a>
+<div
+  class="fb-like"
+  data-href="http://facebook.github.io/fresco{{ page.url }}"
+  data-layout="standard"
+  data-action="like"
+  data-show-faces="true"
+  data-share="true"></div>
+<div id="fb-root"></div>
+<script>(function(d, s, id) {
+  var js, fjs = d.getElementsByTagName(s)[0];
+  if (d.getElementById(id)) return;
+  js = d.createElement(s); js.id = id;
+  js.src = "//connect.facebook.net/en_US/sdk.js#xfbml=1&version=v2.0";
+  fjs.parentNode.insertBefore(js, fjs);
+}(document, 'script', 'facebook-jssdk'));</script>
+<script>!function(d,s,id){
+  var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';
+  if(!d.getElementById(id)){
+  js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';
+  fjs.parentNode.insertBefore(js,fjs);
+  }
+  }(document, 'script', 'twitter-wjs');
+</script>

package/vendor/rocksdb/docs/_includes/ui/button.html ADDED Viewed

	@@ -0,0 +1 @@
1	+ <span class="buttonWrap {{ include.align }}"><a class="button blockButton fbossFontLight pluginBlock margin{{ include.margin }}" target="{{ include.button_target }}" href="{{ include.button_href }}">{{ include.button_text }}</a></span>

package/vendor/rocksdb/docs/_layouts/basic.html ADDED Viewed

@@ -0,0 +1,12 @@
+---
+layout: doc_default
+---
+<div class="mainContainer blogContainer postContainer">
+  <div id="main_wrap" class="wrapper mainWrapper">
+    <div class="post basicPost">
+      {{ content }}
+    </div>
+  </div>
+</div>

package/vendor/rocksdb/docs/_layouts/blog.html ADDED Viewed

@@ -0,0 +1,11 @@
+---
+category: blog
+layout: blog_default
+---
+<div class="mainContainer blogContainer postContainer">
+  <div id="main_wrap" class="wrapper mainWrapper">
+    {{ content }}
+  </div>
+</div>

package/vendor/rocksdb/docs/_layouts/blog_default.html ADDED Viewed

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  {% include head.html %}
+  <body class="docsNavVisible">
+    {% include nav.html alwayson=true %}
+    <div class="navPusher">
+      <div class="docMainWrapper wrapper">
+      {% include nav/collection_nav.html navdata=site.posts type="blog" sectionpath="/blog/" sectiontitle="Blog" %}
+      {{ content }}
+      </div>
+      {% include footer.html %}
+    </div>
+  </body>
+</html>

package/vendor/rocksdb/docs/_layouts/default.html ADDED Viewed

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+  {% include head.html %}
+  <body>
+    {% include nav.html alwayson=true %}
+    <div class="navPusher">
+      {{ content }}
+      {% include footer.html %}
+    </div>
+  </body>
+</html>

package/vendor/rocksdb/docs/_layouts/doc_default.html ADDED Viewed

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  {% include head.html %}
+  <body class="docsNavVisible">
+    {% include nav.html alwayson=true %}
+    <div class="navPusher">
+      <div class="docMainWrapper wrapper">
+      {% include nav/collection_nav.html navdata=site.data.nav_docs type="docs" sectionpath="/docs/" sectiontitle="Docs" data_collection=site.docs %}
+      {{ content }}
+      </div>
+      {% include footer.html %}
+    </div>
+  </body>
+</html>

package/vendor/rocksdb/docs/_layouts/doc_page.html ADDED Viewed

@@ -0,0 +1,10 @@
+---
+layout: doc_default
+---
+<div class="mainContainer documentContainer postContainer">
+  <div id="main_wrap" class="wrapper mainWrapper">
+    {{ content }}
+  </div>
+</div>

package/vendor/rocksdb/docs/_layouts/docs.html ADDED Viewed

@@ -0,0 +1,5 @@
+---
+layout: doc_page
+---
+{% include doc.html %}

package/vendor/rocksdb/docs/_layouts/home.html ADDED Viewed

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+  {% include head.html %}
+  <div class="socialBanner">
+    <div>
+      Support Ukraine 🇺🇦
+      <a href="https://opensource.facebook.com/support-ukraine">
+        Help Provide Humanitarian Aid to Ukraine
+      </a>
+      .
+    </div>
+  </div>
+  <body>
+    {% include nav.html alwayson=true %}
+    <div class="navPusher">
+      {% include home_header.html %}
+      <div class="mainContainer">
+        <div id="main_wrap" class="wrapper mainWrapper">
+          {{ content }}
+        </div>
+        {% include powered_by.html %}
+      </div>
+      {% include footer.html %}
+    </div>
+  </body>
+</html>

package/vendor/rocksdb/docs/_layouts/page.html ADDED Viewed

@@ -0,0 +1,3 @@
+---
+layout: blog
+---

package/vendor/rocksdb/docs/_layouts/plain.html ADDED Viewed

@@ -0,0 +1,10 @@
+---
+layout: default
+---
+<div class="mainContainer blogContainer postContainer">
+  <div id="main_wrap" class="wrapper mainWrapper">
+    {{ content }}
+  </div>
+</div>

package/vendor/rocksdb/docs/_layouts/post.html ADDED Viewed

@@ -0,0 +1,8 @@
+---
+collection: blog
+layout: blog
+---
+<div class="lonePost">
+{% include post.html %}
+</div>

package/vendor/rocksdb/docs/_layouts/redirect.html ADDED Viewed

@@ -0,0 +1,6 @@
+<html>
+<head>
+  <meta http-equiv="refresh" content="0; {{ page.destination }}">
+</head>
+<body></body>
+</html>

package/vendor/rocksdb/docs/_layouts/top-level.html ADDED Viewed

@@ -0,0 +1,10 @@
+---
+layout: default
+---
+<div class="mainContainer blogContainer postContainer">
+  <div id="main_wrap" class="wrapper mainWrapper">
+    {{ content }}
+  </div>
+</div>

package/vendor/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown ADDED Viewed

@@ -0,0 +1,135 @@
+---
+title: How to backup RocksDB?
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+  - /blog/191/how-to-backup-rocksdb/
+---
+In RocksDB, we have implemented an easy way to backup your DB. Here is a simple example:
+    #include "rocksdb/db.h"
+    #include "utilities/backupable_db.h"
+    using namespace rocksdb;
+    DB* db;
+    DB::Open(Options(), "/tmp/rocksdb", &db);
+    BackupableDB* backupable_db = new BackupableDB(db, BackupableDBOptions("/tmp/rocksdb_backup"));
+    backupable_db->Put(...); // do your thing
+    backupable_db->CreateNewBackup();
+    delete backupable_db; // no need to also delete db
+<!--truncate-->
+This simple example will create a backup of your DB in "/tmp/rocksdb_backup". Creating new BackupableDB consumes DB* and you should be calling all the DB methods on object `backupable_db` going forward.
+Restoring is also easy:
+    RestoreBackupableDB* restore = new RestoreBackupableDB(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup"));
+    restore->RestoreDBFromLatestBackup("/tmp/rocksdb", "/tmp/rocksdb");
+    delete restore;
+This code will restore the backup back to "/tmp/rocksdb". The second parameter is the location of log files (In some DBs they are different from DB directory, but usually they are the same. See Options::wal_dir for more info).
+An alternative API for backups is to use BackupEngine directly:
+    #include "rocksdb/db.h"
+    #include "utilities/backupable_db.h"
+    using namespace rocksdb;
+    DB* db;
+    DB::Open(Options(), "/tmp/rocksdb", &db);
+    db->Put(...); // do your thing
+    BackupEngine* backup_engine = BackupEngine::NewBackupEngine(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup"));
+    backup_engine->CreateNewBackup(db);
+    delete db;
+    delete backup_engine;
+Restoring with BackupEngine is similar to RestoreBackupableDB:
+    BackupEngine* backup_engine = BackupEngine::NewBackupEngine(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup"));
+    backup_engine->RestoreDBFromLatestBackup("/tmp/rocksdb", "/tmp/rocksdb");
+    delete backup_engine;
+Backups are incremental. You can create a new backup with `CreateNewBackup()` and only the new data will be copied to backup directory (for more details on what gets copied, see "Under the hood"). Checksum is always calculated for any backuped file (including sst, log, and etc). It is used to make sure files are kept sound in the file system. Checksum is also verified for files from the previous backups even though they do not need to be copied. A checksum mismatch aborts the current backup (see "Under the hood" for more details). Once you have more backups saved, you can issue `GetBackupInfo()` call to get a list of all backups together with information on timestamp of the backup and the size (please note that sum of all backups' sizes is bigger than the actual size of the backup directory because some data is shared by multiple backups). Backups are identified by their always-increasing IDs. `GetBackupInfo()` is available both in `BackupableDB` and `RestoreBackupableDB`.
+You probably want to keep around only small number of backups. To delete old backups, just call `PurgeOldBackups(N)`, where N is how many backups you'd like to keep. All backups except the N newest ones will be deleted. You can also choose to delete arbitrary backup with call `DeleteBackup(id)`.
+`RestoreDBFromLatestBackup()` will restore the DB from the latest consistent backup. An alternative is `RestoreDBFromBackup()` which takes a backup ID and restores that particular backup. Checksum is calculated for any restored file and compared against the one stored during the backup time. If a checksum mismatch is detected, the restore process is aborted and `Status::Corruption` is returned. Very important thing to note here: Let's say you have backups 1, 2, 3, 4. If you restore from backup 2 and start writing more data to your database, newly created backup will delete old backups 3 and 4 and create new backup 3 on top of 2.
+## Advanced usage
+Let's say you want to backup your DB to HDFS. There is an option in `BackupableDBOptions` to set `backup_env`, which will be used for all file I/O related to backup dir (writes when backuping, reads when restoring). If you set it to HDFS Env, all the backups will be stored in HDFS.
+`BackupableDBOptions::info_log` is a Logger object that is used to print out LOG messages if not-nullptr.
+If `BackupableDBOptions::sync` is true, we will sync data to disk after every file write, guaranteeing that backups will be consistent after a reboot or if machine crashes. Setting it to false will speed things up a bit, but some (newer) backups might be inconsistent. In most cases, everything should be fine, though.
+If you set `BackupableDBOptions::destroy_old_data` to true, creating new `BackupableDB` will delete all the old backups in the backup directory.
+`BackupableDB::CreateNewBackup()` method takes a parameter `flush_before_backup`, which is false by default. When `flush_before_backup` is true, `BackupableDB` will first issue a memtable flush and only then copy the DB files to the backup directory. Doing so will prevent log files from being copied to the backup directory (since flush will delete them). If `flush_before_backup` is false, backup will not issue flush before starting the backup. In that case, the backup will also include log files corresponding to live memtables. Backup will be consistent with current state of the database regardless of `flush_before_backup` parameter.
+## Under the hood
+`BackupableDB` implements `DB` interface and adds four methods to it: `CreateNewBackup()`, `GetBackupInfo()`, `PurgeOldBackups()`, `DeleteBackup()`. Any `DB` interface calls will get forwarded to underlying `DB` object.
+When you call `BackupableDB::CreateNewBackup()`, it does the following:
+  1. Disable file deletions
+  2. Get live files (this includes table files, current and manifest file).
+  3. Copy live files to the backup directory. Since table files are immutable and filenames unique, we don't copy a table file that is already present in the backup directory. For example, if there is a file `00050.sst` already backed up and `GetLiveFiles()` returns `00050.sst`, we will not copy that file to the backup directory. However, checksum is calculated for all files regardless if a file needs to be copied or not. If a file is already present, the calculated checksum is compared against previously calculated checksum to make sure nothing crazy happened between backups. If a mismatch is detected, backup is aborted and the system is restored back to the state before `BackupableDB::CreateNewBackup()` is called. One thing to note is that a backup abortion could mean a corruption from a file in backup directory or the corresponding live file in current DB. Both manifest and current files are copied, since they are not immutable.
+  4. If `flush_before_backup` was set to false, we also need to copy log files to the backup directory. We call `GetSortedWalFiles()` and copy all live files to the backup directory.
+  5. Enable file deletions
+Backup IDs are always increasing and we have a file `LATEST_BACKUP` that contains the ID of the latest backup. If we crash in middle of backing up, on a restart we will detect that there are newer backup files than `LATEST_BACKUP` claims there are. In that case, we will delete any backup newer than `LATEST_BACKUP` and clean up all the files since some of the table files might be corrupted. Having corrupted table files in the backup directory is dangerous because of our deduplication strategy.
+## Further reading
+For the API details, see `include/utilities/backupable_db.h`. For the implementation, see `utilities/backupable/backupable_db.cc`.

package/vendor/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown ADDED Viewed

@@ -0,0 +1,54 @@
+---
+title: How to persist in-memory RocksDB database?
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+  - /blog/245/how-to-persist-in-memory-rocksdb-database/
+---
+In recent months, we have focused on optimizing RocksDB for in-memory workloads. With growing RAM sizes and strict low-latency requirements, lots of applications decide to keep their entire data in memory. Running in-memory database with RocksDB is easy -- just mount your RocksDB directory on tmpfs or ramfs [1]. Even if the process crashes, RocksDB can recover all of your data from in-memory filesystem. However, what happens if the machine reboots?
+<!--truncate-->
+In this article we will explain how you can recover your in-memory RocksDB database even after a machine reboot.
+Every update to RocksDB is written to two places - one is an in-memory data structure called memtable and second is write-ahead log. Write-ahead log can be used to completely recover the data in memtable. By default, when we flush the memtable to table file, we also delete the current log, since we don't need it anymore for recovery (the data from the log is "persisted" in the table file -- we say that the log file is obsolete). However, if your table file is stored in in-memory file system, you may need the obsolete write-ahead log to recover the data after the machine reboots. Here's how you can do that.
+Options::wal_dir is the directory where RocksDB stores write-ahead log files. If you configure this directory to be on flash or disk, you will not lose current log file on machine reboot.
+Options::WAL_ttl_seconds is the timeout when we delete the archived log files. If the timeout is non-zero, obsolete log files will be moved to `archive/` directory under Options::wal_dir. Those archived log files will only be deleted after the specified timeout.
+Let's assume Options::wal_dir is a directory on persistent storage and Options::WAL_ttl_seconds is set to one day. To fully recover the DB, we also need to backup the current snapshot of the database (containing table and metadata files) with a frequency of less than one day. RocksDB provides an utility that enables you to easily backup the snapshot of your database. You can learn more about it here: [How to backup RocksDB?](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F)
+You should configure the backup process to avoid backing up log files, since they are already stored in persistent storage. To do that, set BackupableDBOptions::backup_log_files to false.
+Restore process by default cleans up entire DB and WAL directory. Since we didn't include log files in the backup, we need to make sure that restoring the database doesn't delete log files in WAL directory. When restoring, configure RestoreOptions::keep_log_file to true. That option will also move any archived log files back to WAL directory, enabling RocksDB to replay all archived log files and rebuild the in-memory database state.
+To reiterate, here's what you have to do:
+  * Set DB directory to tmpfs or ramfs mounted drive
+  * Set Options::wal_log to a directory on persistent storage
+  * Set Options::WAL_ttl_seconds to T seconds
+  * Backup RocksDB every T/2 seconds, with BackupableDBOptions::backup_log_files = false
+  * When you lose data, restore from backup with RestoreOptions::keep_log_file = true
+[1] You might also want to consider using [PlainTable format](https://github.com/facebook/rocksdb/wiki/PlainTable-Format) for table files

package/vendor/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown ADDED Viewed

@@ -0,0 +1,53 @@
+---
+title: The 1st RocksDB Local Meetup Held on March 27, 2014
+layout: post
+author: xjin
+category: blog
+redirect_from:
+  - /blog/323/the-1st-rocksdb-local-meetup-held-on-march-27-2014/
+---
+On Mar 27, 2014, RocksDB team @ Facebook held the 1st RocksDB local meetup in FB HQ (Menlo Park, California). We invited around 80 guests from 20+ local companies, including LinkedIn, Twitter, Dropbox, Square, Pinterest, MapR, Microsoft and IBM. Finally around 50 guests showed up, totaling around 60% show-up rate.
+<!--truncate-->
+[![Resize of 20140327_200754](/static/images/Resize-of-20140327_200754-300x225.jpg)](/static/images/Resize-of-20140327_200754-300x225.jpg)
+RocksDB team @ Facebook gave four talks about the latest progress and experience on RocksDB:
+  * [Supporting a 1PB In-Memory Workload](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Haobo-RocksDB-In-Memory.pdf)
+  * [Column Families in RocksDB](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Igor-Column-Families.pdf)
+  * ["Lockless" Get() in RocksDB?](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf)
+  * [Prefix Hashing in RocksDB](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Siying-Prefix-Hash.pdf)
+A very interesting question asked by a massive number of guests is: does RocksDB plan to provide replication functionality? Obviously, many applications need a resilient and distributed storage solution, not just single-node storage. We are considering how to approach this issue.
+When will be the next meetup? We haven't decided yet. We will see whether the community is interested in it and how it can help RocksDB grow.
+If you have any questions or feedback for the meetup or RocksDB, please let us know in [our Facebook group](https://www.facebook.com/groups/rocksdb.dev/).
+### Comments
+**[Rajiv](geetasen@gmail.com)**
+Have any of these talks been recorded and if so will they be published?
+**[Igor Canadi](icanadi@fb.com)**
+Yes, I think we plan to publish them soon.

package/vendor/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown ADDED Viewed

@@ -0,0 +1,40 @@
+---
+title: RocksDB 2.8 release
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+  - /blog/371/rocksdb-2-8-release/
+---
+Check out the new RocksDB 2.8 release on [Github](https://github.com/facebook/rocksdb/releases/tag/2.8.fb).
+RocksDB 2.8. is mostly focused on improving performance for in-memory workloads. We are seeing read QPS as high as 5M (we will write a separate blog post on this).
+<!--truncate-->
+Here is the summary of new features:
+  * Added a new table format called PlainTable, which is optimized for RAM storage (ramfs or tmpfs). You can read more details about it on [our wiki](https://github.com/facebook/rocksdb/wiki/PlainTable-Format).
+  * New prefixed memtable format HashLinkedList, which is optimized for cases where there are only a few keys for each prefix.
+  * Merge operator supports a new function PartialMergeMulti() that allows users to do partial merges against multiple operands. This function enables big speedups for workloads that use merge operators.
+  * Added a V2 compaction filter interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB.
+  * Geo-spatial support for locations and radial-search.
+  * Improved read performance using thread local cache for frequently accessed data.
+  * Stability improvements -- we're now ignoring partially written tailing record to MANIFEST or WAL files.
+We have also introduced small incompatible API changes (mostly for advanced users). You can see full release notes in our [HISTORY.my](https://github.com/facebook/rocksdb/blob/2.8.fb/HISTORY.md) file.

package/vendor/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown ADDED Viewed

@@ -0,0 +1,28 @@
+---
+title: Indexing SST Files for Better Lookup Performance
+layout: post
+author: leijin
+category: blog
+redirect_from:
+  - /blog/431/indexing-sst-files-for-better-lookup-performance/
+---
+For a `Get()` request, RocksDB goes through mutable memtable, list of immutable memtables, and SST files to look up the target key. SST files are organized in levels.
+On level 0, files are sorted based on the time they are flushed. Their key range (as defined by FileMetaData.smallest and FileMetaData.largest) are mostly overlapped with each other. So it needs to look up every L0 file.
+<!--truncate-->
+Compaction is scheduled periodically to pick up files from an upper level and merges them with files from lower level. As a result, key/values are moved from L0 down the LSM tree gradually. Compaction sorts key/values and split them into files. From level 1 and below, SST files are sorted based on key. Their key range are mutually exclusive. Instead of scanning through each SST file and checking if a key falls into its range, RocksDB performs a binary search based on FileMetaData.largest to locate a candidate file that can potentially contain the target key. This reduces complexity from O(N) to O(log(N)). However, log(N) can still be large for bottom levels. For a fan-out ratio of 10, level 3 can have 1000 files. That requires 10 comparisons to locate a candidate file. This is a significant cost for an in-memory database when you can do [several million gets per second](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks).
+One observation to this problem is that: after the LSM tree is built, an SST file's position in its level is fixed. Furthermore, its order relative to files from the next level is also fixed. Based on this idea, we can perform [fractional cascading](http://en.wikipedia.org/wiki/Fractional_cascading) kind of optimization to narrow down the binary search range. Here is an example:
+[![tree_example](/static/images/tree_example1.png)](/static/images/tree_example1.png)
+Level 1 has 2 files and level 2 has 8 files. Now, we want to look up key 80. A binary search based FileMetaData.largest tells you file 1 is the candidate. Then key 80 is compared with its FileMetaData.smallest and FileMetaData.largest to decide if it falls into the range. The comparison shows 80 is less than FileMetaData.smallest (100), so file 1 does not possibly contain key 80. We to proceed to check level 2. Usually, we need to do binary search among all 8 files on level 2. But since we already know target key 80 is less than 100 and only file 1 to file 3 can contain key less than 100, we can safely exclude other files from the search. As a result we cut down the search space from 8 files to 3 files.
+Let's look at another example. We want to get key 230. A binary search on level 1 locates to file 2 (this also implies key 230 is larger than file 1's FileMetaData.largest 200). A comparison with file 2's range shows the target key is smaller than file 2's FileMetaData.smallest 300. Even though, we couldn't find key on level 1, we have derived hints that target key is in range between 200 and 300. Any files on level 2 that cannot overlap with [200, 300] can be safely excluded. As a result, we only need to look at file 5 and file 6 on level 2.
+Inspired by this concept, we pre-build pointers at compaction time on level 1 files that point to a range of files on level 2. For example, file 1 on level 1 points to file 3 (on level 2) on the left and file 4 on the right. File 2 will point to level 2 files 6 and 7. At query time, these pointers are used to determine the actual binary search range based on comparison result.
+Our benchmark shows that this optimization improves lookup QPS by ~5% for similar setup mentioned [here](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks).

package/vendor/rocksdb/docs/_posts/2014-05-14-lock.markdown ADDED Viewed

@@ -0,0 +1,88 @@
+---
+title: Reducing Lock Contention in RocksDB
+layout: post
+author: sdong
+category: blog
+redirect_from:
+  - /blog/521/lock/
+---
+In this post, we briefly introduce the recent improvements we did to RocksDB to improve the issue of lock contention costs.
+RocksDB has a simple thread synchronization mechanism (See [RocksDB Architecture Guide](https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide)  to understand terms used below, like SST tables or mem tables). SST tables are immutable after being written and mem tables are lock-free data structures supporting single writer and multiple readers. There is only one single major lock, the DB mutex (DBImpl.mutex_) protecting all the meta operations, including:
+<!--truncate-->
+  * Increase or decrease reference counters of mem tables and SST tables
+  * Change and check meta data structures, before and after finishing compactions, flushes and new mem table creations
+  * Coordinating writers
+This DB mutex used to be scalability bottleneck preventing us from scaling to more than 16 threads. To address the issue, we improved RocksDB in several ways.
+1. Consolidate reference counters and introduce "super version". For every read operation, mutex was acquired, and reference counters for each mem table and each SST table were increased. One such operation is not expensive but if you are building a high throughput server with lots of reads, the lock contention will become the bottleneck. This is especially true if you store all your data in RAM.
+To solve this problem, we created a meta-meta data structure called “[super version](https://reviews.facebook.net/rROCKSDB1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c)”, which holds reference counters to all those mem table and SST tables, so that readers only need to increase the reference counters for this single data structure. In RocksDB, list of live mem tables and SST tables only changes infrequently, which would happen when new mem tables are created or flush/compaction happens. Now, at those times, a new super version is created with their reference counters increased. A super version lists live mem tables and SST tables so a reader only needs acquire the lock in order to find the latest super version and increase its reference counter. From the super version, the reader can find all the mem and SST tables which are safety accessible as long as the reader holds the reference count for the super version.
+2. We replace some reference counters to stc::atomic objects, so that decreasing reference count of an object usually doesn’t need to be inside the mutex any more.
+3. Make fetching super version and reference counting lock-free in read queries. After consolidating reference counting to one single super version and removing the locking for decreasing reference counts, in read case, we only acquire mutex for one thing: fetch the latest super version and increase the reference count for that (dereference the counter is done in an atomic decrease). We designed and implemented a (mostly) lock-free approach to do it. See [details](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf). We will write a separate blog post for that.
+4. Avoid disk I/O inside the mutex. As we know, each disk I/O to hard drives takes several milliseconds. It can be even longer if file system journal is involved or I/Os are queued. Even occasional disk I/O within mutex can cause huge performance outliers.
+We identified in two situations, we might do disk I/O inside mutex and we removed them:
+(1) Opening and closing transactional log files. We moved those operations out of the mutex.
+(2) Information logging. In multiple places we write to logs within mutex. There is a chance that file write will wait for disk I/O to finish before finishing, even if fsync() is not issued, especially in EXT systems. We occasionally see 100+ milliseconds write() latency on EXT. Instead of removing those logging, we came up with a solution of delay logging. When inside mutex, instead of directly writing to the log file, we write to a log buffer, with the timing information. As soon as mutex is released, we flush the log buffer to log files.
+5. Reduce object creation inside the mutex.
+Object creation can be slow because it involves malloc (in our case). Malloc sometimes is slow because it needs to lock some shared data structures. Allocating can also be slow because we sometimes do expensive operations in some of our classes' constructors. For these reasons, we try to reduce object creations inside the mutex. Here are two examples:
+(1) std::vector uses malloc inside. We introduced “[autovector](https://reviews.facebook.net/rROCKSDBc01676e46d3be08c3c140361ef1f5884f47d3b3c)” data structure, in which memory for first a few elements are pre-allocated as members of the autovector class. When an autovector is used as a stack variable, no malloc will be needed unless the pre-allocated buffer is used up. This autovector is quite useful for manipulating those meta data structures. Those meta operations are often locked inside DB mutex.
+(2) When building an iterator, we used to creating iterator of every live men table and SST table within the mutex and a merging iterator on top of them. Besides malloc, some of those iterators can be quite expensive to create, like sorting. Now, instead of doing that, we simply increase the reference counters of them, and release the mutex before creating any iterator.
+6. Deal with mutexes in LRU caches.
+When I said there was only one single major lock, I was lying. In RocksDB, all LRU caches had exclusive mutexes within to protect writes to the LRU lists, which are done in both of read and write operations. LRU caches are used in block cache and table cache. Both of them are accessed more frequently than DB data structures. Lock contention of these two locks are as intense as the DB mutex. Even if LRU cache is sharded into ShardedLRUCache, we can still see lock contentions, especially table caches. We further address this issue in two way:
+(1) Bypassing table caches. A table cache maintains list of SST table’s read handlers. Those handlers contain SST files’ descriptors, table metadata, and possibly data indexes, as well as bloom filters. When the table handler needs to be evicted based on LRU, those information is cleared. When the SST table needs to be read and its table handler is not in LRU cache, the table is opened and those metadata is loaded. In some cases, users want to tune the system in a way that table handler evictions should never happen. It is common for high-throughput, low-latency servers. We introduce a mode where table cache is bypassed in read queries. In this mode, all table handlers are cached and accessed directly, so there is no need to query and adjust table caches for reading the database. It is the users’ responsibility to reserve enough resource for it. This mode can be turned on by setting options.max_open_files=-1.
+(2) [New PlainTable format](//github.com/facebook/rocksdb/wiki/PlainTable-Format) (optimized for SST in ramfs/tmpfs) does not organize data by blocks. Data are located by memory addresses so no block cache is needed.
+With all of those improvements, lock contention is not a bottleneck anymore, which is shown in our [memory-only benchmark](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks) . Furthermore, lock contentions are not causing some huge (50 milliseconds+) latency outliers they used to cause.
+### Comments
+**[Lee Hounshell](lee@apsalar.com)**
+Please post an example of reading the same rocksdb concurrently.
+We are using the latest 3.0 rocksdb; however, when two separate processes
+try and open the same rocksdb for reading, only one of the open requests
+succeed. The other open always fails with “db/LOCK: Resource temporarily unavailable” So far we have not found an option that allows sharing the rocksdb for reads. An example would be most appreciated.
+**[Siying Dong](siying.d@fb.com)**
+Sorry for the delay. We don’t have feature support for this scenario yet. Here is an example you can work around this problem. You can build a snapshot of the DB by doing this:
+1. create a separate directory on the same host for a snapshot of the DB.
+1. call `DB::DisableFileDeletions()`
+1. call `DB::GetLiveFiles()` to get a full list of the files.
+1. for all the files except manifest, add a hardlink file in your new directory pointing to the original file
+1. copy the manifest file and truncate the size (you can read the comments of `DB::GetLiveFiles()` for more information)
+1. call `DB::EnableFileDeletions()`
+1. now you can open the snapshot directory in another process to access those files. Please remember to delete the directory after reading the data to allow those files to be recycled.
+By the way, the best way to ask those questions is in our [facebook group](https://www.facebook.com/groups/rocksdb.dev/). Let us know if you need any further help.
+**[Darshan](darshan.ghumare@gmail.com)**
+Will this consistency problem of RocksDB all occurs in case of single put/write?
+What all ACID properties is supported by RocksDB, only durability irrespective of single or batch write?
+**[Siying Dong](siying.d@fb.com)**
+We recently [introduced optimistic transaction](https://reviews.facebook.net/D33435) which can help you ensure all of ACID.
+This blog post is mainly about optimizations in implementation. The RocksDB consistency semantic is not changed.