rocksdb-native 0.0.0 → 0.0.1

package/vendor/rocksdb/java/src/test/java/org/rocksdb/VerifyChecksumsTest.java

@@ -1,213 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-package org.rocksdb;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class VerifyChecksumsTest {
-  @ClassRule
-  public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE =
-      new RocksNativeLibraryResource();
-
-  @Rule public TemporaryFolder dbFolder = new TemporaryFolder();
-
-  /**
-   * Class to factor out the specific DB operations within the test
-   */
-  abstract static class Operations {
-    final int kv_count;
-    final List<String> elements = new ArrayList<>();
-    final List<String> sortedElements = new ArrayList<>();
-
-    Operations(final int kv_count) {
-      this.kv_count = kv_count;
-      for (int i = 0; i < kv_count; i++) elements.add(MessageFormat.format("{0,number,#}", i));
-      sortedElements.addAll(elements);
-      Collections.sort(sortedElements);
-    }
-
-    void fill(final RocksDB db) throws RocksDBException {
-      for (int i = 0; i < kv_count; i++) {
-        final String key = MessageFormat.format("key{0}", elements.get(i));
-        final String value = MessageFormat.format("value{0}", elements.get(i));
-        // noinspection ObjectAllocationInLoop
-        db.put(key.getBytes(), value.getBytes());
-      }
-      db.flush(new FlushOptions());
-    }
-
-    @SuppressWarnings("ObjectAllocationInLoop")
-    void get(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
-      try (final ReadOptions readOptions = new ReadOptions()) {
-        readOptions.setReadaheadSize(32 * 1024);
-        readOptions.setFillCache(false);
-        readOptions.setVerifyChecksums(verifyFlag);
-
-        for (int i = 0; i < kv_count / 10; i++) {
-          @SuppressWarnings("UnsecureRandomNumberGeneration")
-          final int index = Double.valueOf(Math.random() * kv_count).intValue();
-          final String key = MessageFormat.format("key{0}", sortedElements.get(index));
-          final String expectedValue = MessageFormat.format("value{0}", sortedElements.get(index));
-
-          final byte[] value = db.get(readOptions, key.getBytes());
-          assertThat(value).isEqualTo(expectedValue.getBytes());
-        }
-      }
-    }
-
-    @SuppressWarnings("ObjectAllocationInLoop")
-    void multiGet(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
-      try (final ReadOptions readOptions = new ReadOptions()) {
-        readOptions.setReadaheadSize(32 * 1024);
-        readOptions.setFillCache(false);
-        readOptions.setVerifyChecksums(verifyFlag);
-
-        final List<byte[]> keys = new ArrayList<>();
-        final List<String> expectedValues = new ArrayList<>();
-
-        for (int i = 0; i < kv_count / 10; i++) {
-          @SuppressWarnings("UnsecureRandomNumberGeneration")
-          final int index = Double.valueOf(Math.random() * kv_count).intValue();
-          keys.add(MessageFormat.format("key{0}", sortedElements.get(index)).getBytes());
-
-          expectedValues.add(MessageFormat.format("value{0}", sortedElements.get(index)));
-        }
-
-        final List<byte[]> values = db.multiGetAsList(readOptions, keys);
-        for (int i = 0; i < keys.size(); i++) {
-          assertThat(values.get(i)).isEqualTo(expectedValues.get(i).getBytes());
-        }
-      }
-    }
-
-    void iterate(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
-      final ReadOptions readOptions = new ReadOptions();
-      readOptions.setReadaheadSize(32 * 1024);
-      readOptions.setFillCache(false);
-      readOptions.setVerifyChecksums(verifyFlag);
-      int i = 0;
-      try (final RocksIterator rocksIterator = db.newIterator(readOptions)) {
-        rocksIterator.seekToFirst();
-        rocksIterator.status();
-        while (rocksIterator.isValid()) {
-          final byte[] key = rocksIterator.key();
-          final byte[] value = rocksIterator.value();
-          // noinspection ObjectAllocationInLoop
-          assertThat(key).isEqualTo(
-              (MessageFormat.format("key{0}", sortedElements.get(i))).getBytes());
-          // noinspection ObjectAllocationInLoop
-          assertThat(value).isEqualTo(
-              (MessageFormat.format("value{0}", sortedElements.get(i))).getBytes());
-          rocksIterator.next();
-          rocksIterator.status();
-          i++;
-        }
-      }
-      assertThat(i).isEqualTo(kv_count);
-    }
-
-    abstract void performOperations(final RocksDB db, final boolean verifyFlag)
-        throws RocksDBException;
-  }
-
-  private static final int KV_COUNT = 10000;
-
-  /**
-   * Run some operations and count the TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT before and after
-   * It should GO UP when the read options have checksum verification turned on.
-   * It shoulld REMAIN UNCHANGED when the read options have checksum verification turned off.
-   * As the read options refer only to the read operations, there are still a few checksums
-   * performed outside this (blocks are getting loaded for lots of reasons, not aways directly due
-   * to reads) but this test provides a good enough proxy for whether the flag is being noticed.
-   *
-   * @param operations the DB reading operations to perform which affect the checksum stats
-   *
-   * @throws RocksDBException
-   */
-  private void verifyChecksums(final Operations operations) throws RocksDBException {
-    final String dbPath = dbFolder.getRoot().getAbsolutePath();
-
-    // noinspection SingleStatementInBlock
-    try (final Statistics statistics = new Statistics();
-         final Options options = new Options().setCreateIfMissing(true).setStatistics(statistics)) {
-      try (final RocksDB db = RocksDB.open(options, dbPath)) {
-        // 0
-        System.out.println(MessageFormat.format(
-            "newly open {0}", statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT)));
-        operations.fill(db);
-        //
-        System.out.println(MessageFormat.format(
-            "flushed {0}", statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT)));
-      }
-
-      // 2
-      System.out.println(MessageFormat.format("closed-after-write {0}",
-          statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT)));
-
-      for (final boolean verifyFlag : new boolean[] {false, true, false, true}) {
-        try (final RocksDB db = RocksDB.open(options, dbPath)) {
-          final long beforeOperationsCount =
-              statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT);
-          System.out.println(MessageFormat.format("re-opened {0}", beforeOperationsCount));
-          operations.performOperations(db, verifyFlag);
-          final long afterOperationsCount =
-              statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT);
-          if (verifyFlag) {
-            // We don't need to be exact - we are checking that the checksums happen
-            // exactly how many depends on block size etc etc, so may not be entirely stable
-            System.out.println(MessageFormat.format("verify=true {0}", afterOperationsCount));
-            assertThat(afterOperationsCount).isGreaterThan(beforeOperationsCount + 20);
-          } else {
-            System.out.println(MessageFormat.format("verify=false {0}", afterOperationsCount));
-            assertThat(afterOperationsCount).isEqualTo(beforeOperationsCount);
-          }
-        }
-      }
-    }
-  }
-
-  @Test
-  public void verifyChecksumsInIteration() throws RocksDBException {
-    // noinspection AnonymousInnerClassMayBeStatic
-    verifyChecksums(new Operations(KV_COUNT) {
-      @Override
-      void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
-        iterate(db, verifyFlag);
-      }
-    });
-  }
-
-  @Test
-  public void verifyChecksumsGet() throws RocksDBException {
-    // noinspection AnonymousInnerClassMayBeStatic
-    verifyChecksums(new Operations(KV_COUNT) {
-      @Override
-      void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
-        get(db, verifyFlag);
-      }
-    });
-  }
-
-  @Test
-  public void verifyChecksumsMultiGet() throws RocksDBException {
-    // noinspection AnonymousInnerClassMayBeStatic
-    verifyChecksums(new Operations(KV_COUNT) {
-      @Override
-      void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
-        multiGet(db, verifyFlag);
-      }
-    });
-  }
-}

package/vendor/rocksdb/java/src/test/java/org/rocksdb/WALRecoveryModeTest.java

@@ -1,22 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-package org.rocksdb;
-
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-
-public class WALRecoveryModeTest {
-
-  @Test
-  public void getWALRecoveryMode() {
-    for (final WALRecoveryMode walRecoveryMode : WALRecoveryMode.values()) {
-      assertThat(WALRecoveryMode.getWALRecoveryMode(walRecoveryMode.getValue()))
-          .isEqualTo(walRecoveryMode);
-    }
-  }
-}

package/vendor/rocksdb/java/src/test/java/org/rocksdb/WalFilterTest.java

@@ -1,165 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-package org.rocksdb;
-
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.rocksdb.util.ByteUtil.bytes;
-import static org.rocksdb.util.TestUtil.*;
-
-public class WalFilterTest {
-
-  @ClassRule
-  public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE =
-      new RocksNativeLibraryResource();
-
-  @Rule
-  public TemporaryFolder dbFolder = new TemporaryFolder();
-
-  @Test
-  public void walFilter() throws RocksDBException {
-    // Create 3 batches with two keys each
-    final byte[][][] batchKeys = {
-        new byte[][] {
-            bytes("key1"),
-            bytes("key2")
-        },
-        new byte[][] {
-            bytes("key3"),
-            bytes("key4")
-        },
-        new byte[][] {
-            bytes("key5"),
-            bytes("key6")
-        }
-
-    };
-
-    final List<ColumnFamilyDescriptor> cfDescriptors = Arrays.asList(
-        new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY),
-        new ColumnFamilyDescriptor(bytes("pikachu"))
-    );
-    final List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
-
-    // Test with all WAL processing options
-    for (final WalProcessingOption option : WalProcessingOption.values()) {
-      try (final Options options = optionsForLogIterTest();
-           final DBOptions dbOptions = new DBOptions(options)
-               .setCreateMissingColumnFamilies(true);
-           final RocksDB db = RocksDB.open(dbOptions,
-               dbFolder.getRoot().getAbsolutePath(),
-                cfDescriptors, cfHandles)) {
-        try (final WriteOptions writeOptions = new WriteOptions()) {
-          // Write given keys in given batches
-          for (final byte[][] batchKey : batchKeys) {
-            final WriteBatch batch = new WriteBatch();
-            for (final byte[] bytes : batchKey) {
-              batch.put(cfHandles.get(0), bytes, dummyString(1024));
-            }
-            db.write(writeOptions, batch);
-          }
-        } finally {
-          for (final ColumnFamilyHandle cfHandle : cfHandles) {
-            cfHandle.close();
-          }
-          cfHandles.clear();
-        }
-      }
-
-      // Create a test filter that would apply wal_processing_option at the first
-      // record
-      final int applyOptionForRecordIndex = 1;
-      try (final TestableWalFilter walFilter =
-               new TestableWalFilter(option, applyOptionForRecordIndex)) {
-
-        try (final Options options = optionsForLogIterTest();
-             final DBOptions dbOptions = new DBOptions(options)
-                .setWalFilter(walFilter)) {
-
-          try (final RocksDB db = RocksDB.open(dbOptions,
-              dbFolder.getRoot().getAbsolutePath(),
-              cfDescriptors, cfHandles)) {
-
-            try {
-              assertThat(walFilter.logNumbers).isNotEmpty();
-              assertThat(walFilter.logFileNames).isNotEmpty();
-            } finally {
-              for (final ColumnFamilyHandle cfHandle : cfHandles) {
-                cfHandle.close();
-              }
-              cfHandles.clear();
-            }
-          } catch (final RocksDBException e) {
-            if (option != WalProcessingOption.CORRUPTED_RECORD) {
-              // exception is expected when CORRUPTED_RECORD!
-              throw e;
-            }
-          }
-        }
-      }
-    }
-  }
-
-
-  private static class TestableWalFilter extends AbstractWalFilter {
-    private final WalProcessingOption walProcessingOption;
-    private final int applyOptionForRecordIndex;
-    Map<Integer, Long> cfLognumber;
-    Map<String, Integer> cfNameId;
-    final List<Long> logNumbers = new ArrayList<>();
-    final List<String> logFileNames = new ArrayList<>();
-    private int currentRecordIndex = 0;
-
-    public TestableWalFilter(final WalProcessingOption walProcessingOption,
-        final int applyOptionForRecordIndex) {
-      super();
-      this.walProcessingOption = walProcessingOption;
-      this.applyOptionForRecordIndex = applyOptionForRecordIndex;
-    }
-
-    @Override
-    public void columnFamilyLogNumberMap(final Map<Integer, Long> cfLognumber,
-        final Map<String, Integer> cfNameId) {
-      this.cfLognumber = cfLognumber;
-      this.cfNameId = cfNameId;
-    }
-
-    @Override
-    public LogRecordFoundResult logRecordFound(
-        final long logNumber, final String logFileName, final WriteBatch batch,
-        final WriteBatch newBatch) {
-
-      logNumbers.add(logNumber);
-      logFileNames.add(logFileName);
-
-      final WalProcessingOption optionToReturn;
-      if (currentRecordIndex == applyOptionForRecordIndex) {
-        optionToReturn = walProcessingOption;
-      }
-      else {
-        optionToReturn = WalProcessingOption.CONTINUE_PROCESSING;
-      }
-
-      currentRecordIndex++;
-
-      return new LogRecordFoundResult(optionToReturn, false);
-    }
-
-    @Override
-    public String name() {
-      return "testable-wal-filter";
-    }
-  }
-}

package/vendor/rocksdb/java/src/test/java/org/rocksdb/WriteBatchHandlerTest.java

@@ -1,76 +0,0 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-package org.rocksdb;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.rocksdb.util.CapturingWriteBatchHandler;
-import org.rocksdb.util.CapturingWriteBatchHandler.Event;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.rocksdb.util.CapturingWriteBatchHandler.Action.*;
-
-
-public class WriteBatchHandlerTest {
-  @ClassRule
-  public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE =
-      new RocksNativeLibraryResource();
-
-  @Test
-  public void writeBatchHandler() throws RocksDBException {
-    // setup test data
-    final List<Event> testEvents = Arrays.asList(
-        new Event(DELETE, "k0".getBytes(), null),
-        new Event(PUT, "k1".getBytes(), "v1".getBytes()),
-        new Event(PUT, "k2".getBytes(), "v2".getBytes()),
-        new Event(PUT, "k3".getBytes(), "v3".getBytes()),
-        new Event(LOG, null, "log1".getBytes()),
-        new Event(MERGE, "k2".getBytes(), "v22".getBytes()),
-        new Event(DELETE, "k3".getBytes(), null)
-    );
-
-    // load test data to the write batch
-    try (final WriteBatch batch = new WriteBatch()) {
-      for (final Event testEvent : testEvents) {
-        switch (testEvent.action) {
-
-          case PUT:
-            batch.put(testEvent.key, testEvent.value);
-            break;
-
-          case MERGE:
-            batch.merge(testEvent.key, testEvent.value);
-            break;
-
-          case DELETE:
-            batch.delete(testEvent.key);
-            break;
-
-          case LOG:
-            batch.putLogData(testEvent.value);
-            break;
-        }
-      }
-
-      // attempt to read test data back from the WriteBatch by iterating
-      // with a handler
-      try (final CapturingWriteBatchHandler handler =
-               new CapturingWriteBatchHandler()) {
-        batch.iterate(handler);
-
-        // compare the results to the test data
-        final List<Event> actualEvents =
-            handler.getEvents();
-        assertThat(testEvents.size()).isSameAs(actualEvents.size());
-
-        assertThat(testEvents).isEqualTo(actualEvents);
-      }
-    }
-  }
-}