rocksdb-native 0.0.0 → 0.0.1

package/vendor/rocksdb/tools/run_leveldb.sh

@@ -1,175 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# REQUIRE: benchmark_leveldb.sh exists in the current directory
-# After execution of this script, log files are generated in $output_dir.
-# report.txt provides a high level statistics
-#
-# This should be used with the LevelDB fork listed here to use additional test options.
-# For more details on the changes see the blog post listed below.
-#   https://github.com/mdcallag/leveldb-1
-#   http://smalldatum.blogspot.com/2015/04/comparing-leveldb-and-rocksdb-take-2.html
-#
-# This should be run from the parent of the tools directory. The command line is:
-#   [$env_vars] tools/run_flash_bench.sh [list-of-threads]
-#
-# This runs a sequence of tests in the following sequence:
-#   step 1) load - bulkload, compact, fillseq, overwrite
-#   step 2) read-only for each number of threads
-#   step 3) read-write for each number of threads
-#
-# The list of threads is optional and when not set is equivalent to "24". 
-# Were list-of-threads specified as "1 2 4" then the tests in steps 2, 3 and
-# 4 above would be repeated for 1, 2 and 4 threads. The tests in step 1 are
-# only run for 1 thread.
-
-# Test output is written to $OUTPUT_DIR, currently /tmp/output. The performance
-# summary is in $OUTPUT_DIR/report.txt. There is one file in $OUTPUT_DIR per
-# test and the tests are listed below.
-#
-# The environment variables are also optional. The variables are:
-#   NKEYS         - number of key/value pairs to load
-#   NWRITESPERSEC - the writes/second rate limit for the *whilewriting* tests.
-#                   If this is too large then the non-writer threads can get
-#                   starved.
-#   VAL_SIZE      - the length of the value in the key/value pairs loaded.
-#                   You can estimate the size of the test database from this,
-#                   NKEYS and the compression rate (--compression_ratio) set
-#                   in tools/benchmark_leveldb.sh
-#   BLOCK_LENGTH  - value for db_bench --block_size
-#   CACHE_BYTES   - the size of the RocksDB block cache in bytes
-#   DATA_DIR      - directory in which to create database files
-#   DO_SETUP      - when set to 0 then a backup of the database is copied from
-#                   $DATA_DIR.bak to $DATA_DIR and the load tests from step 1
-#                   This allows tests from steps 2, 3 to be repeated faster.
-#   SAVE_SETUP    - saves a copy of the database at the end of step 1 to
-#                   $DATA_DIR.bak.
-
-# Size constants
-K=1024
-M=$((1024 * K))
-G=$((1024 * M))
-
-num_keys=${NKEYS:-$((1 * G))}
-wps=${NWRITESPERSEC:-$((10 * K))}
-vs=${VAL_SIZE:-400}
-cs=${CACHE_BYTES:-$(( 1 * G ))}
-bs=${BLOCK_LENGTH:-4096}
-
-# If no command line arguments then run for 24 threads.
-if [[ $# -eq 0 ]]; then
-  nthreads=( 24 )
-else
-  nthreads=( "$@" )
-fi
-
-for num_thr in "${nthreads[@]}" ; do
-  echo Will run for $num_thr threads
-done
-
-# Update these parameters before execution !!!
-db_dir=${DATA_DIR:-"/tmp/rocksdb/"}
-
-do_setup=${DO_SETUP:-1}
-save_setup=${SAVE_SETUP:-0}
-
-output_dir="${TMPDIR:-/tmp}/output"
-
-ARGS="\
-OUTPUT_DIR=$output_dir \
-NUM_KEYS=$num_keys \
-DB_DIR=$db_dir \
-VALUE_SIZE=$vs \
-BLOCK_SIZE=$bs \
-CACHE_SIZE=$cs"
-
-mkdir -p $output_dir
-echo -e "ops/sec\tmb/sec\tusec/op\tavg\tp50\tTest" \
-  > $output_dir/report.txt
-
-# Notes on test sequence:
-#   step 1) Setup database via sequential fill followed by overwrite to fragment it.
-#           Done without setting DURATION to make sure that overwrite does $num_keys writes
-#   step 2) read-only tests for all levels of concurrency requested
-#   step 3) non read-only tests for all levels of concurrency requested
-
-###### Setup the database
-
-if [[ $do_setup != 0 ]]; then
-  echo Doing setup
-
-  # Test 2a: sequential fill with large values to get peak ingest
-  #          adjust NUM_KEYS given the use of larger values
-  env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
-       ./tools/benchmark_leveldb.sh fillseq
-
-  # Test 2b: sequential fill with the configured value size
-  env $ARGS ./tools/benchmark_leveldb.sh fillseq
-
-  # Test 3: single-threaded overwrite
-  env $ARGS NUM_THREADS=1 DB_BENCH_NO_SYNC=1 ./tools/benchmark_leveldb.sh overwrite
-
-else
-  echo Restoring from backup
-
-  rm -rf $db_dir
-
-  if [ ! -d ${db_dir}.bak ]; then
-    echo Database backup does not exist at ${db_dir}.bak
-    exit -1
-  fi
-
-  echo Restore database from ${db_dir}.bak
-  cp -p -r ${db_dir}.bak $db_dir
-fi
-
-if [[ $save_setup != 0 ]]; then
-  echo Save database to ${db_dir}.bak
-  cp -p -r $db_dir ${db_dir}.bak
-fi
-
-###### Read-only tests
-
-for num_thr in "${nthreads[@]}" ; do
-  # Test 4: random read
-  env $ARGS NUM_THREADS=$num_thr ./tools/benchmark_leveldb.sh readrandom
-
-done
-
-###### Non read-only tests
-
-for num_thr in "${nthreads[@]}" ; do
-  # Test 7: overwrite with sync=0
-  env $ARGS NUM_THREADS=$num_thr DB_BENCH_NO_SYNC=1 \
-    ./tools/benchmark_leveldb.sh overwrite
-
-  # Test 8: overwrite with sync=1
-  # Not run for now because LevelDB db_bench doesn't have an option to limit the
-  # test run to X seconds and doing sync-per-commit for --num can take too long.
-  # env $ARGS NUM_THREADS=$num_thr ./tools/benchmark_leveldb.sh overwrite
-
-  # Test 11: random read while writing
-  env $ARGS NUM_THREADS=$num_thr WRITES_PER_SECOND=$wps \
-    ./tools/benchmark_leveldb.sh readwhilewriting
-
-done
-
-echo bulkload > $output_dir/report2.txt
-head -1 $output_dir/report.txt >> $output_dir/report2.txt
-grep bulkload $output_dir/report.txt >> $output_dir/report2.txt
-echo fillseq >> $output_dir/report2.txt
-head -1 $output_dir/report.txt >> $output_dir/report2.txt
-grep fillseq $output_dir/report.txt >> $output_dir/report2.txt
-echo overwrite sync=0 >> $output_dir/report2.txt
-head -1 $output_dir/report.txt >> $output_dir/report2.txt
-grep overwrite $output_dir/report.txt | grep \.s0  >> $output_dir/report2.txt
-echo overwrite sync=1 >> $output_dir/report2.txt
-head -1 $output_dir/report.txt >> $output_dir/report2.txt
-grep overwrite $output_dir/report.txt | grep \.s1  >> $output_dir/report2.txt
-echo readrandom >> $output_dir/report2.txt
-head -1 $output_dir/report.txt >> $output_dir/report2.txt
-grep readrandom $output_dir/report.txt  >> $output_dir/report2.txt
-echo readwhile >> $output_dir/report2.txt >> $output_dir/report2.txt
-head -1 $output_dir/report.txt >> $output_dir/report2.txt
-grep readwhilewriting $output_dir/report.txt >> $output_dir/report2.txt
-
-cat $output_dir/report2.txt

package/vendor/rocksdb/tools/verify_random_db.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# A shell script to verify DB generated by generate_random_db.sh cannot opened and read correct data.
-# ./ldb needs to be avaible to be executed.
-#
-# Usage: <SCRIPT> <DB Path>
-
-scriptpath=`dirname $BASH_SOURCE`
-if [ "$#" -lt 2 ]; then
-  echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name] [if_try_load_options] [if_ignore_unknown_options]"
-  exit 1
-fi
-
-db_dir=$1
-base_db_dir=$2
-dump_file_name=${3:-"dump_file.txt"}
-try_load_options=${4:-"1"}
-ignore_unknown_options=${5:-"0"}
-db_dump=$db_dir"/"$dump_file_name
-base_db_dump=$base_db_dir"/"$dump_file_name
-extra_params=
-
-if [ "$try_load_options" = "0" ]; then
- extra_params=" --try_load_options=false"
-elif [ "$try_load_options" = "1" ]; then
- extra_params=" --try_load_options=true"
-fi
-
-if [ "$ignore_unknown_options" = "1" ]; then
- extra_params="$extra_params --ignore_unknown_options"
-fi
-
-set -e
-echo == Dumping data from $db_dir to $db_dump
-./ldb dump --db=$db_dir $extra_params > $db_dump
-
-echo == Dumping data from $base_db_dir to $base_db_dump
-./ldb dump --db=$base_db_dir $extra_params > $base_db_dump
-
-diff $db_dump $base_db_dump

package/vendor/rocksdb/tools/write_external_sst.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-#
-
-if [ "$#" -lt 3 ]; then
-  echo "usage: $BASH_SOURCE <input_data_path> <DB Path> <extern SST dir>"
-  exit 1
-fi
-
-input_data_dir=$1
-db_dir=$2
-extern_sst_dir=$3
-rm -rf $db_dir
-mkdir -p $extern_sst_dir
-
-set -e
-
-n=0
-
-for f in `find $input_data_dir -name sorted_data*`
-do
-  echo == Writing external SST file $f to $extern_sst_dir/extern_sst${n}
-  ./ldb --db=$db_dir --create_if_missing write_extern_sst $extern_sst_dir/extern_sst${n} < $f
-  let "n = n + 1"
-done

package/vendor/rocksdb/tools/write_stress_runner.py

@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-import argparse
-import random
-
-import subprocess
-import sys
-import time
-
-
-def generate_runtimes(total_runtime):
-    # combination of short runtimes and long runtimes, with heavier
-    # weight on short runtimes
-    possible_runtimes_sec = list(range(1, 10)) + list(range(1, 20)) + [100, 1000]
-    runtimes = []
-    while total_runtime > 0:
-        chosen = random.choice(possible_runtimes_sec)
-        chosen = min(chosen, total_runtime)
-        runtimes.append(chosen)
-        total_runtime -= chosen
-    return runtimes
-
-
-def main(args):
-    runtimes = generate_runtimes(int(args.runtime_sec))
-    print(
-        "Going to execute write stress for " + str(runtimes)
-    )  # noqa: E999 T25377293 Grandfathered in
-    first_time = True
-
-    for runtime in runtimes:
-        kill = random.choice([False, True])
-
-        cmd = "./write_stress --runtime_sec=" + ("-1" if kill else str(runtime))
-
-        if len(args.db) > 0:
-            cmd = cmd + " --db=" + args.db
-
-        if first_time:
-            first_time = False
-        else:
-            # use current db
-            cmd = cmd + " --destroy_db=false"
-        if random.choice([False, True]):
-            cmd = cmd + " --delete_obsolete_files_with_fullscan=true"
-        if random.choice([False, True]):
-            cmd = cmd + " --low_open_files_mode=true"
-
-        print(
-            "Running write_stress for %d seconds (%s): %s"
-            % (runtime, ("kill-mode" if kill else "clean-shutdown-mode"), cmd)
-        )
-
-        child = subprocess.Popen([cmd], shell=True)
-        killtime = time.time() + runtime
-        while not kill or time.time() < killtime:
-            time.sleep(1)
-            if child.poll() is not None:
-                if child.returncode == 0:
-                    break
-                else:
-                    print(
-                        "ERROR: write_stress died with exitcode=%d\n" % child.returncode
-                    )
-                    sys.exit(1)
-        if kill:
-            child.kill()
-        # breathe
-        time.sleep(3)
-
-
-if __name__ == "__main__":
-    random.seed(time.time())
-    parser = argparse.ArgumentParser(
-        description="This script runs and kills \
-        write_stress multiple times"
-    )
-    parser.add_argument("--runtime_sec", default="1000")
-    parser.add_argument("--db", default="")
-    args = parser.parse_args()
-    main(args)

package/vendor/rocksdb/util/crc32c_ppc.c

@@ -1,94 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  Copyright (c) 2017 International Business Machines Corp.
-//  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#define CRC_TABLE
-#include <stdint.h>
-#include <stdlib.h>
-#include <strings.h>
-#include "util/crc32c_ppc_constants.h"
-
-#define VMX_ALIGN 16
-#define VMX_ALIGN_MASK (VMX_ALIGN - 1)
-
-#ifdef REFLECT
-static unsigned int crc32_align(unsigned int crc, unsigned char const *p,
-                                unsigned long len) {
-  while (len--) crc = crc_table[(crc ^ *p++) & 0xff] ^ (crc >> 8);
-  return crc;
-}
-#endif
-
-#ifdef HAVE_POWER8
-unsigned int __crc32_vpmsum(unsigned int crc, unsigned char const *p,
-                            unsigned long len);
-
-static uint32_t crc32_vpmsum(uint32_t crc, unsigned char const *data,
-                             size_t len) {
-  unsigned int prealign;
-  unsigned int tail;
-
-#ifdef CRC_XOR
-  crc ^= 0xffffffff;
-#endif
-
-  if (len < VMX_ALIGN + VMX_ALIGN_MASK) {
-    crc = crc32_align(crc, data, (unsigned long)len);
-    goto out;
-  }
-
-  if ((unsigned long)data & VMX_ALIGN_MASK) {
-    prealign = VMX_ALIGN - ((unsigned long)data & VMX_ALIGN_MASK);
-    crc = crc32_align(crc, data, prealign);
-    len -= prealign;
-    data += prealign;
-  }
-
-  crc = __crc32_vpmsum(crc, data, (unsigned long)len & ~VMX_ALIGN_MASK);
-
-  tail = len & VMX_ALIGN_MASK;
-  if (tail) {
-    data += len & ~VMX_ALIGN_MASK;
-    crc = crc32_align(crc, data, tail);
-  }
-
-out:
-#ifdef CRC_XOR
-  crc ^= 0xffffffff;
-#endif
-
-  return crc;
-}
-
-/* This wrapper function works around the fact that crc32_vpmsum
- * does not gracefully handle the case where the data pointer is NULL.  There
- * may be room for performance improvement here.
- */
-uint32_t crc32c_ppc(uint32_t crc, unsigned char const *data, size_t len) {
-  unsigned char *buf2;
-
-  if (!data) {
-    buf2 = (unsigned char *)malloc(len);
-    bzero(buf2, len);
-    crc = crc32_vpmsum(crc, buf2, len);
-    free(buf2);
-  } else {
-    crc = crc32_vpmsum(crc, data, (unsigned long)len);
-  }
-  return crc;
-}
-
-#else /* HAVE_POWER8 */
-
-/* This symbol has to exist on non-ppc architectures (and on legacy
- * ppc systems using power7 or below) in order to compile properly
- * there, even though it won't be called.
- */
-uint32_t crc32c_ppc(uint32_t crc, unsigned char const *data, size_t len) {
-  return 0;
-}
-
-#endif /* HAVE_POWER8 */