rocksdb-native 0.0.0 → 0.0.1

package/vendor/rocksdb/tools/check_all_python.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-import glob
-
-# Checks that all python files in the repository are at least free of syntax
-# errors. This provides a minimal pre-/post-commit check for python file
-# modifications.
-
-filenames = []
-# Avoid scanning all of ./ because there might be other external repos
-# linked in.
-for base in ["buckifier", "build_tools", "coverage", "tools"]:
-    # Clean this up when we finally upgrade to Python 3
-    for suff in ["*", "*/*", "*/*/*"]:
-        filenames += glob.glob(base + "/" + suff + ".py")
-
-for filename in filenames:
-    source = open(filename, "r").read() + "\n"
-    # Parses and syntax checks the file, throwing on error. (No pyc written.)
-    _ = compile(source, filename, "exec")
-
-print("No syntax errors in {0} .py files".format(len(filenames)))

package/vendor/rocksdb/tools/check_format_compatible.sh

@@ -1,379 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# A shell script to build and run different versions of ldb to check for
-# expected forward and backward compatibility with "current" version. The
-# working copy must have no uncommitted changes.
-#
-# Usage: <SCRIPT> [ref_for_current]
-# `ref_for_current` can be a revision, tag, commit or branch name. Default is HEAD.
-#
-# Return value 0 means all regression tests pass. 1 if not pass.
-#
-# Environment options:
-#  SHORT_TEST=1 - Test only the oldest branch for each kind of test. This is
-#    a good choice for PR validation as it is relatively fast and will find
-#    most issues.
-#  USE_SSH=1 - Connect to GitHub with ssh instead of https
-
-if ! git diff-index --quiet HEAD; then
-  echo "You have uncommitted changes. Aborting."
-  exit 1
-fi
-
-current_checkout_name=${1:-HEAD}
-# This allows the script to work even if with transient refs like "HEAD"
-current_checkout_hash="$(git rev-parse --quiet --verify $current_checkout_name)"
-
-if [ "$current_checkout_hash" == "" ]; then
-  echo "Not a recognized ref: $current_checkout_name"
-  exit 1
-fi
-
-# To restore to prior branch at the end
-orig_branch="$(git rev-parse --abbrev-ref HEAD)"
-tmp_branch=_tmp_format_compatible
-tmp_origin=_tmp_origin
-
-# Don't depend on what current "origin" might be
-set -e
-git remote remove $tmp_origin 2>/dev/null || true
-if [ "$USE_SSH" ]; then
-  git remote add $tmp_origin "git@github.com:facebook/rocksdb.git"
-else
-  git remote add $tmp_origin "https://github.com/facebook/rocksdb.git"
-fi
-git fetch $tmp_origin
-
-# Used in building some ancient RocksDB versions where by default it tries to
-# use a precompiled libsnappy.a checked in to the repo.
-export SNAPPY_LDFLAGS=-lsnappy
-
-cleanup() {
-  echo "== Cleaning up"
-  git reset --hard || true
-  git checkout "$orig_branch" || true
-  git branch -D $tmp_branch || true
-  git remote remove $tmp_origin || true
-}
-trap cleanup EXIT # Always clean up, even on failure or Ctrl+C
-
-scriptpath=`dirname ${BASH_SOURCE[0]}`
-
-test_dir=${TEST_TMPDIR:-"/tmp"}"/rocksdb_format_compatible_$USER"
-rm -rf ${test_dir:?}
-
-# Prevent 'make clean' etc. from wiping out test_dir
-export TEST_TMPDIR=$test_dir"/misc"
-
-# For saving current version of scripts as we checkout different versions to test
-script_copy_dir=$test_dir"/script_copy"
-mkdir -p $script_copy_dir
-cp -f $scriptpath/*.sh $script_copy_dir
-
-# For shared raw input data
-input_data_path=$test_dir"/test_data_input"
-mkdir -p $input_data_path
-# For external sst ingestion test
-ext_test_dir=$test_dir"/ext"
-mkdir -p $ext_test_dir
-# For DB dump test
-db_test_dir=$test_dir"/db"
-mkdir -p $db_test_dir
-# For backup/restore test (uses DB test)
-bak_test_dir=$test_dir"/bak"
-mkdir -p $bak_test_dir
-
-python_bin=$(which python3 || which python || echo python3)
-
-# Generate random files.
-for i in {1..6}
-do
-  input_data[$i]=$input_data_path/data$i
-  echo == Generating random input file ${input_data[$i]}
-  $python_bin - <<EOF
-import random
-random.seed($i)
-symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
-with open('${input_data[$i]}', 'w') as f:
-  for i in range(1,1024):
-    k = ""
-    for j in range(1, random.randint(1,32)):
-      k=k + symbols[random.randint(0, len(symbols) - 1)]
-    vb = ""
-    for j in range(1, random.randint(0,128)):
-      vb = vb + symbols[random.randint(0, len(symbols) - 1)]
-    v = ""
-    for j in range(1, random.randint(1, 5)):
-      v = v + vb
-    print(k + " ==> " + v, file=f)
-EOF
-done
-
-# Generate file(s) with sorted keys.
-sorted_input_data=$input_data_path/sorted_data
-echo == Generating file with sorted keys ${sorted_input_data}
-$python_bin - <<EOF
-with open('${sorted_input_data}', 'w') as f:
-  for i in range(0,10):
-    k = str(i)
-    v = "value" + k
-    print(k + " ==> " + v, file=f)
-EOF
-
-# db_backward_only_refs defined below the rest
-
-# To check for DB forward compatibility with loading options (old version
-# reading data from new), as well as backward compatibility
-declare -a db_forward_with_options_refs=("6.27.fb" "6.28.fb" "6.29.fb" "7.0.fb" "7.1.fb" "7.2.fb" "7.3.fb" "7.4.fb" "7.5.fb" "7.6.fb" "7.7.fb" "7.8.fb" "7.9.fb" "7.10.fb" "8.0.fb" "8.1.fb" "8.2.fb")
-# To check for DB forward compatibility without loading options (in addition
-# to the "with loading options" set), as well as backward compatibility
-declare -a db_forward_no_options_refs=() # N/A at the moment
-
-# To check for SST ingestion backward compatibility (new version reading
-# data from old) (ldb ingest_extern_sst added in 5.16.x, back-ported to
-# 5.14.x, 5.15.x)
-declare -a ext_backward_only_refs=("5.14.fb" "5.15.fb" "5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb" "6.6.fb" "6.7.fb" "6.8.fb" "6.9.fb" "6.10.fb" "6.11.fb" "6.12.fb" "6.13.fb" "6.14.fb" "6.15.fb" "6.16.fb" "6.17.fb" "6.18.fb" "6.19.fb" "6.20.fb" "6.21.fb" "6.22.fb" "6.23.fb" "6.24.fb" "6.25.fb" "6.26.fb")
-# To check for SST ingestion forward compatibility (old version reading
-# data from new) as well as backward compatibility
-declare -a ext_forward_refs=("${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}")
-
-# To check for backup backward compatibility (new version reading data
-# from old) (ldb backup/restore added in 4.11.x)
-declare -a bak_backward_only_refs=("4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb" "5.11.fb" "5.12.fb" "5.13.fb" "${ext_backward_only_refs[@]}")
-# To check for backup forward compatibility (old version reading data
-# from new) as well as backward compatibility
-declare -a bak_forward_refs=("${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}")
-
-# Branches (git refs) to check for DB backward compatibility (new version
-# reading data from old) (in addition to the "forward compatible" list)
-# NOTE: 2.7.fb.branch shows assertion violation in some configurations
-declare -a db_backward_only_refs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "${bak_backward_only_refs[@]}")
-
-if [ "$SHORT_TEST" ]; then
-  # Use only the first (if exists) of each list
-  db_backward_only_refs=(${db_backward_only_refs[0]})
-  db_forward_no_options_refs=(${db_forward_no_options_refs[0]})
-  db_forward_with_options_refs=(${db_forward_with_options_refs[0]})
-  ext_backward_only_refs=(${ext_backward_only_refs[0]})
-  ext_forward_refs=(${ext_forward_refs[0]})
-  bak_backward_only_refs=(${bak_backward_only_refs[0]})
-  bak_forward_refs=(${bak_forward_refs[0]})
-fi
-
-# De-duplicate & accumulate
-declare -a checkout_refs=()
-for checkout_ref in "${db_backward_only_refs[@]}" "${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}" "${ext_backward_only_refs[@]}" "${ext_forward_refs[@]}" "${bak_backward_only_refs[@]}" "${bak_forward_refs[@]}"
-do
-  if [ ! -e $db_test_dir/$checkout_ref ]; then
-    mkdir -p $db_test_dir/$checkout_ref
-    checkout_refs+=($checkout_ref)
-  fi
-done
-
-generate_db()
-{
-    set +e
-    bash "$script_copy_dir"/generate_random_db.sh "$1" "$2"
-    if [ $? -ne 0 ]; then
-        echo ==== Error loading data from $2 to $1 ====
-        exit 1
-    fi
-    set -e
-}
-
-compare_db()
-{
-    set +e
-    bash "$script_copy_dir"/verify_random_db.sh "$1" "$2" "$3" "$4" "$5"
-    if [ $? -ne 0 ]; then
-        echo ==== Read different content from $1 and $2 or error happened. ====
-        exit 1
-    fi
-    set -e
-}
-
-write_external_sst()
-{
-    set +e
-    bash "$script_copy_dir"/write_external_sst.sh "$1" "$2" "$3"
-    if [ $? -ne 0 ]; then
-        echo ==== Error writing external SST file using data from $1 to $3 ====
-        exit 1
-    fi
-    set -e
-}
-
-ingest_external_sst()
-{
-    set +e
-    bash "$script_copy_dir"/ingest_external_sst.sh "$1" "$2"
-    if [ $? -ne 0 ]; then
-        echo ==== Error ingesting external SST in $2 to DB at $1 ====
-        exit 1
-    fi
-    set -e
-}
-
-backup_db()
-{
-    set +e
-    bash "$script_copy_dir"/backup_db.sh "$1" "$2"
-    if [ $? -ne 0 ]; then
-        echo ==== Error backing up DB $1 to $2 ====
-        exit 1
-    fi
-    set -e
-}
-
-restore_db()
-{
-    set +e
-    bash "$script_copy_dir"/restore_db.sh "$1" "$2"
-    if [ $? -ne 0 ]; then
-        echo ==== Error restoring from $1 to $2 ====
-        exit 1
-    fi
-    set -e
-}
-
-member_of_array()
-{
-  local e match="$1"
-  shift
-  for e; do [[ "$e" == "$match" ]] && return 0; done
-  return 1
-}
-
-force_no_fbcode()
-{
-  # Not all branches recognize ROCKSDB_NO_FBCODE and we should not need
-  # to patch old branches for changes to available FB compilers.
-  sed -i -e 's|-d /mnt/gvfs/third-party|"$ROCKSDB_FORCE_FBCODE"|' build_tools/build_detect_platform
-}
-
-# General structure from here:
-# * Check out, build, and do stuff with the "current" branch.
-# * For each older branch under consideration,
-#   * Check out, build, and do stuff with it, potentially using data
-#     generated from "current" branch.
-# * (Again) check out, build, and do (other) stuff with the "current"
-#    branch, potentially using data from older branches.
-#
-# This way, we only do at most n+1 checkout+build steps, without the
-# need to stash away executables.
-
-# Decorate name
-current_checkout_name="$current_checkout_name ($current_checkout_hash)"
-
-echo "== Building $current_checkout_name debug"
-git checkout -B $tmp_branch $current_checkout_hash
-force_no_fbcode
-make clean
-DISABLE_WARNING_AS_ERROR=1 make ldb -j32
-
-echo "== Using $current_checkout_name, generate DB with extern SST and ingest"
-current_ext_test_dir=$ext_test_dir"/current"
-write_external_sst $input_data_path ${current_ext_test_dir}_pointless $current_ext_test_dir
-ingest_external_sst ${current_ext_test_dir}_ingest $current_ext_test_dir
-
-echo "== Generating DB from $current_checkout_name ..."
-current_db_test_dir=$db_test_dir"/current"
-generate_db $input_data_path $current_db_test_dir
-
-echo "== Creating backup of DB from $current_checkout_name ..."
-current_bak_test_dir=$bak_test_dir"/current"
-backup_db $current_db_test_dir $current_bak_test_dir
-
-for checkout_ref in "${checkout_refs[@]}"
-do
-  echo "== Building $checkout_ref debug"
-  git reset --hard $tmp_origin/$checkout_ref
-  force_no_fbcode
-  make clean
-  DISABLE_WARNING_AS_ERROR=1 make ldb -j32
-
-  # We currently assume DB backward compatibility for every branch listed
-  echo "== Use $checkout_ref to generate a DB ..."
-  generate_db $input_data_path $db_test_dir/$checkout_ref
-
-  if member_of_array "$checkout_ref" "${ext_backward_only_refs[@]}" ||
-    member_of_array "$checkout_ref" "${ext_forward_refs[@]}"
-  then
-    echo "== Use $checkout_ref to generate DB with extern SST file"
-    write_external_sst $input_data_path $ext_test_dir/${checkout_ref}_pointless $ext_test_dir/$checkout_ref
-  fi
-
-  if member_of_array "$checkout_ref" "${ext_forward_refs[@]}"
-  then
-    echo "== Use $checkout_ref to ingest extern SST file and compare vs. $current_checkout_name"
-    ingest_external_sst $ext_test_dir/${checkout_ref}_ingest $ext_test_dir/$checkout_ref
-    compare_db $ext_test_dir/${checkout_ref}_ingest ${current_ext_test_dir}_ingest db_dump.txt 1 1
-
-    rm -rf ${ext_test_dir:?}/${checkout_ref}_ingest
-    echo "== Use $checkout_ref to ingest extern SST file from $current_checkout_name"
-    ingest_external_sst $ext_test_dir/${checkout_ref}_ingest $current_ext_test_dir
-    compare_db $ext_test_dir/${checkout_ref}_ingest ${current_ext_test_dir}_ingest db_dump.txt 1 1
-  fi
-
-  if member_of_array "$checkout_ref" "${db_forward_no_options_refs[@]}" ||
-    member_of_array "$checkout_ref" "${db_forward_with_options_refs[@]}"
-  then
-    echo "== Use $checkout_ref to open DB generated using $current_checkout_name..."
-    compare_db $db_test_dir/$checkout_ref $current_db_test_dir forward_${checkout_ref}_dump.txt 0
-  fi
-
-  if member_of_array "$checkout_ref" "${db_forward_with_options_refs[@]}"
-  then
-    echo "== Use $checkout_ref to open DB generated using $current_checkout_name with its options..."
-    compare_db $db_test_dir/$checkout_ref $current_db_test_dir forward_${checkout_ref}_dump.txt 1 1
-  fi
-
-  if member_of_array "$checkout_ref" "${bak_backward_only_refs[@]}" ||
-    member_of_array "$checkout_ref" "${bak_forward_refs[@]}"
-  then
-    echo "== Use $checkout_ref to backup DB"
-    backup_db $db_test_dir/$checkout_ref $bak_test_dir/$checkout_ref
-  fi
-
-  if member_of_array "$checkout_ref" "${bak_forward_refs[@]}"
-  then
-    echo "== Use $checkout_ref to restore DB from $current_checkout_name"
-    rm -rf ${db_test_dir:?}/$checkout_ref
-    restore_db $current_bak_test_dir $db_test_dir/$checkout_ref
-    compare_db $db_test_dir/$checkout_ref $current_db_test_dir forward_${checkout_ref}_dump.txt 0
-  fi
-done
-
-echo "== Building $current_checkout_name debug (again, final)"
-git reset --hard $current_checkout_hash
-force_no_fbcode
-make clean
-DISABLE_WARNING_AS_ERROR=1 make ldb -j32
-
-for checkout_ref in "${checkout_refs[@]}"
-do
-  # We currently assume DB backward compatibility for every branch listed
-  echo "== Use $current_checkout_name to open DB generated using $checkout_ref..."
-  compare_db $db_test_dir/$checkout_ref $current_db_test_dir db_dump.txt 1 0
-
-  if member_of_array "$checkout_ref" "${ext_backward_only_refs[@]}" ||
-    member_of_array "$checkout_ref" "${ext_forward_refs[@]}"
-  then
-    rm -rf ${ext_test_dir:?}/${checkout_ref}_ingest
-    echo "== Use $current_checkout_name to ingest extern SST file from $checkout_ref"
-    ingest_external_sst $ext_test_dir/${checkout_ref}_ingest $current_ext_test_dir
-    compare_db $ext_test_dir/${checkout_ref}_ingest ${current_ext_test_dir}_ingest db_dump.txt 1 1
-  fi
-
-  if member_of_array "$checkout_ref" "${bak_backward_only_refs[@]}" ||
-    member_of_array "$checkout_ref" "${bak_forward_refs[@]}"
-  then
-    echo "== Use $current_checkout_name to restore DB from $checkout_ref"
-    rm -rf ${db_test_dir:?}/$checkout_ref
-    restore_db $bak_test_dir/$checkout_ref $db_test_dir/$checkout_ref
-    compare_db $db_test_dir/$checkout_ref $current_db_test_dir db_dump.txt 1 0
-  fi
-done
-
-echo ==== Compatibility Test PASSED ====