rocksdb-native 0.0.0 → 0.0.1

package/vendor/rocksdb/tools/advisor/test/test_rule_parser.py

@@ -1,226 +0,0 @@
-# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-#  This source code is licensed under both the GPLv2 (found in the
-#  COPYING file in the root directory) and Apache 2.0 License
-#  (found in the LICENSE.Apache file in the root directory).
-
-import os
-import unittest
-
-from advisor.db_log_parser import DatabaseLogs, DataSource
-from advisor.db_options_parser import DatabaseOptions
-from advisor.rule_parser import RulesSpec
-
-RuleToSuggestions = {
-    "stall-too-many-memtables": ["inc-bg-flush", "inc-write-buffer"],
-    "stall-too-many-L0": [
-        "inc-max-subcompactions",
-        "inc-max-bg-compactions",
-        "inc-write-buffer-size",
-        "dec-max-bytes-for-level-base",
-        "inc-l0-slowdown-writes-trigger",
-    ],
-    "stop-too-many-L0": [
-        "inc-max-bg-compactions",
-        "inc-write-buffer-size",
-        "inc-l0-stop-writes-trigger",
-    ],
-    "stall-too-many-compaction-bytes": [
-        "inc-max-bg-compactions",
-        "inc-write-buffer-size",
-        "inc-hard-pending-compaction-bytes-limit",
-        "inc-soft-pending-compaction-bytes-limit",
-    ],
-    "level0-level1-ratio": ["l0-l1-ratio-health-check"],
-}
-
-
-class TestAllRulesTriggered(unittest.TestCase):
-    def setUp(self):
-        # load the Rules
-        this_path = os.path.abspath(os.path.dirname(__file__))
-        ini_path = os.path.join(this_path, "input_files/triggered_rules.ini")
-        self.db_rules = RulesSpec(ini_path)
-        self.db_rules.load_rules_from_spec()
-        self.db_rules.perform_section_checks()
-        # load the data sources: LOG and OPTIONS
-        log_path = os.path.join(this_path, "input_files/LOG-0")
-        options_path = os.path.join(this_path, "input_files/OPTIONS-000005")
-        db_options_parser = DatabaseOptions(options_path)
-        self.column_families = db_options_parser.get_column_families()
-        db_logs_parser = DatabaseLogs(log_path, self.column_families)
-        self.data_sources = {
-            DataSource.Type.DB_OPTIONS: [db_options_parser],
-            DataSource.Type.LOG: [db_logs_parser],
-        }
-
-    def test_triggered_conditions(self):
-        conditions_dict = self.db_rules.get_conditions_dict()
-        rules_dict = self.db_rules.get_rules_dict()
-        # Make sure none of the conditions is triggered beforehand
-        for cond in conditions_dict.values():
-            self.assertFalse(cond.is_triggered(), repr(cond))
-        for rule in rules_dict.values():
-            self.assertFalse(
-                rule.is_triggered(conditions_dict, self.column_families), repr(rule)
-            )
-
-        # # Trigger the conditions as per the data sources.
-        # trigger_conditions(, conditions_dict)
-
-        # Get the set of rules that have been triggered
-        triggered_rules = self.db_rules.get_triggered_rules(
-            self.data_sources, self.column_families
-        )
-
-        # Make sure each condition and rule is triggered
-        for cond in conditions_dict.values():
-            if cond.get_data_source() is DataSource.Type.TIME_SERIES:
-                continue
-            self.assertTrue(cond.is_triggered(), repr(cond))
-
-        for rule in rules_dict.values():
-            self.assertIn(rule, triggered_rules)
-            # Check the suggestions made by the triggered rules
-            for sugg in rule.get_suggestions():
-                self.assertIn(sugg, RuleToSuggestions[rule.name])
-
-        for rule in triggered_rules:
-            self.assertIn(rule, rules_dict.values())
-            for sugg in RuleToSuggestions[rule.name]:
-                self.assertIn(sugg, rule.get_suggestions())
-
-
-class TestConditionsConjunctions(unittest.TestCase):
-    def setUp(self):
-        # load the Rules
-        this_path = os.path.abspath(os.path.dirname(__file__))
-        ini_path = os.path.join(this_path, "input_files/test_rules.ini")
-        self.db_rules = RulesSpec(ini_path)
-        self.db_rules.load_rules_from_spec()
-        self.db_rules.perform_section_checks()
-        # load the data sources: LOG and OPTIONS
-        log_path = os.path.join(this_path, "input_files/LOG-1")
-        options_path = os.path.join(this_path, "input_files/OPTIONS-000005")
-        db_options_parser = DatabaseOptions(options_path)
-        self.column_families = db_options_parser.get_column_families()
-        db_logs_parser = DatabaseLogs(log_path, self.column_families)
-        self.data_sources = {
-            DataSource.Type.DB_OPTIONS: [db_options_parser],
-            DataSource.Type.LOG: [db_logs_parser],
-        }
-
-    def test_condition_conjunctions(self):
-        conditions_dict = self.db_rules.get_conditions_dict()
-        rules_dict = self.db_rules.get_rules_dict()
-        # Make sure none of the conditions is triggered beforehand
-        for cond in conditions_dict.values():
-            self.assertFalse(cond.is_triggered(), repr(cond))
-        for rule in rules_dict.values():
-            self.assertFalse(
-                rule.is_triggered(conditions_dict, self.column_families), repr(rule)
-            )
-
-        # Trigger the conditions as per the data sources.
-        self.db_rules.trigger_conditions(self.data_sources)
-
-        # Check for the conditions
-        conds_triggered = ["log-1-true", "log-2-true", "log-3-true"]
-        conds_not_triggered = ["log-4-false", "options-1-false"]
-        for cond in conds_triggered:
-            self.assertTrue(conditions_dict[cond].is_triggered(), repr(cond))
-        for cond in conds_not_triggered:
-            self.assertFalse(conditions_dict[cond].is_triggered(), repr(cond))
-
-        # Check for the rules
-        rules_triggered = ["multiple-conds-true"]
-        rules_not_triggered = [
-            "single-condition-false",
-            "multiple-conds-one-false",
-            "multiple-conds-all-false",
-        ]
-        for rule_name in rules_triggered:
-            rule = rules_dict[rule_name]
-            self.assertTrue(
-                rule.is_triggered(conditions_dict, self.column_families), repr(rule)
-            )
-        for rule_name in rules_not_triggered:
-            rule = rules_dict[rule_name]
-            self.assertFalse(
-                rule.is_triggered(conditions_dict, self.column_families), repr(rule)
-            )
-
-
-class TestSanityChecker(unittest.TestCase):
-    def setUp(self):
-        this_path = os.path.abspath(os.path.dirname(__file__))
-        ini_path = os.path.join(this_path, "input_files/rules_err1.ini")
-        db_rules = RulesSpec(ini_path)
-        db_rules.load_rules_from_spec()
-        self.rules_dict = db_rules.get_rules_dict()
-        self.conditions_dict = db_rules.get_conditions_dict()
-        self.suggestions_dict = db_rules.get_suggestions_dict()
-
-    def test_rule_missing_suggestions(self):
-        regex = ".*rule must have at least one suggestion.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.rules_dict["missing-suggestions"].perform_checks()
-
-    def test_rule_missing_conditions(self):
-        regex = ".*rule must have at least one condition.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.rules_dict["missing-conditions"].perform_checks()
-
-    def test_condition_missing_regex(self):
-        regex = ".*provide regex for log condition.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.conditions_dict["missing-regex"].perform_checks()
-
-    def test_condition_missing_options(self):
-        regex = ".*options missing in condition.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.conditions_dict["missing-options"].perform_checks()
-
-    def test_condition_missing_expression(self):
-        regex = ".*expression missing in condition.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.conditions_dict["missing-expression"].perform_checks()
-
-    def test_suggestion_missing_option(self):
-        regex = ".*provide option or description.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.suggestions_dict["missing-option"].perform_checks()
-
-    def test_suggestion_missing_description(self):
-        regex = ".*provide option or description.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            self.suggestions_dict["missing-description"].perform_checks()
-
-
-class TestParsingErrors(unittest.TestCase):
-    def setUp(self):
-        self.this_path = os.path.abspath(os.path.dirname(__file__))
-
-    def test_condition_missing_source(self):
-        ini_path = os.path.join(self.this_path, "input_files/rules_err2.ini")
-        db_rules = RulesSpec(ini_path)
-        regex = ".*provide source for condition.*"
-        with self.assertRaisesRegex(NotImplementedError, regex):
-            db_rules.load_rules_from_spec()
-
-    def test_suggestion_missing_action(self):
-        ini_path = os.path.join(self.this_path, "input_files/rules_err3.ini")
-        db_rules = RulesSpec(ini_path)
-        regex = ".*provide action for option.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            db_rules.load_rules_from_spec()
-
-    def test_section_no_name(self):
-        ini_path = os.path.join(self.this_path, "input_files/rules_err4.ini")
-        db_rules = RulesSpec(ini_path)
-        regex = "Parsing error: needed section header:.*"
-        with self.assertRaisesRegex(ValueError, regex):
-            db_rules.load_rules_from_spec()
-
-
-if __name__ == "__main__":
-    unittest.main()

package/vendor/rocksdb/tools/analyze_txn_stress_test.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# Usage:
-# 1. Enable ROCKS_LOG_DETAILS in util/logging.h
-# 2. Run ./transaction_test --gtest_filter="MySQLStyleTransactionTest/MySQLStyleTransactionTest.TransactionStressTest/*" --gtest_break_on_failure
-# 3. SET=1 # 2 or 3
-# 4. LOG=/dev/shm/transaction_testdb_8600601584148590297/LOG
-# 5. grep RandomTransactionVerify $LOG | cut -d' ' -f 12 | sort -n # to find verify snapshots
-# 5. vn=1345
-# 6. vn_1=1340
-# 4. . tools/tools/analyze_txn_stress_test.sh
-echo Input params:
-# The rocksdb LOG path
-echo $LOG
-# Snapshot at which we got RandomTransactionVerify failure
-echo $vn
-# The snapshot before that where RandomTransactionVerify passed
-echo $vn_1
-# The stress tests use 3 sets, one or more might have shown inconsistent results.
-SET=${SET-1} # 1 or 2 or 3
-echo Checking set number $SET
-
-# Find the txns that committed between the two snapshots, and gather their changes made by them in /tmp/changes.txt
-# 2019/02/28-15:25:51.655477 7fffec9ff700 [DEBUG] [ilities/transactions/write_prepared_txn_db.cc:416] Txn 68497 Committing with 68498
-grep Committing $LOG | awk '{if ($9 <= vn && $9 > vn_1) print $0}' vn=$vn vn_1=${vn_1} > /tmp/txn.txt
-# 2019/02/28-15:25:49.046464 7fffe81f5700 [DEBUG] [il/transaction_test_util.cc:216] Commit of 65541 OK (txn12936193128775589751-9089)
-for i in `cat /tmp/txn.txt | awk '{print $6}'`; do grep "Commit of $i " $LOG; done > /tmp/names.txt
-for n in `cat /tmp/names.txt | awk '{print $9}'`; do grep $n $LOG; done > /tmp/changes.txt
-echo "Sum of the changes:"
-cat /tmp/changes.txt | grep Insert | awk '{print $12}' | cut -d= -f1 | cut -d+ -f2 | awk '{sum+=$1} END{print sum}'
-
-# Gather read values at each snapshot
-# 2019/02/28-15:25:51.655926 7fffebbff700 [DEBUG] [il/transaction_test_util.cc:347] VerifyRead at 67972 (67693): 000230 value: 15983
-grep "VerifyRead at ${vn_1} (.*): 000${SET}" $LOG | cut -d' ' -f 9- > /tmp/va.txt
-grep "VerifyRead at ${vn} (.*): 000${SET}" $LOG | cut -d' ' -f 9- > /tmp/vb.txt
-
-# For each key in the 2nd snapshot, find the value read by 1st, do the adds, and see if the results match.
-IFS=$'\n'
-for l in `cat /tmp/vb.txt`; 
-do
-  grep $l /tmp/va.txt > /dev/null ; 
-  if [[ $? -ne 0 ]]; then 
-    #echo $l
-    k=`echo $l | awk '{print $1}'`;
-    v=`echo $l | awk '{print $3}'`;
-    # 2019/02/28-15:25:19.350111 7fffe81f5700 [DEBUG] [il/transaction_test_util.cc:194] Insert (txn12936193128775589751-2298) OK snap: 16289 key:000219 value: 3772+95=3867
-    exp=`grep "\<$k\>" /tmp/changes.txt | tail -1 | cut -d= -f2`;
-    if [[ $v -ne $exp ]]; then echo $l; fi
-  else
-    k=`echo $l | awk '{print $1}'`;
-    grep "\<$k\>" /tmp/changes.txt
-  fi;
-done
-
-# Check that all the keys read in the 1st snapshot are still visible in the 2nd
-for l in `cat /tmp/va.txt`; 
-do
-  k=`echo $l | awk '{print $1}'`;
-  grep "\<$k\>" /tmp/vb.txt > /dev/null
-  if [[ $? -ne 0 ]]; then
-    echo missing key $k
-  fi
-done
-
-# The following found a bug in ValidateSnapshot. It checks if the adds on each key match up.
-grep Insert /tmp/changes.txt | cut -d' ' -f 10 | sort | uniq > /tmp/keys.txt
-for k in `cat /tmp/keys.txt`;
-do
-  grep "\<$k\>" /tmp/changes.txt > /tmp/adds.txt;
-  # 2019/02/28-15:25:19.350111 7fffe81f5700 [DEBUG] [il/transaction_test_util.cc:194] Insert (txn12936193128775589751-2298) OK snap: 16289 key:000219 value: 3772+95=3867
-  START=`head -1 /tmp/adds.txt | cut -d' ' -f 12 | cut -d+ -f1`
-  END=`tail -1 /tmp/adds.txt | cut -d' ' -f 12 | cut -d= -f2`
-  ADDS=`cat /tmp/adds.txt | grep Insert | awk '{print $12}' | cut -d= -f1 | cut -d+ -f2 | awk '{sum+=$1} END{print sum}'`
-  EXP=$((START+ADDS))
-  # If first + all the adds != last then there was an issue with ValidateSnapshot.
-  if [[ $END -ne $EXP ]]; then echo inconsistent txn: $k $START+$ADDS=$END; cat /tmp/adds.txt; return 1; fi
-done

package/vendor/rocksdb/tools/auto_sanity_test.sh

@@ -1,93 +0,0 @@
-# shellcheck disable=SC2148
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test"
-
-if [ "$#" -lt 2 ]; then
-  echo "usage: ./auto_sanity_test.sh [new_commit] [old_commit]"
-  echo "Missing either [new_commit] or [old_commit], perform sanity check with the latest and 10th latest commits."
-  recent_commits=`git log | grep -e "^commit [a-z0-9]\+$"| head -n10 | sed -e 's/commit //g'`
-  commit_new=`echo "$recent_commits" | head -n1`
-  commit_old=`echo "$recent_commits" | tail -n1`
-  echo "the most recent commits are:"
-  echo "$recent_commits"
-else
-  commit_new=$1
-  commit_old=$2
-fi
-
-if [ ! -d $TMP_DIR ]; then
-  mkdir $TMP_DIR
-fi
-dir_new="${TMP_DIR}/${commit_new}"
-dir_old="${TMP_DIR}/${commit_old}"
-
-function makestuff() {
-  echo "make clean"
-  make clean > /dev/null
-  echo "make db_sanity_test -j32"
-  make db_sanity_test -j32 > /dev/null
-  if [ $? -ne 0 ]; then
-    echo "[ERROR] Failed to perform 'make db_sanity_test'"
-    exit 1
-  fi
-}
-
-rm -r -f $dir_new
-rm -r -f $dir_old
-
-echo "Running db sanity check with commits $commit_new and $commit_old."
-
-echo "============================================================="
-echo "Making build $commit_new"
-git checkout $commit_new
-if [ $? -ne 0 ]; then
-  echo "[ERROR] Can't checkout $commit_new"
-  exit 1
-fi
-makestuff
-mv db_sanity_test new_db_sanity_test
-echo "Creating db based on the new commit --- $commit_new"
-./new_db_sanity_test $dir_new create
-cp ./tools/db_sanity_test.cc $dir_new
-cp ./tools/auto_sanity_test.sh $dir_new
-
-echo "============================================================="
-echo "Making build $commit_old"
-git checkout $commit_old
-if [ $? -ne 0 ]; then
-  echo "[ERROR] Can't checkout $commit_old"
-  exit 1
-fi
-cp -f $dir_new/db_sanity_test.cc ./tools/.
-cp -f $dir_new/auto_sanity_test.sh ./tools/.
-makestuff
-mv db_sanity_test old_db_sanity_test
-echo "Creating db based on the old commit --- $commit_old"
-./old_db_sanity_test $dir_old create
-
-echo "============================================================="
-echo "[Backward Compatibility Check]"
-echo "Verifying old db $dir_old using the new commit --- $commit_new"
-./new_db_sanity_test $dir_old verify
-if [ $? -ne 0 ]; then
-  echo "[ERROR] Backward Compatibility Check fails:"
-  echo "    Verification of $dir_old using commit $commit_new failed."
-  exit 2
-fi
-
-echo "============================================================="
-echo "[Forward Compatibility Check]"
-echo "Verifying new db $dir_new using the old commit --- $commit_old"
-./old_db_sanity_test $dir_new verify
-if [ $? -ne 0 ]; then
-  echo "[ERROR] Forward Compatibility Check fails:"
-  echo "    $dir_new using commit $commit_old failed."
-  exit 2
-fi
-
-rm old_db_sanity_test
-rm new_db_sanity_test
-rm -rf $dir_new
-rm -rf $dir_old
-
-echo "Auto sanity test passed!"

package/vendor/rocksdb/tools/backup_db.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-#
-
-if [ "$#" -lt 2 ]; then
-  echo "usage: ${BASH_SOURCE[0]} <DB Path> <Backup Dir>"
-  exit 1
-fi
-
-db_dir="$1"
-backup_dir="$2"
-
-echo "== Backing up DB $db_dir to $backup_dir"
-./ldb backup --db="$db_dir" --backup_dir="$backup_dir"