redis-benchmarks-specification 0.1.255__tar.gz → 0.1.257__tar.gz

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.255
+Version: 0.1.257
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.255"
+version = "0.1.257"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/__cli__/stats.py

@@ -500,3 +500,8 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
             conn.sadd(tested_groups_key, group)
         for command in list(tracked_commands_json.keys()):
             conn.sadd(tested_commands_key, command)
+
+    logging.info(f"There is a total of : {len(tracked_groups)} tracked command groups.")
+    logging.info(
+        f"There is a total of : {len(list(tracked_commands_json.keys()))} tracked commands."
+    )

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/__compare__/args.py

@@ -8,6 +8,10 @@
 import datetime
 import os
 
+from redis_benchmarks_specification.__common__.env import (
+    SPECS_PATH_TEST_SUITES,
+)
+
 
 def get_start_time_vars(start_time=None):
     if start_time is None:
@@ -30,6 +34,12 @@ START_TIME_LAST_SIX_MONTHS_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=18
 
 
 def create_compare_arguments(parser):
+    parser.add_argument(
+        "--test-suites-folder",
+        type=str,
+        default=SPECS_PATH_TEST_SUITES,
+        help="Test suites folder, containing the different test variations",
+    )
     parser.add_argument(
         "--test",
         type=str,

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/__compare__/compare.py

@@ -27,6 +27,7 @@ from redis_benchmarks_specification.__common__.github import (
 )
 from redis_benchmarks_specification.__compare__.args import create_compare_arguments
 
+from redis_benchmarks_specification.__common__.runner import get_benchmark_specs
 
 from redis_benchmarks_specification.__common__.package import (
     get_version_string,
@@ -268,6 +269,40 @@ def compare_command_logic(args, project_name, project_version):
             )
         )
 
+    testsuites_folder = os.path.abspath(args.test_suites_folder)
+    logging.info("Using test-suites folder dir {}".format(testsuites_folder))
+    testsuite_spec_files = get_benchmark_specs(testsuites_folder)
+    logging.info(
+        "There are a total of {} test-suites being run in folder {}".format(
+            len(testsuite_spec_files), testsuites_folder
+        )
+    )
+    tests_with_config = {}
+    for test_file in testsuite_spec_files:
+        if args.defaults_filename in test_file:
+            continue
+        benchmark_config = {}
+        with open(test_file, "r") as stream:
+            try:
+                benchmark_config = yaml.safe_load(stream)
+                test_name = benchmark_config["name"]
+                tests_with_config[test_name] = benchmark_config
+                if "tested-groups" in benchmark_config:
+                    origin_tested_groups = benchmark_config["tested-groups"]
+                else:
+                    logging.warn("dont have test groups in {}".format(test_name))
+                if "tested-commands" in benchmark_config:
+                    origin_tested_commands = benchmark_config["tested-commands"]
+                else:
+                    logging.warn("dont have test commands in {}".format(test_name))
+            except Exception as e:
+                logging.error(
+                    "while loading file {} and error was returned: {}".format(
+                        test_file, e.__str__()
+                    )
+                )
+                pass
+
     fn = check_regression_comment
     (
         contains_regression_comment,
@@ -330,6 +365,7 @@ def compare_command_logic(args, project_name, project_version):
         comparison_github_org,
         args.regression_str,
         args.improvement_str,
+        tests_with_config,
     )
     total_regressions = len(regressions_list)
     total_improvements = len(improvements_list)
@@ -588,6 +624,7 @@ def compute_regression_table(
     comparison_github_org="redis",
     regression_str="REGRESSION",
     improvement_str="IMPROVEMENT",
+    tests_with_config={},
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -668,6 +705,8 @@ def compute_regression_table(
         baseline_only_list,
         comparison_only_list,
         no_datapoints_list,
+        group_change,
+        command_change,
     ) = from_rts_to_regression_table(
         baseline_deployment_name,
         comparison_deployment_name,
@@ -698,6 +737,7 @@ def compute_regression_table(
         comparison_github_org,
         regression_str,
         improvement_str,
+        tests_with_config,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -712,6 +752,22 @@ def compute_regression_table(
         baseline_deployment_name,
     )
 
+    table_output += "<details>\n  <summary>By GROUP change csv:</summary>\n\n"
+    table_output += "\ncommand_group,min_change,max_change  \n"
+    for group_name, changes_list in group_change.items():
+        max_change = max(changes_list)
+        min_change = min(changes_list)
+        table_output += f"{group_name},{min_change:.3f},{max_change:.3f}\n"
+    table_output += "\n</details>\n"
+    table_output += "\n\n"
+    table_output += "<details>\n  <summary>By COMMAND change csv:</summary>\n\n"
+    table_output += "\ncommand,min_change,max_change  \n"
+    for command_name, changes_list in command_change.items():
+        max_change = max(changes_list)
+        min_change = min(changes_list)
+        table_output += f"{command_name},{min_change:.3f},{max_change:.3f}\n"
+    table_output += "\n</details>\n"
+
     if total_unstable > 0:
         old_stdout = sys.stdout
         sys.stdout = mystdout = StringIO()
@@ -819,7 +875,7 @@ def compute_regression_table(
     if len_no_datapoints > 0:
         table_output += f"\n  WARNING: There were {len_no_datapoints} benchmarks with NO datapoints for both baseline and comparison.\n\n"
         no_datapoints_test_names_str = "|".join([l for l in no_datapoints_list])
-        
+
         table_output += (
             f"  NO DATAPOINTS test regexp names: {no_datapoints_test_names_str}\n\n"
         )
@@ -999,6 +1055,7 @@ def from_rts_to_regression_table(
     comparison_github_org="redis",
     regression_str="REGRESSION",
     improvement_str="IMPROVEMENT",
+    tests_with_config={},
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -1022,8 +1079,20 @@ def from_rts_to_regression_table(
     no_datapoints_list = []
     no_datapoints_baseline_list = []
     no_datapoints_comparison_list = []
+    group_change = {}
+    command_change = {}
     original_metric_mode = metric_mode
     for test_name in test_names:
+        tested_groups = []
+        tested_commands = []
+        if test_name in tests_with_config:
+            test_spec = tests_with_config[test_name]
+            if "tested-groups" in test_spec:
+                tested_groups = test_spec["tested-groups"]
+            if "tested-commands" in test_spec:
+                tested_commands = test_spec["tested-commands"]
+        else:
+            logging.error(f"Test does not contain spec info: {test_name}")
         metric_mode = original_metric_mode
         compare_version = "main"
         # GE
@@ -1257,6 +1326,16 @@ def from_rts_to_regression_table(
                     if simplify_table is False:
                         note = note + " No Change"
 
+            for test_group in tested_groups:
+                if test_group not in group_change:
+                    group_change[test_group] = []
+                group_change[test_group].append(percentage_change)
+
+            for test_command in tested_commands:
+                if test_command not in command_change:
+                    command_change[test_command] = []
+                command_change[test_command].append(percentage_change)
+
             if (
                 detected_improvement is False
                 and detected_regression is False
@@ -1345,6 +1424,8 @@ def from_rts_to_regression_table(
         baseline_only_list,
         comparison_only_list,
         no_datapoints_list,
+        group_change,
+        command_change,
     )
 
 

redis_benchmarks_specification-0.1.257/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys int encoded, checking STRLEN command performance.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 999998
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: ' --command "SET __key__ __key__" --key-prefix "" --command-key-pattern="S"  --key-minimum=2 --key-maximum 1000000 -c 1 -t 1 --pipeline 100 --hide-histogram -n allkeys'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- strlen
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command "STRLEN __key__" --key-prefix "" --command-key-pattern="G"  --key-minimum=2 --key-maximum 1000000 -c 50 -t 4 --pipeline 10 --hide-histogram --test-time 60
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 98

redis_benchmarks_specification-0.1.255/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values copy.yml → redis_benchmarks_specification-0.1.257/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values-pipeline-10.yml

@@ -1,5 +1,5 @@
 version: 0.4
-name: memtier_benchmark-3Mkeys-load-string-with-512B-values
+name: memtier_benchmark-3Mkeys-load-string-with-512B-values-pipeline-10
 description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands.
 dbconfig:
   configuration-parameters:
@@ -17,15 +17,13 @@ redis-topologies:
 - oss-standalone-04-io-threads
 - oss-standalone-08-io-threads
 - oss-standalone-16-io-threads
-- oss-standalone-32-io-threads
-- oss-standalone-64-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--data-size" "512" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram'
+  arguments: '"--data-size" "512" --pipeline 10 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram'
   resources:
     requests:
       cpus: '13'

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml

@@ -17,8 +17,6 @@ redis-topologies:
 - oss-standalone-04-io-threads
 - oss-standalone-08-io-threads
 - oss-standalone-16-io-threads
-- oss-standalone-32-io-threads
-- oss-standalone-64-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-2000_conns.yml

@@ -17,6 +17,10 @@ tested-commands:
 - set
 redis-topologies:
 - oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-400_conns.yml

@@ -17,6 +17,10 @@ tested-commands:
 - set
 redis-topologies:
 - oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-40_conns.yml

@@ -17,6 +17,10 @@ tested-commands:
 - set
 redis-topologies:
 - oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-2000_conns.yml

@@ -17,6 +17,10 @@ tested-commands:
 - set
 redis-topologies:
 - oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-400_conns.yml

@@ -17,6 +17,10 @@ tested-commands:
 - set
 redis-topologies:
 - oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub

{redis_benchmarks_specification-0.1.255 → redis_benchmarks_specification-0.1.257}/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-5200_conns.yml

@@ -17,6 +17,10 @@ tested-commands:
 - set
 redis-topologies:
 - oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
 build-variants:
 - gcc:8.5.0-amd64-debian-buster-default
 - dockerhub