redis-benchmarks-specification 0.1.256__py3-none-any.whl → 0.1.257__py3-none-any.whl

redis_benchmarks_specification/__cli__/stats.py

@@ -500,3 +500,8 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
             conn.sadd(tested_groups_key, group)
         for command in list(tracked_commands_json.keys()):
             conn.sadd(tested_commands_key, command)
+
+    logging.info(f"There is a total of : {len(tracked_groups)} tracked command groups.")
+    logging.info(
+        f"There is a total of : {len(list(tracked_commands_json.keys()))} tracked commands."
+    )

redis_benchmarks_specification/__compare__/args.py

@@ -8,6 +8,10 @@
 import datetime
 import os
 
+from redis_benchmarks_specification.__common__.env import (
+    SPECS_PATH_TEST_SUITES,
+)
+
 
 def get_start_time_vars(start_time=None):
     if start_time is None:
@@ -30,6 +34,12 @@ START_TIME_LAST_SIX_MONTHS_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=18
 
 
 def create_compare_arguments(parser):
+    parser.add_argument(
+        "--test-suites-folder",
+        type=str,
+        default=SPECS_PATH_TEST_SUITES,
+        help="Test suites folder, containing the different test variations",
+    )
     parser.add_argument(
         "--test",
         type=str,

redis_benchmarks_specification/__compare__/compare.py

@@ -27,6 +27,7 @@ from redis_benchmarks_specification.__common__.github import (
 )
 from redis_benchmarks_specification.__compare__.args import create_compare_arguments
 
+from redis_benchmarks_specification.__common__.runner import get_benchmark_specs
 
 from redis_benchmarks_specification.__common__.package import (
     get_version_string,
@@ -268,6 +269,40 @@ def compare_command_logic(args, project_name, project_version):
             )
         )
 
+    testsuites_folder = os.path.abspath(args.test_suites_folder)
+    logging.info("Using test-suites folder dir {}".format(testsuites_folder))
+    testsuite_spec_files = get_benchmark_specs(testsuites_folder)
+    logging.info(
+        "There are a total of {} test-suites being run in folder {}".format(
+            len(testsuite_spec_files), testsuites_folder
+        )
+    )
+    tests_with_config = {}
+    for test_file in testsuite_spec_files:
+        if args.defaults_filename in test_file:
+            continue
+        benchmark_config = {}
+        with open(test_file, "r") as stream:
+            try:
+                benchmark_config = yaml.safe_load(stream)
+                test_name = benchmark_config["name"]
+                tests_with_config[test_name] = benchmark_config
+                if "tested-groups" in benchmark_config:
+                    origin_tested_groups = benchmark_config["tested-groups"]
+                else:
+                    logging.warn("dont have test groups in {}".format(test_name))
+                if "tested-commands" in benchmark_config:
+                    origin_tested_commands = benchmark_config["tested-commands"]
+                else:
+                    logging.warn("dont have test commands in {}".format(test_name))
+            except Exception as e:
+                logging.error(
+                    "while loading file {} and error was returned: {}".format(
+                        test_file, e.__str__()
+                    )
+                )
+                pass
+
     fn = check_regression_comment
     (
         contains_regression_comment,
@@ -330,6 +365,7 @@ def compare_command_logic(args, project_name, project_version):
         comparison_github_org,
         args.regression_str,
         args.improvement_str,
+        tests_with_config,
     )
     total_regressions = len(regressions_list)
     total_improvements = len(improvements_list)
@@ -588,6 +624,7 @@ def compute_regression_table(
     comparison_github_org="redis",
     regression_str="REGRESSION",
     improvement_str="IMPROVEMENT",
+    tests_with_config={},
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -668,6 +705,8 @@ def compute_regression_table(
         baseline_only_list,
         comparison_only_list,
         no_datapoints_list,
+        group_change,
+        command_change,
     ) = from_rts_to_regression_table(
         baseline_deployment_name,
         comparison_deployment_name,
@@ -698,6 +737,7 @@ def compute_regression_table(
         comparison_github_org,
         regression_str,
         improvement_str,
+        tests_with_config,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -712,6 +752,22 @@ def compute_regression_table(
         baseline_deployment_name,
     )
 
+    table_output += "<details>\n  <summary>By GROUP change csv:</summary>\n\n"
+    table_output += "\ncommand_group,min_change,max_change  \n"
+    for group_name, changes_list in group_change.items():
+        max_change = max(changes_list)
+        min_change = min(changes_list)
+        table_output += f"{group_name},{min_change:.3f},{max_change:.3f}\n"
+    table_output += "\n</details>\n"
+    table_output += "\n\n"
+    table_output += "<details>\n  <summary>By COMMAND change csv:</summary>\n\n"
+    table_output += "\ncommand,min_change,max_change  \n"
+    for command_name, changes_list in command_change.items():
+        max_change = max(changes_list)
+        min_change = min(changes_list)
+        table_output += f"{command_name},{min_change:.3f},{max_change:.3f}\n"
+    table_output += "\n</details>\n"
+
     if total_unstable > 0:
         old_stdout = sys.stdout
         sys.stdout = mystdout = StringIO()
@@ -819,7 +875,7 @@ def compute_regression_table(
     if len_no_datapoints > 0:
         table_output += f"\n  WARNING: There were {len_no_datapoints} benchmarks with NO datapoints for both baseline and comparison.\n\n"
         no_datapoints_test_names_str = "|".join([l for l in no_datapoints_list])
-        
+
         table_output += (
             f"  NO DATAPOINTS test regexp names: {no_datapoints_test_names_str}\n\n"
         )
@@ -999,6 +1055,7 @@ def from_rts_to_regression_table(
     comparison_github_org="redis",
     regression_str="REGRESSION",
     improvement_str="IMPROVEMENT",
+    tests_with_config={},
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -1022,8 +1079,20 @@ def from_rts_to_regression_table(
     no_datapoints_list = []
     no_datapoints_baseline_list = []
     no_datapoints_comparison_list = []
+    group_change = {}
+    command_change = {}
     original_metric_mode = metric_mode
     for test_name in test_names:
+        tested_groups = []
+        tested_commands = []
+        if test_name in tests_with_config:
+            test_spec = tests_with_config[test_name]
+            if "tested-groups" in test_spec:
+                tested_groups = test_spec["tested-groups"]
+            if "tested-commands" in test_spec:
+                tested_commands = test_spec["tested-commands"]
+        else:
+            logging.error(f"Test does not contain spec info: {test_name}")
         metric_mode = original_metric_mode
         compare_version = "main"
         # GE
@@ -1257,6 +1326,16 @@ def from_rts_to_regression_table(
                     if simplify_table is False:
                         note = note + " No Change"
 
+            for test_group in tested_groups:
+                if test_group not in group_change:
+                    group_change[test_group] = []
+                group_change[test_group].append(percentage_change)
+
+            for test_command in tested_commands:
+                if test_command not in command_change:
+                    command_change[test_command] = []
+                command_change[test_command].append(percentage_change)
+
             if (
                 detected_improvement is False
                 and detected_regression is False
@@ -1345,6 +1424,8 @@ def from_rts_to_regression_table(
         baseline_only_list,
         comparison_only_list,
         no_datapoints_list,
+        group_change,
+        command_change,
     )
 
 

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys int encoded, checking STRLEN command performance.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 999998
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: ' --command "SET __key__ __key__" --key-prefix "" --command-key-pattern="S"  --key-minimum=2 --key-maximum 1000000 -c 1 -t 1 --pipeline 100 --hide-histogram -n allkeys'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- strlen
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command "STRLEN __key__" --key-prefix "" --command-key-pattern="G"  --key-minimum=2 --key-maximum 1000000 -c 50 -t 4 --pipeline 10 --hide-histogram --test-time 60
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 98

{redis_benchmarks_specification-0.1.256.dist-info → redis_benchmarks_specification-0.1.257.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.256
+Version: 0.1.257
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.256.dist-info → redis_benchmarks_specification-0.1.257.dist-info}/RECORD

@@ -9,7 +9,7 @@ redis_benchmarks_specification/__builder__/schema.py,sha256=1wcmyVJBcWrBvK58pghN
 redis_benchmarks_specification/__cli__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__cli__/args.py,sha256=uZkk1Jom9i0xJ_OpVMrIWbw_70jFo7IswLV2EtKTKEA,7210
 redis_benchmarks_specification/__cli__/cli.py,sha256=6tt0Ai-JIFEF3ykWFU2_g5ZrzKVIoyLLXUmyzYpVDF4,21843
-redis_benchmarks_specification/__cli__/stats.py,sha256=wahzZRbpfokv8dQU8O4BH5JFrOZk-l6k8LWdKfue9_0,20204
+redis_benchmarks_specification/__cli__/stats.py,sha256=QbxiJj_F5Hu7ktstBcS45xekPE9CmEl-niYiZcX8dZA,20411
 redis_benchmarks_specification/__common__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb5qj9jzsBhLVNO7K_W2Clumj4pxrkG8,5938
 redis_benchmarks_specification/__common__/env.py,sha256=kvJ8Ll-fvI_Tc0vynrzUEr22TqnJizzvJ4Lu9RjNr_M,3119
@@ -19,8 +19,8 @@ redis_benchmarks_specification/__common__/runner.py,sha256=7DBI09eu_4RibK6MwcYyI
 redis_benchmarks_specification/__common__/spec.py,sha256=eTF5559epBB0FrJPx-jRDQVeP_ZVOgyC7Vjxr2xk6fo,3262
 redis_benchmarks_specification/__common__/timeseries.py,sha256=_LJFtC5sVP7DTaLZaIzv5g7wRxPTQZRwFIYvWX4p4N8,50533
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
-redis_benchmarks_specification/__compare__/args.py,sha256=-vQco6WG73DJtUFA7s-5bNk05vwHjOc94puVKZMZUYE,6513
-redis_benchmarks_specification/__compare__/compare.py,sha256=9O5tiQR5rGpuHawPmVJbpb6_mjN_LpNMHr9jtqzIRxA,53480
+redis_benchmarks_specification/__compare__/args.py,sha256=FlKD1wutBoKxeahpXw1gY2H_1FOPH5y-o5QsIPfFsT0,6802
+redis_benchmarks_specification/__compare__/compare.py,sha256=brLymkKXa1ZzV--27LOFuzvFUQwGWKA-lKN7Bnbvlzg,57029
 redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gqDKgzvNJnkGNKY,491
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=lYvbPd_3ppHZv4f2sRwXcF-fcBrwRSn3H2RMmNVkojY,7221
@@ -254,6 +254,7 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrb
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrby.yml,sha256=jU9FuUVTYNruLmBwS1GDA6oUzi8iBi5lgVOLJABTsew,707
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat-pipeline-10.yml,sha256=ocKvmopOhCmhyJm9RiTbVVS3gBA1ug63tMrDa0FApys,754
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-incrbyfloat.yml,sha256=4UnmF-vx8At2ah1YjFUMHgvXFpfi4EVVXsAWjN42M78,728
+redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-int-encoding-strlen-pipeline-10.yml,sha256=wKXG73RiNnIGKv_YCSvJ0jnKar7YjkRpvhnIOdK62cI,1100
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-mget-1KiB.yml,sha256=s1PLNH__vexG6bE-GL_xPAqVelupxY73AWWE5siYIYE,999
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-set-with-ex-100B-pipeline-10.yml,sha256=bmFN61JPtQ6TWCGzUMmnhNcjgA5lMurAMlBSL9jBvX0,1035
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10.yml,sha256=W_h2zXnSXLBfgXwyY2m0RowiWJUffHNRHMbmpnp7kAo,1030
@@ -345,8 +346,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed
 redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello-pipeline-10.yml,sha256=y8c0KsJ-SYmEfvW8m5rQg0hd9boh-FWkzAZTma4OYaI,734
 redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml,sha256=VIFNaaAqRQMeDvtXmKGDpJTnu658Lv5i_oAju5uSi_c,708
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
-redis_benchmarks_specification-0.1.256.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.256.dist-info/METADATA,sha256=0aGL9-HYDVTsuoE6fKAqfoVvAF4dsHAFsTrKb7rZ2BA,22726
-redis_benchmarks_specification-0.1.256.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.256.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.256.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.257.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.257.dist-info/METADATA,sha256=nqTiM_A0pnwYqWIIIPI3fQNvL6mx9cvKHQ4wXod8kYg,22726
+redis_benchmarks_specification-0.1.257.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.257.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.257.dist-info/RECORD,,