redis-benchmarks-specification 0.1.224__tar.gz → 0.1.225__tar.gz

{redis_benchmarks_specification-0.1.224 → redis_benchmarks_specification-0.1.225}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.224
+Version: 0.1.225
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.224 → redis_benchmarks_specification-0.1.225}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.224"
+version = "0.1.225"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.224 → redis_benchmarks_specification-0.1.225}/redis_benchmarks_specification/__compare__/args.py

@@ -123,6 +123,12 @@ def create_compare_arguments(parser):
     parser.add_argument("--simple-table", type=bool, default=False)
     parser.add_argument("--use_metric_context_path", type=bool, default=False)
     parser.add_argument("--testname_regex", type=str, default=".*", required=False)
+    parser.add_argument(
+        "--regression_str", type=str, default="REGRESSION", required=False
+    )
+    parser.add_argument(
+        "--improvement_str", type=str, default="IMPROVEMENT", required=False
+    )
     parser.add_argument(
         "--regressions-percent-lower-limit",
         type=float,

{redis_benchmarks_specification-0.1.224 → redis_benchmarks_specification-0.1.225}/redis_benchmarks_specification/__compare__/compare.py

@@ -285,7 +285,7 @@ def compare_command_logic(args, project_name, project_version):
         detected_regressions,
         table_output,
         total_improvements,
-        total_regressions,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -328,7 +328,10 @@ def compare_command_logic(args, project_name, project_version):
         comparison_target_branch,
         baseline_github_org,
         comparison_github_org,
+        args.regression_str,
+        args.improvement_str,
     )
+    total_regressions = len(regressions_list)
     prepare_regression_comment(
         auto_approve,
         baseline_branch,
@@ -355,6 +358,7 @@ def compare_command_logic(args, project_name, project_version):
         total_unstable,
         verbose,
         args.regressions_percent_lower_limit,
+        regressions_list,
     )
     return (
         detected_regressions,
@@ -393,6 +397,7 @@ def prepare_regression_comment(
     total_unstable,
     verbose,
     regressions_percent_lower_limit,
+    regressions_list=[],
 ):
     if total_comparison_points > 0:
         comment_body = "### Automated performance analysis summary\n\n"
@@ -423,6 +428,14 @@ def prepare_regression_comment(
             comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format(
                 total_regressions, regressions_percent_lower_limit
             )
+            if len(regressions_list) > 0:
+                regression_values = [l[1] for l in regressions_list]
+                regression_df = pd.DataFrame(regression_values)
+                median_regression = round(float(regression_df.median().iloc[0]), 2)
+                max_regression = round(float(regression_df.max().iloc[0]), 2)
+                min_regression = round(float(regression_df.min().iloc[0]), 2)
+
+                comparison_summary += f"   - Median/Common-Case regression was {median_regression}%% and ranged from [{min_regression},{max_regression}] %%.\n"
 
         comment_body += comparison_summary
         comment_body += "\n"
@@ -561,6 +574,8 @@ def compute_regression_table(
     comparison_target_branch=None,
     baseline_github_org="redis",
     comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -665,6 +680,8 @@ def compute_regression_table(
         comparison_github_repo,
         baseline_github_org,
         comparison_github_org,
+        regression_str,
+        improvement_str,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -697,7 +714,7 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
-        test_names_str = "|".join(regressions_list)
+        test_names_str = "|".join([l[0] for l in regressions_list])
         table_output += f"Regressions test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
@@ -750,7 +767,7 @@ def compute_regression_table(
         detected_regressions,
         table_output,
         total_improvements,
-        total_regressions,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -919,6 +936,8 @@ def from_rts_to_regression_table(
     comparison_github_repo="redis",
     baseline_github_org="redis",
     comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -1103,27 +1122,27 @@ def from_rts_to_regression_table(
         if baseline_v != "N/A" or comparison_v != "N/A":
             detected_regression = False
             detected_improvement = False
-            if percentage_change < 0.0 and not unstable:
+            if percentage_change < 0.0:
                 if -waterline >= percentage_change:
                     detected_regression = True
                     total_regressions = total_regressions + 1
-                    note = note + " REGRESSION"
+                    note = note + f" {regression_str}"
                     detected_regressions.append(test_name)
                 elif percentage_change < -noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential REGRESSION"
+                        note = note + f" potential {regression_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
 
-            if percentage_change > 0.0 and not unstable:
+            if percentage_change > 0.0:
                 if percentage_change > waterline:
                     detected_improvement = True
                     total_improvements = total_improvements + 1
-                    note = note + " IMPROVEMENT"
+                    note = note + f" {improvement_str}"
                 elif percentage_change > noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential IMPROVEMENT"
+                        note = note + f" potential {improvement_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
@@ -1147,7 +1166,7 @@ def from_rts_to_regression_table(
                 test_link,
             )
             if detected_regression:
-                regressions_list.append(test_name)
+                regressions_list.append([test_name, percentage_change])
                 table_regressions.append(line)
 
             if detected_improvement:

{redis_benchmarks_specification-0.1.224 → redis_benchmarks_specification-0.1.225}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -1375,7 +1375,7 @@ def process_self_contained_coordinator_stream(
                             detected_regressions,
                             table_output,
                             total_improvements,
-                            total_regressions,
+                            regressions_list,
                             total_stable,
                             total_unstable,
                             total_comparison_points,
@@ -1409,6 +1409,7 @@ def process_self_contained_coordinator_stream(
                             use_metric_context_path,
                             running_platform,
                         )
+                        total_regressions = len(regressions_list)
                         auto_approve = True
                         grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks"
 
@@ -1438,6 +1439,7 @@ def process_self_contained_coordinator_stream(
                             total_unstable,
                             verbose,
                             regressions_percent_lower_limit,
+                            regressions_list,
                         )
                     logging.info(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"

redis_benchmarks_specification-0.1.225/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- sadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --pipeline 10 --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- set
+priority: 12

redis_benchmarks_specification-0.1.225/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- sadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- set
+priority: 12