redis-benchmarks-specification 0.1.224__py3-none-any.whl → 0.1.225__py3-none-any.whl

redis_benchmarks_specification/__compare__/args.py

@@ -123,6 +123,12 @@ def create_compare_arguments(parser):
     parser.add_argument("--simple-table", type=bool, default=False)
     parser.add_argument("--use_metric_context_path", type=bool, default=False)
     parser.add_argument("--testname_regex", type=str, default=".*", required=False)
+    parser.add_argument(
+        "--regression_str", type=str, default="REGRESSION", required=False
+    )
+    parser.add_argument(
+        "--improvement_str", type=str, default="IMPROVEMENT", required=False
+    )
     parser.add_argument(
         "--regressions-percent-lower-limit",
         type=float,

redis_benchmarks_specification/__compare__/compare.py

@@ -285,7 +285,7 @@ def compare_command_logic(args, project_name, project_version):
         detected_regressions,
         table_output,
         total_improvements,
-        total_regressions,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -328,7 +328,10 @@ def compare_command_logic(args, project_name, project_version):
         comparison_target_branch,
         baseline_github_org,
         comparison_github_org,
+        args.regression_str,
+        args.improvement_str,
     )
+    total_regressions = len(regressions_list)
     prepare_regression_comment(
         auto_approve,
         baseline_branch,
@@ -355,6 +358,7 @@ def compare_command_logic(args, project_name, project_version):
         total_unstable,
         verbose,
         args.regressions_percent_lower_limit,
+        regressions_list,
     )
     return (
         detected_regressions,
@@ -393,6 +397,7 @@ def prepare_regression_comment(
     total_unstable,
     verbose,
     regressions_percent_lower_limit,
+    regressions_list=[],
 ):
     if total_comparison_points > 0:
         comment_body = "### Automated performance analysis summary\n\n"
@@ -423,6 +428,14 @@ def prepare_regression_comment(
             comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format(
                 total_regressions, regressions_percent_lower_limit
             )
+            if len(regressions_list) > 0:
+                regression_values = [l[1] for l in regressions_list]
+                regression_df = pd.DataFrame(regression_values)
+                median_regression = round(float(regression_df.median().iloc[0]), 2)
+                max_regression = round(float(regression_df.max().iloc[0]), 2)
+                min_regression = round(float(regression_df.min().iloc[0]), 2)
+
+                comparison_summary += f"   - Median/Common-Case regression was {median_regression}%% and ranged from [{min_regression},{max_regression}] %%.\n"
 
         comment_body += comparison_summary
         comment_body += "\n"
@@ -561,6 +574,8 @@ def compute_regression_table(
     comparison_target_branch=None,
     baseline_github_org="redis",
     comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -665,6 +680,8 @@ def compute_regression_table(
         comparison_github_repo,
         baseline_github_org,
         comparison_github_org,
+        regression_str,
+        improvement_str,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -697,7 +714,7 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
-        test_names_str = "|".join(regressions_list)
+        test_names_str = "|".join([l[0] for l in regressions_list])
         table_output += f"Regressions test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
@@ -750,7 +767,7 @@ def compute_regression_table(
         detected_regressions,
         table_output,
         total_improvements,
-        total_regressions,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -919,6 +936,8 @@ def from_rts_to_regression_table(
     comparison_github_repo="redis",
     baseline_github_org="redis",
     comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -1103,27 +1122,27 @@ def from_rts_to_regression_table(
         if baseline_v != "N/A" or comparison_v != "N/A":
             detected_regression = False
             detected_improvement = False
-            if percentage_change < 0.0 and not unstable:
+            if percentage_change < 0.0:
                 if -waterline >= percentage_change:
                     detected_regression = True
                     total_regressions = total_regressions + 1
-                    note = note + " REGRESSION"
+                    note = note + f" {regression_str}"
                     detected_regressions.append(test_name)
                 elif percentage_change < -noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential REGRESSION"
+                        note = note + f" potential {regression_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
 
-            if percentage_change > 0.0 and not unstable:
+            if percentage_change > 0.0:
                 if percentage_change > waterline:
                     detected_improvement = True
                     total_improvements = total_improvements + 1
-                    note = note + " IMPROVEMENT"
+                    note = note + f" {improvement_str}"
                 elif percentage_change > noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential IMPROVEMENT"
+                        note = note + f" potential {improvement_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
@@ -1147,7 +1166,7 @@ def from_rts_to_regression_table(
                 test_link,
             )
             if detected_regression:
-                regressions_list.append(test_name)
+                regressions_list.append([test_name, percentage_change])
                 table_regressions.append(line)
 
             if detected_improvement:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -1375,7 +1375,7 @@ def process_self_contained_coordinator_stream(
                             detected_regressions,
                             table_output,
                             total_improvements,
-                            total_regressions,
+                            regressions_list,
                             total_stable,
                             total_unstable,
                             total_comparison_points,
@@ -1409,6 +1409,7 @@ def process_self_contained_coordinator_stream(
                             use_metric_context_path,
                             running_platform,
                         )
+                        total_regressions = len(regressions_list)
                         auto_approve = True
                         grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks"
 
@@ -1438,6 +1439,7 @@ def process_self_contained_coordinator_stream(
                             total_unstable,
                             verbose,
                             regressions_percent_lower_limit,
+                            regressions_list,
                         )
                     logging.info(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- sadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --pipeline 10 --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- set
+priority: 12

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-set-intset-with-100-elements
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading SETs with 100 integer elements.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- sadd
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 180 -c 50 -t 4 --command "SADD __key__ 436858 768277 10664 860016 865292 841848 313147 896678 386308 977048 203069 283373 593503 457419 169542 391186 130304 916639 453967 273773 589383 657683 182813 641487 580095 195884 372142 774005 768470 95729 556839 771113 432139 613472 528415 174665 155343 215707 753892 81624 199200 877732 358153 337683 24664 481128 356011 28176 535519 724263 780697 435661 3623 6998 985912 696150 913066 171765 808279 608289 187563 300496 104333 684239 434669 111037 928553 714514 606168 190478 772363 393219 243921 549032 624950 743846 957491 864902 345918 846332 42207 243593 125622 915607 523476 226892 773003 840853 792775 582050 702242 194856 270254 617201 454742 543414 960526 136178 43558 172112" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 4g
+
+tested-groups:
+- set
+priority: 12

{redis_benchmarks_specification-0.1.224.dist-info → redis_benchmarks_specification-0.1.225.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.224
+Version: 0.1.225
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.224.dist-info → redis_benchmarks_specification-0.1.225.dist-info}/RECORD

@@ -19,8 +19,8 @@ redis_benchmarks_specification/__common__/runner.py,sha256=7DBI09eu_4RibK6MwcYyI
 redis_benchmarks_specification/__common__/spec.py,sha256=eTF5559epBB0FrJPx-jRDQVeP_ZVOgyC7Vjxr2xk6fo,3262
 redis_benchmarks_specification/__common__/timeseries.py,sha256=_LJFtC5sVP7DTaLZaIzv5g7wRxPTQZRwFIYvWX4p4N8,50533
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
-redis_benchmarks_specification/__compare__/args.py,sha256=L7Ap7A7qRv8ebIehDSleeXhK2UgWHL2YXeVyIFpRW-k,6299
-redis_benchmarks_specification/__compare__/compare.py,sha256=UiW4nbox4n4xneG8fWIfTkbnpRi9zbpDOiMiEaGgCMU,45899
+redis_benchmarks_specification/__compare__/args.py,sha256=-vQco6WG73DJtUFA7s-5bNk05vwHjOc94puVKZMZUYE,6513
+redis_benchmarks_specification/__compare__/compare.py,sha256=XbHglfawnQhHUVu3KhL4npDzuu-blaTV5T3GFWJotkE,46845
 redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gqDKgzvNJnkGNKY,491
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=lYvbPd_3ppHZv4f2sRwXcF-fcBrwRSn3H2RMmNVkojY,7221
@@ -34,7 +34,7 @@ redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py,sha256=s
 redis_benchmarks_specification/__self_contained_coordinator__/docker.py,sha256=Alf9Y1dfuOMoD4u_Dv3jTodkwZfSrlo2_YceevaWNFo,2757
 redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py,sha256=qB1rwqkROfuyFotB7MfUQiYS4Gzafd8dd2ca7lT4l2I,2909
 redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=FqVVvbXPsmq2I7pSH-JEklb1SDRdS7rG6ZHc6xTCUE0,28611
-redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=xyUex4kGWFP7vWtdrPQF92Pcs2ujCsTTPmtvTUMWFPg,74733
+redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=qfK60P1pvYb1cDCiU6i_R0fYTL3D0TTFHwgl5pIGdVg,74844
 redis_benchmarks_specification/__setups__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__setups__/topologies.py,sha256=xQ1IJkcTji_ZjLiJd3vOxZpvbNtBLZw9cPkw5hGJKHU,481
 redis_benchmarks_specification/__spec__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
@@ -83,6 +83,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-hash-hm
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-with-100B-values.yml,sha256=Y0FiaI0e35wrz9Ddet0y0e0UKjfANJxjnx2SkwsVBYw,817
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-with-10B-values.yml,sha256=uK2uepy114KHiu3M4-ezrrTPawUjnWg1PhFyiJIFTLQ,814
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-with-1KiB-values.yml,sha256=ZN8tfGgk-3eXllIUgSAZkxZurFgZTXOkTyD6e0-0-to,820
+redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements-pipeline-10.yml,sha256=Ne308Kylwr_y8qLPBB0MbDBhRWAWwcfBNyIMoaYBrTo,1482
+redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-set-intset-with-100-elements.yml,sha256=NO2WjwdF5Nnur7yoODMQWFX_liGO9eRiG_iJ_PTPofw,1456
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values-pipeline-10.yml,sha256=1hAvDhK_AbJUTggts9yqnvjwuonV1h49_pVGD0whMgQ,879
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values.yml,sha256=HD-42L4ubNCBx5qyKsC3tMU5f693Z-GjyHO7igyg7Oc,849
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values-pipeline-10.yml,sha256=XEv6OUKCkK-aCKRzYa5DVNwl6ir77J-eoEnuqaxSv8w,949
@@ -168,8 +170,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-stream-5-entr
 redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml,sha256=XAIFlbR6VJnmQRwedLGBGenbIsMC_I3uA35Mz_bkTTc,1028
 redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml,sha256=VIFNaaAqRQMeDvtXmKGDpJTnu658Lv5i_oAju5uSi_c,708
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
-redis_benchmarks_specification-0.1.224.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.224.dist-info/METADATA,sha256=xc_Fe0CVZl4pmny7ic9ZhtsaePCG3E0sapvX08_Oink,22726
-redis_benchmarks_specification-0.1.224.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.224.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.224.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.225.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.225.dist-info/METADATA,sha256=oy56bG7M57BjELEZ3rZcVwtCASeYVzdMwwW9DMXta_8,22726
+redis_benchmarks_specification-0.1.225.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.225.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.225.dist-info/RECORD,,