redis-benchmarks-specification 0.1.208__py3-none-any.whl → 0.1.210__py3-none-any.whl

redis_benchmarks_specification/__compare__/compare.py

@@ -16,6 +16,9 @@ import os
 from tqdm import tqdm
 import argparse
 
+from io import StringIO
+import sys
+
 from redis_benchmarks_specification.__common__.github import (
     update_comment_if_needed,
     create_new_pr_comment,
@@ -245,7 +248,6 @@ def compare_command_logic(args, project_name, project_version):
     testname_regex = args.testname_regex
     auto_approve = args.auto_approve
     running_platform = args.running_platform
-    grafana_base_dashboard = args.grafana_base_dashboard
 
     if running_platform is not None:
         logging.info(
@@ -583,7 +585,11 @@ def compute_regression_table(
         )
     (
         detected_regressions,
-        table,
+        table_full,
+        table_stable,
+        table_unstable,
+        table_improvements,
+        table_regressions,
         total_improvements,
         total_regressions,
         total_stable,
@@ -619,13 +625,60 @@ def compute_regression_table(
             baseline_str, comparison_str
         )
     )
-    writer = MarkdownTableWriter(
-        table_name="Comparison between {} and {}.\n\nTime Period from {}. (environment used: {})\n".format(
-            baseline_str,
-            comparison_str,
-            from_human_str,
-            baseline_deployment_name,
-        ),
+
+    table_output = "# Comparison between {} and {}.\n\nTime Period from {}. (environment used: {})\n\n".format(
+        baseline_str,
+        comparison_str,
+        from_human_str,
+        baseline_deployment_name,
+    )
+
+    if total_regressions > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Regressions Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                "Baseline {} (median obs. +- std.dev)".format(baseline_str),
+                "Comparison {} (median obs. +- std.dev)".format(comparison_str),
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_regressions,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    if total_improvements > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Improvements Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                "Baseline {} (median obs. +- std.dev)".format(baseline_str),
+                "Comparison {} (median obs. +- std.dev)".format(comparison_str),
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_improvements,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    old_stdout = sys.stdout
+    sys.stdout = mystdout = StringIO()
+    writer_full = MarkdownTableWriter(
+        table_name="",
         headers=[
             "Test Case",
             "Baseline {} (median obs. +- std.dev)".format(baseline_str),
@@ -633,21 +686,15 @@ def compute_regression_table(
             "% change ({})".format(metric_mode),
             "Note",
         ],
-        value_matrix=table,
+        value_matrix=table_full,
     )
-    table_output = ""
-
-    from io import StringIO
-    import sys
-
-    old_stdout = sys.stdout
-    sys.stdout = mystdout = StringIO()
+    table_output += "<details>\n  <summary>Full Results table:</summary>\n\n"
 
-    writer.dump(mystdout, False)
+    writer_full.dump(mystdout, False)
 
     sys.stdout = old_stdout
-
-    table_output = mystdout.getvalue()
+    table_output += mystdout.getvalue()
+    table_output += "\n</details>\n"
 
     return (
         detected_regressions,
@@ -742,7 +789,11 @@ def from_rts_to_regression_table(
     running_platform=None,
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
-    table = []
+    table_full = []
+    table_unstable = []
+    table_stable = []
+    table_regressions = []
+    table_improvements = []
     detected_regressions = []
     total_improvements = 0
     total_stable = 0
@@ -752,6 +803,10 @@ def from_rts_to_regression_table(
     noise_waterline = 3
     progress = tqdm(unit="benchmark time-series", total=len(test_names))
     for test_name in test_names:
+        compare_version = "v0.1.208"
+        github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
+        test_path = f"redis_benchmarks_specification/test-suites/{test_name}.yml"
+        test_link = f"[{test_name}]({github_link}/{compare_version}/{test_path})"
         multi_value_baseline = check_multi_value_filter(baseline_str)
         multi_value_comparison = check_multi_value_filter(comparison_str)
 
@@ -940,6 +995,25 @@ def from_rts_to_regression_table(
                 total_unstable += 1
 
             should_add_line = False
+            line = get_line(
+                baseline_v_str,
+                comparison_v_str,
+                note,
+                percentage_change,
+                test_link,
+            )
+            if detected_regression:
+                table_regressions.append(line)
+
+            if detected_improvement:
+                table_improvements.append(line)
+
+            if unstable:
+                table_unstable.append(line)
+            else:
+                if not detected_regression and not detected_improvement:
+                    table_stable.append(line)
+
             if print_regressions_only and detected_regression:
                 should_add_line = True
             if print_improvements_only and detected_improvement:
@@ -951,17 +1025,14 @@ def from_rts_to_regression_table(
 
             if should_add_line:
                 total_comparison_points = total_comparison_points + 1
-                add_line(
-                    baseline_v_str,
-                    comparison_v_str,
-                    note,
-                    percentage_change,
-                    table,
-                    test_name,
-                )
+                table_full.append(line)
     return (
         detected_regressions,
-        table,
+        table_full,
+        table_stable,
+        table_unstable,
+        table_improvements,
+        table_regressions,
         total_improvements,
         total_regressions,
         total_stable,
@@ -1037,6 +1108,23 @@ def get_test_names_from_db(rts, tags_regex_string, test_names, used_key):
     return test_names
 
 
+def get_line(
+    baseline_v_str,
+    comparison_v_str,
+    note,
+    percentage_change,
+    test_name,
+):
+    percentage_change_str = "{:.1f}% ".format(percentage_change)
+    return [
+        test_name,
+        baseline_v_str,
+        comparison_v_str,
+        percentage_change_str,
+        note.strip(),
+    ]
+
+
 def add_line(
     baseline_v_str,
     comparison_v_str,
@@ -1045,15 +1133,14 @@ def add_line(
     table,
     test_name,
 ):
-    percentage_change_str = "{:.1f}% ".format(percentage_change)
     table.append(
-        [
-            test_name,
+        get_line(
             baseline_v_str,
             comparison_v_str,
-            percentage_change_str,
-            note.strip(),
-        ]
+            note,
+            percentage_change,
+            test_name,
+        )
     )
 
 

redis_benchmarks_specification/__runner__/runner.py

@@ -211,7 +211,7 @@ def prepare_memtier_benchmark_parameters(
     server,
     password,
     local_benchmark_output_filename,
-    oss_cluster_api_enabled,
+    oss_cluster_api_enabled=False,
     tls_enabled=False,
     tls_skip_verify=False,
     tls_cert=None,
@@ -382,6 +382,7 @@ def process_self_contained_coordinator_stream(
     defaults_filename = args.defaults_filename
     override_test_runs = args.override_test_runs
     (
+        _,
         _,
         default_metrics,
         _,
@@ -396,7 +397,7 @@ def process_self_contained_coordinator_stream(
 
         with open(test_file, "r") as stream:
             _, benchmark_config, test_name = get_final_benchmark_config(
-                None, stream, ""
+                None, None, stream, ""
             )
 
             if tls_enabled:
@@ -410,6 +411,7 @@ def process_self_contained_coordinator_stream(
             for topology_spec_name in benchmark_config["redis-topologies"]:
                 test_result = False
                 benchmark_tool_global = ""
+                full_result_path = None
                 try:
                     current_cpu_pos = args.cpuset_start_pos
                     temporary_dir_client = tempfile.mkdtemp(dir=home)
@@ -756,7 +758,10 @@ def process_self_contained_coordinator_stream(
                     profiler_frequency = 99
 
                     # start the profile
-                    (profiler_name, profilers_map,) = profilers_start_if_required(
+                    (
+                        profiler_name,
+                        profilers_map,
+                    ) = profilers_start_if_required(
                         profilers_enabled,
                         profilers_list,
                         redis_pids,
@@ -821,7 +826,10 @@ def process_self_contained_coordinator_stream(
                             benchmark_end_time, benchmark_start_time
                         )
                     )
-                    (_, overall_tabular_data_map,) = profilers_stop_if_required(
+                    (
+                        _,
+                        overall_tabular_data_map,
+                    ) = profilers_stop_if_required(
                         datasink_push_results_redistimeseries,
                         benchmark_duration_seconds,
                         collection_summary_str,
@@ -1067,7 +1075,10 @@ def print_results_table_stdout(
     cpu_usage=None,
 ):
     # check which metrics to extract
-    (_, metrics,) = merge_default_and_config_metrics(
+    (
+        _,
+        metrics,
+    ) = merge_default_and_config_metrics(
         benchmark_config,
         default_metrics,
         None,
@@ -1092,7 +1103,10 @@ def prepare_overall_total_test_results(
     benchmark_config, default_metrics, results_dict, test_name, overall_results_matrix
 ):
     # check which metrics to extract
-    (_, metrics,) = merge_default_and_config_metrics(
+    (
+        _,
+        metrics,
+    ) = merge_default_and_config_metrics(
         benchmark_config,
         default_metrics,
         None,

redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -1,5 +1,4 @@
 import argparse
-import datetime
 import os
 from redis_benchmarks_specification.__common__.env import (
     MACHINE_CPU_COUNT,
@@ -20,10 +19,6 @@ from redis_benchmarks_specification.__common__.env import (
     PROFILERS_DEFAULT,
     ALLOWED_PROFILERS,
 )
-from redis_benchmarks_specification.__compare__.args import (
-    START_TIME_NOW_UTC,
-    START_TIME_LAST_SIX_MONTHS_UTC,
-)
 
 PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
 

redis_benchmarks_specification/__self_contained_coordinator__/build_info.py

@@ -10,11 +10,13 @@ def extract_build_info_from_streamdata(testDetails):
     arch = "amd64"
     use_git_timestamp = False
     git_timestamp_ms = None
-    metadata = None
+    metadata = {}
     build_variant_name = None
     fields = [fieldname.decode() for fieldname in testDetails.keys()]
-    logging.info("Fields on stream {}".format(fields))
-    git_hash = testDetails[b"git_hash"]
+    logging.info("Fields on stream {}".format(testDetails))
+    git_hash = None
+    if b"git_hash" in testDetails:
+        git_hash = testDetails[b"git_hash"].decode()
     if b"use_git_timestamp" in testDetails:
         use_git_timestamp = bool(testDetails[b"use_git_timestamp"].decode())
     if b"git_timestamp_ms" in testDetails:

redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -18,9 +18,9 @@ def generate_standalone_redis_server_args(
         "no",
         "--port",
         "{}".format(port),
-        "--dir",
-        dbdir,
     ]
+    if dbdir != "":
+        command.extend(["--dbdir", dbdir])
     if configuration_parameters is not None:
         for parameter, parameter_value in configuration_parameters.items():
             if parameter not in added_params:

redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py

@@ -43,7 +43,10 @@ def data_prepopulation_step(
     full_benchmark_path = "/usr/local/bin/{}".format(preload_tool)
     client_mnt_point = "/mnt/client/"
     if "memtier_benchmark" in preload_tool:
-        (_, preload_command_str,) = prepare_memtier_benchmark_parameters(
+        (
+            _,
+            preload_command_str,
+        ) = prepare_memtier_benchmark_parameters(
             benchmark_config["dbconfig"]["preload_tool"],
             full_benchmark_path,
             port,

redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -120,6 +120,7 @@ def process_self_contained_coordinator_stream(
     stream_id = "n/a"
     overall_result = False
     total_test_suite_runs = 0
+    full_result_path = None
     try:
         stream_id, testDetails = newTestInfo[0][1][0]
         stream_id = stream_id.decode()
@@ -163,7 +164,7 @@ def process_self_contained_coordinator_stream(
 
                 with open(test_file, "r") as stream:
                     result, benchmark_config, test_name = get_final_benchmark_config(
-                        None, stream, ""
+                        None, None, stream, ""
                     )
                     if result is False:
                         logging.error(
@@ -456,7 +457,10 @@ def process_self_contained_coordinator_stream(
                                 )
                             r.shutdown(save=False)
 
-                            (_, overall_tabular_data_map,) = profilers_stop_if_required(
+                            (
+                                _,
+                                overall_tabular_data_map,
+                            ) = profilers_stop_if_required(
                                 datasink_push_results_redistimeseries,
                                 benchmark_duration_seconds,
                                 collection_summary_str,