redis-benchmarks-specification 0.1.102__py3-none-any.whl → 0.1.150__py3-none-any.whl

redis_benchmarks_specification/__compare__/compare.py

@@ -16,6 +16,9 @@ import os
 from tqdm import tqdm
 import argparse
 
+from io import StringIO
+import sys
+
 from redis_benchmarks_specification.__common__.github import (
     update_comment_if_needed,
     create_new_pr_comment,
@@ -200,6 +203,8 @@ def compare_command_logic(args, project_name, project_version):
             )
         )
         baseline_branch = default_baseline_branch
+    if baseline_branch == "":
+        baseline_branch = None
     comparison_branch = args.comparison_branch
     simplify_table = args.simple_table
     print_regressions_only = args.print_regressions_only
@@ -245,7 +250,16 @@ def compare_command_logic(args, project_name, project_version):
     testname_regex = args.testname_regex
     auto_approve = args.auto_approve
     running_platform = args.running_platform
-    grafana_base_dashboard = args.grafana_base_dashboard
+    baseline_target_version = args.baseline_target_version
+    comparison_target_version = args.comparison_target_version
+    baseline_target_branch = args.baseline_target_branch
+    comparison_target_branch = args.comparison_target_branch
+    baseline_github_repo = args.baseline_github_repo
+    comparison_github_repo = args.comparison_github_repo
+    baseline_github_org = args.baseline_github_org
+    comparison_github_org = args.comparison_github_org
+    baseline_hash = args.baseline_hash
+    comparison_hash = args.comparison_hash
 
     if running_platform is not None:
         logging.info(
@@ -270,8 +284,8 @@ def compare_command_logic(args, project_name, project_version):
     (
         detected_regressions,
         table_output,
-        total_improvements,
-        total_regressions,
+        improvements_list,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
@@ -304,7 +318,21 @@ def compare_command_logic(args, project_name, project_version):
         to_ts_ms,
         use_metric_context_path,
         running_platform,
+        baseline_target_version,
+        comparison_target_version,
+        baseline_hash,
+        comparison_hash,
+        baseline_github_repo,
+        comparison_github_repo,
+        baseline_target_branch,
+        comparison_target_branch,
+        baseline_github_org,
+        comparison_github_org,
+        args.regression_str,
+        args.improvement_str,
     )
+    total_regressions = len(regressions_list)
+    total_improvements = len(improvements_list)
     prepare_regression_comment(
         auto_approve,
         baseline_branch,
@@ -331,6 +359,8 @@ def compare_command_logic(args, project_name, project_version):
         total_unstable,
         verbose,
         args.regressions_percent_lower_limit,
+        regressions_list,
+        improvements_list,
     )
     return (
         detected_regressions,
@@ -369,6 +399,8 @@ def prepare_regression_comment(
     total_unstable,
     verbose,
     regressions_percent_lower_limit,
+    regressions_list=[],
+    improvements_list=[],
 ):
     if total_comparison_points > 0:
         comment_body = "### Automated performance analysis summary\n\n"
@@ -395,10 +427,27 @@ def prepare_regression_comment(
             comparison_summary += "- Detected a total of {} improvements above the improvement water line.\n".format(
                 total_improvements
             )
+            if len(improvements_list) > 0:
+                regression_values = [l[1] for l in improvements_list]
+                regression_df = pd.DataFrame(regression_values)
+                median_regression = round(float(regression_df.median().iloc[0]), 1)
+                max_regression = round(float(regression_df.max().iloc[0]), 1)
+                min_regression = round(float(regression_df.min().iloc[0]), 1)
+
+                comparison_summary += f"   - Median/Common-Case improvement was {median_regression}% and ranged from [{min_regression}%,{max_regression}%].\n"
+
         if total_regressions > 0:
             comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format(
                 total_regressions, regressions_percent_lower_limit
             )
+            if len(regressions_list) > 0:
+                regression_values = [l[1] for l in regressions_list]
+                regression_df = pd.DataFrame(regression_values)
+                median_regression = round(float(regression_df.median().iloc[0]), 1)
+                max_regression = round(float(regression_df.max().iloc[0]), 1)
+                min_regression = round(float(regression_df.min().iloc[0]), 1)
+
+                comparison_summary += f"   - Median/Common-Case regression was {median_regression}% and ranged from [{min_regression}%,{max_regression}%].\n"
 
         comment_body += comparison_summary
         comment_body += "\n"
@@ -505,7 +554,7 @@ def compute_regression_table(
     tf_triggering_env,
     metric_name,
     comparison_branch,
-    baseline_branch="master",
+    baseline_branch="unstable",
     baseline_tag=None,
     comparison_tag=None,
     baseline_deployment_name="oss-standalone",
@@ -527,6 +576,18 @@ def compute_regression_table(
     to_ts_ms=None,
     use_metric_context_path=None,
     running_platform=None,
+    baseline_target_version=None,
+    comparison_target_version=None,
+    comparison_hash=None,
+    baseline_hash=None,
+    baseline_github_repo="redis",
+    comparison_github_repo="redis",
+    baseline_target_branch=None,
+    comparison_target_branch=None,
+    baseline_github_org="redis",
+    comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -550,7 +611,15 @@ def compute_regression_table(
         comparison_branch,
         baseline_tag,
         comparison_tag,
+        baseline_target_version,
+        comparison_target_version,
+        comparison_hash,
+        baseline_hash,
+        baseline_target_branch,
+        comparison_target_branch,
     )
+    logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}")
+    logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}")
     (
         prefix,
         testcases_setname,
@@ -583,12 +652,22 @@ def compute_regression_table(
         )
     (
         detected_regressions,
-        table,
+        table_full,
+        table_stable,
+        table_unstable,
+        table_improvements,
+        table_regressions,
         total_improvements,
         total_regressions,
         total_stable,
         total_unstable,
         total_comparison_points,
+        regressions_list,
+        improvements_list,
+        unstable_list,
+        baseline_only_list,
+        comparison_only_list,
+        no_datapoints_list,
     ) = from_rts_to_regression_table(
         baseline_deployment_name,
         comparison_deployment_name,
@@ -613,58 +692,164 @@ def compute_regression_table(
         tf_triggering_env,
         verbose,
         running_platform,
+        baseline_github_repo,
+        comparison_github_repo,
+        baseline_github_org,
+        comparison_github_org,
+        regression_str,
+        improvement_str,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
             baseline_str, comparison_str
         )
     )
-    writer = MarkdownTableWriter(
-        table_name="Comparison between {} and {}.\n\nTime Period from {}. (environment used: {})\n".format(
-            baseline_str,
-            comparison_str,
-            from_human_str,
-            baseline_deployment_name,
-        ),
+
+    table_output = "# Comparison between {} and {}.\n\nTime Period from {}. (environment used: {})\n\n".format(
+        baseline_str,
+        comparison_str,
+        from_human_str,
+        baseline_deployment_name,
+    )
+
+    if total_unstable > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Unstable Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                f"Baseline {baseline_github_org}/{baseline_github_repo} {baseline_str} (median obs. +- std.dev)",
+                f"Comparison {comparison_github_org}/{comparison_github_repo} {comparison_str} (median obs. +- std.dev)",
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_unstable,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        test_names_str = "|".join([l[0] for l in unstable_list])
+        table_output += f"Unstable test regexp names: {test_names_str}\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    if total_regressions > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Regressions Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                f"Baseline {baseline_github_org}/{baseline_github_repo} {baseline_str} (median obs. +- std.dev)",
+                f"Comparison {comparison_github_org}/{comparison_github_repo} {comparison_str} (median obs. +- std.dev)",
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_regressions,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        test_names_str = "|".join([l[0] for l in regressions_list])
+        table_output += f"Regressions test regexp names: {test_names_str}\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    if total_improvements > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Improvements Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                f"Baseline {baseline_github_org}/{baseline_github_repo} {baseline_str} (median obs. +- std.dev)",
+                f"Comparison {comparison_github_org}/{comparison_github_repo} {comparison_str} (median obs. +- std.dev)",
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_improvements,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        test_names_str = "|".join([l[0] for l in improvements_list])
+        table_output += f"Improvements test regexp names: {test_names_str}\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    old_stdout = sys.stdout
+    sys.stdout = mystdout = StringIO()
+    writer_full = MarkdownTableWriter(
+        table_name="",
         headers=[
             "Test Case",
-            "Baseline {} (median obs. +- std.dev)".format(baseline_str),
-            "Comparison {} (median obs. +- std.dev)".format(comparison_str),
+            f"Baseline {baseline_github_org}/{baseline_github_repo} {baseline_str} (median obs. +- std.dev)",
+            f"Comparison {comparison_github_org}/{comparison_github_repo} {comparison_str} (median obs. +- std.dev)",
             "% change ({})".format(metric_mode),
             "Note",
         ],
-        value_matrix=table,
+        value_matrix=table_full,
     )
-    table_output = ""
-
-    from io import StringIO
-    import sys
+    table_output += "<details>\n  <summary>Full Results table:</summary>\n\n"
 
-    old_stdout = sys.stdout
-    sys.stdout = mystdout = StringIO()
-
-    writer.dump(mystdout, False)
+    writer_full.dump(mystdout, False)
 
     sys.stdout = old_stdout
-
-    table_output = mystdout.getvalue()
+    table_output += mystdout.getvalue()
+    table_output += "\n</details>\n"
+    len_baseline_only_list = len(baseline_only_list)
+    if len_baseline_only_list > 0:
+        table_output += f"\n  WARNING: There were {len_baseline_only_list} benchmarks with datapoints only on baseline.\n\n"
+        baseline_only_test_names_str = "|".join([l[0] for l in baseline_only_list])
+        table_output += (
+            f"  Baseline only test regexp names: {baseline_only_test_names_str}\n\n"
+        )
+    len_comparison_only_list = len(comparison_only_list)
+    if len_comparison_only_list > 0:
+        table_output += f"\n  WARNING: There were {len_comparison_only_list} benchmarks with datapoints only on comparison.\n\n"
+        comparison_only_test_names_str = "|".join([l[0] for l in comparison_only_list])
+        table_output += (
+            f"  Comparison only test regexp names: {comparison_only_test_names_str}\n\n"
+        )
+    len_no_datapoints = len(no_datapoints_list)
+    if len_no_datapoints > 0:
+        table_output += f"\n  WARNING: There were {len_no_datapoints} benchmarks with NO datapoints for both baseline and comparison.\n\n"
+        no_datapoints_test_names_str = "|".join([l[0] for l in no_datapoints_list])
+        table_output += (
+            f"  NO DATAPOINTS test regexp names: {no_datapoints_test_names_str}\n\n"
+        )
 
     return (
         detected_regressions,
         table_output,
-        total_improvements,
-        total_regressions,
+        improvements_list,
+        regressions_list,
         total_stable,
         total_unstable,
         total_comparison_points,
     )
 
 
+def get_by_error(name, by_str_arr):
+    by_string = ",".join(by_str_arr)
+    return f"--{name}-branch, --{name}-tag, --{name}-target-branch, --{name}-hash, and --{name}-target-version are mutually exclusive. You selected a total of {len(by_str_arr)}: {by_string}. Pick one..."
+
+
 def get_by_strings(
     baseline_branch,
     comparison_branch,
     baseline_tag,
     comparison_tag,
+    baseline_target_version=None,
+    comparison_target_version=None,
+    baseline_hash=None,
+    comparison_hash=None,
+    baseline_target_branch=None,
+    comparison_target_branch=None,
 ):
     baseline_covered = False
     comparison_covered = False
@@ -672,45 +857,112 @@ def get_by_strings(
     by_str_comparison = ""
     baseline_str = ""
     comparison_str = ""
+    baseline_by_arr = []
+    comparison_by_arr = []
+
+    ################# BASELINE BY ....
+
     if baseline_branch is not None:
-        baseline_covered = True
         by_str_baseline = "branch"
+        baseline_covered = True
         baseline_str = baseline_branch
-    if comparison_branch is not None:
-        comparison_covered = True
-        by_str_comparison = "branch"
-        comparison_str = comparison_branch
+        baseline_by_arr.append(by_str_baseline)
 
     if baseline_tag is not None:
-        if comparison_covered:
-            logging.error(
-                "--baseline-branch and --baseline-tag are mutually exclusive. Pick one..."
-            )
+        by_str_baseline = "version"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
             exit(1)
         baseline_covered = True
-        by_str_baseline = "version"
         baseline_str = baseline_tag
 
+    if baseline_target_version is not None:
+        by_str_baseline = "target+version"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
+            exit(1)
+        baseline_covered = True
+        baseline_str = baseline_target_version
+
+    if baseline_hash is not None:
+        by_str_baseline = "hash"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
+            exit(1)
+        baseline_covered = True
+        baseline_str = baseline_hash
+    if baseline_target_branch is not None:
+        by_str_baseline = "target+branch"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
+            exit(1)
+        baseline_covered = True
+        baseline_str = baseline_target_branch
+
+    ################# COMPARISON BY ....
+
+    if comparison_branch is not None:
+        by_str_comparison = "branch"
+        comparison_covered = True
+        comparison_str = comparison_branch
+
     if comparison_tag is not None:
         # check if we had already covered comparison
         if comparison_covered:
             logging.error(
-                "--comparison-branch and --comparison-tag are mutually exclusive. Pick one..."
+                "--comparison-branch and --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
             )
             exit(1)
         comparison_covered = True
         by_str_comparison = "version"
         comparison_str = comparison_tag
+    if comparison_target_version is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "target+version"
+        comparison_str = comparison_target_version
+
+    if comparison_target_branch is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "target+branch"
+        comparison_str = comparison_target_branch
+
+    if comparison_hash is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "hash"
+        comparison_str = comparison_hash
 
     if baseline_covered is False:
         logging.error(
-            "You need to provider either " + "( --baseline-branch or --baseline-tag ) "
+            "You need to provider either "
+            + "( --baseline-branch, --baseline-tag, --baseline-hash, --baseline-target-branch or --baseline-target-version ) "
         )
         exit(1)
     if comparison_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --comparison-branch or --comparison-tag ) "
+            + "( --comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch or --comparison-target-version ) "
         )
         exit(1)
     return baseline_str, by_str_baseline, comparison_str, by_str_comparison
@@ -740,9 +992,19 @@ def from_rts_to_regression_table(
     tf_triggering_env,
     verbose,
     running_platform=None,
+    baseline_github_repo="redis",
+    comparison_github_repo="redis",
+    baseline_github_org="redis",
+    comparison_github_org="redis",
+    regression_str="REGRESSION",
+    improvement_str="IMPROVEMENT",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
-    table = []
+    table_full = []
+    table_unstable = []
+    table_stable = []
+    table_regressions = []
+    table_improvements = []
     detected_regressions = []
     total_improvements = 0
     total_stable = 0
@@ -751,7 +1013,22 @@ def from_rts_to_regression_table(
     total_comparison_points = 0
     noise_waterline = 3
     progress = tqdm(unit="benchmark time-series", total=len(test_names))
+    regressions_list = []
+    improvements_list = []
+    unstable_list = []
+    baseline_only_list = []
+    comparison_only_list = []
+    no_datapoints_list = []
+    no_datapoints_baseline_list = []
+    no_datapoints_comparison_list = []
+    original_metric_mode = metric_mode
     for test_name in test_names:
+        metric_mode = original_metric_mode
+        compare_version = "main"
+        # GE
+        github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
+        test_path = f"redis_benchmarks_specification/test-suites/{test_name}.yml"
+        test_link = f"[{test_name}]({github_link}/{compare_version}/{test_path})"
         multi_value_baseline = check_multi_value_filter(baseline_str)
         multi_value_comparison = check_multi_value_filter(comparison_str)
 
@@ -760,8 +1037,11 @@ def from_rts_to_regression_table(
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(baseline_deployment_name),
+            "github_repo={}".format(baseline_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
+        if baseline_github_org != "":
+            filters_baseline.append(f"github_org={baseline_github_org}")
         if running_platform is not None:
             filters_baseline.append("running_platform={}".format(running_platform))
         filters_comparison = [
@@ -769,8 +1049,15 @@ def from_rts_to_regression_table(
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(comparison_deployment_name),
+            "github_repo={}".format(comparison_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
+        if comparison_github_org != "":
+            filters_comparison.append(f"github_org={comparison_github_org}")
+        if "hash" not in by_str_baseline:
+            filters_baseline.append("hash==")
+        if "hash" not in by_str_comparison:
+            filters_comparison.append("hash==")
         if running_platform is not None:
             filters_comparison.append("running_platform={}".format(running_platform))
         baseline_timeseries = rts.ts().queryindex(filters_baseline)
@@ -794,6 +1081,22 @@ def from_rts_to_regression_table(
         if len(baseline_timeseries) > 1 and multi_value_baseline is False:
             baseline_timeseries = get_only_Totals(baseline_timeseries)
 
+        if len(baseline_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for baseline timeseries {baseline_timeseries}"
+            )
+            no_datapoints_baseline_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
+        if len(comparison_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for comparison timeseries {comparison_timeseries}"
+            )
+            no_datapoints_comparison_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
         if len(baseline_timeseries) != 1 and multi_value_baseline is False:
             if verbose:
                 logging.warning(
@@ -870,20 +1173,40 @@ def from_rts_to_regression_table(
             )
 
             waterline = regressions_percent_lower_limit
-            if regressions_percent_lower_limit < largest_variance:
-                note = "waterline={:.1f}%.".format(largest_variance)
-                waterline = largest_variance
+            # if regressions_percent_lower_limit < largest_variance:
+            #     note = "waterline={:.1f}%.".format(largest_variance)
+            #     waterline = largest_variance
 
-        except redis.exceptions.ResponseError:
+        except redis.exceptions.ResponseError as e:
+            logging.error(
+                "Detected a redis.exceptions.ResponseError. {}".format(e.__str__())
+            )
             pass
         except ZeroDivisionError as e:
             logging.error("Detected a ZeroDivisionError. {}".format(e.__str__()))
             pass
         unstable = False
-        if baseline_v != "N/A" and comparison_v != "N/A":
+
+        if baseline_v != "N/A" and comparison_v == "N/A":
+            logging.warning(
+                "Baseline contains datapoints but comparison not for test: {test_name}"
+            )
+            baseline_only_list.append(test_name)
+        if comparison_v != "N/A" and baseline_v == "N/A":
+            logging.warning(
+                "Comparison contains datapoints but baseline not for test: {test_name}"
+            )
+            comparison_only_list.append(test_name)
+        if (
+            baseline_v != "N/A"
+            and comparison_pct_change != "N/A"
+            and comparison_v != "N/A"
+            and baseline_pct_change != "N/A"
+        ):
             if comparison_pct_change > 10.0 or baseline_pct_change > 10.0:
                 note = "UNSTABLE (very high variance)"
                 unstable = True
+                unstable_list.append([test_name, "n/a"])
 
             baseline_v_str = prepare_value_str(
                 baseline_pct_change, baseline_v, baseline_values, simplify_table
@@ -899,32 +1222,36 @@ def from_rts_to_regression_table(
             else:
                 # lower-better
                 percentage_change = (
-                    float(baseline_v) / float(comparison_v) - 1
+                    -(float(baseline_v) - float(comparison_v)) / float(baseline_v)
                 ) * 100.0
+        else:
+            logging.warn(
+                f"Missing data for test {test_name}. baseline_v={baseline_v} (pct_change={baseline_pct_change}), comparison_v={comparison_v} (pct_change={comparison_pct_change}) "
+            )
         if baseline_v != "N/A" or comparison_v != "N/A":
             detected_regression = False
             detected_improvement = False
-            if percentage_change < 0.0 and not unstable:
+            if percentage_change < 0.0:
                 if -waterline >= percentage_change:
                     detected_regression = True
                     total_regressions = total_regressions + 1
-                    note = note + " REGRESSION"
+                    note = note + f" {regression_str}"
                     detected_regressions.append(test_name)
                 elif percentage_change < -noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential REGRESSION"
+                        note = note + f" potential {regression_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
 
-            if percentage_change > 0.0 and not unstable:
+            if percentage_change > 0.0:
                 if percentage_change > waterline:
                     detected_improvement = True
                     total_improvements = total_improvements + 1
-                    note = note + " IMPROVEMENT"
+                    note = note + f" {improvement_str}"
                 elif percentage_change > noise_waterline:
                     if simplify_table is False:
-                        note = note + " potential IMPROVEMENT"
+                        note = note + f" potential {improvement_str}"
                 else:
                     if simplify_table is False:
                         note = note + " No Change"
@@ -940,6 +1267,27 @@ def from_rts_to_regression_table(
                 total_unstable += 1
 
             should_add_line = False
+            line = get_line(
+                baseline_v_str,
+                comparison_v_str,
+                note,
+                percentage_change,
+                test_link,
+            )
+            if detected_regression:
+                regressions_list.append([test_name, percentage_change])
+                table_regressions.append(line)
+
+            if detected_improvement:
+                improvements_list.append([test_name, percentage_change])
+                table_improvements.append(line)
+
+            if unstable:
+                table_unstable.append(line)
+            else:
+                if not detected_regression and not detected_improvement:
+                    table_stable.append(line)
+
             if print_regressions_only and detected_regression:
                 should_add_line = True
             if print_improvements_only and detected_improvement:
@@ -951,22 +1299,51 @@ def from_rts_to_regression_table(
 
             if should_add_line:
                 total_comparison_points = total_comparison_points + 1
-                add_line(
-                    baseline_v_str,
-                    comparison_v_str,
-                    note,
-                    percentage_change,
-                    table,
-                    test_name,
-                )
+                table_full.append(line)
+        else:
+            logging.warning(
+                "There were no datapoints both for baseline and comparison for test: {test_name}"
+            )
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+    logging.warning(
+        f"There is a total of {len(no_datapoints_list)} tests without datapoints for baseline AND comparison"
+    )
+    logging.info(
+        f"There is a total of {len(comparison_only_list)} tests without datapoints for baseline"
+    )
+    print(
+        "No datapoint baseline regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_baseline_list)
+        )
+    )
+    logging.info(
+        f"There is a total of {len(baseline_only_list)} tests without datapoints for comparison"
+    )
+    print(
+        "No datapoint comparison regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_comparison_list)
+        )
+    )
+    logging.info(f"There is a total of {len(unstable_list)} UNSTABLE tests")
     return (
         detected_regressions,
-        table,
+        table_full,
+        table_stable,
+        table_unstable,
+        table_improvements,
+        table_regressions,
         total_improvements,
         total_regressions,
         total_stable,
         total_unstable,
         total_comparison_points,
+        regressions_list,
+        improvements_list,
+        unstable_list,
+        baseline_only_list,
+        comparison_only_list,
+        no_datapoints_list,
     )
 
 
@@ -1037,6 +1414,23 @@ def get_test_names_from_db(rts, tags_regex_string, test_names, used_key):
     return test_names
 
 
+def get_line(
+    baseline_v_str,
+    comparison_v_str,
+    note,
+    percentage_change,
+    test_name,
+):
+    percentage_change_str = "{:.1f}% ".format(percentage_change)
+    return [
+        test_name,
+        baseline_v_str,
+        comparison_v_str,
+        percentage_change_str,
+        note.strip(),
+    ]
+
+
 def add_line(
     baseline_v_str,
     comparison_v_str,
@@ -1045,15 +1439,14 @@ def add_line(
     table,
     test_name,
 ):
-    percentage_change_str = "{:.1f}% ".format(percentage_change)
     table.append(
-        [
-            test_name,
+        get_line(
             baseline_v_str,
             comparison_v_str,
-            percentage_change_str,
-            note.strip(),
-        ]
+            note,
+            percentage_change,
+            test_name,
+        )
     )
 
 
@@ -1071,23 +1464,25 @@ def get_v_pct_change_and_largest_var(
         _, comparison_v = comparison_datapoints[0]
         for tuple in comparison_datapoints:
             if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n):
-                comparison_values.append(tuple[1])
-        comparison_df = pd.DataFrame(comparison_values)
-        comparison_median = float(comparison_df.median())
-        comparison_v = comparison_median
-        comparison_std = float(comparison_df.std())
-        if verbose:
-            logging.info(
-                "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(
-                    comparison_datapoints,
-                    comparison_v,
-                    comparison_std,
-                    comparison_median,
+                if tuple[1] > 0.0:
+                    comparison_values.append(tuple[1])
+        if len(comparison_values) > 0:
+            comparison_df = pd.DataFrame(comparison_values)
+            comparison_median = float(comparison_df.median().iloc[0])
+            comparison_v = comparison_median
+            comparison_std = float(comparison_df.std().iloc[0])
+            if verbose:
+                logging.info(
+                    "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(
+                        comparison_datapoints,
+                        comparison_v,
+                        comparison_std,
+                        comparison_median,
+                    )
                 )
-            )
-        comparison_pct_change = (comparison_std / comparison_median) * 100.0
-        if comparison_pct_change > largest_variance:
-            largest_variance = comparison_pct_change
+            comparison_pct_change = (comparison_std / comparison_median) * 100.0
+            if comparison_pct_change > largest_variance:
+                largest_variance = comparison_pct_change
     return comparison_pct_change, comparison_v, largest_variance