redis-benchmarks-specification 0.1.281__py3-none-any.whl → 0.1.282__py3-none-any.whl

redis_benchmarks_specification/__common__/timeseries.py

@@ -147,7 +147,11 @@ def extract_results_table(
 
                     # For precision_summary metrics, construct the full resolved path for display
                     display_path = metric_jsonpath
-                    if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath and use_metric_context_path:
+                    if (
+                        "precision_summary" in metric_jsonpath
+                        and "*" in metric_jsonpath
+                        and use_metric_context_path
+                    ):
                         # Replace the wildcard with the actual precision level
                         display_path = metric_jsonpath.replace("*", metric_context_path)
 
@@ -156,7 +160,10 @@ def extract_results_table(
                         metric_name = metric_jsonpath
                     else:
                         # For display purposes, use the resolved path for precision_summary
-                        if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath:
+                        if (
+                            "precision_summary" in metric_jsonpath
+                            and "*" in metric_jsonpath
+                        ):
                             metric_name = display_path
                         else:
                             # Clean up the metric name for other cases
@@ -167,7 +174,11 @@ def extract_results_table(
                             metric_name = metric_name.replace(" ", "_")
 
                     # Apply standard cleaning to all metric names
-                    if not ("precision_summary" in metric_jsonpath and "*" in metric_jsonpath and use_metric_context_path):
+                    if not (
+                        "precision_summary" in metric_jsonpath
+                        and "*" in metric_jsonpath
+                        and use_metric_context_path
+                    ):
                         metric_name = metric_name.replace("'", "")
                         metric_name = metric_name.replace('"', "")
                         metric_name = metric_name.replace("(", "")

redis_benchmarks_specification/__compare__/args.py

@@ -57,6 +57,18 @@ def create_compare_arguments(parser):
     parser.add_argument("--baseline_github_org", type=str, default="")
     parser.add_argument("--comparison_github_org", type=str, default="")
     parser.add_argument("--triggering_env", type=str, default="ci")
+    parser.add_argument(
+        "--triggering_env_baseline",
+        type=str,
+        default=None,
+        help="Triggering environment for baseline data. If not specified, falls back to --triggering_env",
+    )
+    parser.add_argument(
+        "--triggering_env_comparison",
+        type=str,
+        default=None,
+        help="Triggering environment for comparison data. If not specified, falls back to --triggering_env",
+    )
     parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
     parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="")
     parser.add_argument("--deployment_name", type=str, default="oss-standalone")
@@ -67,6 +79,18 @@ def create_compare_arguments(parser):
     parser.add_argument(
         "--running_platform", type=str, default="intel64-ubuntu22.04-redis-icx1"
     )
+    parser.add_argument(
+        "--running_platform_baseline",
+        type=str,
+        default=None,
+        help="Platform for baseline data. If not specified, falls back to --running_platform",
+    )
+    parser.add_argument(
+        "--running_platform_comparison",
+        type=str,
+        default=None,
+        help="Platform for comparison data. If not specified, falls back to --running_platform",
+    )
     parser.add_argument("--extra-filter", type=str, default=None)
     parser.add_argument(
         "--last_n",

redis_benchmarks_specification/__compare__/compare.py

@@ -251,6 +251,16 @@ def compare_command_logic(args, project_name, project_version):
     testname_regex = args.testname_regex
     auto_approve = args.auto_approve
     running_platform = args.running_platform
+
+    # Handle separate baseline and comparison platform/environment arguments
+    # Fall back to general arguments if specific ones are not provided
+    running_platform_baseline = args.running_platform_baseline or args.running_platform
+    running_platform_comparison = (
+        args.running_platform_comparison or args.running_platform
+    )
+    triggering_env_baseline = args.triggering_env_baseline or args.triggering_env
+    triggering_env_comparison = args.triggering_env_comparison or args.triggering_env
+
     baseline_target_version = args.baseline_target_version
     comparison_target_version = args.comparison_target_version
     baseline_target_branch = args.baseline_target_branch
@@ -262,10 +272,31 @@ def compare_command_logic(args, project_name, project_version):
     baseline_hash = args.baseline_hash
     comparison_hash = args.comparison_hash
 
-    if running_platform is not None:
+    # Log platform and environment information
+    if running_platform_baseline == running_platform_comparison:
+        if running_platform_baseline is not None:
+            logging.info(
+                "Using platform named: {} for both baseline and comparison.\n\n".format(
+                    running_platform_baseline
+                )
+            )
+    else:
+        logging.info(
+            "Using platform named: {} for baseline and {} for comparison.\n\n".format(
+                running_platform_baseline, running_platform_comparison
+            )
+        )
+
+    if triggering_env_baseline == triggering_env_comparison:
+        logging.info(
+            "Using triggering environment: {} for both baseline and comparison.".format(
+                triggering_env_baseline
+            )
+        )
+    else:
         logging.info(
-            "Using platform named: {} to do the comparison.\n\n".format(
-                running_platform
+            "Using triggering environment: {} for baseline and {} for comparison.".format(
+                triggering_env_baseline, triggering_env_comparison
             )
         )
 
@@ -328,7 +359,8 @@ def compare_command_logic(args, project_name, project_version):
         rts,
         tf_github_org,
         tf_github_repo,
-        tf_triggering_env,
+        triggering_env_baseline,
+        triggering_env_comparison,
         metric_name,
         comparison_branch,
         baseline_branch,
@@ -352,7 +384,8 @@ def compare_command_logic(args, project_name, project_version):
         to_date,
         to_ts_ms,
         use_metric_context_path,
-        running_platform,
+        running_platform_baseline,
+        running_platform_comparison,
         baseline_target_version,
         comparison_target_version,
         baseline_hash,
@@ -383,11 +416,13 @@ def compare_command_logic(args, project_name, project_version):
         pr_link,
         regression_comment,
         rts,
-        running_platform,
+        running_platform_baseline,
+        running_platform_comparison,
         table_output,
         tf_github_org,
         tf_github_repo,
-        tf_triggering_env,
+        triggering_env_baseline,
+        triggering_env_comparison,
         total_comparison_points,
         total_improvements,
         total_regressions,
@@ -423,11 +458,13 @@ def prepare_regression_comment(
     pr_link,
     regression_comment,
     rts,
-    running_platform,
+    running_platform_baseline,
+    running_platform_comparison,
     table_output,
     tf_github_org,
     tf_github_repo,
-    tf_triggering_env,
+    triggering_env_baseline,
+    triggering_env_comparison,
     total_comparison_points,
     total_improvements,
     total_regressions,
@@ -441,9 +478,25 @@ def prepare_regression_comment(
     if total_comparison_points > 0:
         comment_body = "### Automated performance analysis summary\n\n"
         comment_body += "This comment was automatically generated given there is performance data available.\n\n"
-        if running_platform is not None:
-            comment_body += "Using platform named: {} to do the comparison.\n\n".format(
-                running_platform
+        # Add platform information to comment
+        if running_platform_baseline == running_platform_comparison:
+            if running_platform_baseline is not None:
+                comment_body += "Using platform named: {} for both baseline and comparison.\n\n".format(
+                    running_platform_baseline
+                )
+        else:
+            comment_body += "Using platform named: {} for baseline and {} for comparison.\n\n".format(
+                running_platform_baseline, running_platform_comparison
+            )
+
+        # Add triggering environment information to comment
+        if triggering_env_baseline == triggering_env_comparison:
+            comment_body += "Using triggering environment: {} for both baseline and comparison.\n\n".format(
+                triggering_env_baseline
+            )
+        else:
+            comment_body += "Using triggering environment: {} for baseline and {} for comparison.\n\n".format(
+                triggering_env_baseline, triggering_env_comparison
             )
         comparison_summary = "In summary:\n"
         if total_stable > 0:
@@ -507,7 +560,7 @@ def prepare_regression_comment(
 
         if is_actionable_pr:
             zset_project_pull_request = get_project_compare_zsets(
-                tf_triggering_env,
+                triggering_env_baseline,
                 tf_github_org,
                 tf_github_repo,
             )
@@ -516,16 +569,22 @@ def prepare_regression_comment(
                     zset_project_pull_request, comparison_branch
                 )
             )
-            _, start_time_ms, _ = get_start_time_vars()
-            res = rts.zadd(
-                zset_project_pull_request,
-                {comparison_branch: start_time_ms},
-            )
-            logging.info(
-                "Result of Populating the pull request performance ZSETs: {} with branch {}: {}".format(
-                    zset_project_pull_request, comparison_branch, res
+            # Only add to Redis sorted set if comparison_branch is not None
+            if comparison_branch is not None:
+                _, start_time_ms, _ = get_start_time_vars()
+                res = rts.zadd(
+                    zset_project_pull_request,
+                    {comparison_branch: start_time_ms},
+                )
+                logging.info(
+                    "Result of Populating the pull request performance ZSETs: {} with branch {}: {}".format(
+                        zset_project_pull_request, comparison_branch, res
+                    )
+                )
+            else:
+                logging.warning(
+                    "Skipping Redis ZADD operation because comparison_branch is None"
                 )
-            )
 
             if contains_regression_comment:
                 update_comment_if_needed(
@@ -587,7 +646,8 @@ def compute_regression_table(
     rts,
     tf_github_org,
     tf_github_repo,
-    tf_triggering_env,
+    tf_triggering_env_baseline,
+    tf_triggering_env_comparison,
     metric_name,
     comparison_branch,
     baseline_branch="unstable",
@@ -611,7 +671,8 @@ def compute_regression_table(
     to_date=None,
     to_ts_ms=None,
     use_metric_context_path=None,
-    running_platform=None,
+    running_platform_baseline=None,
+    running_platform_comparison=None,
     baseline_target_version=None,
     comparison_target_version=None,
     comparison_hash=None,
@@ -672,7 +733,9 @@ def compute_regression_table(
         _,
         _,
         _,
-    ) = get_overall_dashboard_keynames(tf_github_org, tf_github_repo, tf_triggering_env)
+    ) = get_overall_dashboard_keynames(
+        tf_github_org, tf_github_repo, tf_triggering_env_baseline
+    )
     test_names = []
     used_key = testcases_setname
     test_filter = "test_name"
@@ -728,9 +791,11 @@ def compute_regression_table(
         simplify_table,
         test_filter,
         test_names,
-        tf_triggering_env,
+        tf_triggering_env_baseline,
+        tf_triggering_env_comparison,
         verbose,
-        running_platform,
+        running_platform_baseline,
+        running_platform_comparison,
         baseline_github_repo,
         comparison_github_repo,
         baseline_github_org,
@@ -1047,9 +1112,11 @@ def from_rts_to_regression_table(
     simplify_table,
     test_filter,
     test_names,
-    tf_triggering_env,
+    tf_triggering_env_baseline,
+    tf_triggering_env_comparison,
     verbose,
-    running_platform=None,
+    running_platform_baseline=None,
+    running_platform_comparison=None,
     baseline_github_repo="redis",
     comparison_github_repo="redis",
     baseline_github_org="redis",
@@ -1109,19 +1176,21 @@ def from_rts_to_regression_table(
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(baseline_deployment_name),
             "github_repo={}".format(baseline_github_repo),
-            "triggering_env={}".format(tf_triggering_env),
+            "triggering_env={}".format(tf_triggering_env_baseline),
         ]
         if baseline_github_org != "":
             filters_baseline.append(f"github_org={baseline_github_org}")
-        if running_platform is not None:
-            filters_baseline.append("running_platform={}".format(running_platform))
+        if running_platform_baseline is not None:
+            filters_baseline.append(
+                "running_platform={}".format(running_platform_baseline)
+            )
         filters_comparison = [
             "{}={}".format(by_str_comparison, comparison_str),
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(comparison_deployment_name),
             "github_repo={}".format(comparison_github_repo),
-            "triggering_env={}".format(tf_triggering_env),
+            "triggering_env={}".format(tf_triggering_env_comparison),
         ]
         if comparison_github_org != "":
             filters_comparison.append(f"github_org={comparison_github_org}")
@@ -1129,8 +1198,10 @@ def from_rts_to_regression_table(
             filters_baseline.append("hash==")
         if "hash" not in by_str_comparison:
             filters_comparison.append("hash==")
-        if running_platform is not None:
-            filters_comparison.append("running_platform={}".format(running_platform))
+        if running_platform_comparison is not None:
+            filters_comparison.append(
+                "running_platform={}".format(running_platform_comparison)
+            )
         baseline_timeseries = rts.ts().queryindex(filters_baseline)
         comparison_timeseries = rts.ts().queryindex(filters_comparison)
 
@@ -1302,30 +1373,61 @@ def from_rts_to_regression_table(
         if baseline_v != "N/A" or comparison_v != "N/A":
             detected_regression = False
             detected_improvement = False
-            if percentage_change < 0.0:
-                if -waterline >= percentage_change:
-                    detected_regression = True
-                    total_regressions = total_regressions + 1
-                    note = note + f" {regression_str}"
-                    detected_regressions.append(test_name)
-                elif percentage_change < -noise_waterline:
-                    if simplify_table is False:
-                        note = note + f" potential {regression_str}"
-                else:
-                    if simplify_table is False:
-                        note = note + " No Change"
-
-            if percentage_change > 0.0:
-                if percentage_change > waterline:
-                    detected_improvement = True
-                    total_improvements = total_improvements + 1
-                    note = note + f" {improvement_str}"
-                elif percentage_change > noise_waterline:
-                    if simplify_table is False:
-                        note = note + f" potential {improvement_str}"
-                else:
-                    if simplify_table is False:
-                        note = note + " No Change"
+
+            # For higher-better metrics: negative change = regression, positive change = improvement
+            # For lower-better metrics: positive change = regression, negative change = improvement
+            if metric_mode == "higher-better":
+                # Higher is better: negative change is bad (regression), positive change is good (improvement)
+                if percentage_change < 0.0:
+                    if -waterline >= percentage_change:
+                        detected_regression = True
+                        total_regressions = total_regressions + 1
+                        note = note + f" {regression_str}"
+                        detected_regressions.append(test_name)
+                    elif percentage_change < -noise_waterline:
+                        if simplify_table is False:
+                            note = note + f" potential {regression_str}"
+                    else:
+                        if simplify_table is False:
+                            note = note + " No Change"
+
+                if percentage_change > 0.0:
+                    if percentage_change > waterline:
+                        detected_improvement = True
+                        total_improvements = total_improvements + 1
+                        note = note + f" {improvement_str}"
+                    elif percentage_change > noise_waterline:
+                        if simplify_table is False:
+                            note = note + f" potential {improvement_str}"
+                    else:
+                        if simplify_table is False:
+                            note = note + " No Change"
+            else:
+                # Lower is better: positive change is bad (regression), negative change is good (improvement)
+                if percentage_change > 0.0:
+                    if percentage_change >= waterline:
+                        detected_regression = True
+                        total_regressions = total_regressions + 1
+                        note = note + f" {regression_str}"
+                        detected_regressions.append(test_name)
+                    elif percentage_change > noise_waterline:
+                        if simplify_table is False:
+                            note = note + f" potential {regression_str}"
+                    else:
+                        if simplify_table is False:
+                            note = note + " No Change"
+
+                if percentage_change < 0.0:
+                    if -percentage_change > waterline:
+                        detected_improvement = True
+                        total_improvements = total_improvements + 1
+                        note = note + f" {improvement_str}"
+                    elif -percentage_change > noise_waterline:
+                        if simplify_table is False:
+                            note = note + f" potential {improvement_str}"
+                    else:
+                        if simplify_table is False:
+                            note = note + " No Change"
 
             for test_group in tested_groups:
                 if test_group not in group_change:

redis_benchmarks_specification/__runner__/args.py

@@ -20,6 +20,12 @@ def create_client_runner_args(project_name):
         description=project_name,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=project_name,
+        help="Show version information and exit",
+    )
     parser.add_argument(
         "--platform-name",
         type=str,