redis-benchmarks-specification 0.1.94__tar.gz → 0.1.96__tar.gz

{redis_benchmarks_specification-0.1.94 → redis_benchmarks_specification-0.1.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.94
+Version: 0.1.96
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.94 → redis_benchmarks_specification-0.1.96}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.94"
+version = "0.1.96"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.94 → redis_benchmarks_specification-0.1.96}/redis_benchmarks_specification/__common__/github.py

@@ -2,6 +2,17 @@ import logging
 from github import Github
 
 
+def check_regression_comment(comments):
+    res = False
+    pos = -1
+    for n, comment in enumerate(comments):
+        body = comment.body
+        if "Comparison between" in body and "Time Period from" in body:
+            res = True
+            pos = n
+    return res, pos
+
+
 def generate_build_started_pr_comment(
     build_datetime,
     commit_datetime,
@@ -21,6 +32,8 @@ def generate_build_started_pr_comment(
     )
     comment_body += f"Started building at {build_datetime}\n"
     comment_body += "You can check each build/benchmark progress in grafana:\n"
+    if not isinstance(git_hash, str):
+        git_hash = git_hash.decode()
     comment_body += f"   - git hash: {git_hash}\n"
     comment_body += f"   - git branch: {git_branch}\n"
     comment_body += f"   - commit date and time: {commit_datetime}\n"
@@ -57,6 +70,8 @@ def generate_build_finished_pr_comment(
     )
     comment_body += f"Started building at {build_start_datetime} and took {build_duration_seconds} seconds.\n"
     comment_body += "You can check each build/benchmark progress in grafana:\n"
+    if not isinstance(git_hash, str):
+        git_hash = git_hash.decode()
     comment_body += f"   - git hash: {git_hash}\n"
     comment_body += f"   - git branch: {git_branch}\n"
     comment_body += f"   - commit date and time: {commit_datetime}\n"
@@ -237,7 +252,7 @@ def generate_benchmark_started_pr_comment(
     )
 
     comment_body += f"Started benchmark suite at {benchmark_suite_start_datetime} and took {benchmark_suite_duration_secs} seconds "
-    if total_pending == 0:
+    if total_pending > 0:
         comment_body += "up until now.\n"
     else:
         comment_body += "to finish.\n"

{redis_benchmarks_specification-0.1.94 → redis_benchmarks_specification-0.1.96}/redis_benchmarks_specification/__compare__/compare.py

@@ -20,6 +20,7 @@ from redis_benchmarks_specification.__common__.github import (
     update_comment_if_needed,
     create_new_pr_comment,
     check_github_available_and_actionable,
+    check_regression_comment,
 )
 from redis_benchmarks_specification.__compare__.args import create_compare_arguments
 
@@ -165,45 +166,9 @@ def compare_command_logic(args, project_name, project_version):
         username=args.redistimeseries_user,
     )
     rts.ping()
-    default_baseline_branch = None
-    default_metrics_str = ""
-    if args.defaults_filename != "" and os.path.exists(args.defaults_filename):
-        logging.info(
-            "Loading configuration from defaults file: {}".format(
-                args.defaults_filename
-            )
-        )
-        with open(args.defaults_filename) as yaml_fd:
-            defaults_dict = yaml.safe_load(yaml_fd)
-            if "exporter" in defaults_dict:
-                exporter_dict = defaults_dict["exporter"]
-                if "comparison" in exporter_dict:
-                    comparison_dict = exporter_dict["comparison"]
-                    if "metrics" in comparison_dict:
-                        metrics = comparison_dict["metrics"]
-                        logging.info("Detected defaults metrics info. reading metrics")
-                        default_metrics = []
-
-                        for metric in metrics:
-                            if metric.startswith("$."):
-                                metric = metric[2:]
-                            logging.info("Will use metric: {}".format(metric))
-                            default_metrics.append(metric)
-                        if len(default_metrics) == 1:
-                            default_metrics_str = default_metrics[0]
-                        if len(default_metrics) > 1:
-                            default_metrics_str = "({})".format(
-                                ",".join(default_metrics)
-                            )
-                        logging.info("Default metrics: {}".format(default_metrics_str))
-
-                    if "baseline-branch" in comparison_dict:
-                        default_baseline_branch = comparison_dict["baseline-branch"]
-                        logging.info(
-                            "Detected baseline branch in defaults file. {}".format(
-                                default_baseline_branch
-                            )
-                        )
+    default_baseline_branch, default_metrics_str = extract_default_branch_and_metric(
+        args.defaults_filename
+    )
 
     tf_github_org = args.github_org
     tf_github_repo = args.github_repo
@@ -300,25 +265,7 @@ def compare_command_logic(args, project_name, project_version):
     ) = check_github_available_and_actionable(
         fn, github_token, pull_request, tf_github_org, tf_github_repo, verbose
     )
-
-    grafana_dashboards_uids = {
-        "redisgraph": "SH9_rQYGz",
-        "redisbloom": "q4-5sRR7k",
-        "redisearch": "3Ejv2wZnk",
-        "redisjson": "UErSC0jGk",
-        "redistimeseries": "2WMw61UGz",
-    }
-    uid = None
-    if tf_github_repo.lower() in grafana_dashboards_uids:
-        uid = grafana_dashboards_uids[tf_github_repo.lower()]
-    grafana_link_base = None
-    if uid is not None:
-        grafana_link_base = "{}/{}".format(grafana_base_dashboard, uid)
-        logging.info(
-            "There is a grafana dashboard for this repo. Base link: {}".format(
-                grafana_link_base
-            )
-        )
+    grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks"
 
     (
         detected_regressions,
@@ -358,7 +305,71 @@ def compare_command_logic(args, project_name, project_version):
         use_metric_context_path,
         running_platform,
     )
-    comment_body = ""
+    prepare_regression_comment(
+        auto_approve,
+        baseline_branch,
+        baseline_tag,
+        comparison_branch,
+        comparison_tag,
+        contains_regression_comment,
+        github_pr,
+        grafana_link_base,
+        is_actionable_pr,
+        old_regression_comment_body,
+        pr_link,
+        regression_comment,
+        rts,
+        running_platform,
+        table_output,
+        tf_github_org,
+        tf_github_repo,
+        tf_triggering_env,
+        total_comparison_points,
+        total_improvements,
+        total_regressions,
+        total_stable,
+        total_unstable,
+        verbose,
+        args.regressions_percent_lower_limit,
+    )
+    return (
+        detected_regressions,
+        "",
+        total_improvements,
+        total_regressions,
+        total_stable,
+        total_unstable,
+        total_comparison_points,
+    )
+
+
+def prepare_regression_comment(
+    auto_approve,
+    baseline_branch,
+    baseline_tag,
+    comparison_branch,
+    comparison_tag,
+    contains_regression_comment,
+    github_pr,
+    grafana_link_base,
+    is_actionable_pr,
+    old_regression_comment_body,
+    pr_link,
+    regression_comment,
+    rts,
+    running_platform,
+    table_output,
+    tf_github_org,
+    tf_github_repo,
+    tf_triggering_env,
+    total_comparison_points,
+    total_improvements,
+    total_regressions,
+    total_stable,
+    total_unstable,
+    verbose,
+    regressions_percent_lower_limit,
+):
     if total_comparison_points > 0:
         comment_body = "### Automated performance analysis summary\n\n"
         comment_body += "This comment was automatically generated given there is performance data available.\n\n"
@@ -386,7 +397,7 @@ def compare_command_logic(args, project_name, project_version):
             )
         if total_regressions > 0:
             comparison_summary += "- Detected a total of {} regressions bellow the regression water line {}.\n".format(
-                total_regressions, args.regressions_percent_lower_limit
+                total_regressions, regressions_percent_lower_limit
             )
 
         comment_body += comparison_summary
@@ -430,19 +441,6 @@ def compare_command_logic(args, project_name, project_version):
                     zset_project_pull_request, comparison_branch, res
                 )
             )
-            user_input = "n"
-            html_url = "n/a"
-            (
-                baseline_str,
-                by_str_baseline,
-                comparison_str,
-                by_str_comparison,
-            ) = get_by_strings(
-                baseline_branch,
-                comparison_branch,
-                baseline_tag,
-                comparison_tag,
-            )
 
             if contains_regression_comment:
                 update_comment_if_needed(
@@ -457,26 +455,47 @@ def compare_command_logic(args, project_name, project_version):
 
     else:
         logging.error("There was no comparison points to produce a table...")
-    return (
-        detected_regressions,
-        comment_body,
-        total_improvements,
-        total_regressions,
-        total_stable,
-        total_unstable,
-        total_comparison_points,
-    )
 
 
-def check_regression_comment(comments):
-    res = False
-    pos = -1
-    for n, comment in enumerate(comments):
-        body = comment.body
-        if "Comparison between" in body and "Time Period from" in body:
-            res = True
-            pos = n
-    return res, pos
+def extract_default_branch_and_metric(defaults_filename):
+    default_baseline_branch = None
+    default_metrics_str = ""
+    if defaults_filename != "" and os.path.exists(defaults_filename):
+        logging.info(
+            "Loading configuration from defaults file: {}".format(defaults_filename)
+        )
+        with open(defaults_filename) as yaml_fd:
+            defaults_dict = yaml.safe_load(yaml_fd)
+            if "exporter" in defaults_dict:
+                exporter_dict = defaults_dict["exporter"]
+                if "comparison" in exporter_dict:
+                    comparison_dict = exporter_dict["comparison"]
+                    if "metrics" in comparison_dict:
+                        metrics = comparison_dict["metrics"]
+                        logging.info("Detected defaults metrics info. reading metrics")
+                        default_metrics = []
+
+                        for metric in metrics:
+                            if metric.startswith("$."):
+                                metric = metric[2:]
+                            logging.info("Will use metric: {}".format(metric))
+                            default_metrics.append(metric)
+                        if len(default_metrics) == 1:
+                            default_metrics_str = default_metrics[0]
+                        if len(default_metrics) > 1:
+                            default_metrics_str = "({})".format(
+                                ",".join(default_metrics)
+                            )
+                        logging.info("Default metrics: {}".format(default_metrics_str))
+
+                    if "baseline-branch" in comparison_dict:
+                        default_baseline_branch = comparison_dict["baseline-branch"]
+                        logging.info(
+                            "Detected baseline branch in defaults file. {}".format(
+                                default_baseline_branch
+                            )
+                        )
+    return default_baseline_branch, default_metrics_str
 
 
 def compute_regression_table(

{redis_benchmarks_specification-0.1.94 → redis_benchmarks_specification-0.1.96}/redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -1,4 +1,5 @@
 import argparse
+import datetime
 import os
 from redis_benchmarks_specification.__common__.env import (
     MACHINE_CPU_COUNT,
@@ -19,6 +20,10 @@ from redis_benchmarks_specification.__common__.env import (
     PROFILERS_DEFAULT,
     ALLOWED_PROFILERS,
 )
+from redis_benchmarks_specification.__compare__.args import (
+    START_TIME_NOW_UTC,
+    START_TIME_LAST_SIX_MONTHS_UTC,
+)
 
 PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)