redis-benchmarks-specification 0.1.79__tar.gz → 0.1.80__tar.gz

{redis_benchmarks_specification-0.1.79 → redis_benchmarks_specification-0.1.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.79
+Version: 0.1.80
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.79 → redis_benchmarks_specification-0.1.80}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.79"
+version = "0.1.80"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.79 → redis_benchmarks_specification-0.1.80}/redis_benchmarks_specification/__common__/github.py

@@ -90,6 +90,7 @@ def check_github_available_and_actionable(
 
 
 def create_new_pr_comment(auto_approve, comment_body, github_pr, pr_link):
+    regression_comment = None
     user_input = "n"
     if auto_approve:
         print("auto approving...")
@@ -100,6 +101,7 @@ def create_new_pr_comment(auto_approve, comment_body, github_pr, pr_link):
         regression_comment = github_pr.create_comment(comment_body)
         html_url = regression_comment.html_url
         print("created comment. Access it via {}".format(html_url))
+    return regression_comment
 
 
 def update_comment_if_needed(
@@ -141,3 +143,60 @@ def update_comment_if_needed(
             print(
                 "Updated comment. Access it via {}".format(regression_comment.html_url)
             )
+
+
+def check_benchmark_build_comment(comments):
+    res = False
+    pos = -1
+    for n, comment in enumerate(comments):
+        body = comment.body
+        if "CE Performance Automation : step" in body:
+            res = True
+            pos = n
+    return res, pos
+
+
+def check_benchmark_running_comment(comments):
+    res = False
+    pos = -1
+    for n, comment in enumerate(comments):
+        body = comment.body
+        if "CE Performance Automation : step" in body:
+            res = True
+            pos = n
+    return res, pos
+
+
+def markdown_progress_bar(current, total, bar_length=40):
+    progress = current / total
+    block = int(round(bar_length * progress))
+    bar = "#" * block + "-" * (bar_length - block)
+    percentage = round(progress * 100, 2)
+    return f"[{bar}] {percentage}%"
+
+
+def generate_benchmark_started_pr_comment(
+    benchmark_stream_ids,
+    total_pending,
+    total_benchmarks,
+):
+    comment_body = (
+        "### CE Performance Automation : step 2 of 2 (benchmark) RUNNING...\n\n"
+    )
+    comment_body += (
+        "This comment was automatically generated given a benchmark was triggered.\n\n"
+    )
+
+    completed = total_benchmarks - total_pending
+    comment_body += (
+        f"Status: {markdown_progress_bar(completed,total_benchmarks)} pending.\n\n"
+    )
+    comment_body += f"In total will run {total_benchmarks} benchmarks.\n"
+    comment_body += f"    - {total_pending} pending.\n"
+    comment_body += f"    - {total_pending} completed.\n"
+
+    for benchmark_stream_id in benchmark_stream_ids:
+        benchmark_stream_id = benchmark_stream_id.decode()
+        grafana_benchmark_status_link = f"https://benchmarksredisio.grafana.net/d/edsxdsrbexhc0f/ce-benchmark-run-status?orgId=1&var-benchmark_work_stream={benchmark_stream_id}"
+        comment_body += f"You can check a the status in detail via the [grafana link]({grafana_benchmark_status_link})"
+    return comment_body

{redis_benchmarks_specification-0.1.79 → redis_benchmarks_specification-0.1.80}/redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -1,5 +1,5 @@
 import argparse
-
+import os
 from redis_benchmarks_specification.__common__.env import (
     MACHINE_CPU_COUNT,
     SPECS_PATH_SETUPS,
@@ -20,6 +20,8 @@ from redis_benchmarks_specification.__common__.env import (
     ALLOWED_PROFILERS,
 )
 
+PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
+
 
 def create_self_contained_coordinator_args(project_name):
     parser = argparse.ArgumentParser(
@@ -30,6 +32,7 @@ def create_self_contained_coordinator_args(project_name):
     parser.add_argument("--event_stream_port", type=int, default=GH_REDIS_SERVER_PORT)
     parser.add_argument("--event_stream_pass", type=str, default=GH_REDIS_SERVER_AUTH)
     parser.add_argument("--event_stream_user", type=str, default=GH_REDIS_SERVER_USER)
+    parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
     parser.add_argument(
         "--cpu-count",
         type=int,

{redis_benchmarks_specification-0.1.79 → redis_benchmarks_specification-0.1.80}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -25,6 +25,13 @@ from redis_benchmarks_specification.__common__.env import (
     REDIS_SOCKET_TIMEOUT,
     REDIS_BINS_EXPIRE_SECS,
 )
+from redis_benchmarks_specification.__common__.github import (
+    check_github_available_and_actionable,
+    check_benchmark_running_comment,
+    update_comment_if_needed,
+    create_new_pr_comment,
+    generate_benchmark_started_pr_comment,
+)
 from redis_benchmarks_specification.__common__.package import (
     get_version_string,
     populate_with_poetry_data,
@@ -219,6 +226,11 @@ def main():
     arch = args.arch
     logging.info("Running for arch: {}".format(arch))
 
+    # Github token
+    github_token = args.github_token
+    if github_token is not None:
+        logging.info("Detected GITHUB token. will push PR comments with updates")
+
     # Docker air gap usage
     docker_air_gap = args.docker_air_gap
     if docker_air_gap:
@@ -270,6 +282,7 @@ def main():
             override_memtier_test_time,
             default_metrics,
             arch,
+            github_token,
         )
 
 
@@ -293,6 +306,7 @@ def self_contained_coordinator_blocking_read(
     override_test_time=None,
     default_metrics=None,
     arch="amd64",
+    github_token=None,
 ):
     num_process_streams = 0
     num_process_test_suites = 0
@@ -339,6 +353,7 @@ def self_contained_coordinator_blocking_read(
             None,
             default_metrics,
             arch,
+            github_token,
         )
         num_process_streams = num_process_streams + 1
         num_process_test_suites = num_process_test_suites + total_test_suite_runs
@@ -411,10 +426,20 @@ def process_self_contained_coordinator_stream(
     override_test_time=None,
     default_metrics=[],
     arch="amd64",
+    github_token=None,
 ):
     stream_id = "n/a"
     overall_result = False
     total_test_suite_runs = 0
+    # github updates
+    is_actionable_pr = False
+    contains_regression_comment = False
+    github_pr = None
+    old_regression_comment_body = ""
+    pr_link = ""
+    regression_comment = None
+    pull_request = None
+    auto_approve_github = True
     try:
         stream_id, testDetails = newTestInfo[0][1][0]
         stream_id = stream_id.decode()
@@ -434,6 +459,24 @@ def process_self_contained_coordinator_stream(
                 run_arch,
             ) = extract_build_info_from_streamdata(testDetails)
 
+            if b"pull_request" in testDetails:
+                pull_request = testDetails[b"pull_request"].decode()
+                logging.info(
+                    f"detected a pull_request definition on the streamdata {pull_request}"
+                )
+                verbose = True
+                fn = check_benchmark_running_comment
+                (
+                    contains_regression_comment,
+                    github_pr,
+                    is_actionable_pr,
+                    old_regression_comment_body,
+                    pr_link,
+                    regression_comment,
+                ) = check_github_available_and_actionable(
+                    fn, github_token, pull_request, "redis", "redis", verbose
+                )
+
             test_regexp = ".*"
             if b"test_regexp" in testDetails:
                 test_regexp = testDetails[b"test_regexp"]
@@ -529,6 +572,24 @@ def process_self_contained_coordinator_stream(
                             f"Added test named {test_name} to the pending test list in key {stream_test_list_pending}"
                         )
                     filtered_test_files.append(test_file)
+                pending_tests = len(filtered_test_files)
+                comment_body = generate_benchmark_started_pr_comment(
+                    stream_id, pending_tests, len(filtered_test_files)
+                )
+                # update on github if needed
+                if is_actionable_pr:
+                    if contains_regression_comment:
+                        update_comment_if_needed(
+                            auto_approve_github,
+                            comment_body,
+                            old_regression_comment_body,
+                            regression_comment,
+                            verbose,
+                        )
+                    else:
+                        regression_comment = create_new_pr_comment(
+                            auto_approve_github, comment_body, github_pr, pr_link
+                        )
 
                 for test_file in filtered_test_files:
                     redis_containers = []
@@ -1033,6 +1094,23 @@ def process_self_contained_coordinator_stream(
                     conn.lrem(stream_test_list_running, 1, test_name)
                     conn.lpush(stream_test_list_completed, test_name)
                     conn.expire(stream_test_list_completed, REDIS_BINS_EXPIRE_SECS)
+                    pending_tests = pending_tests - 1
+
+                    # update on github if needed
+                    if is_actionable_pr:
+                        comment_body = generate_benchmark_started_pr_comment(
+                            stream_id, pending_tests, len(filtered_test_files)
+                        )
+                        update_comment_if_needed(
+                            auto_approve_github,
+                            comment_body,
+                            old_regression_comment_body,
+                            regression_comment,
+                            verbose,
+                        )
+                        logging.info(
+                            f"Updated github comment with latest test info {regression_comment.html_url}"
+                        )
                     logging.info(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"
                     )