redis-benchmarks-specification 0.1.78__tar.gz → 0.1.79__tar.gz

{redis_benchmarks_specification-0.1.78 → redis_benchmarks_specification-0.1.79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.78
+Version: 0.1.79
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com
@@ -9,7 +9,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: Flask (>=2.0.3,<3.0.0)
 Requires-Dist: Flask-HTTPAuth (>=4.4.0,<5.0.0)
 Requires-Dist: GitPython (>=3.1.20,<4.0.0)

{redis_benchmarks_specification-0.1.78 → redis_benchmarks_specification-0.1.79}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.78"
+version = "0.1.79"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.78 → redis_benchmarks_specification-0.1.79}/redis_benchmarks_specification/__api__/app.py

@@ -86,11 +86,13 @@ def create_app(conn, user, test_config=None):
             event_type = "Ignored event from webhook"
             use_event = False
             # Pull request labeled
+            pull_request_number = None
             trigger_label = PULL_REQUEST_TRIGGER_LABEL
             if "pull_request" in request_data:
                 action = request_data["action"]
                 if should_action(action):
                     pull_request_dict = request_data["pull_request"]
+
                     head_dict = pull_request_dict["head"]
                     repo_dict = head_dict["repo"]
                     labels = []
@@ -107,9 +109,8 @@ def create_app(conn, user, test_config=None):
                         label_name = label["name"]
                         if trigger_label == label_name:
                             use_event = True
-                            event_type = "Pull request labeled with '{}'".format(
-                                trigger_label
-                            )
+                            pull_request_number = request_data["number"]
+                            event_type = f"Pull request #{pull_request_number} labeled with '{trigger_label}'"
                             detected_label = True
                     if detected_label is False:
                         app.logger.info(
@@ -161,6 +162,8 @@ def create_app(conn, user, test_config=None):
                     "gh_repo": gh_repo,
                     "gh_org": gh_org,
                 }
+                if pull_request_number is not None:
+                    fields_after["pull_request_number"] = pull_request_number
                 app.logger.info(
                     "Using event {} to trigger benchmark. final fields: {}".format(
                         event_type, fields_after
@@ -174,7 +177,6 @@ def create_app(conn, user, test_config=None):
                         event_type, response_data
                     )
                 )
-
             else:
                 app.logger.info(
                     "{}. input json was: {}".format(event_type, request_data)

{redis_benchmarks_specification-0.1.78 → redis_benchmarks_specification-0.1.79}/redis_benchmarks_specification/__builder__/builder.py

@@ -31,11 +31,19 @@ from redis_benchmarks_specification.__common__.env import (
     REDIS_SOCKET_TIMEOUT,
     REDIS_BINS_EXPIRE_SECS,
 )
+from redis_benchmarks_specification.__common__.github import (
+    check_github_available_and_actionable,
+    generate_build_finished_pr_comment,
+    update_comment_if_needed,
+    create_new_pr_comment,
+)
 from redis_benchmarks_specification.__common__.package import (
     populate_with_poetry_data,
     get_version_string,
 )
 
+PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
+
 
 class ZipFileWithPermissions(ZipFile):
     def _extract_member(self, member, targetpath, pwd):
@@ -80,6 +88,8 @@ def main():
         action="store_true",
         help="Store the docker images in redis keys.",
     )
+    parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
+    parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="")
     args = parser.parse_args()
     if args.logname is not None:
         print("Writting log to {}".format(args.logname))
@@ -138,7 +148,8 @@ def main():
     build_spec_image_prefetch(builders_folder, different_build_specs)
 
     builder_consumer_group_create(conn)
-
+    if args.github_token is not None:
+        logging.info("detected a github token. will update as much as possible!!! =)")
     previous_id = args.consumer_start_id
     while True:
         previous_id, new_builds_count, _ = builder_process_stream(
@@ -148,6 +159,7 @@ def main():
             previous_id,
             args.docker_air_gap,
             arch,
+            args.github_token,
         )
 
 
@@ -172,6 +184,17 @@ def builder_consumer_group_create(conn, id="$"):
         )
 
 
+def check_benchmark_build_comment(comments):
+    res = False
+    pos = -1
+    for n, comment in enumerate(comments):
+        body = comment.body
+        if "CE Performance Automation : step 1 of 2" in body:
+            res = True
+            pos = n
+    return res, pos
+
+
 def builder_process_stream(
     builders_folder,
     conn,
@@ -179,8 +202,10 @@ def builder_process_stream(
     previous_id,
     docker_air_gap=False,
     arch="amd64",
+    github_token=None,
 ):
     new_builds_count = 0
+    auto_approve_github_comments = True
     build_stream_fields_arr = []
     logging.info("Entering blocking read waiting for work.")
     consumer_name = "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1")
@@ -216,6 +241,12 @@ def builder_process_stream(
             buffer = conn.get(binary_zip_key)
             git_timestamp_ms = None
             use_git_timestamp = False
+            commit_datetime = "n/a"
+            if b"commit_datetime" in testDetails:
+                commit_datetime = int(testDetails[b"commit_datetime"].decode())
+            commit_summary = "n/a"
+            if b"commit_summary" in testDetails:
+                commit_summary = int(testDetails[b"commit_summary"].decode())
             git_branch, git_version = get_branch_version_from_test_details(testDetails)
             if b"use_git_timestamp" in testDetails:
                 use_git_timestamp = bool(testDetails[b"use_git_timestamp"])
@@ -238,6 +269,30 @@ def builder_process_stream(
             if b"tests_groups_regexp" in testDetails:
                 tests_groups_regexp = testDetails[b"tests_groups_regexp"].decode()
 
+            # github updates
+            is_actionable_pr = False
+            contains_regression_comment = False
+            github_pr = None
+            old_regression_comment_body = ""
+            pr_link = ""
+            regression_comment = ""
+            pull_request = None
+            if b"pull_request" in testDetails:
+                pull_request = testDetails[b"pull_request"].decode()
+                logging.info(f"Detected PR info in builder. PR: {pull_request}")
+                verbose = True
+
+                fn = check_benchmark_build_comment
+                (
+                    contains_regression_comment,
+                    github_pr,
+                    is_actionable_pr,
+                    old_regression_comment_body,
+                    pr_link,
+                    regression_comment,
+                ) = check_github_available_and_actionable(
+                    fn, github_token, pull_request, "redis", "redis", verbose
+                )
             for build_spec in different_build_specs:
                 build_config, id = get_build_config(builders_folder + "/" + build_spec)
                 build_config_metadata = get_build_config_metadata(build_config)
@@ -355,6 +410,8 @@ def builder_process_stream(
                     "tests_priority_lower_limit": tests_priority_lower_limit,
                     "tests_groups_regexp": tests_groups_regexp,
                 }
+                if pull_request is not None:
+                    build_stream_fields["pull_request"] = pull_request
                 if git_branch is not None:
                     build_stream_fields["git_branch"] = git_branch
                 if git_version is not None:
@@ -398,6 +455,34 @@ def builder_process_stream(
                     logging.info(
                         f"Adding information of build->benchmark stream info in list {builder_list_completed}. Adding benchmark stream id: {benchmark_stream_id_decoded}"
                     )
+                    benchmark_stream_ids = [benchmark_stream_id_decoded]
+                    comment_body = generate_build_finished_pr_comment(
+                        benchmark_stream_ids,
+                        commit_datetime,
+                        commit_summary,
+                        git_branch,
+                        git_hash,
+                        tests_groups_regexp,
+                        tests_priority_lower_limit,
+                        tests_priority_upper_limit,
+                        tests_regexp,
+                    )
+                    if is_actionable_pr:
+                        if contains_regression_comment:
+                            update_comment_if_needed(
+                                auto_approve_github_comments,
+                                comment_body,
+                                old_regression_comment_body,
+                                regression_comment,
+                                verbose,
+                            )
+                        else:
+                            create_new_pr_comment(
+                                auto_approve_github_comments,
+                                comment_body,
+                                github_pr,
+                                pr_link,
+                            )
                 shutil.rmtree(temporary_dir, ignore_errors=True)
                 new_builds_count = new_builds_count + 1
                 build_stream_fields_arr.append(build_stream_fields)

{redis_benchmarks_specification-0.1.78 → redis_benchmarks_specification-0.1.79}/redis_benchmarks_specification/__cli__/args.py

@@ -4,7 +4,7 @@
 #  All rights reserved.
 #
 import datetime
-
+import os
 
 from redis_benchmarks_specification.__common__.env import (
     GH_REDIS_SERVER_HOST,
@@ -21,6 +21,7 @@ START_TIME_NOW_UTC, _, _ = get_start_time_vars()
 START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
 CLI_TOOL_STATS = "stats"
 CLI_TOOL_TRIGGER = "trigger"
+PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
 
 
 def spec_cli_args(parser):
@@ -69,6 +70,15 @@ def spec_cli_args(parser):
         action="store_true",
         help="Include modules statistics on commandstats.",
     )
+    parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
+    parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="")
+    parser.add_argument(
+        "--auto-approve",
+        required=False,
+        default=False,
+        action="store_true",
+        help="Skip interactive approval of changes to github before applying.",
+    )
     parser.add_argument("--summary-csv", type=str, default="")
     parser.add_argument("--group-csv", type=str, default="")
     parser.add_argument("--commands-json-file", type=str, default="./commands.json")

{redis_benchmarks_specification-0.1.78 → redis_benchmarks_specification-0.1.79}/redis_benchmarks_specification/__cli__/cli.py

@@ -17,8 +17,16 @@ import packaging
 import redis
 from packaging import version
 import time
+from github import Github
 
 
+from redis_benchmarks_specification.__common__.github import (
+    update_comment_if_needed,
+    create_new_pr_comment,
+    check_github_available_and_actionable,
+    generate_build_finished_pr_comment,
+)
+
 from redis_benchmarks_specification.__cli__.args import spec_cli_args
 from redis_benchmarks_specification.__cli__.stats import (
     generate_stats_cli_command_logic,
@@ -165,6 +173,28 @@ def get_repo(args):
     return redisDirPath, cleanUp
 
 
+def check_benchmark_run_comment(comments):
+    res = False
+    pos = -1
+    for n, comment in enumerate(comments):
+        body = comment.body
+        if "CE Performance Automation" in body and "Triggered a benchmark" in body:
+            res = True
+            pos = n
+    return res, pos
+
+
+def check_benchmark_build_comment(comments):
+    res = False
+    pos = -1
+    for n, comment in enumerate(comments):
+        body = comment.body
+        if "CE Performance Automation : step 1 of 2" in body:
+            res = True
+            pos = n
+    return res, pos
+
+
 def trigger_tests_cli_command_logic(args, project_name, project_version):
     logging.info(
         "Using: {project_name} {project_version}".format(
@@ -176,6 +206,23 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
         logging.error("You must specify either --use-tags or --use-branch flag")
         sys.exit(1)
 
+    github_token = args.github_token
+    pull_request = args.pull_request
+    verbose = True
+    auto_approve = args.auto_approve
+
+    fn = check_benchmark_build_comment
+    (
+        contains_regression_comment,
+        github_pr,
+        is_actionable_pr,
+        old_regression_comment_body,
+        pr_link,
+        regression_comment,
+    ) = check_github_available_and_actionable(
+        fn, github_token, pull_request, "redis", "redis", verbose
+    )
+
     redisDirPath, cleanUp = get_repo(args)
     repo = git.Repo(redisDirPath)
 
@@ -267,10 +314,20 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             )
             if args.platform:
                 commit_dict["platform"] = args.platform
-            commit_dict["tests_priority_upper_limit"] = args.tests_priority_upper_limit
-            commit_dict["tests_priority_lower_limit"] = args.tests_priority_lower_limit
-            commit_dict["tests_regexp"] = args.tests_regexp
-            commit_dict["tests_groups_regexp"] = args.tests_groups_regexp
+            tests_priority_upper_limit = args.tests_priority_upper_limit
+            tests_priority_lower_limit = args.tests_priority_lower_limit
+            tests_regexp = args.tests_regexp
+            tests_groups_regexp = args.tests_groups_regexp
+            commit_dict["tests_priority_upper_limit"] = tests_priority_upper_limit
+            commit_dict["tests_priority_lower_limit"] = tests_priority_lower_limit
+            commit_dict["tests_regexp"] = tests_regexp
+            commit_dict["tests_groups_regexp"] = tests_groups_regexp
+            git_hash = cdict["git_hash"]
+            git_branch = "n/a"
+            if "git_branch" in cdict:
+                git_branch = cdict["git_branch"]
+            commit_datetime = cdict["commit_datetime"]
+            commit_summary = cdict["commit_summary"]
             if result is True:
                 stream_id = "n/a"
                 if args.dry_run is False:
@@ -292,6 +349,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
                             reply_fields,
                         )
                     )
+
                     if args.wait_build is True:
                         decoded_stream_id = stream_id.decode()
                         builder_list_streams = (
@@ -352,6 +410,31 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
                             f"FINAL total of {len_list} already build benchmark stream ids for this build: {benchmark_stream_ids}"
                         )
 
+                        comment_body = generate_build_finished_pr_comment(
+                            benchmark_stream_ids,
+                            commit_datetime,
+                            commit_summary,
+                            git_branch,
+                            git_hash,
+                            tests_groups_regexp,
+                            tests_priority_lower_limit,
+                            tests_priority_upper_limit,
+                            tests_regexp,
+                        )
+                        if is_actionable_pr:
+                            if contains_regression_comment:
+                                update_comment_if_needed(
+                                    auto_approve,
+                                    comment_body,
+                                    old_regression_comment_body,
+                                    regression_comment,
+                                    verbose,
+                                )
+                            else:
+                                create_new_pr_comment(
+                                    auto_approve, comment_body, github_pr, pr_link
+                                )
+
                 else:
                     logging.info(
                         "DRY-RUN: build for commit: {}. Date: {} Full commited info: {}".format(

redis_benchmarks_specification-0.1.79/redis_benchmarks_specification/__common__/github.py

@@ -0,0 +1,143 @@
+import logging
+from github import Github
+
+
+def generate_build_finished_pr_comment(
+    benchmark_stream_ids,
+    commit_datetime,
+    commit_summary,
+    git_branch,
+    git_hash,
+    tests_groups_regexp,
+    tests_priority_lower_limit,
+    tests_priority_upper_limit,
+    tests_regexp,
+):
+    comment_body = "### CE Performance Automation : step 1 of 2 (build) done\n\n"
+    comment_body += (
+        "This comment was automatically generated given a benchmark was triggered.\n"
+    )
+    comment_body += "You can check each build/benchmark progress in grafana:\n"
+    comment_body += f"   - git hash: {git_hash}\n"
+    comment_body += f"   - git branch: {git_branch}\n"
+    comment_body += f"   - commit date and time: {commit_datetime}\n"
+    comment_body += f"   - commit summary: {commit_summary}\n"
+    comment_body += f"   - test filters:\n"
+    comment_body += (
+        f"       - command priority lower limit: {tests_priority_lower_limit}\n"
+    )
+    comment_body += (
+        f"       - command priority upper limit: {tests_priority_upper_limit}\n"
+    )
+    comment_body += f"       - test name regex: {tests_regexp}\n"
+    comment_body += f"       - command group regex: {tests_groups_regexp}\n\n"
+    for benchmark_stream_id in benchmark_stream_ids:
+        benchmark_stream_id = benchmark_stream_id.decode()
+        grafana_benchmark_status_link = f"https://benchmarksredisio.grafana.net/d/edsxdsrbexhc0f/ce-benchmark-run-status?orgId=1&var-benchmark_work_stream={benchmark_stream_id}"
+        print("=============================================================")
+        print(f"Check benchmark run status in: {grafana_benchmark_status_link}")
+        comment_body += f"You can check a comparison in detail via the [grafana link]({grafana_benchmark_status_link})"
+    return comment_body
+
+
+def check_github_available_and_actionable(
+    fn, github_token, pull_request, tf_github_org, tf_github_repo, verbose
+):
+    # using an access token
+    is_actionable_pr = False
+    contains_regression_comment = False
+    regression_comment = None
+    github_pr = None
+    old_regression_comment_body = ""
+    if github_token is not None:
+        logging.info("Detected github token")
+        g = Github(github_token)
+        if pull_request is not None and pull_request != "":
+            pull_request_n = int(pull_request)
+            github_pr = (
+                g.get_user(tf_github_org)
+                .get_repo(tf_github_repo)
+                .get_issue(pull_request_n)
+            )
+            comments = github_pr.get_comments()
+            pr_link = github_pr.html_url
+            logging.info("Working on github PR already: {}".format(pr_link))
+            is_actionable_pr = True
+            contains_regression_comment, pos = fn(comments)
+            if contains_regression_comment:
+                regression_comment = comments[pos]
+                old_regression_comment_body = regression_comment.body
+                logging.info(
+                    "Already contains PR comment. Link: {}".format(
+                        regression_comment.html_url
+                    )
+                )
+                if verbose:
+                    logging.info("Printing old PR comment:")
+                    print("".join(["-" for x in range(1, 80)]))
+                    print(regression_comment.body)
+                    print("".join(["-" for x in range(1, 80)]))
+            else:
+                logging.info("Does not contain PR comment")
+    return (
+        contains_regression_comment,
+        github_pr,
+        is_actionable_pr,
+        old_regression_comment_body,
+        pr_link,
+        regression_comment,
+    )
+
+
+def create_new_pr_comment(auto_approve, comment_body, github_pr, pr_link):
+    user_input = "n"
+    if auto_approve:
+        print("auto approving...")
+    else:
+        user_input = input("Do you wish to add a comment in {} (y/n): ".format(pr_link))
+    if user_input.lower() == "y" or auto_approve:
+        print("creating an comment in PR {}".format(pr_link))
+        regression_comment = github_pr.create_comment(comment_body)
+        html_url = regression_comment.html_url
+        print("created comment. Access it via {}".format(html_url))
+
+
+def update_comment_if_needed(
+    auto_approve, comment_body, old_regression_comment_body, regression_comment, verbose
+):
+    same_comment = False
+    user_input = "n"
+    if comment_body == old_regression_comment_body:
+        logging.info(
+            "The old regression comment is the same as the new comment. skipping..."
+        )
+        same_comment = True
+    else:
+        logging.info(
+            "The old regression comment is different from the new comment. updating it..."
+        )
+        comment_body_arr = comment_body.split("\n")
+        old_regression_comment_body_arr = old_regression_comment_body.split("\n")
+        if verbose:
+            DF = [
+                x for x in comment_body_arr if x not in old_regression_comment_body_arr
+            ]
+            print("---------------------")
+            print(DF)
+            print("---------------------")
+    if same_comment is False:
+        if auto_approve:
+            print("auto approving...")
+        else:
+            user_input = input(
+                "Do you wish to update the comment {} (y/n): ".format(
+                    regression_comment.html_url
+                )
+            )
+        if user_input.lower() == "y" or auto_approve:
+            print("Updating comment {}".format(regression_comment.html_url))
+            regression_comment.edit(comment_body)
+            html_url = regression_comment.html_url
+            print(
+                "Updated comment. Access it via {}".format(regression_comment.html_url)
+            )