redis-benchmarks-specification 0.1.74__tar.gz → 0.1.76__tar.gz

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.74
+Version: 0.1.76
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com
@@ -9,13 +9,14 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: Flask (>=2.0.3,<3.0.0)
 Requires-Dist: Flask-HTTPAuth (>=4.4.0,<5.0.0)
 Requires-Dist: GitPython (>=3.1.20,<4.0.0)
 Requires-Dist: PyGithub (>=1.55,<2.0)
 Requires-Dist: PyYAML (>=6.0,<7.0)
 Requires-Dist: argparse (>=1.4.0,<2.0.0)
-Requires-Dist: docker (>=5.0.0,<6.0.0)
+Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: flask-restx (>=0.5.0,<0.6.0)
 Requires-Dist: marshmallow (>=3.12.2,<4.0.0)
 Requires-Dist: node-semver (>=0.8.1,<0.9.0)

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.74"
+version = "0.1.76"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"
@@ -14,7 +14,7 @@ marshmallow = "^3.12.2"
 argparse = "^1.4.0"
 Flask-HTTPAuth = "^4.4.0"
 PyYAML = "^6.0"
-docker = "^5.0.0"
+docker = "^7.1.0"
 redisbench-admin = "^0.9.23"
 psutil = "^5.9.4"
 PyGithub = "^1.55"
@@ -29,8 +29,7 @@ pytest = "^7.2.0"
 pytest-cov = "^4.0.0"
 black = "22.10.0"
 flake8 = "^6.0.0"
-tox-poetry-installer = {extras = ["poetry"], version = "^0.10.2"}
-tox-docker = {extras = ["poetry"], version = "^3.1.0"}
+tox-poetry-installer = {extras = ["poetry"], version = "^0.10.3"}
 
 [build-system]
 requires = ["poetry_core>=1.0.0"]

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/redis_benchmarks_specification/__builder__/builder.py

@@ -141,7 +141,7 @@ def main():
 
     previous_id = args.consumer_start_id
     while True:
-        previous_id, new_builds_count = builder_process_stream(
+        previous_id, new_builds_count, _ = builder_process_stream(
             builders_folder,
             conn,
             different_build_specs,
@@ -181,6 +181,7 @@ def builder_process_stream(
     arch="amd64",
 ):
     new_builds_count = 0
+    build_stream_fields_arr = []
     logging.info("Entering blocking read waiting for work.")
     consumer_name = "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1")
     newTestInfo = conn.xreadgroup(
@@ -196,6 +197,7 @@ def builder_process_stream(
     else:
         streamId, testDetails = newTestInfo[0][1][0]
         logging.info("Received work . Stream id {}.".format(streamId))
+        conn.lpush("benchmarks:{streamId}")
         # commit = None
         # commited_date = ""
         # tag = ""
@@ -220,6 +222,22 @@ def builder_process_stream(
                 use_git_timestamp = bool(testDetails[b"use_git_timestamp"])
             if b"git_timestamp_ms" in testDetails:
                 git_timestamp_ms = int(testDetails[b"git_timestamp_ms"].decode())
+            tests_regexp = ".*"
+            if b"tests_regexp" in testDetails:
+                tests_regexp = testDetails[b"tests_regexp"].decode()
+            tests_priority_upper_limit = 10000
+            if b"tests_priority_upper_limit" in testDetails:
+                tests_priority_upper_limit = int(
+                    testDetails[b"tests_priority_upper_limit"].decode()
+                )
+            tests_priority_lower_limit = 0
+            if b"tests_priority_lower_limit" in testDetails:
+                tests_priority_lower_limit = int(
+                    testDetails[b"tests_priority_lower_limit"].decode()
+                )
+            tests_groups_regexp = ".*"
+            if b"tests_groups_regexp" in testDetails:
+                tests_groups_regexp = testDetails[b"tests_groups_regexp"].decode()
 
             for build_spec in different_build_specs:
                 build_config, id = get_build_config(builders_folder + "/" + build_spec)
@@ -333,11 +351,18 @@ def builder_process_stream(
                     "build_command": build_command,
                     "metadata": json.dumps(build_config_metadata),
                     "build_artifacts": ",".join(build_artifacts),
+                    "tests_regexp": tests_regexp,
+                    "tests_priority_upper_limit": tests_priority_upper_limit,
+                    "tests_priority_lower_limit": tests_priority_lower_limit,
+                    "tests_groups_regexp": tests_groups_regexp,
                 }
                 if git_branch is not None:
                     build_stream_fields["git_branch"] = git_branch
                 if git_version is not None:
                     build_stream_fields["git_version"] = git_version
+                if git_timestamp_ms is not None:
+                    build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
+
                 if git_timestamp_ms is not None:
                     build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
                 for artifact in build_artifacts:
@@ -356,16 +381,23 @@ def builder_process_stream(
                 if b"platform" in testDetails:
                     build_stream_fields["platform"] = testDetails[b"platform"]
                 if result is True:
-                    stream_id = conn.xadd(
+                    benchmark_stream_id = conn.xadd(
                         STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields
                     )
                     logging.info(
                         "sucessfully built build variant {} for redis git_sha {}. Stream id: {}".format(
-                            id, git_hash, stream_id
+                            id, git_hash, benchmark_stream_id
                         )
                     )
+                    builder_list_completed = f"builder:{streamId}:builds_completed"
+                    conn.lpush(builder_list_completed, benchmark_stream_id)
+                    conn.expire(builder_list_completed, REDIS_BINS_EXPIRE_SECS)
+                    logging.info(
+                        f"Adding information of build->benchmark stream info in list {builder_list_completed}. Adding benchmark stream id: {benchmark_stream_id}"
+                    )
                 shutil.rmtree(temporary_dir, ignore_errors=True)
                 new_builds_count = new_builds_count + 1
+                build_stream_fields_arr.append(build_stream_fields)
             ack_reply = conn.xack(
                 STREAM_KEYNAME_GH_EVENTS_COMMIT,
                 STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
@@ -387,7 +419,7 @@ def builder_process_stream(
                 )
         else:
             logging.error("Missing commit information within received message.")
-    return previous_id, new_builds_count
+    return previous_id, new_builds_count, build_stream_fields_arr
 
 
 def build_spec_image_prefetch(builders_folder, different_build_specs):

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/redis_benchmarks_specification/__cli__/args.py

@@ -30,6 +30,30 @@ def spec_cli_args(parser):
         default=SPECS_PATH_TEST_SUITES,
         help="Test suites folder, containing the different test variations",
     )
+    parser.add_argument(
+        "--tests-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter test names",
+    )
+    parser.add_argument(
+        "--tests-groups-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter test group names",
+    )
+    parser.add_argument(
+        "--tests-priority-lower-limit",
+        type=int,
+        default=0,
+        help="Run a subset of the tests based uppon a preset priority. By default runs all tests.",
+    )
+    parser.add_argument(
+        "--tests-priority-upper-limit",
+        type=int,
+        default=100000,
+        help="Run a subset of the tests based uppon a preset priority. By default runs all tests.",
+    )
     parser.add_argument(
         "--defaults_filename",
         type=str,
@@ -93,6 +117,8 @@ def spec_cli_args(parser):
         default=START_TIME_NOW_UTC,
     )
     parser.add_argument("--redis_repo", type=str, default=None)
+    parser.add_argument("--gh_org", type=str, default="redis")
+    parser.add_argument("--gh_repo", type=str, default="redis")
     parser.add_argument("--trigger-unstable-commits", type=bool, default=True)
     parser.add_argument(
         "--use-tags",
@@ -136,4 +162,16 @@ def spec_cli_args(parser):
         default="",
         help="Only trigger tests on the specified platform.",
     )
+    parser.add_argument(
+        "--wait-build",
+        default=False,
+        action="store_true",
+        help="Wait for build to be finished.",
+    )
+    parser.add_argument(
+        "--wait-build-timeout",
+        type=int,
+        default=-1,
+        help="Wait x sections for build. If -1, waits forever.",
+    )
     return parser

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/redis_benchmarks_specification/__cli__/cli.py

@@ -16,6 +16,7 @@ import git
 import packaging
 import redis
 from packaging import version
+import time
 
 
 from redis_benchmarks_specification.__cli__.args import spec_cli_args
@@ -26,7 +27,11 @@ from redis_benchmarks_specification.__common__.builder_schema import (
     get_commit_dict_from_sha,
     request_build_from_commit_info,
 )
-from redis_benchmarks_specification.__common__.env import REDIS_BINS_EXPIRE_SECS
+from redis_benchmarks_specification.__common__.env import (
+    REDIS_BINS_EXPIRE_SECS,
+    STREAM_KEYNAME_GH_EVENTS_COMMIT,
+    STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
+)
 from redis_benchmarks_specification.__common__.package import (
     get_version_string,
     populate_with_poetry_data,
@@ -142,14 +147,11 @@ def get_repo(args):
     if redisDirPath is None:
         cleanUp = True
         redisDirPath = tempfile.mkdtemp()
+        remote_url = f"https://github.com/{args.gh_org}/{args.gh_repo}"
         logging.info(
-            "Retrieving redis repo from remote into {}. Using branch {}.".format(
-                redisDirPath, args.branch
-            )
-        )
-        cmd = "git clone https://github.com/redis/redis {} --branch {}\n".format(
-            redisDirPath, args.branch
+            f"Retrieving redis repo from remote {remote_url} into {redisDirPath}. Using branch {args.branch}."
         )
+        cmd = f"git clone {remote_url} {redisDirPath} --branch {args.branch}\n"
         process = subprocess.Popen(
             "/bin/bash", stdin=subprocess.PIPE, stdout=subprocess.PIPE
         )
@@ -231,57 +233,135 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             )
             filtered_hash_commits.append(cdict)
 
-    if args.dry_run is False:
-        conn = redis.StrictRedis(
-            host=args.redis_host,
-            port=args.redis_port,
-            password=args.redis_pass,
-            username=args.redis_user,
-            decode_responses=False,
+    logging.info(
+        "Checking connection to redis with user: {}, host: {}, port: {}".format(
+            args.redis_user,
+            args.redis_host,
+            args.redis_port,
         )
-
-        for rep in range(0, 1):
-            for cdict in filtered_hash_commits:
-                (
-                    result,
-                    error_msg,
-                    commit_dict,
-                    _,
-                    binary_key,
-                    binary_value,
-                ) = get_commit_dict_from_sha(
-                    cdict["git_hash"], "redis", "redis", cdict, True, args.gh_token
-                )
-                if args.platform:
-                    commit_dict["platform"] = args.platform
-                if result is True:
-                    stream_id = "n/a"
-                    if args.dry_run is False:
-                        (
-                            result,
-                            reply_fields,
-                            error_msg,
-                        ) = request_build_from_commit_info(
-                            conn,
-                            commit_dict,
-                            {},
-                            binary_key,
-                            binary_value,
-                            REDIS_BINS_EXPIRE_SECS,
-                        )
-                        stream_id = reply_fields["id"]
+    )
+    conn = redis.StrictRedis(
+        host=args.redis_host,
+        port=args.redis_port,
+        password=args.redis_pass,
+        username=args.redis_user,
+        decode_responses=False,
+    )
+    conn.ping()
+    for rep in range(0, 1):
+        for cdict in filtered_hash_commits:
+            (
+                result,
+                error_msg,
+                commit_dict,
+                _,
+                binary_key,
+                binary_value,
+            ) = get_commit_dict_from_sha(
+                cdict["git_hash"],
+                args.gh_org,
+                args.gh_repo,
+                cdict,
+                True,
+                args.gh_token,
+            )
+            if args.platform:
+                commit_dict["platform"] = args.platform
+            commit_dict["tests_priority_upper_limit"] = args.tests_priority_upper_limit
+            commit_dict["tests_priority_lower_limit"] = args.tests_priority_lower_limit
+            commit_dict["tests_regexp"] = args.tests_regexp
+            commit_dict["tests_groups_regexp"] = args.tests_groups_regexp
+            if result is True:
+                stream_id = "n/a"
+                if args.dry_run is False:
+                    (result, reply_fields, error_msg,) = request_build_from_commit_info(
+                        conn,
+                        commit_dict,
+                        {},
+                        binary_key,
+                        binary_value,
+                        REDIS_BINS_EXPIRE_SECS,
+                    )
+                    stream_id = reply_fields["id"]
                     logging.info(
-                        "Successfully requested a build for commit: {}. Date: {} Request stream id: {}.".format(
+                        "Successfully requested a build for commit: {}. Date: {} Request stream id: {}. full commited info: {}. Reply fields: {}".format(
                             cdict["git_hash"],
                             cdict["commit_datetime"],
                             stream_id,
+                            commit_dict,
+                            reply_fields,
                         )
                     )
-                else:
-                    logging.error(error_msg)
+                    if args.wait_build is True:
+                        decoded_stream_id = stream_id.decode()
+                        builder_list_streams = (
+                            f"builder:{decoded_stream_id}:builds_completed"
+                        )
+                        len_list = 0
+                        stream_ack = False
+                        sleep_secs = 10
+                        benchmark_stream_ids = []
+                        while len_list == 0 or stream_ack is False:
 
-    else:
-        logging.info("Skipping actual work trigger ( dry-run )")
+                            logging.info(
+                                f"checking benchmark streams info in key: {builder_list_streams}"
+                            )
+                            benchmark_stream_ids = conn.lrange(
+                                builder_list_streams, 0, -1
+                            )
+                            len_list = len(benchmark_stream_ids)
+                            logging.info(
+                                f"There is a total of {len_list} already build benchmark stream ids for this build: {benchmark_stream_ids}"
+                            )
+
+                            if len_list > 0:
+                                pending_build_streams = conn.xpending_range(
+                                    STREAM_KEYNAME_GH_EVENTS_COMMIT,
+                                    STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
+                                    "-",
+                                    "+",
+                                    1000,
+                                )
+                                len_pending = len(pending_build_streams)
+                                logging.info(
+                                    f"There is a total of {len_pending} pending builds for stream {STREAM_KEYNAME_GH_EVENTS_COMMIT} and cg {STREAM_GH_EVENTS_COMMIT_BUILDERS_CG}. Checking for stream id: {stream_id}"
+                                )
+                                found_id = False
+                                for pending_try in pending_build_streams:
+                                    logging.info(f"pending entry: {pending_try}")
+                                    pending_id = pending_try["message_id"]
+                                    if stream_id == pending_id:
+                                        found_id = True
+                                        logging.info(
+                                            f"Found the stream id {stream_id} as part of pending entry list. Waiting for it to be ack."
+                                        )
+
+                                if found_id is True:
+                                    logging.info(
+                                        f"Sleeping for {sleep_secs} before checking pending list again."
+                                    )
+                                    time.sleep(sleep_secs)
+                                else:
+                                    stream_ack = True
+                            else:
+                                logging.info(
+                                    f"Sleeping for {sleep_secs} before checking builds again."
+                                )
+                                time.sleep(sleep_secs)
+                        logging.info(
+                            f"FINAL total of {len_list} already build benchmark stream ids for this build: {benchmark_stream_ids}"
+                        )
+
+                else:
+                    logging.info(
+                        "DRY-RUN: build for commit: {}. Date: {} Full commited info: {}".format(
+                            cdict["git_hash"],
+                            cdict["commit_datetime"],
+                            commit_dict,
+                        )
+                    )
+            else:
+                logging.error(error_msg)
     if cleanUp is True:
         logging.info("Removing temporary redis dir {}.".format(redisDirPath))
         shutil.rmtree(redisDirPath)

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/redis_benchmarks_specification/__compare__/compare.py

@@ -15,7 +15,6 @@ import datetime as dt
 import os
 from tqdm import tqdm
 from github import Github
-from slack_sdk.webhook import WebhookClient
 import argparse
 from redis_benchmarks_specification.__compare__.args import create_compare_arguments
 
@@ -282,9 +281,6 @@ def compare_command_logic(args, project_name, project_version):
     contains_regression_comment = False
     regression_comment = None
     github_pr = None
-    # slack related
-    webhook_notifications_active = False
-    webhook_client_slack = None
     if running_platform is not None:
         logging.info(
             "Using platform named: {} to do the comparison.\n\n".format(
@@ -455,7 +451,6 @@ def compare_command_logic(args, project_name, project_version):
             )
             user_input = "n"
             html_url = "n/a"
-            regression_count = len(detected_regressions)
             (
                 baseline_str,
                 by_str_baseline,
@@ -801,7 +796,6 @@ def from_rts_to_regression_table(
     total_comparison_points = 0
     noise_waterline = 3
     progress = tqdm(unit="benchmark time-series", total=len(test_names))
-    at_comparison = 0
     for test_name in test_names:
         multi_value_baseline = check_multi_value_filter(baseline_str)
         multi_value_comparison = check_multi_value_filter(comparison_str)

{redis_benchmarks_specification-0.1.74 → redis_benchmarks_specification-0.1.76}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -5,7 +5,7 @@ import pathlib
 import shutil
 import tempfile
 import traceback
-
+import re
 import docker
 import redis
 import os
@@ -23,6 +23,7 @@ from redis_benchmarks_specification.__common__.env import (
     LOG_LEVEL,
     REDIS_HEALTH_CHECK_INTERVAL,
     REDIS_SOCKET_TIMEOUT,
+    REDIS_BINS_EXPIRE_SECS,
 )
 from redis_benchmarks_specification.__common__.package import (
     get_version_string,
@@ -433,6 +434,13 @@ def process_self_contained_coordinator_stream(
                 run_arch,
             ) = extract_build_info_from_streamdata(testDetails)
 
+            test_regexp = ".*"
+            if b"test_regexp" in testDetails:
+                test_regexp = testDetails[b"test_regexp"]
+                logging.info(
+                    f"detected a regexp definition on the streamdata {test_regexp}"
+                )
+
             skip_test = False
             if b"platform" in testDetails:
                 platform = testDetails[b"platform"]
@@ -466,11 +474,37 @@ def process_self_contained_coordinator_stream(
                     images_loaded = docker_client.images.load(airgap_docker_image_bin)
                     logging.info("Successfully loaded images {}".format(images_loaded))
 
+                filtered_test_files = []
+
+                stream_test_list_pending = (
+                    f"{stream_id}:{running_platform}:tests_pending"
+                )
+                stream_test_list_running = (
+                    f"{stream_id}:{running_platform}:tests_running"
+                )
+                stream_test_list_completed = (
+                    f"{stream_id}:{running_platform}:tests_completed"
+                )
                 for test_file in testsuite_spec_files:
                     if defaults_filename in test_file:
                         continue
-                    redis_containers = []
-                    client_containers = []
+
+                    if test_regexp != ".*":
+                        logging.info(
+                            "Filtering all tests via a regular expression: {}".format(
+                                test_regexp
+                            )
+                        )
+                        tags_regex_string = re.compile(test_regexp)
+
+                        match_obj = re.search(tags_regex_string, test_file)
+                        if match_obj is None:
+                            logging.info(
+                                "Skipping {} given it does not match regex {}".format(
+                                    test_file, test_regexp
+                                )
+                            )
+                            continue
 
                     with open(test_file, "r") as stream:
                         (
@@ -485,6 +519,28 @@ def process_self_contained_coordinator_stream(
                                 )
                             )
                             continue
+                        conn.lpush(stream_test_list_pending, test_name)
+                        conn.expire(stream_test_list_pending, REDIS_BINS_EXPIRE_SECS)
+                        logging.info(
+                            f"Added test named {test_name} to the pending test list in key {stream_test_list_pending}"
+                        )
+                    filtered_test_files.append(test_file)
+
+                for test_file in filtered_test_files:
+                    redis_containers = []
+                    client_containers = []
+                    with open(test_file, "r") as stream:
+                        (
+                            _,
+                            benchmark_config,
+                            test_name,
+                        ) = get_final_benchmark_config(None, stream, "")
+                        conn.lrem(stream_test_list_pending, 1, test_name)
+                        conn.lpush(stream_test_list_running, test_name)
+                        conn.expire(stream_test_list_running, REDIS_BINS_EXPIRE_SECS)
+                        logging.info(
+                            f"Added test named {test_name} to the pending test list in key {stream_test_list_running}"
+                        )
                         (
                             _,
                             _,
@@ -970,6 +1026,12 @@ def process_self_contained_coordinator_stream(
 
                             overall_result &= test_result
 
+                    conn.lrem(stream_test_list_running, 1, test_name)
+                    conn.lpush(stream_test_list_completed, test_name)
+                    conn.expire(stream_test_list_completed, REDIS_BINS_EXPIRE_SECS)
+                    logging.info(
+                        f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"
+                    )
         else:
             logging.error("Missing commit information within received message.")
     except: