redis-benchmarks-specification 0.1.102__tar.gz → 0.1.150__tar.gz

{redis_benchmarks_specification-0.1.102 → redis_benchmarks_specification-0.1.150}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.102
+Version: 0.1.150
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com
-Requires-Python: >=3.8.1,<4.0
+Requires-Python: >=3.10.0,<4.0.0
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: Flask (>=2.0.3,<3.0.0)
 Requires-Dist: Flask-HTTPAuth (>=4.4.0,<5.0.0)
 Requires-Dist: GitPython (>=3.1.20,<4.0.0)
@@ -17,12 +17,15 @@ Requires-Dist: PyYAML (>=6.0,<7.0)
 Requires-Dist: argparse (>=1.4.0,<2.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: flask-restx (>=0.5.0,<0.6.0)
+Requires-Dist: jsonpath-ng (>=1.6.1,<2.0.0)
 Requires-Dist: marshmallow (>=3.12.2,<4.0.0)
 Requires-Dist: node-semver (>=0.8.1,<0.9.0)
+Requires-Dist: numpy (>=2.0.0,<3.0.0)
 Requires-Dist: oyaml (>=1.0,<2.0)
+Requires-Dist: pandas (>=2.1.2,<3.0.0)
 Requires-Dist: psutil (>=5.9.4,<6.0.0)
 Requires-Dist: redis (>=4.2.0,<5.0.0)
-Requires-Dist: redisbench-admin (>=0.9.23,<0.10.0)
+Requires-Dist: redisbench-admin (>=0.11.1,<0.12.0)
 Requires-Dist: semver (>=2.13.0,<3.0.0)
 Requires-Dist: typed-ast (>=1.5.0,<2.0.0)
 Description-Content-Type: text/markdown
@@ -418,6 +421,12 @@ To run a specific test:
 $ tox -- utils/tests/test_runner.py
 ```
 
+To run a specific test with verbose logging:
+
+```sh
+$ tox -- -vv --log-cli-level=INFO utils/tests/test_runner.py
+```
+
 ## License
 
 redis-benchmarks-specification is distributed under the BSD3 license - see [LICENSE](LICENSE)

{redis_benchmarks_specification-0.1.102 → redis_benchmarks_specification-0.1.150}/Readme.md

@@ -389,6 +389,12 @@ To run a specific test:
 $ tox -- utils/tests/test_runner.py
 ```
 
+To run a specific test with verbose logging:
+
+```sh
+$ tox -- -vv --log-cli-level=INFO utils/tests/test_runner.py
+```
+
 ## License
 
 redis-benchmarks-specification is distributed under the BSD3 license - see [LICENSE](LICENSE)

{redis_benchmarks_specification-0.1.102 → redis_benchmarks_specification-0.1.150}/pyproject.toml

@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.102"
+version = "0.1.150"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<4.0"
+python = "^3.10.0"
 Flask = "^2.0.3"
 flask-restx = "^0.5.0"
 redis = "^4.2.0"
@@ -15,7 +15,7 @@ argparse = "^1.4.0"
 Flask-HTTPAuth = "^4.4.0"
 PyYAML = "^6.0"
 docker = "^7.1.0"
-redisbench-admin = "^0.9.23"
+redisbench-admin = "^0.11.1"
 psutil = "^5.9.4"
 PyGithub = "^1.55"
 GitPython = "^3.1.20"
@@ -23,13 +23,33 @@ semver = "^2.13.0"
 node-semver = "^0.8.1"
 typed-ast = "^1.5.0"
 oyaml = "^1.0"
+pandas = "^2.1.2"
+numpy = "^2.0.0"
+jsonpath-ng = "^1.6.1"
 
 [tool.poetry.dev-dependencies]
-pytest = "^7.2.0"
-pytest-cov = "^4.0.0"
-black = "22.10.0"
-flake8 = "^6.0.0"
-tox-poetry-installer = {extras = ["poetry"], version = "^0.10.3"}
+click = "8.1.7"
+black = "24.4.2"
+flake8 = "7.1.0"
+flynt = "~0.69.0"
+isort = "5.10.1"
+mock = "4.0.3"
+packaging = ">=20.4"
+pytest = "8.2.2"
+pytest-timeout = "2.3.1"
+pytest-asyncio = ">=0.23.7"
+tox = ">=4.16.0"
+tox-docker = ">=5.0.0"
+invoke = "2.2.0"
+pytest-cov = ">=5.0.0"
+vulture = ">=2.3.0"
+ujson = ">=4.2.0"
+wheel = ">=0.43.0"
+docker = ">=7.1.0"
+
+[tool.poetry.group.dev.dependencies]
+#tox-poetry-installer = {extras = ["poetry"], version = "^0.10.3"}
+docker = "^7.1.0"
 
 [build-system]
 requires = ["poetry_core>=1.0.0"]

{redis_benchmarks_specification-0.1.102 → redis_benchmarks_specification-0.1.150}/redis_benchmarks_specification/__builder__/builder.py

@@ -234,6 +234,7 @@ def builder_process_stream(
         if b"git_hash" in testDetails:
             git_hash = testDetails[b"git_hash"]
             logging.info("Received commit hash specifier {}.".format(git_hash))
+            logging.info(f"Received the following build stream: {testDetails}.")
             binary_zip_key = testDetails[b"zip_archive_key"]
             logging.info(
                 "Retriving zipped source from key {}.".format(
@@ -245,10 +246,10 @@ def builder_process_stream(
             use_git_timestamp = False
             commit_datetime = "n/a"
             if b"commit_datetime" in testDetails:
-                commit_datetime = int(testDetails[b"commit_datetime"].decode())
+                commit_datetime = testDetails[b"commit_datetime"].decode()
             commit_summary = "n/a"
             if b"commit_summary" in testDetails:
-                commit_summary = int(testDetails[b"commit_summary"].decode())
+                commit_summary = testDetails[b"commit_summary"].decode()
             git_branch, git_version = get_branch_version_from_test_details(testDetails)
             if b"use_git_timestamp" in testDetails:
                 use_git_timestamp = bool(testDetails[b"use_git_timestamp"])
@@ -271,6 +272,16 @@ def builder_process_stream(
             if b"tests_groups_regexp" in testDetails:
                 tests_groups_regexp = testDetails[b"tests_groups_regexp"].decode()
 
+            github_org = "redis"
+            if b"github_org" in testDetails:
+                github_org = testDetails[b"github_org"].decode()
+                logging.info(f"detected github_org info on build stream {github_org}")
+
+            github_repo = "redis"
+            if b"github_repo" in testDetails:
+                github_repo = testDetails[b"github_repo"].decode()
+                logging.info(f"detected github_repo info on build stream {github_repo}")
+
             # github updates
             is_actionable_pr = False
             contains_regression_comment = False
@@ -312,28 +323,7 @@ def builder_process_stream(
                 if "run_image" in build_config:
                     run_image = build_config["run_image"]
                 if docker_air_gap:
-                    airgap_key = "docker:air-gap:{}".format(run_image)
-                    logging.info(
-                        "DOCKER AIR GAP: storing run image named: {} in redis key {}".format(
-                            run_image, airgap_key
-                        )
-                    )
-                    run_image_binary_stream = io.BytesIO()
-                    run_image_docker = docker_client.images.get(run_image)
-                    for chunk in run_image_docker.save():
-                        run_image_binary_stream.write(chunk)
-                    # 7 days expire
-                    binary_exp_secs = 24 * 60 * 60 * 7
-                    res_airgap = conn.set(
-                        airgap_key,
-                        run_image_binary_stream.getbuffer(),
-                        ex=binary_exp_secs,
-                    )
-                    logging.info(
-                        "DOCKER AIR GAP: result of set bin data to {}: {}".format(
-                            airgap_key, res_airgap
-                        )
-                    )
+                    store_airgap_image_redis(conn, docker_client, run_image)
 
                 compiler = build_config["compiler"]
                 cpp_compiler = build_config["cpp_compiler"]
@@ -342,6 +332,14 @@ def builder_process_stream(
                 build_artifacts = ["redis-server"]
                 if "build_artifacts" in build_config:
                     build_artifacts = build_config["build_artifacts"]
+                if b"build_artifacts" in testDetails:
+                    new_build_artifacts = (
+                        testDetails[b"build_artifacts"].decode().split(",")
+                    )
+                    logging.info(
+                        f"overriding default build artifacts {build_artifacts} by {new_build_artifacts}"
+                    )
+                    build_artifacts = new_build_artifacts
                 build_vars_str = ""
                 if "env" in build_config:
                     if build_config["env"] is not None:
@@ -364,22 +362,31 @@ def builder_process_stream(
                     "linenoise",
                     "lua",
                 ]
+                if "fast_float" in deps_dir:
+                    deps_list.append("fast_float")
                 if "hdr_histogram" in deps_dir:
                     deps_list.append("hdr_histogram")
                 if "fpconv" in deps_dir:
                     deps_list.append("fpconv")
                 redis_temporary_dir = temporary_dir + "/" + redis_dir + "/"
                 logging.info("Using redis temporary dir {}".format(redis_temporary_dir))
-                build_command = "bash -c 'make Makefile.dep && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j'".format(
-                    cpp_compiler,
-                    compiler,
-                    " ".join(deps_list),
-                    build_vars_str,
-                    cpp_compiler,
-                    compiler,
-                    "redis-server",
-                    build_vars_str,
-                )
+                # build_command = "bash -c 'make Makefile.dep && cd ./deps && CXX={} CC={} make {} {} -j && cd .. && CXX={} CC={} make {} {} -j'".format(
+                #     cpp_compiler,
+                #     compiler,
+                #     " ".join(deps_list),
+                #     build_vars_str,
+                #     cpp_compiler,
+                #     compiler,
+                #     "redis-server",
+                #     build_vars_str,
+                # )
+                build_command = "sh -c 'make -j'"
+                if b"build_command" in testDetails:
+                    build_command = testDetails[b"build_command"].decode()
+                server_name = "redis"
+                if b"server_name" in testDetails:
+                    server_name = testDetails[b"server_name"].decode()
+
                 build_start_datetime = datetime.datetime.utcnow()
                 logging.info(
                     "Using the following build command {}.".format(build_command)
@@ -429,48 +436,35 @@ def builder_process_stream(
                 build_duration = build_end_datetime - build_start_datetime
                 build_duration_secs = build_duration.total_seconds()
 
-                build_stream_fields = {
-                    "id": id,
-                    "git_hash": git_hash,
-                    "use_git_timestamp": str(use_git_timestamp),
-                    "build_image": build_image,
-                    "run_image": run_image,
-                    "compiler": compiler,
-                    "cpp_compiler": cpp_compiler,
-                    "os": build_os,
-                    "arch": build_arch,
-                    "build_vars": build_vars_str,
-                    "build_command": build_command,
-                    "metadata": json.dumps(build_config_metadata),
-                    "build_artifacts": ",".join(build_artifacts),
-                    "tests_regexp": tests_regexp,
-                    "tests_priority_upper_limit": tests_priority_upper_limit,
-                    "tests_priority_lower_limit": tests_priority_lower_limit,
-                    "tests_groups_regexp": tests_groups_regexp,
-                }
-                if pull_request is not None:
-                    build_stream_fields["pull_request"] = pull_request
-                if git_branch is not None:
-                    build_stream_fields["git_branch"] = git_branch
-                if git_version is not None:
-                    build_stream_fields["git_version"] = git_version
-                if git_timestamp_ms is not None:
-                    build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
-                for artifact in build_artifacts:
-                    bin_key = "zipped:artifacts:{}:{}.zip".format(id, artifact)
-                    bin_artifact = open(
-                        "{}src/{}".format(redis_temporary_dir, artifact), "rb"
-                    ).read()
-                    bin_artifact_len = len(bytes(bin_artifact))
-                    assert bin_artifact_len > 0
-                    conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
-                    build_stream_fields[artifact] = bin_key
-                    build_stream_fields[
-                        "{}_len_bytes".format(artifact)
-                    ] = bin_artifact_len
-                result = True
-                if b"platform" in testDetails:
-                    build_stream_fields["platform"] = testDetails[b"platform"]
+                build_stream_fields, result = generate_benchmark_stream_request(
+                    id,
+                    conn,
+                    run_image,
+                    build_arch,
+                    testDetails,
+                    build_os,
+                    build_artifacts,
+                    build_command,
+                    build_config_metadata,
+                    build_image,
+                    build_vars_str,
+                    compiler,
+                    cpp_compiler,
+                    git_branch,
+                    git_hash,
+                    git_timestamp_ms,
+                    git_version,
+                    pull_request,
+                    redis_temporary_dir,
+                    tests_groups_regexp,
+                    tests_priority_lower_limit,
+                    tests_priority_upper_limit,
+                    tests_regexp,
+                    use_git_timestamp,
+                    server_name,
+                    github_org,
+                    github_repo,
+                )
                 if result is True:
                     benchmark_stream_id = conn.xadd(
                         STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields
@@ -551,6 +545,122 @@ def builder_process_stream(
     return previous_id, new_builds_count, build_stream_fields_arr
 
 
+def store_airgap_image_redis(conn, docker_client, run_image):
+    airgap_key = "docker:air-gap:{}".format(run_image)
+    logging.info(
+        "DOCKER AIR GAP: storing run image named: {} in redis key {}".format(
+            run_image, airgap_key
+        )
+    )
+    # 7 days expire
+    binary_exp_secs = 24 * 60 * 60 * 7
+    if conn.exists(airgap_key):
+        logging.info(
+            f"DOCKER AIRGAP KEY ALREADY EXISTS: {airgap_key}. Updating only the expire time"
+        )
+        conn.expire(airgap_key, binary_exp_secs)
+    else:
+        run_image_binary_stream = io.BytesIO()
+        run_image_docker = docker_client.images.get(run_image)
+        for chunk in run_image_docker.save():
+            run_image_binary_stream.write(chunk)
+        res_airgap = conn.set(
+            airgap_key,
+            run_image_binary_stream.getbuffer(),
+            ex=binary_exp_secs,
+        )
+        logging.info(
+            "DOCKER AIR GAP: result of set bin data to {}: {}".format(
+                airgap_key, res_airgap
+            )
+        )
+
+
+def generate_benchmark_stream_request(
+    id,
+    conn,
+    run_image,
+    build_arch,
+    testDetails,
+    build_os,
+    build_artifacts=[],
+    build_command=None,
+    build_config_metadata=None,
+    build_image=None,
+    build_vars_str=None,
+    compiler=None,
+    cpp_compiler=None,
+    git_branch=None,
+    git_hash=None,
+    git_timestamp_ms=None,
+    git_version=None,
+    pull_request=None,
+    redis_temporary_dir=None,
+    tests_groups_regexp=".*",
+    tests_priority_lower_limit=0,
+    tests_priority_upper_limit=10000,
+    tests_regexp=".*",
+    use_git_timestamp=False,
+    server_name="redis",
+    github_org="redis",
+    github_repo="redis",
+):
+    build_stream_fields = {
+        "id": id,
+        "use_git_timestamp": str(use_git_timestamp),
+        "run_image": run_image,
+        "os": build_os,
+        "arch": build_arch,
+        "build_artifacts": ",".join(build_artifacts),
+        "tests_regexp": tests_regexp,
+        "tests_priority_upper_limit": tests_priority_upper_limit,
+        "tests_priority_lower_limit": tests_priority_lower_limit,
+        "tests_groups_regexp": tests_groups_regexp,
+        "server_name": server_name,
+        "github_org": github_org,
+        "github_repo": github_repo,
+    }
+    if build_config_metadata is not None:
+        build_stream_fields["metadata"] = json.dumps(build_config_metadata)
+    if compiler is not None:
+        build_stream_fields["compiler"] = compiler
+    if cpp_compiler is not None:
+        build_stream_fields["cpp_compiler"] = cpp_compiler
+    if build_vars_str is not None:
+        build_stream_fields["build_vars"] = build_vars_str
+    if build_command is not None:
+        logging.info(f"adding build_command: {build_command}")
+        build_stream_fields["build_command"] = build_command
+    if build_image is not None:
+        build_stream_fields["build_image"] = build_image
+    else:
+        build_stream_fields["build_image"] = run_image
+    if git_hash is not None:
+        build_stream_fields["git_hash"] = git_hash
+    if pull_request is not None:
+        build_stream_fields["pull_request"] = pull_request
+    if git_branch is not None:
+        build_stream_fields["git_branch"] = git_branch
+    if git_version is not None:
+        build_stream_fields["git_version"] = git_version
+    if git_timestamp_ms is not None:
+        build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
+
+    prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
+    for artifact in build_artifacts:
+        bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip"
+        bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read()
+        bin_artifact_len = len(bytes(bin_artifact))
+        assert bin_artifact_len > 0
+        conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
+        build_stream_fields[artifact] = bin_key
+        build_stream_fields["{}_len_bytes".format(artifact)] = bin_artifact_len
+    result = True
+    if b"platform" in testDetails:
+        build_stream_fields["platform"] = testDetails[b"platform"]
+    return build_stream_fields, result
+
+
 def build_spec_image_prefetch(builders_folder, different_build_specs):
     logging.info("checking build spec requirements")
     already_checked_images = []

{redis_benchmarks_specification-0.1.102 → redis_benchmarks_specification-0.1.150}/redis_benchmarks_specification/__cli__/args.py

@@ -5,7 +5,7 @@
 #
 import datetime
 import os
-
+from distutils.util import strtobool
 from redis_benchmarks_specification.__common__.env import (
     GH_REDIS_SERVER_HOST,
     GH_TOKEN,
@@ -18,9 +18,10 @@ from redis_benchmarks_specification.__common__.env import (
 from redisbench_admin.run.common import get_start_time_vars
 
 START_TIME_NOW_UTC, _, _ = get_start_time_vars()
-START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
+START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=90)
 CLI_TOOL_STATS = "stats"
 CLI_TOOL_TRIGGER = "trigger"
+CLI_TOOL_DOCKERHUB = "dockerhub"
 PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
 
 
@@ -70,6 +71,12 @@ def spec_cli_args(parser):
         action="store_true",
         help="Include modules statistics on commandstats.",
     )
+    parser.add_argument(
+        "--use-git-timestamp",
+        type=lambda x: bool(strtobool(x)),
+        default=True,
+        help="Use git timestamp",
+    )
     parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
     parser.add_argument("--pull-request", type=str, default=None, nargs="?", const="")
     parser.add_argument(
@@ -109,7 +116,7 @@ def spec_cli_args(parser):
         type=str,
         default=CLI_TOOL_TRIGGER,
         help="subtool to use. One of '{}' ".format(
-            ",".join([CLI_TOOL_STATS, CLI_TOOL_TRIGGER])
+            ",".join([CLI_TOOL_STATS, CLI_TOOL_TRIGGER, CLI_TOOL_DOCKERHUB])
         ),
     )
     parser.add_argument("--gh_token", type=str, default=GH_TOKEN)
@@ -129,7 +136,18 @@ def spec_cli_args(parser):
     parser.add_argument("--redis_repo", type=str, default=None)
     parser.add_argument("--gh_org", type=str, default="redis")
     parser.add_argument("--gh_repo", type=str, default="redis")
+    parser.add_argument("--server_name", type=str, default=None)
+    parser.add_argument("--run_image", type=str, default="redis")
+    parser.add_argument("--build_arch", type=str, default="amd64")
+    parser.add_argument("--id", type=str, default="dockerhub")
+    parser.add_argument("--mnt_point", type=str, default="")
     parser.add_argument("--trigger-unstable-commits", type=bool, default=True)
+    parser.add_argument(
+        "--docker-dont-air-gap",
+        default=False,
+        action="store_true",
+        help="Dont store the docker images in redis keys.",
+    )
     parser.add_argument(
         "--use-tags",
         default=False,
@@ -154,6 +172,21 @@ def spec_cli_args(parser):
         action="store_true",
         help="Iterate over the git commits.",
     )
+    parser.add_argument(
+        "--build_artifacts",
+        type=str,
+        default="",
+    )
+    parser.add_argument(
+        "--build_command",
+        type=str,
+        default="",
+    )
+    parser.add_argument(
+        "--git_hash",
+        type=str,
+        default="",
+    )
     parser.add_argument(
         "--dry-run",
         default=False,
@@ -163,8 +196,8 @@ def spec_cli_args(parser):
     parser.add_argument(
         "--last_n",
         type=int,
-        default=-1,
-        help="Use the last N samples. by default will use all available values",
+        default=1,
+        help="Use the last N samples. by default will use last commit",
     )
     parser.add_argument(
         "--platform",