redis-benchmarks-specification 0.2.42__py3-none-any.whl

redis_benchmarks_specification/__setups__/topologies.py

@@ -0,0 +1,17 @@
+import os
+import yaml
+
+
+def get_topologies(usecase_filename):
+    full_filename = os.path.abspath(usecase_filename)
+    topologies_map = {}
+    with open(full_filename) as stream:
+        build_config = yaml.safe_load(stream)
+        spec = build_config["spec"]
+        setups = spec["setups"]
+        for topology in setups:
+            topology_name = topology["name"]
+            topologies_map[topology_name] = topology
+        # print(build_config)
+
+    return topologies_map

redis_benchmarks_specification/__spec__/__init__.py

@@ -0,0 +1,5 @@
+#  Apache 2 License
+#
+#  Copyright (c) 2021., Redis Labs
+#  All rights reserved.
+#

redis_benchmarks_specification/__spec__/args.py

@@ -0,0 +1,78 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import datetime
+
+
+from redis_benchmarks_specification.__common__.env import (
+    GH_REDIS_SERVER_HOST,
+    GH_TOKEN,
+    GH_REDIS_SERVER_PORT,
+    GH_REDIS_SERVER_AUTH,
+    GH_REDIS_SERVER_USER,
+)
+
+from redisbench_admin.run.common import get_start_time_vars
+
+START_TIME_NOW_UTC, _, _ = get_start_time_vars()
+START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
+
+
+def spec_cli_args(parser):
+    parser.add_argument("--redis_host", type=str, default=GH_REDIS_SERVER_HOST)
+    parser.add_argument("--branch", type=str, default="unstable")
+    parser.add_argument("--gh_token", type=str, default=GH_TOKEN)
+    parser.add_argument("--redis_port", type=int, default=GH_REDIS_SERVER_PORT)
+    parser.add_argument("--redis_pass", type=str, default=GH_REDIS_SERVER_AUTH)
+    parser.add_argument("--redis_user", type=str, default=GH_REDIS_SERVER_USER)
+    parser.add_argument(
+        "--from-date",
+        type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
+        default=START_TIME_LAST_YEAR_UTC,
+    )
+    parser.add_argument(
+        "--to-date",
+        type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"),
+        default=START_TIME_NOW_UTC,
+    )
+    parser.add_argument("--redis_repo", type=str, default=None)
+    parser.add_argument("--trigger-unstable-commits", type=bool, default=True)
+    parser.add_argument(
+        "--use-tags",
+        default=False,
+        action="store_true",
+        help="Iterate over the git tags.",
+    )
+    parser.add_argument(
+        "--tags-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter tag names",
+    )
+    parser.add_argument(
+        "--hash-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter commit hashes",
+    )
+    parser.add_argument(
+        "--use-branch",
+        default=False,
+        action="store_true",
+        help="Iterate over the git commits.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        default=False,
+        action="store_true",
+        help="Only check how many benchmarks we would trigger. Don't request benchmark runs at the end.",
+    )
+    parser.add_argument(
+        "--last_n",
+        type=int,
+        default=-1,
+        help="Use the last N samples. by default will use all available values",
+    )
+    return parser

redis_benchmarks_specification/__spec__/cli.py

@@ -0,0 +1,259 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+
+import argparse
+import datetime
+import logging
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import git
+import packaging
+import redis
+from packaging import version
+
+
+from redis_benchmarks_specification.__cli__.args import spec_cli_args
+from redis_benchmarks_specification.__common__.builder_schema import (
+    get_commit_dict_from_sha,
+    request_build_from_commit_info,
+)
+from redis_benchmarks_specification.__common__.env import REDIS_BINS_EXPIRE_SECS
+from redis_benchmarks_specification.__common__.package import (
+    get_version_string,
+    populate_with_poetry_data,
+)
+
+# logging settings
+logging.basicConfig(
+    format="%(asctime)s %(levelname)-4s %(message)s",
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+def main():
+    _, _, project_version = populate_with_poetry_data()
+    project_name = "redis-benchmarks-spec-cli"
+    parser = argparse.ArgumentParser(
+        description=get_version_string(project_name, project_version),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser = spec_cli_args(parser)
+    args = parser.parse_args()
+
+    cli_command_logic(args, project_name, project_version)
+
+
+def cli_command_logic(args, project_name, project_version):
+    logging.info(
+        "Using: {project_name} {project_version}".format(
+            project_name=project_name, project_version=project_version
+        )
+    )
+    if args.use_branch is False and args.use_tags is False:
+        logging.error("You must specify either --use-tags or --use-branch flag")
+        sys.exit(1)
+    redisDirPath = args.redis_repo
+    cleanUp = False
+    if redisDirPath is None:
+        cleanUp = True
+        redisDirPath = tempfile.mkdtemp()
+        logging.info(
+            "Retrieving redis repo from remote into {}. Using branch {}.".format(
+                redisDirPath, args.branch
+            )
+        )
+        cmd = "git clone https://github.com/redis/redis {} --branch {}\n".format(
+            redisDirPath, args.branch
+        )
+        process = subprocess.Popen(
+            "/bin/bash", stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        )
+        process.communicate(cmd.encode())
+    else:
+        logging.info(
+            "Using the following redis repo to retrieve versions info {}. No need to fetch remote data.".format(
+                redisDirPath
+            )
+        )
+    logging.info(
+        "Using the following timeframe: from {} to {}".format(
+            args.from_date, args.to_date
+        )
+    )
+    repo = git.Repo(redisDirPath)
+    commits = []
+    total_commits = 0
+    if args.use_branch:
+        for commit in repo.iter_commits():
+
+            commit_datetime_utc = datetime.datetime.utcfromtimestamp(
+                commit.committed_datetime.timestamp()
+            )
+            git_timestamp_ms = int(commit_datetime_utc.timestamp() * 1000)
+            if args.from_date <= commit_datetime_utc <= args.to_date:
+                if (
+                    args.last_n > 0 and total_commits < args.last_n
+                ) or args.last_n == -1:
+                    total_commits = total_commits + 1
+                    print(commit.summary)
+                    commits.append(
+                        {
+                            "git_hash": commit.hexsha,
+                            "git_branch": repo.active_branch.name,
+                            "commit_summary": commit.summary,
+                            "git_timestamp_ms": git_timestamp_ms,
+                        }
+                    )
+    if args.use_tags:
+        tags_regexp = args.tags_regexp
+        if tags_regexp == ".*":
+            logging.info(
+                "Acception all tags that follow semver between the timeframe. If you need further filter specify a regular expression via --tags-regexp"
+            )
+        else:
+            logging.info(
+                "Filtering all tags via a regular expression: {}".format(tags_regexp)
+            )
+        tags_regex_string = re.compile(tags_regexp)
+
+        tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+        for tag in tags:
+            if (
+                args.from_date
+                <= datetime.datetime.utcfromtimestamp(
+                    tag.commit.committed_datetime.timestamp()
+                )
+                <= args.to_date
+            ):
+
+                try:
+                    version.Version(tag.name)
+                    match_obj = re.search(tags_regex_string, tag.name)
+                    if match_obj is None:
+                        logging.info(
+                            "Skipping {} given it does not match regex {}".format(
+                                tag.name, tags_regexp
+                            )
+                        )
+                    else:
+                        git_version = tag.name
+                        print(
+                            "Commit summary: {}. Extract semver: {}".format(
+                                tag.commit.summary, git_version
+                            )
+                        )
+                        commits.append(
+                            {
+                                "git_hash": tag.commit.hexsha,
+                                "git_version": git_version,
+                                "commit_summary": tag.commit.summary,
+                            }
+                        )
+                except packaging.version.InvalidVersion:
+                    logging.info(
+                        "Ignoring tag {} given we were not able to extract commit or version info from it.".format(
+                            tag.name
+                        )
+                    )
+                    pass
+    by_description = "n/a"
+    if args.use_branch:
+        by_description = "from branch {}".format(repo.active_branch.name)
+    if args.use_tags:
+        by_description = "by tags"
+    logging.info(
+        "Will trigger {} distinct tests {}.".format(len(commits), by_description)
+    )
+
+    hash_regexp = args.hash_regexp
+    if hash_regexp == ".*":
+        logging.info(
+            "Acception all commit hashes. If you need further filter specify a regular expression via --hash-regexp"
+        )
+    else:
+        logging.info(
+            "Filtering all commit hashes via a regular expression: {}".format(
+                hash_regexp
+            )
+        )
+    hash_regexp_string = re.compile(hash_regexp)
+    filtered_hash_commits = []
+    for cdict in commits:
+        commit_hash = cdict["git_hash"]
+        commit_summary = cdict["commit_summary"]
+        match_obj = re.search(hash_regexp_string, commit_hash)
+        if match_obj is None:
+            logging.info(
+                "Skipping {} given it does not match regex {}".format(
+                    commit_hash, hash_regexp_string
+                )
+            )
+        else:
+            print(
+                "Commit with hash: {} added. summary: {}".format(
+                    commit_hash, commit_summary
+                )
+            )
+            filtered_hash_commits.append(cdict)
+
+    if args.dry_run is False:
+        conn = redis.StrictRedis(
+            host=args.redis_host,
+            port=args.redis_port,
+            password=args.redis_pass,
+            username=args.redis_user,
+            decode_responses=False,
+        )
+
+        for rep in range(0, 1):
+            for cdict in filtered_hash_commits:
+
+                # Pass local repository path if using local repo
+                local_repo_path = redisDirPath if args.redis_repo is not None else None
+
+                (
+                    result,
+                    error_msg,
+                    commit_dict,
+                    _,
+                    binary_key,
+                    binary_value,
+                ) = get_commit_dict_from_sha(
+                    cdict["git_hash"],
+                    "redis",
+                    "redis",
+                    cdict,
+                    True,
+                    args.gh_token,
+                    None,
+                    local_repo_path,
+                )
+                if result is True:
+                    result, reply_fields, error_msg = request_build_from_commit_info(
+                        conn,
+                        commit_dict,
+                        {},
+                        binary_key,
+                        binary_value,
+                        REDIS_BINS_EXPIRE_SECS,
+                    )
+                    logging.info(
+                        "Successfully requested a build for commit: {}. Request stream id: {}.".format(
+                            cdict["git_hash"], reply_fields["id"]
+                        )
+                    )
+                else:
+                    logging.error(error_msg)
+
+    else:
+        logging.info("Skipping actual work trigger ( dry-run )")
+    if cleanUp is True:
+        logging.info("Removing temporary redis dir {}.".format(redisDirPath))
+        shutil.rmtree(redisDirPath)

redis_benchmarks_specification/__watchdog__/__init__.py

@@ -0,0 +1,5 @@
+#  Apache 2 License
+#
+#  Copyright (c) 2021., Redis Labs
+#  All rights reserved.
+#

redis_benchmarks_specification/__watchdog__/args.py

@@ -0,0 +1,54 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+import datetime
+
+
+from redis_benchmarks_specification.__common__.env import (
+    GH_REDIS_SERVER_HOST,
+    GH_REDIS_SERVER_PORT,
+    GH_REDIS_SERVER_AUTH,
+    GH_REDIS_SERVER_USER,
+    STREAM_KEYNAME_GH_EVENTS_COMMIT,
+    STREAM_KEYNAME_NEW_BUILD_EVENTS,
+)
+
+from redisbench_admin.run.common import get_start_time_vars
+
+START_TIME_NOW_UTC, _, _ = get_start_time_vars()
+START_TIME_LAST_YEAR_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=7)
+
+
+def spec_watchdog_args(parser):
+    # events related Redis
+    parser.add_argument("--redis_host", type=str, default=GH_REDIS_SERVER_HOST)
+    parser.add_argument("--redis_port", type=int, default=GH_REDIS_SERVER_PORT)
+    parser.add_argument("--redis_pass", type=str, default=GH_REDIS_SERVER_AUTH)
+    parser.add_argument("--redis_user", type=str, default=GH_REDIS_SERVER_USER)
+    # event stream from github
+    parser.add_argument(
+        "--events_stream_keyname_commits",
+        type=str,
+        default=STREAM_KEYNAME_GH_EVENTS_COMMIT,
+    )
+    # build events stream. This is the stream read by the coordinators to kickoff benchmark variations
+    parser.add_argument(
+        "--events_stream_keyname_builds",
+        type=str,
+        default=STREAM_KEYNAME_NEW_BUILD_EVENTS,
+    )
+    parser.add_argument(
+        "--update-interval",
+        type=int,
+        default=60,
+        help="watchdog update interval in seconds",
+    )
+    parser.add_argument(
+        "--dry-run",
+        default=False,
+        action="store_true",
+        help="Only check how many benchmarks we would trigger. Don't request benchmark runs at the end.",
+    )
+    return parser

redis_benchmarks_specification/__watchdog__/watchdog.py

@@ -0,0 +1,175 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2021., Redis Labs Modules
+#  All rights reserved.
+#
+
+import argparse
+import datetime
+import logging
+import time
+
+import redis
+
+# logging settings
+from redisbench_admin.run.common import get_start_time_vars
+from redistimeseries.client import Client
+
+from redis_benchmarks_specification.__watchdog__.args import spec_watchdog_args
+from redis_benchmarks_specification.__common__.package import (
+    get_version_string,
+    populate_with_poetry_data,
+)
+
+logging.basicConfig(
+    format="%(asctime)s %(levelname)-4s %(message)s",
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+def main():
+    _, _, project_version = populate_with_poetry_data()
+    project_name = "redis-benchmarks-spec-watchdog"
+    parser = argparse.ArgumentParser(
+        description=get_version_string(project_name, project_version),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser = spec_watchdog_args(parser)
+    args = parser.parse_args()
+
+    cli_command_logic(args, project_name, project_version)
+
+
+def cli_command_logic(args, project_name, project_version):
+    logging.info(
+        "Using: {project_name} {project_version}".format(
+            project_name=project_name, project_version=project_version
+        )
+    )
+    logging.info("Checking connection to RedisTimeSeries.")
+    rts = Client(
+        host=args.redis_host,
+        port=args.redis_port,
+        password=args.redis_pass,
+    )
+
+    # tsname_overall_sc_coordinator_running = "{}/sc-coordinator/state:running".format(
+    #     prefix
+    # )
+    # tsname_overall_api_running = "{}/api/state:running".format(prefix)
+    # tsname_overall_builder_running = "{}/builder/state:running".format(prefix)
+
+    rts.redis.ping()
+    update_interval = args.update_interval
+    logging.info(
+        "Entering watching loop. Ticking every {} secs".format(update_interval)
+    )
+    while True:
+        starttime, start_time_ms, _ = get_start_time_vars()
+        try:
+
+            prefix = "mgt-timeseries::benchmarks.redis.io/redis/redis::"
+            create_consumer_group_rts_metrics(
+                args.dry_run,
+                prefix,
+                args.events_stream_keyname_commits,
+                "event stream from github webhook",
+                rts,
+                start_time_ms,
+            )
+            create_consumer_group_rts_metrics(
+                args.dry_run,
+                prefix,
+                args.events_stream_keyname_builds,
+                "event stream from built artifacts",
+                rts,
+                start_time_ms,
+            )
+        except redis.exceptions.ConnectionError as e:
+            logging.error(
+                "Detected an error while writing data to rts: {}".format(e.__str__())
+            )
+        sleep_time_secs = float(update_interval) - (
+            (datetime.datetime.now() - starttime).total_seconds()
+            % float(update_interval)
+        )
+        logging.info("Sleeping for {} secs".format(sleep_time_secs))
+        time.sleep(sleep_time_secs)
+
+
+def create_consumer_group_rts_metrics(
+    dry_run, prefix, stream_name, event_stream_desc, rts, start_time_ms
+):
+    groups = rts.redis.xinfo_groups(stream_name)
+    stream_info = rts.redis.xinfo_stream(stream_name)
+    total_events = stream_info["length"]
+    total_cgroups = stream_info["groups"]
+
+    labels_dict = {
+        "metric": "consumer_groups:total",
+        "type": "mgt-timeseries",
+        "stream": stream_name,
+        "github_org_repo": "redis/redis",
+    }
+    tsname_overall_cgrougs_gh_events = "{}stream={}:consumer_groups:total".format(
+        prefix, stream_name
+    )
+    logging.info(
+        "stream={}; consumer_groups:total={}".format(stream_name, total_cgroups)
+    )
+    if dry_run is False:
+        rts.add(
+            tsname_overall_cgrougs_gh_events,
+            start_time_ms,
+            total_cgroups,
+            labels=labels_dict,
+        )
+    labels_dict["metric"] = "total_events"
+    tsname_total_events_gh_events = "{}stream={}:events:total".format(
+        prefix, stream_name
+    )
+    logging.info("stream={}; total_events={}".format(stream_name, total_events))
+    if dry_run is False:
+        rts.add(
+            tsname_total_events_gh_events,
+            start_time_ms,
+            total_events,
+            labels=labels_dict,
+        )
+
+    for consumer_group in groups:
+        consumer_group_name = consumer_group["name"]
+        labels_dict["consumer_group"] = consumer_group_name
+        events_prefix = "{}stream={}:consumer_group={}:events:".format(
+            prefix, stream_name, consumer_group_name
+        )
+        consumer_group_pending = consumer_group["pending"]
+        lag = total_events - consumer_group_pending
+        tsname_pending_gh_events = "{}pending".format(events_prefix)
+        tsname_total_gh_events = "{}total".format(events_prefix)
+        logging.info("cg={}; total_events={}".format(consumer_group_name, total_events))
+        logging.info(
+            "cg={}; pending={}".format(consumer_group_name, consumer_group_pending)
+        )
+        if dry_run is False:
+            labels_dict["metric"] = "events:pending"
+            labels_dict["metric+consumer_group"] = "{} {}".format(
+                "events:pending", consumer_group_name
+            )
+            rts.add(
+                tsname_pending_gh_events,
+                start_time_ms,
+                consumer_group_pending,
+                labels=labels_dict,
+            )
+            labels_dict["metric"] = "events:total"
+            labels_dict["metric+consumer_group"] = "{} {}".format(
+                "events:total", consumer_group_name
+            )
+            rts.add(
+                tsname_total_gh_events,
+                start_time_ms,
+                lag,
+                labels=labels_dict,
+            )

redis_benchmarks_specification/commands/commands.py

@@ -0,0 +1,15 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2022., Redis Performance Group <performance at redis dot com>
+#  All rights reserved.
+#
+
+
+def generate_command_groups(commands_json):
+    groups = {}
+    for command_name, command_description in commands_json.items():
+        group_name = command_description["group"]
+        if group_name not in groups:
+            groups[group_name] = []
+        groups[group_name].append(command_name)
+    return groups

redis_benchmarks_specification/setups/builders/gcc:15.2.0-amd64-debian-bookworm-default.yml

@@ -0,0 +1,20 @@
+version: 0.1
+id: gcc:15.2.0-amd64-debian-bookworm-default
+os: debian-bookworm
+arch: amd64
+compiler: "gcc"
+cpp_compiler: "g++"
+kind: docker
+build_image: gcc:15.2.0-bookworm
+run_image: gcc:15.2.0-bookworm
+description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description)
+              pre-configured environment with all the tools required to build with gcc."
+metadata:
+  compiler: "gcc"
+  compiler_version: "15.2.0"
+  os: debian-bookworm
+  arch: amd64
+
+env:
+  REDIS_CFLAGS: "-g -fno-omit-frame-pointer"
+

redis_benchmarks_specification/setups/builders/gcc:15.2.0-arm64-debian-bookworm-default.yml

@@ -0,0 +1,20 @@
+version: 0.1
+id: gcc:15.2.0-arm64-debian-bookworm-default
+os: debian-bookworm
+arch: arm64
+compiler: "gcc"
+cpp_compiler: "g++"
+kind: docker
+build_image: gcc:15.2.0-bookworm
+run_image: gcc:15.2.0-bookworm
+description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description)
+              pre-configured environment with all the tools required to build with gcc."
+metadata:
+  compiler: "gcc"
+  compiler_version: "15.2.0"
+  os: debian-bookworm
+  arch: arm64
+
+env:
+  REDIS_CFLAGS: "-g -fno-omit-frame-pointer"
+

redis_benchmarks_specification/setups/platforms/aws-ec2-1node-c5.4xlarge.yml

@@ -0,0 +1,27 @@
+kind: VM
+
+infrastructure_as_code:
+  trigger_mode: "provision"
+  provisioning:
+    setup_teardown_required: true
+    definition_language: "hcl"
+    definition_contributor: redislabs
+    definition: https://github.com/RedisLabsModules/testing-infrastructure/tree/master/terraform/oss-1node-c5.4xlarge
+    units: 1
+  provider: "aws"
+  os: "ubuntu18.04"
+  os_arch: "linux/amd64"
+  user: "ubuntu"
+  pem: "benchmarks.redislabs.pem"
+
+metadata:
+  labels:
+    provider: "aws"
+    instance_type: "c5.4xlarge"
+    os: "ubuntu18.04"
+    os_arch: "linux/amd64"
+    definition_contributor: redislabs
+
+unit-specs:
+  cpu: 16
+  memory: 32