redis-benchmarks-specification 0.2.42__py3-none-any.whl

redis_benchmarks_specification/__self_contained_coordinator__/__init__.py

@@ -0,0 +1,5 @@
+#  Apache 2 License
+#
+#  Copyright (c) 2021., Redis Labs
+#  All rights reserved.
+#

redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -0,0 +1,210 @@
+import argparse
+import os
+from redis_benchmarks_specification.__common__.env import (
+    MACHINE_CPU_COUNT,
+    SPECS_PATH_SETUPS,
+    SPECS_PATH_TEST_SUITES,
+    DATASINK_RTS_HOST,
+    DATASINK_RTS_PORT,
+    DATASINK_RTS_AUTH,
+    DATASINK_RTS_USER,
+    DATASINK_RTS_PUSH,
+    MACHINE_NAME,
+    GH_REDIS_SERVER_HOST,
+    GH_REDIS_SERVER_PORT,
+    GH_REDIS_SERVER_AUTH,
+    GH_REDIS_SERVER_USER,
+    PROFILERS_ENABLED,
+    PROFILERS,
+    PROFILERS_DEFAULT,
+    ALLOWED_PROFILERS,
+)
+
+PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
+
+
+def create_self_contained_coordinator_args(project_name):
+    parser = argparse.ArgumentParser(
+        description=project_name,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=project_name,
+        help="Show version information and exit",
+    )
+    parser.add_argument("--event_stream_host", type=str, default=GH_REDIS_SERVER_HOST)
+    parser.add_argument("--event_stream_port", type=int, default=GH_REDIS_SERVER_PORT)
+    parser.add_argument("--event_stream_pass", type=str, default=GH_REDIS_SERVER_AUTH)
+    parser.add_argument("--event_stream_user", type=str, default=GH_REDIS_SERVER_USER)
+    parser.add_argument("--github_token", type=str, default=PERFORMANCE_GH_TOKEN)
+    parser.add_argument(
+        "--cpu-count",
+        type=int,
+        default=MACHINE_CPU_COUNT,
+        help="Specify how much of the available CPU resources the coordinator can use.",
+    )
+    parser.add_argument("--redis_proc_start_port", type=int, default=6379)
+    parser.add_argument("--cpuset_start_pos", type=int, default=0)
+    parser.add_argument(
+        "--platform-name",
+        type=str,
+        default=MACHINE_NAME,
+        help="Specify the running platform name. By default it will use the machine name.",
+    )
+    parser.add_argument(
+        "--logname", type=str, default=None, help="logname to write the logs to"
+    )
+    parser.add_argument(
+        "--consumer-start-id",
+        type=str,
+        default=">",
+    )
+    parser.add_argument(
+        "--consumer-id",
+        type=int,
+        default=1,
+        help="Consumer id for consumer group for platform",
+    )
+    parser.add_argument(
+        "--consumer-pos",
+        type=int,
+        default=1,
+        help="Consumer pos for consumer group for platform",
+    )
+
+    parser.add_argument(
+        "--setups-folder",
+        type=str,
+        default=SPECS_PATH_SETUPS,
+        help="Setups folder, containing the build environment variations sub-folder that we use to trigger different build artifacts",
+    )
+    parser.add_argument(
+        "--test-suites-folder",
+        type=str,
+        default=SPECS_PATH_TEST_SUITES,
+        help="Test suites folder, containing the different test variations",
+    )
+    parser.add_argument(
+        "--test",
+        type=str,
+        default="",
+        help="specify a test to run. By default will run all the tests"
+        + " present in the folder specified in --test-suites-folder.",
+    )
+    parser.add_argument(
+        "--tests-regexp",
+        type=str,
+        default=".*",
+        help="Interpret PATTERN as a regular expression to filter test names",
+    )
+    parser.add_argument(
+        "--datasink_redistimeseries_host", type=str, default=DATASINK_RTS_HOST
+    )
+    parser.add_argument(
+        "--datasink_redistimeseries_port", type=int, default=DATASINK_RTS_PORT
+    )
+    parser.add_argument(
+        "--datasink_redistimeseries_pass", type=str, default=DATASINK_RTS_AUTH
+    )
+    parser.add_argument(
+        "--datasink_redistimeseries_user", type=str, default=DATASINK_RTS_USER
+    )
+    parser.add_argument(
+        "--datasink_push_results_redistimeseries",
+        default=DATASINK_RTS_PUSH,
+        action="store_true",
+        help="uploads the results to RedisTimeSeries. Proper credentials are required",
+    )
+    parser.add_argument("--profilers", type=str, default=PROFILERS)
+    parser.add_argument(
+        "--enable-profilers",
+        default=PROFILERS_ENABLED,
+        action="store_true",
+        help="Enable Identifying On-CPU and Off-CPU Time using perf/ebpf/vtune tooling. "
+        + "By default the chosen profilers are {}".format(PROFILERS_DEFAULT)
+        + "Full list of profilers: {}".format(ALLOWED_PROFILERS)
+        + "Only available on x86 Linux platform and kernel version >= 4.9",
+    )
+    parser.add_argument(
+        "--grafana-profile-dashboard",
+        type=str,
+        default="https://benchmarksredisio.grafana.net/d/uRPZar57k/ci-profiler-viewer",
+    )
+    parser.add_argument(
+        "--docker-air-gap",
+        default=False,
+        action="store_true",
+        help="Read the docker images from redis keys.",
+    )
+    parser.add_argument(
+        "--verbose",
+        default=False,
+        action="store_true",
+        help="Run in verbose mode.",
+    )
+    parser.add_argument(
+        "--override-memtier-test-time",
+        default=0,
+        type=int,
+        help="override memtier test-time for each benchmark. By default will preserve test time specified in test spec",
+    )
+    parser.add_argument(
+        "--defaults_filename",
+        type=str,
+        default="{}/defaults.yml".format(SPECS_PATH_TEST_SUITES),
+        help="specify the defaults file containing spec topologies, common metric extractions,etc...",
+    )
+    parser.add_argument(
+        "--arch", type=str, default="amd64", help="arch to build artifacts"
+    )
+    parser.add_argument(
+        "--tests-priority-lower-limit",
+        type=int,
+        default=0,
+        help="Run a subset of the tests based uppon a preset priority. By default runs all tests.",
+    )
+    parser.add_argument(
+        "--tests-priority-upper-limit",
+        type=int,
+        default=100000,
+        help="Run a subset of the tests based uppon a preset priority. By default runs all tests.",
+    )
+    parser.add_argument(
+        "--topology",
+        type=str,
+        default="",
+        help="Filter tests to run only with the specified topology (e.g. oss-standalone)",
+    )
+    parser.add_argument(
+        "--exclusive-hardware",
+        default=False,
+        action="store_true",
+        help="Enable exclusive hardware mode. Kills all memtier processes and stops all docker containers before and after each test.",
+    )
+    parser.add_argument(
+        "--http-port",
+        type=int,
+        default=8080,
+        help="Port for HTTP server endpoints (/ping health check and /reset-queue POST endpoint).",
+    )
+    parser.add_argument(
+        "--http-auth-username",
+        type=str,
+        default=None,
+        help="Username for HTTP endpoint authentication. HTTP server is disabled if not provided.",
+    )
+    parser.add_argument(
+        "--http-auth-password",
+        type=str,
+        default=None,
+        help="Password for HTTP endpoint authentication. HTTP server is disabled if not provided.",
+    )
+    parser.add_argument(
+        "--skip-clear-pending-on-startup",
+        default=False,
+        action="store_true",
+        help="Skip automatically clearing pending messages and resetting consumer group position on startup. By default, pending messages are cleared and consumer group is reset to latest position to skip old work and recover from crashes.",
+    )
+    return parser

redis_benchmarks_specification/__self_contained_coordinator__/artifacts.py

@@ -0,0 +1,27 @@
+import logging
+import os
+
+
+def restore_build_artifacts_from_test_details(
+    build_artifacts, conn, temporary_dir, testDetails
+):
+    for build_artifact in build_artifacts:
+        build_artifact_key = "{}".format(build_artifact).encode()
+        if build_artifact_key in testDetails:
+            buffer_key = testDetails[build_artifact_key]
+            logging.info(
+                "Reading artifact binary {} from key {}".format(
+                    build_artifact, buffer_key
+                )
+            )
+            buffer = bytes(conn.get(buffer_key))
+            artifact_fname = "{}/{}".format(temporary_dir, build_artifact)
+            with open(artifact_fname, "wb") as fd:
+                fd.write(buffer)
+                os.chmod(artifact_fname, 755)
+
+            logging.info(
+                "Successfully restored {} into {}".format(
+                    build_artifact, artifact_fname
+                )
+            )

redis_benchmarks_specification/__self_contained_coordinator__/build_info.py

@@ -0,0 +1,61 @@
+import json
+import logging
+
+from redis_benchmarks_specification.__common__.builder_schema import (
+    get_branch_version_from_test_details,
+)
+
+
+def extract_build_info_from_streamdata(testDetails):
+    arch = "amd64"
+    use_git_timestamp = False
+    git_timestamp_ms = None
+    metadata = {}
+    build_variant_name = None
+    fields = [fieldname.decode() for fieldname in testDetails.keys()]
+    logging.info("Fields on stream {}".format(testDetails))
+    git_hash = None
+    if b"git_hash" in testDetails:
+        git_hash = testDetails[b"git_hash"].decode()
+    if b"use_git_timestamp" in testDetails:
+        use_git_timestamp = bool(testDetails[b"use_git_timestamp"].decode())
+    if b"git_timestamp_ms" in testDetails:
+        git_timestamp_ms = int(testDetails[b"git_timestamp_ms"].decode())
+    if b"id" in testDetails:
+        build_variant_name = testDetails[b"id"]
+        if type(build_variant_name) == bytes:
+            build_variant_name = build_variant_name.decode()
+    git_branch, git_version = get_branch_version_from_test_details(testDetails)
+    if type(git_hash) == bytes:
+        git_hash = git_hash.decode()
+    logging.info("Received commit hash specifier {}.".format(git_hash))
+    build_artifacts_str = "redis-server"
+    build_image = testDetails[b"build_image"].decode()
+    run_image = build_image
+    if b"arch" in testDetails:
+        arch = testDetails[b"arch"].decode()
+        logging.info("detected arch info {}.".format(arch))
+    else:
+        logging.info("using default arch info {}.".format(arch))
+    if b"run_image" in testDetails:
+        run_image = testDetails[b"run_image"].decode()
+        logging.info("detected run image info {}.".format(run_image))
+    else:
+        logging.info("using build image info {}.".format(build_image))
+    if b"build_artifacts" in testDetails:
+        build_artifacts_str = testDetails[b"build_artifacts"].decode()
+    build_artifacts = build_artifacts_str.split(",")
+    if b"metadata" in testDetails:
+        metadata = json.loads(testDetails[b"metadata"].decode())
+    return (
+        build_variant_name,
+        metadata,
+        build_artifacts,
+        git_hash,
+        git_branch,
+        git_version,
+        run_image,
+        use_git_timestamp,
+        git_timestamp_ms,
+        arch,
+    )

redis_benchmarks_specification/__self_contained_coordinator__/clients.py

@@ -0,0 +1,58 @@
+import logging
+
+
+def prepare_memtier_benchmark_parameters(
+    clientconfig,
+    full_benchmark_path,
+    port,
+    server,
+    local_benchmark_output_filename,
+    oss_cluster_api_enabled,
+    password=None,
+):
+    benchmark_command = [
+        full_benchmark_path,
+        "--json-out-file",
+        local_benchmark_output_filename,
+        "--port",
+        "{}".format(port),
+        "--server",
+        "{}".format(server),
+    ]
+
+    # Add password authentication if provided
+    if password is not None and password != "":
+        benchmark_command.extend(["--authenticate", password])
+        logging.info("Memtier benchmark will use password authentication")
+
+    if oss_cluster_api_enabled is True:
+        benchmark_command.append("--cluster-mode")
+    benchmark_command_str = " ".join(benchmark_command)
+    if "arguments" in clientconfig:
+        benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"]
+
+    return None, benchmark_command_str
+
+
+def prepare_vector_db_benchmark_parameters(
+    clientconfig, full_benchmark_path, port, server, password, client_mnt_point
+):
+    benchmark_command = []
+    # if port is not None:
+    #     benchmark_command.extend(["REDIS_PORT={}".format(port)])
+    # if password is not None:
+    #     benchmark_command.extend(["REDIS_AUTH={}".format(password)])
+    benchmark_command.extend(
+        [
+            full_benchmark_path,
+            "--host",
+            f"{server}",
+        ]
+    )
+    benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-test")])
+    benchmark_command.extend(
+        ["--datasets", clientconfig.get("datasets", "glove-100-angular")]
+    )
+    benchmark_command_str = " ".join(benchmark_command)
+    benchmark_command_str = f"bash -c 'ITERATIONS=1 {benchmark_command_str} && mv /code/results {client_mnt_point}.'"
+    return None, benchmark_command_str

redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py

@@ -0,0 +1,17 @@
+import math
+
+
+def generate_cpuset_cpus(ceil_db_cpu_limit, current_cpu_pos):
+    previous_cpu_pos = current_cpu_pos
+    current_cpu_pos = current_cpu_pos + int(ceil_db_cpu_limit)
+    db_cpuset_cpus = ",".join(
+        [str(x) for x in range(previous_cpu_pos, current_cpu_pos)]
+    )
+    return db_cpuset_cpus, current_cpu_pos
+
+
+def extract_db_cpu_limit(topologies_map, topology_spec_name):
+    topology_spec = topologies_map[topology_spec_name]
+    db_cpu_limit = topology_spec["resources"]["requests"]["cpus"]
+    ceil_db_cpu_limit = math.ceil(float(db_cpu_limit))
+    return ceil_db_cpu_limit

redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -0,0 +1,108 @@
+import logging
+
+import docker
+
+from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import (
+    generate_cpuset_cpus,
+)
+
+
+def generate_standalone_redis_server_args(
+    binary,
+    port,
+    dbdir,
+    configuration_parameters=None,
+    redis_arguments="",
+    password=None,
+):
+    added_params = ["port", "protected-mode", "dir", "requirepass", "logfile"]
+    # start redis-server
+    command = [
+        binary,
+        "--protected-mode",
+        "no",
+        "--port",
+        "{}".format(port),
+    ]
+
+    # Add password authentication if provided
+    if password is not None and password != "":
+        command.extend(["--requirepass", password])
+        logging.info("Redis server will be started with password authentication")
+    if dbdir != "":
+        command.extend(["--dir", dbdir])
+        command.extend(["--logfile", f"{dbdir}redis.log"])
+    if configuration_parameters is not None:
+        for parameter, parameter_value in configuration_parameters.items():
+            if parameter not in added_params:
+                command.extend(
+                    [
+                        "--{}".format(parameter),
+                        parameter_value,
+                    ]
+                )
+    if redis_arguments != "":
+        redis_arguments_arr = redis_arguments.split(" ")
+        logging.info(f"adding redis arguments {redis_arguments_arr}")
+        command.extend(redis_arguments_arr)
+    return command
+
+
+def teardown_containers(redis_containers, container_type):
+    for container in redis_containers:
+        try:
+            container.stop()
+        except docker.errors.NotFound:
+            logging.info(
+                "When trying to stop {} container with id {} and image {} it was already stopped".format(
+                    container_type, container.id, container.image
+                )
+            )
+            pass
+
+
+def spin_docker_standalone_redis(
+    ceil_db_cpu_limit,
+    current_cpu_pos,
+    docker_client,
+    redis_configuration_parameters,
+    redis_containers,
+    redis_proc_start_port,
+    run_image,
+    temporary_dir,
+    password=None,
+):
+    mnt_point = "/mnt/redis/"
+    command = generate_standalone_redis_server_args(
+        "{}redis-server".format(mnt_point),
+        redis_proc_start_port,
+        mnt_point,
+        redis_configuration_parameters,
+        "",
+        password,
+    )
+    command_str = " ".join(command)
+    db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
+        ceil_db_cpu_limit, current_cpu_pos
+    )
+    logging.info(
+        "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
+            run_image, db_cpuset_cpus, command_str
+        )
+    )
+    container = docker_client.containers.run(
+        image=run_image,
+        volumes={
+            temporary_dir: {"bind": mnt_point, "mode": "rw"},
+        },
+        auto_remove=True,
+        privileged=True,
+        working_dir=mnt_point,
+        command=command_str,
+        network_mode="host",
+        detach=True,
+        cpuset_cpus=db_cpuset_cpus,
+        pid_mode="host",
+    )
+    redis_containers.append(container)
+    return current_cpu_pos

redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py

@@ -0,0 +1,19 @@
+import os
+import json
+
+
+def post_process_vector_db(temporary_dir):
+    results_dir = os.path.join(temporary_dir, "results")
+    results = {}
+    for file in os.listdir(results_dir):
+        if "upload" in file:
+            with open(os.path.join(results_dir, file), "r") as f:
+                upload_results = json.load(f)
+                results["upload_time"] = upload_results["results"]["upload_time"]
+        else:
+            with open(os.path.join(results_dir, file), "r") as f:
+                query_results = json.load(f)
+                results["rps"] = query_results["results"]["rps"]
+                results["precision"] = query_results["results"]["mean_precisions"]
+                results["total_time"] = query_results["results"]["total_time"]
+    return results

redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py

@@ -0,0 +1,96 @@
+import datetime
+import logging
+
+from redisbench_admin.run.common import get_start_time_vars
+from redisbench_admin.run.run import calculate_client_tool_duration_and_check
+from redisbench_admin.utils.local import get_local_run_full_filename
+
+from redis_benchmarks_specification.__common__.spec import (
+    extract_client_container_image,
+    extract_client_tool,
+)
+from redis_benchmarks_specification.__self_contained_coordinator__.clients import (
+    prepare_memtier_benchmark_parameters,
+)
+
+
+def data_prepopulation_step(
+    benchmark_config,
+    benchmark_tool_workdir,
+    client_cpuset_cpus,
+    docker_client,
+    git_hash,
+    port,
+    temporary_dir,
+    test_name,
+    redis_password,
+):
+    # setup the benchmark
+    (
+        start_time,
+        start_time_ms,
+        start_time_str,
+    ) = get_start_time_vars()
+    local_benchmark_output_filename = get_local_run_full_filename(
+        start_time_str,
+        git_hash,
+        "preload__" + test_name,
+        "oss-standalone",
+    )
+    preload_image = extract_client_container_image(
+        benchmark_config["dbconfig"], "preload_tool"
+    )
+    preload_tool = extract_client_tool(benchmark_config["dbconfig"], "preload_tool")
+    full_benchmark_path = "/usr/local/bin/{}".format(preload_tool)
+    client_mnt_point = "/mnt/client/"
+    if "memtier_benchmark" in preload_tool:
+        (
+            _,
+            preload_command_str,
+        ) = prepare_memtier_benchmark_parameters(
+            benchmark_config["dbconfig"]["preload_tool"],
+            full_benchmark_path,
+            port,
+            "localhost",
+            local_benchmark_output_filename,
+            False,
+            redis_password,
+        )
+
+        logging.info(
+            "Using docker image {} as benchmark PRELOAD image (cpuset={}) with the following args: {}".format(
+                preload_image,
+                client_cpuset_cpus,
+                preload_command_str,
+            )
+        )
+        # run the benchmark
+        preload_start_time = datetime.datetime.now()
+
+        client_container_stdout = docker_client.containers.run(
+            image=preload_image,
+            volumes={
+                temporary_dir: {
+                    "bind": client_mnt_point,
+                    "mode": "rw",
+                },
+            },
+            auto_remove=True,
+            privileged=True,
+            working_dir=benchmark_tool_workdir,
+            command=preload_command_str,
+            network_mode="host",
+            detach=False,
+            cpuset_cpus=client_cpuset_cpus,
+        )
+
+        preload_end_time = datetime.datetime.now()
+        preload_duration_seconds = calculate_client_tool_duration_and_check(
+            preload_end_time, preload_start_time, "Preload", False
+        )
+        logging.info(
+            "Tool {} seconds to load data. Output {}".format(
+                preload_duration_seconds,
+                client_container_stdout,
+            )
+        )