redis-benchmarks-specification 0.1.262__tar.gz → 0.1.264__tar.gz

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: redis-benchmarks-specification
-Version: 0.1.262
+Version: 0.1.264
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: Flask (>=2.0.3,<3.0.0)
 Requires-Dist: Flask-HTTPAuth (>=4.4.0,<5.0.0)
 Requires-Dist: GitPython (>=3.1.20,<4.0.0)

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.262"
+version = "0.1.264"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/redis_benchmarks_specification/__builder__/builder.py

@@ -381,6 +381,8 @@ def builder_process_stream(
                 #     build_vars_str,
                 # )
                 build_command = "sh -c 'make -j'"
+                if "build_command" in build_config:
+                    build_command = build_config["build_command"]
                 if b"build_command" in testDetails:
                     build_command = testDetails[b"build_command"].decode()
                 server_name = "redis"
@@ -649,7 +651,13 @@ def generate_benchmark_stream_request(
     prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
     for artifact in build_artifacts:
         bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip"
-        bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read()
+        if artifact == "redisearch.so":
+            bin_artifact = open(
+                f"{redis_temporary_dir}modules/redisearch/src/bin/linux-x64-release/search-community/{artifact}",
+                "rb",
+            ).read()
+        else:
+            bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read()
         bin_artifact_len = len(bytes(bin_artifact))
         assert bin_artifact_len > 0
         conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/redis_benchmarks_specification/__cli__/stats.py

@@ -108,6 +108,8 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
         benchmark_config = {}
         requires_override = False
         test_result = True
+        tested_groups_match_origin = True
+
         with open(test_file, "r") as stream:
 
             try:
@@ -291,7 +293,7 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                     )
 
                 if tested_groups != origin_tested_groups:
-                    requires_override = True
+                    tested_groups_match_origin = False
                     benchmark_config["tested-groups"] = tested_groups
                     logging.warn(
                         "there is a difference between specified test-groups in the yaml (name={}) and the ones we've detected {}!={}".format(
@@ -312,7 +314,15 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
             test_result = False
         overall_result &= test_result
 
-        if requires_override and override_enabled:
+        if not tested_groups_match_origin:
+            if len(tested_groups) > 0:
+                overall_result = False
+            else:
+                logging.warn(
+                    "difference between specified and detected test-groups was ignored since command info is not available in this benchmark version"
+                )
+
+        if (requires_override or not tested_groups_match_origin) and override_enabled:
             logging.info(
                 "Saving a new version of the file {} with the overrided data".format(
                     test_file

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -165,4 +165,10 @@ def create_self_contained_coordinator_args(project_name):
         default=100000,
         help="Run a subset of the tests based uppon a preset priority. By default runs all tests.",
     )
+    parser.add_argument(
+        "--topology",
+        type=str,
+        default="",
+        help="Filter tests to run only with the specified topology (e.g. oss-standalone)",
+    )
     return parser

redis_benchmarks_specification-0.1.264/redis_benchmarks_specification/__self_contained_coordinator__/clients.py

@@ -0,0 +1,48 @@
+def prepare_memtier_benchmark_parameters(
+    clientconfig,
+    full_benchmark_path,
+    port,
+    server,
+    local_benchmark_output_filename,
+    oss_cluster_api_enabled,
+):
+    benchmark_command = [
+        full_benchmark_path,
+        "--json-out-file",
+        local_benchmark_output_filename,
+        "--port",
+        "{}".format(port),
+        "--server",
+        "{}".format(server),
+    ]
+    if oss_cluster_api_enabled is True:
+        benchmark_command.append("--cluster-mode")
+    benchmark_command_str = " ".join(benchmark_command)
+    if "arguments" in clientconfig:
+        benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"]
+
+    return None, benchmark_command_str
+
+
+def prepare_vector_db_benchmark_parameters(
+    clientconfig, full_benchmark_path, port, server, password, client_mnt_point
+):
+    benchmark_command = []
+    # if port is not None:
+    #     benchmark_command.extend(["REDIS_PORT={}".format(port)])
+    # if password is not None:
+    #     benchmark_command.extend(["REDIS_AUTH={}".format(password)])
+    benchmark_command.extend(
+        [
+            full_benchmark_path,
+            "--host",
+            f"{server}",
+        ]
+    )
+    benchmark_command.extend(["--engines", clientconfig.get("engines", "redis-test")])
+    benchmark_command.extend(
+        ["--datasets", clientconfig.get("datasets", "glove-100-angular")]
+    )
+    benchmark_command_str = " ".join(benchmark_command)
+    benchmark_command_str = f"bash -c 'ITERATIONS=1 {benchmark_command_str} && mv /code/results {client_mnt_point}.'"
+    return None, benchmark_command_str

redis_benchmarks_specification-0.1.264/redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py

@@ -0,0 +1,19 @@
+import os
+import json
+
+
+def post_process_vector_db(temporary_dir):
+    results_dir = os.path.join(temporary_dir, "results")
+    results = {}
+    for file in os.listdir(results_dir):
+        if "upload" in file:
+            with open(os.path.join(results_dir, file), "r") as f:
+                upload_results = json.load(f)
+                results["upload_time"] = upload_results["results"]["upload_time"]
+        else:
+            with open(os.path.join(results_dir, file), "r") as f:
+                query_results = json.load(f)
+                results["rps"] = query_results["results"]["rps"]
+                results["precision"] = query_results["results"]["mean_precisions"]
+                results["total_time"] = query_results["results"]["total_time"]
+    return results

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -56,6 +56,7 @@ from redis_benchmarks_specification.__self_contained_coordinator__.build_info im
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.clients import (
     prepare_memtier_benchmark_parameters,
+    prepare_vector_db_benchmark_parameters,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import (
     extract_db_cpu_limit,
@@ -347,9 +348,12 @@ def process_self_contained_coordinator_stream(
                             # backwards compatible
                             if benchmark_tool is None:
                                 benchmark_tool = "redis-benchmark"
-                            full_benchmark_path = "/usr/local/bin/{}".format(
-                                benchmark_tool
-                            )
+                            if benchmark_tool == "vector_db_benchmark":
+                                full_benchmark_path = "python /code/run.py"
+                            else:
+                                full_benchmark_path = "/usr/local/bin/{}".format(
+                                    benchmark_tool
+                                )
 
                             # setup the benchmark
                             (
@@ -370,32 +374,42 @@ def process_self_contained_coordinator_stream(
                                     local_benchmark_output_filename
                                 )
                             )
-                            if "memtier_benchmark" not in benchmark_tool:
-                                # prepare the benchmark command
+                            if "memtier_benchmark" in benchmark_tool:
                                 (
-                                    benchmark_command,
+                                    _,
                                     benchmark_command_str,
-                                ) = prepare_benchmark_parameters(
-                                    benchmark_config,
+                                ) = prepare_memtier_benchmark_parameters(
+                                    benchmark_config["clientconfig"],
                                     full_benchmark_path,
                                     redis_proc_start_port,
                                     "localhost",
                                     local_benchmark_output_filename,
-                                    False,
                                     benchmark_tool_workdir,
-                                    False,
                                 )
-                            else:
+                            elif "vector_db_benchmark" in benchmark_tool:
                                 (
                                     _,
                                     benchmark_command_str,
-                                ) = prepare_memtier_benchmark_parameters(
+                                ) = prepare_vector_db_benchmark_parameters(
                                     benchmark_config["clientconfig"],
                                     full_benchmark_path,
                                     redis_proc_start_port,
                                     "localhost",
+                                )
+                            else:
+                                # prepare the benchmark command
+                                (
+                                    benchmark_command,
+                                    benchmark_command_str,
+                                ) = prepare_benchmark_parameters(
+                                    benchmark_config,
+                                    full_benchmark_path,
+                                    redis_proc_start_port,
+                                    "localhost",
                                     local_benchmark_output_filename,
+                                    False,
                                     benchmark_tool_workdir,
+                                    False,
                                 )
 
                             client_container_image = extract_client_container_image(

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -7,6 +7,7 @@ import tempfile
 import traceback
 import re
 import docker
+import docker.errors
 import redis
 import os
 from pathlib import Path
@@ -14,6 +15,9 @@ import sys
 import time
 
 from docker.models.containers import Container
+from redis_benchmarks_specification.__self_contained_coordinator__.post_processing import (
+    post_process_vector_db,
+)
 from redisbench_admin.profilers.profilers_local import (
     check_compatible_system_and_kernel_and_prepare_profile,
 )
@@ -107,6 +111,9 @@ from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import
     extract_db_cpu_limit,
     generate_cpuset_cpus,
 )
+from redis_benchmarks_specification.__self_contained_coordinator__.clients import (
+    prepare_vector_db_benchmark_parameters,
+)
 from redis_benchmarks_specification.__self_contained_coordinator__.docker import (
     generate_standalone_redis_server_args,
 )
@@ -364,9 +371,17 @@ def self_contained_coordinator_blocking_read(
         count=1,
         block=0,
     )
+    logging.info(f"New test info: {newTestInfo}")
     if len(newTestInfo[0]) < 2 or len(newTestInfo[0][1]) < 1:
         stream_id = ">"
     else:
+        # Create args object with topology parameter
+        class Args:
+            def __init__(self):
+                self.topology = ""
+
+        args = Args()
+
         (
             stream_id,
             overall_result,
@@ -398,6 +413,7 @@ def self_contained_coordinator_blocking_read(
             default_metrics_str,
             docker_keep_env,
             restore_build_artifacts_default,
+            args,
         )
         num_process_streams = num_process_streams + 1
         num_process_test_suites = num_process_test_suites + total_test_suite_runs
@@ -478,6 +494,7 @@ def process_self_contained_coordinator_stream(
     default_metrics_str="ALL_STATS.Totals.Ops/sec",
     docker_keep_env=False,
     restore_build_artifacts_default=True,
+    args=None,
 ):
     stream_id = "n/a"
     overall_result = False
@@ -549,7 +566,7 @@ def process_self_contained_coordinator_stream(
                 )
                 new_executable = f"{mnt_point}{server_name}-server"
                 logging.info(
-                    "changing executable from {executable} to {new_executable}"
+                    f"changing executable from {executable} to {new_executable}"
                 )
                 executable = new_executable
 
@@ -764,6 +781,18 @@ def process_self_contained_coordinator_stream(
                         for topology_spec_name in benchmark_config["redis-topologies"]:
                             setup_name = topology_spec_name
                             setup_type = "oss-standalone"
+
+                            # Filter by topology if specified
+                            if (
+                                args is not None
+                                and args.topology
+                                and topology_spec_name != args.topology
+                            ):
+                                logging.info(
+                                    f"Skipping topology {topology_spec_name} as it doesn't match the requested topology {args.topology}"
+                                )
+                                continue
+
                             if topology_spec_name in topologies_map:
                                 topology_spec = topologies_map[topology_spec_name]
                                 setup_type = topology_spec["type"]
@@ -906,9 +935,12 @@ def process_self_contained_coordinator_stream(
                                 # backwards compatible
                                 if benchmark_tool is None:
                                     benchmark_tool = "redis-benchmark"
-                                full_benchmark_path = "/usr/local/bin/{}".format(
-                                    benchmark_tool
-                                )
+                                if benchmark_tool == "vector_db_benchmark":
+                                    full_benchmark_path = "python /code/run.py"
+                                else:
+                                    full_benchmark_path = "/usr/local/bin/{}".format(
+                                        benchmark_tool
+                                    )
 
                                 # setup the benchmark
                                 (
@@ -929,41 +961,53 @@ def process_self_contained_coordinator_stream(
                                         local_benchmark_output_filename
                                     )
                                 )
-                                if "memtier_benchmark" not in benchmark_tool:
+                                if "memtier_benchmark" in benchmark_tool:
                                     # prepare the benchmark command
                                     (
-                                        benchmark_command,
+                                        _,
                                         benchmark_command_str,
-                                    ) = prepare_benchmark_parameters(
-                                        benchmark_config,
+                                        arbitrary_command,
+                                    ) = prepare_memtier_benchmark_parameters(
+                                        benchmark_config["clientconfig"],
                                         full_benchmark_path,
                                         redis_proc_start_port,
                                         "localhost",
+                                        None,
                                         local_benchmark_output_filename,
                                         False,
-                                        benchmark_tool_workdir,
                                         False,
+                                        False,
+                                        None,
+                                        None,
+                                        None,
+                                        None,
+                                        override_test_time,
                                     )
-                                else:
+                                elif "vector_db_benchmark" in benchmark_tool:
                                     (
                                         _,
                                         benchmark_command_str,
-                                        arbitrary_command,
-                                    ) = prepare_memtier_benchmark_parameters(
+                                    ) = prepare_vector_db_benchmark_parameters(
                                         benchmark_config["clientconfig"],
                                         full_benchmark_path,
                                         redis_proc_start_port,
                                         "localhost",
                                         None,
+                                        client_mnt_point,
+                                    )
+                                else:
+                                    (
+                                        benchmark_command,
+                                        benchmark_command_str,
+                                    ) = prepare_benchmark_parameters(
+                                        benchmark_config,
+                                        full_benchmark_path,
+                                        redis_proc_start_port,
+                                        "localhost",
                                         local_benchmark_output_filename,
                                         False,
+                                        benchmark_tool_workdir,
                                         False,
-                                        False,
-                                        None,
-                                        None,
-                                        None,
-                                        None,
-                                        override_test_time,
                                     )
 
                                 client_container_image = extract_client_container_image(
@@ -995,23 +1039,37 @@ def process_self_contained_coordinator_stream(
                                 )
                                 # run the benchmark
                                 benchmark_start_time = datetime.datetime.now()
-
-                                client_container_stdout = docker_client.containers.run(
-                                    image=client_container_image,
-                                    volumes={
-                                        temporary_dir_client: {
-                                            "bind": client_mnt_point,
-                                            "mode": "rw",
-                                        },
-                                    },
-                                    auto_remove=True,
-                                    privileged=True,
-                                    working_dir=benchmark_tool_workdir,
-                                    command=benchmark_command_str,
-                                    network_mode="host",
-                                    detach=False,
-                                    cpuset_cpus=client_cpuset_cpus,
-                                )
+                                try:
+                                    client_container_stdout = (
+                                        docker_client.containers.run(
+                                            image=client_container_image,
+                                            volumes={
+                                                temporary_dir_client: {
+                                                    "bind": client_mnt_point,
+                                                    "mode": "rw",
+                                                },
+                                            },
+                                            auto_remove=True,
+                                            privileged=True,
+                                            working_dir=benchmark_tool_workdir,
+                                            command=benchmark_command_str,
+                                            network_mode="host",
+                                            detach=False,
+                                            cpuset_cpus=client_cpuset_cpus,
+                                        )
+                                    )
+                                except docker.errors.ContainerError as e:
+                                    logging.info(
+                                        "stdout: {}".format(
+                                            e.container.logs(stdout=True)
+                                        )
+                                    )
+                                    logging.info(
+                                        "stderr: {}".format(
+                                            e.container.logs(stderr=True)
+                                        )
+                                    )
+                                    raise e
 
                                 benchmark_end_time = datetime.datetime.now()
                                 benchmark_duration_seconds = (
@@ -1133,39 +1191,44 @@ def process_self_contained_coordinator_stream(
                                     and git_timestamp_ms is not None
                                 ):
                                     datapoint_time_ms = git_timestamp_ms
-                                post_process_benchmark_results(
-                                    benchmark_tool,
-                                    local_benchmark_output_filename,
-                                    datapoint_time_ms,
-                                    start_time_str,
-                                    client_container_stdout,
-                                    None,
-                                )
-                                full_result_path = local_benchmark_output_filename
-                                if "memtier_benchmark" in benchmark_tool:
-                                    full_result_path = "{}/{}".format(
-                                        temporary_dir_client,
+                                if "vector_db_benchmark" in benchmark_tool:
+                                    results_dict = post_process_vector_db(
+                                        temporary_dir_client
+                                    )
+                                else:
+                                    post_process_benchmark_results(
+                                        benchmark_tool,
                                         local_benchmark_output_filename,
+                                        datapoint_time_ms,
+                                        start_time_str,
+                                        client_container_stdout,
+                                        None,
                                     )
-                                logging.info(
-                                    "Reading results json from {}".format(
-                                        full_result_path
+                                    full_result_path = local_benchmark_output_filename
+                                    if "memtier_benchmark" in benchmark_tool:
+                                        full_result_path = "{}/{}".format(
+                                            temporary_dir_client,
+                                            local_benchmark_output_filename,
+                                        )
+                                    logging.info(
+                                        "Reading results json from {}".format(
+                                            full_result_path
+                                        )
                                     )
-                                )
 
-                                with open(
-                                    full_result_path,
-                                    "r",
-                                ) as json_file:
-                                    results_dict = json.load(json_file)
-                                    print_results_table_stdout(
-                                        benchmark_config,
-                                        default_metrics,
-                                        results_dict,
-                                        setup_type,
-                                        test_name,
-                                        None,
-                                    )
+                                    with open(
+                                        full_result_path,
+                                        "r",
+                                    ) as json_file:
+                                        results_dict = json.load(json_file)
+                                        print_results_table_stdout(
+                                            benchmark_config,
+                                            default_metrics,
+                                            results_dict,
+                                            setup_type,
+                                            test_name,
+                                            None,
+                                        )
 
                                 dataset_load_duration_seconds = 0
                                 try:

redis_benchmarks_specification-0.1.264/redis_benchmarks_specification/setups/builders/gcc:10.5.0-amd64-debian-bullseye-redisearch.yml

@@ -0,0 +1,24 @@
+version: 0.1
+id: gcc:10.5.0-amd64-debian-bullseye-redisearch
+os: debian-bullseye
+arch: amd64
+compiler: "gcc"
+cpp_compiler: "g++"
+kind: docker
+build_image: gcc:10.5.0-bullseye 
+run_image: debian:bullseye
+description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description)
+              pre-configured environment with all the tools required to build with gcc."
+build_artifacts:
+  - redisearch.so
+  - redis-server
+build_command: "sh -c 'apt update && apt install python3-pip -y && BUILD_WITH_MODULES=yes make --ignore-errors -j && cd modules/redisearch/src && make setup && make build && cd -'"
+metadata:
+  compiler: "gcc"
+  compiler_version: "10.5.0"
+  os: debian-bullseye
+  arch: amd64
+
+env:
+  REDIS_CFLAGS: "-g -fno-omit-frame-pointer"
+

{redis_benchmarks_specification-0.1.262 → redis_benchmarks_specification-0.1.264}/redis_benchmarks_specification/setups/topologies/topologies.yml

@@ -10,6 +10,17 @@ spec:
         cpus: "1"
         memory: "10g"
 
+  - name: oss-standalone-with-redisearch
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --loadmodule /mnt/redis/redisearch.so
+    resources:
+      requests:
+        cpus: "1"
+        memory: "10g"
+
   - name: oss-standalone-02-io-threads
     type: oss-standalone
     redis_topology:

redis_benchmarks_specification-0.1.264/redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-string-get-10B-pipeline-100-nokeyprefix.yml

@@ -0,0 +1,28 @@
+version: 0.4
+name: memtier_benchmark-10Mkeys-string-get-10B-pipeline-100-nokeyprefix
+description: memtier_benchmark, 10M keys, string GET, 10B value size, pipeline=100, no key prefix
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 10000000
+  resources:
+    requests:
+      memory: 10g
+tested-groups:
+- string
+tested-commands:
+- get
+redis-topologies:
+- oss-standalone
+- oss-cluster-3-primaries
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "10" --command "GET __key__" --command-key-pattern="R" --key-prefix="" -c 50 -t 2 --pipeline 100 --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: '3'
+      memory: 2g
+
+priority: 33

redis_benchmarks_specification-0.1.264/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100-nokeyprefix.yml

@@ -0,0 +1,28 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-string-with-10B-values-pipeline-100-nokeyprefix
+description: memtier_benchmark, 1M keys, string SET, 10B value size, pipeline=100, no key prefix
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- string
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+- oss-cluster-3-primaries
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "10" --command "SET __key__ __data__" --command-key-pattern="R" --key-prefix="" -c 50 -t 2 --pipeline 100 --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: '3'
+      memory: 2g
+
+priority: 33