redis-benchmarks-specification 0.1.214__tar.gz → 0.1.216__tar.gz

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.214
+Version: 0.1.216
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.214"
+version = "0.1.216"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/redis_benchmarks_specification/__builder__/builder.py

@@ -343,6 +343,8 @@ def builder_process_stream(
                     "linenoise",
                     "lua",
                 ]
+                if "fast_float" in deps_dir:
+                    deps_list.append("fast_float")
                 if "hdr_histogram" in deps_dir:
                     deps_list.append("hdr_histogram")
                 if "fpconv" in deps_dir:

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/redis_benchmarks_specification/__cli__/cli.py

@@ -87,6 +87,23 @@ def trigger_tests_dockerhub_cli_command_logic(args, project_name, project_versio
         args.build_arch,
         testDetails,
         "n/a",
+        [],
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        None,
+        ".*",
+        0,
+        10000,
+        args.tests_regexp,
     )
     build_stream_fields["github_repo"] = args.gh_repo
     build_stream_fields["github_org"] = args.gh_org

redis_benchmarks_specification-0.1.216/redis_benchmarks_specification/__common__/spec.py

@@ -0,0 +1,82 @@
+import math
+import logging
+
+
+def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
+    redis_configuration_parameters = {}
+    modules_configuration_parameters_map = {}
+    dataset_load_timeout_secs = 120
+    dataset_name = None
+    dbconfig_present = False
+    if dbconfig_keyname in benchmark_config:
+        dbconfig_present = True
+        if type(benchmark_config[dbconfig_keyname]) == list:
+            for k in benchmark_config[dbconfig_keyname]:
+                if "configuration-parameters" in k:
+                    cp = k["configuration-parameters"]
+                    for item in cp:
+                        for k, v in item.items():
+                            redis_configuration_parameters[k] = v
+                if "dataset_load_timeout_secs" in k:
+                    dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
+                if "dataset_name" in k:
+                    dataset_name = k["dataset_name"]
+        if type(benchmark_config[dbconfig_keyname]) == dict:
+            if "configuration-parameters" in benchmark_config[dbconfig_keyname]:
+                cp = benchmark_config[dbconfig_keyname]["configuration-parameters"]
+                for k, v in cp.items():
+                    redis_configuration_parameters[k] = v
+            if "dataset_load_timeout_secs" in benchmark_config[dbconfig_keyname]:
+                dataset_load_timeout_secs = benchmark_config[dbconfig_keyname][
+                    "dataset_load_timeout_secs"
+                ]
+            if "dataset_name" in benchmark_config[dbconfig_keyname]:
+                dataset_name = benchmark_config[dbconfig_keyname]["dataset_name"]
+
+    return (
+        dbconfig_present,
+        dataset_name,
+        redis_configuration_parameters,
+        dataset_load_timeout_secs,
+        modules_configuration_parameters_map,
+    )
+
+
+def extract_redis_configuration_from_topology(topologies_map, topology_spec_name):
+    redis_arguments = ""
+    topology_spec = topologies_map[topology_spec_name]
+    if "redis_arguments" in topology_spec:
+        redis_arguments = topology_spec["redis_arguments"]
+        logging.info(
+            f"extracted redis_arguments: {redis_arguments} from topology: {topology_spec_name}"
+        )
+    return redis_arguments
+
+
+def extract_client_cpu_limit(benchmark_config):
+    db_cpu_limit = benchmark_config["clientconfig"]["resources"]["requests"]["cpus"]
+    ceil_db_cpu_limit = math.ceil(float(db_cpu_limit))
+    return ceil_db_cpu_limit
+
+
+def extract_build_variant_variations(benchmark_config, keyname="build-variants"):
+    result = None
+    if keyname in benchmark_config:
+        result = benchmark_config[keyname]
+    return result
+
+
+def extract_client_container_image(benchmark_config, keyname="clientconfig"):
+    client_container_image = None
+    if keyname in benchmark_config:
+        if "run_image" in benchmark_config[keyname]:
+            client_container_image = benchmark_config[keyname]["run_image"]
+    return client_container_image
+
+
+def extract_client_tool(benchmark_config, keyname="clientconfig"):
+    client_tool = None
+    if keyname in benchmark_config:
+        if "tool" in benchmark_config[keyname]:
+            client_tool = benchmark_config[keyname]["tool"]
+    return client_tool

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/redis_benchmarks_specification/__compare__/args.py

@@ -92,8 +92,14 @@ def create_compare_arguments(parser):
     )
     parser.add_argument("--baseline-branch", type=str, default=None, required=False)
     parser.add_argument("--baseline-tag", type=str, default=None, required=False)
+    parser.add_argument(
+        "--baseline-target-version", type=str, default=None, required=False
+    )
     parser.add_argument("--comparison-branch", type=str, default=None, required=False)
     parser.add_argument("--comparison-tag", type=str, default=None, required=False)
+    parser.add_argument(
+        "--comparison-target-version", type=str, default=None, required=False
+    )
     parser.add_argument("--print-regressions-only", type=bool, default=False)
     parser.add_argument("--print-improvements-only", type=bool, default=False)
     parser.add_argument("--skip-unstable", type=bool, default=False)

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/redis_benchmarks_specification/__compare__/compare.py

@@ -248,6 +248,8 @@ def compare_command_logic(args, project_name, project_version):
     testname_regex = args.testname_regex
     auto_approve = args.auto_approve
     running_platform = args.running_platform
+    baseline_target_version = args.baseline_target_version
+    comparison_target_version = args.comparison_target_version
 
     if running_platform is not None:
         logging.info(
@@ -306,6 +308,8 @@ def compare_command_logic(args, project_name, project_version):
         to_ts_ms,
         use_metric_context_path,
         running_platform,
+        baseline_target_version,
+        comparison_target_version,
     )
     prepare_regression_comment(
         auto_approve,
@@ -529,6 +533,8 @@ def compute_regression_table(
     to_ts_ms=None,
     use_metric_context_path=None,
     running_platform=None,
+    baseline_target_version=None,
+    comparison_target_version=None,
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -552,7 +558,11 @@ def compute_regression_table(
         comparison_branch,
         baseline_tag,
         comparison_tag,
+        baseline_target_version,
+        comparison_target_version,
     )
+    logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}")
+    logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}")
     (
         prefix,
         testcases_setname,
@@ -712,6 +722,8 @@ def get_by_strings(
     comparison_branch,
     baseline_tag,
     comparison_tag,
+    baseline_target_version=None,
+    comparison_target_version=None,
 ):
     baseline_covered = False
     comparison_covered = False
@@ -738,6 +750,16 @@ def get_by_strings(
         by_str_baseline = "version"
         baseline_str = baseline_tag
 
+    if baseline_target_version is not None:
+        if comparison_covered:
+            logging.error(
+                "--baseline-branch, --baseline-tag and --baseline-target-version are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        baseline_covered = True
+        by_str_baseline = "target+version"
+        baseline_str = baseline_target_version
+
     if comparison_tag is not None:
         # check if we had already covered comparison
         if comparison_covered:
@@ -748,16 +770,27 @@ def get_by_strings(
         comparison_covered = True
         by_str_comparison = "version"
         comparison_str = comparison_tag
+    if comparison_target_version is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "target+version"
+        comparison_str = comparison_target_version
 
     if baseline_covered is False:
         logging.error(
-            "You need to provider either " + "( --baseline-branch or --baseline-tag ) "
+            "You need to provider either "
+            + "( --baseline-branch, --baseline-tag, or --baseline-target-version ) "
         )
         exit(1)
     if comparison_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --comparison-branch or --comparison-tag ) "
+            + "( --comparison-branch, --comparison-tag, or --comparison-target-version ) "
         )
         exit(1)
     return baseline_str, by_str_baseline, comparison_str, by_str_comparison
@@ -822,6 +855,7 @@ def from_rts_to_regression_table(
         filters_comparison = [
             "{}={}".format(by_str_comparison, comparison_str),
             "metric={}".format(metric_name),
+            "hash==",
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(comparison_deployment_name),
             "triggering_env={}".format(tf_triggering_env),

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -8,7 +8,7 @@ from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import
 
 
 def generate_standalone_redis_server_args(
-    binary, port, dbdir, configuration_parameters=None
+    binary, port, dbdir, configuration_parameters=None, redis_arguments=""
 ):
     added_params = ["port", "protected-mode", "dir"]
     # start redis-server
@@ -20,7 +20,7 @@ def generate_standalone_redis_server_args(
         "{}".format(port),
     ]
     if dbdir != "":
-        command.extend(["--dbdir", dbdir])
+        command.extend(["--dir", dbdir])
     if configuration_parameters is not None:
         for parameter, parameter_value in configuration_parameters.items():
             if parameter not in added_params:
@@ -30,6 +30,10 @@ def generate_standalone_redis_server_args(
                         parameter_value,
                     ]
                 )
+    if redis_arguments != "":
+        redis_arguments_arr = redis_arguments.split(" ")
+        logging.info(f"adding redis arguments {redis_arguments_arr}")
+        command.extend(redis_arguments_arr)
     return command
 
 

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.216}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -80,7 +80,6 @@ from redisbench_admin.run.grafana import generate_artifacts_table_grafana_redis
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
 from redisbench_admin.utils.benchmark_config import (
     get_final_benchmark_config,
-    extract_redis_dbconfig_parameters,
     get_defaults,
 )
 from redisbench_admin.utils.local import get_local_run_full_filename
@@ -95,6 +94,8 @@ from redis_benchmarks_specification.__common__.spec import (
     extract_client_cpu_limit,
     extract_client_tool,
     extract_client_container_image,
+    extract_redis_dbconfig_parameters,
+    extract_redis_configuration_from_topology,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import (
     restore_build_artifacts_from_test_details,
@@ -756,6 +757,14 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
                         for topology_spec_name in benchmark_config["redis-topologies"]:
+                            setup_name = topology_spec_name
+                            setup_type = "oss-standalone"
+                            if topology_spec_name in topologies_map:
+                                topology_spec = topologies_map[topology_spec_name]
+                                setup_type = topology_spec["type"]
+                            logging.info(
+                                f"Running topology named {topology_spec_name} of type {setup_type}"
+                            )
                             test_result = False
                             redis_container = None
                             try:
@@ -763,6 +772,11 @@ def process_self_contained_coordinator_stream(
                                 ceil_db_cpu_limit = extract_db_cpu_limit(
                                     topologies_map, topology_spec_name
                                 )
+                                redis_arguments = (
+                                    extract_redis_configuration_from_topology(
+                                        topologies_map, topology_spec_name
+                                    )
+                                )
                                 temporary_dir = tempfile.mkdtemp(dir=home)
                                 temporary_dir_client = tempfile.mkdtemp(dir=home)
                                 logging.info(
@@ -776,8 +790,6 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
 
-                                setup_name = "oss-standalone"
-                                setup_type = "oss-standalone"
                                 tf_triggering_env = "ci"
                                 github_actor = "{}-{}".format(
                                     tf_triggering_env, running_platform
@@ -814,44 +826,22 @@ def process_self_contained_coordinator_stream(
                                     redis_proc_start_port,
                                     mnt_point,
                                     redis_configuration_parameters,
+                                    redis_arguments,
                                 )
                                 command_str = " ".join(command)
                                 db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
                                     ceil_db_cpu_limit, current_cpu_pos
                                 )
-                                logging.info(
-                                    "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
-                                        run_image, db_cpuset_cpus, command_str
-                                    )
-                                )
-                                volumes = {}
-                                working_dir = "/"
-                                if mnt_point != "":
-                                    volumes = {
-                                        temporary_dir: {
-                                            "bind": mnt_point,
-                                            "mode": "rw",
-                                        },
-                                    }
-                                    working_dir = mnt_point
-                                redis_container = docker_client.containers.run(
-                                    image=run_image,
-                                    volumes=volumes,
-                                    auto_remove=True,
-                                    privileged=True,
-                                    working_dir=mnt_point,
-                                    command=command_str,
-                                    network_mode="host",
-                                    detach=True,
-                                    cpuset_cpus=db_cpuset_cpus,
-                                    pid_mode="host",
-                                    publish_all_ports=True,
+                                redis_container = start_redis_container(
+                                    command_str,
+                                    db_cpuset_cpus,
+                                    docker_client,
+                                    mnt_point,
+                                    redis_containers,
+                                    run_image,
+                                    temporary_dir,
                                 )
 
-                                time.sleep(5)
-
-                                redis_containers.append(redis_container)
-
                                 r = redis.StrictRedis(port=redis_proc_start_port)
                                 r.ping()
                                 redis_conns = [r]
@@ -926,7 +916,7 @@ def process_self_contained_coordinator_stream(
                                         start_time_str,
                                         git_hash,
                                         test_name,
-                                        "oss-standalone",
+                                        setup_name,
                                     )
                                 )
                                 logging.info(
@@ -1238,9 +1228,10 @@ def process_self_contained_coordinator_stream(
                                                 stdout=True, stderr=True
                                             )
                                         )
+                                        redis_container.remove()
                                     except docker.errors.NotFound:
                                         logging.info(
-                                            "When trying to stop DB container with id {} and image {} it was already stopped".format(
+                                            "When trying to fetch logs from DB container with id {} and image {} it was already stopped".format(
                                                 redis_container.id,
                                                 redis_container.image,
                                             )
@@ -1256,6 +1247,7 @@ def process_self_contained_coordinator_stream(
                                 for redis_container in redis_containers:
                                     try:
                                         redis_container.stop()
+                                        redis_container.remove()
                                     except docker.errors.NotFound:
                                         logging.info(
                                             "When trying to stop DB container with id {} and image {} it was already stopped".format(
@@ -1269,6 +1261,7 @@ def process_self_contained_coordinator_stream(
                                     if type(redis_container) == Container:
                                         try:
                                             redis_container.stop()
+                                            redis_container.remove()
                                         except docker.errors.NotFound:
                                             logging.info(
                                                 "When trying to stop Client container with id {} and image {} it was already stopped".format(
@@ -1282,6 +1275,7 @@ def process_self_contained_coordinator_stream(
                                         temporary_dir, temporary_dir_client
                                     )
                                 )
+
                                 shutil.rmtree(temporary_dir, ignore_errors=True)
                                 shutil.rmtree(temporary_dir_client, ignore_errors=True)
 
@@ -1462,6 +1456,50 @@ def process_self_contained_coordinator_stream(
     return stream_id, overall_result, total_test_suite_runs
 
 
+def start_redis_container(
+    command_str,
+    db_cpuset_cpus,
+    docker_client,
+    mnt_point,
+    redis_containers,
+    run_image,
+    temporary_dir,
+    auto_remove=False,
+):
+    logging.info(
+        "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
+            run_image, db_cpuset_cpus, command_str
+        )
+    )
+    volumes = {}
+    working_dir = "/"
+    if mnt_point != "":
+        volumes = {
+            temporary_dir: {
+                "bind": mnt_point,
+                "mode": "rw",
+            },
+        }
+        logging.info(f"setting volume as follow: {volumes}. working_dir={mnt_point}")
+        working_dir = mnt_point
+    redis_container = docker_client.containers.run(
+        image=run_image,
+        volumes=volumes,
+        auto_remove=auto_remove,
+        privileged=True,
+        working_dir=mnt_point,
+        command=command_str,
+        network_mode="host",
+        detach=True,
+        cpuset_cpus=db_cpuset_cpus,
+        pid_mode="host",
+        publish_all_ports=True,
+    )
+    time.sleep(5)
+    redis_containers.append(redis_container)
+    return redis_container
+
+
 def filter_test_files(
     defaults_filename,
     priority_lower_limit,

redis_benchmarks_specification-0.1.216/redis_benchmarks_specification/setups/topologies/topologies.yml

@@ -0,0 +1,142 @@
+spec:
+  setups:
+  - name: oss-standalone
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    resources:
+      requests:
+        cpus: "1"
+        memory: "10g"
+
+  - name: oss-standalone-02-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 2 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "3"
+        memory: "10g"
+
+  - name: oss-standalone-04-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 4 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "5"
+        memory: "10g"
+
+  - name: oss-standalone-08-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 8 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "9"
+        memory: "10g"
+
+  - name: oss-standalone-16-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 16 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "17"
+        memory: "10g"
+
+  - name: oss-standalone-32-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 32 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "33"
+        memory: "10g"
+
+  - name: oss-standalone-64-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 64 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "65"
+        memory: "10g"
+  - name: oss-standalone-1replica
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 1
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "2"
+        memory: "10g"
+
+  - name: oss-cluster-3-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 3
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "3"
+        memory: "30g"
+
+  - name: oss-cluster-5-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 5
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "5"
+        memory: "50g"
+
+  - name: oss-cluster-9-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 9
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "10"
+        memory: "90g"
+
+  - name: oss-cluster-15-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 15
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "15"
+        memory: "150g"
+
+  - name: oss-cluster-30-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 30
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "30"
+        memory: "300g"

redis_benchmarks_specification-0.1.216/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-load-string-with-512B-values
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
+- oss-standalone-32-io-threads
+- oss-standalone-64-io-threads
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "512" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram'
+  resources:
+    requests:
+      cpus: '13'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 17