redis-benchmarks-specification 0.1.214__tar.gz → 0.1.215__tar.gz

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.215}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.214
+Version: 0.1.215
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.215}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.214"
+version = "0.1.215"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

redis_benchmarks_specification-0.1.215/redis_benchmarks_specification/__common__/spec.py

@@ -0,0 +1,82 @@
+import math
+import logging
+
+
+def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):
+    redis_configuration_parameters = {}
+    modules_configuration_parameters_map = {}
+    dataset_load_timeout_secs = 120
+    dataset_name = None
+    dbconfig_present = False
+    if dbconfig_keyname in benchmark_config:
+        dbconfig_present = True
+        if type(benchmark_config[dbconfig_keyname]) == list:
+            for k in benchmark_config[dbconfig_keyname]:
+                if "configuration-parameters" in k:
+                    cp = k["configuration-parameters"]
+                    for item in cp:
+                        for k, v in item.items():
+                            redis_configuration_parameters[k] = v
+                if "dataset_load_timeout_secs" in k:
+                    dataset_load_timeout_secs = k["dataset_load_timeout_secs"]
+                if "dataset_name" in k:
+                    dataset_name = k["dataset_name"]
+        if type(benchmark_config[dbconfig_keyname]) == dict:
+            if "configuration-parameters" in benchmark_config[dbconfig_keyname]:
+                cp = benchmark_config[dbconfig_keyname]["configuration-parameters"]
+                for k, v in cp.items():
+                    redis_configuration_parameters[k] = v
+            if "dataset_load_timeout_secs" in benchmark_config[dbconfig_keyname]:
+                dataset_load_timeout_secs = benchmark_config[dbconfig_keyname][
+                    "dataset_load_timeout_secs"
+                ]
+            if "dataset_name" in benchmark_config[dbconfig_keyname]:
+                dataset_name = benchmark_config[dbconfig_keyname]["dataset_name"]
+
+    return (
+        dbconfig_present,
+        dataset_name,
+        redis_configuration_parameters,
+        dataset_load_timeout_secs,
+        modules_configuration_parameters_map,
+    )
+
+
+def extract_redis_configuration_from_topology(topologies_map, topology_spec_name):
+    redis_arguments = ""
+    topology_spec = topologies_map[topology_spec_name]
+    if "redis_arguments" in topology_spec:
+        redis_arguments = topology_spec["redis_arguments"]
+        logging.info(
+            f"extracted redis_arguments: {redis_arguments} from topology: {topology_spec_name}"
+        )
+    return redis_arguments
+
+
+def extract_client_cpu_limit(benchmark_config):
+    db_cpu_limit = benchmark_config["clientconfig"]["resources"]["requests"]["cpus"]
+    ceil_db_cpu_limit = math.ceil(float(db_cpu_limit))
+    return ceil_db_cpu_limit
+
+
+def extract_build_variant_variations(benchmark_config, keyname="build-variants"):
+    result = None
+    if keyname in benchmark_config:
+        result = benchmark_config[keyname]
+    return result
+
+
+def extract_client_container_image(benchmark_config, keyname="clientconfig"):
+    client_container_image = None
+    if keyname in benchmark_config:
+        if "run_image" in benchmark_config[keyname]:
+            client_container_image = benchmark_config[keyname]["run_image"]
+    return client_container_image
+
+
+def extract_client_tool(benchmark_config, keyname="clientconfig"):
+    client_tool = None
+    if keyname in benchmark_config:
+        if "tool" in benchmark_config[keyname]:
+            client_tool = benchmark_config[keyname]["tool"]
+    return client_tool

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.215}/redis_benchmarks_specification/__compare__/args.py

@@ -92,8 +92,14 @@ def create_compare_arguments(parser):
     )
     parser.add_argument("--baseline-branch", type=str, default=None, required=False)
     parser.add_argument("--baseline-tag", type=str, default=None, required=False)
+    parser.add_argument(
+        "--baseline-target-version", type=str, default=None, required=False
+    )
     parser.add_argument("--comparison-branch", type=str, default=None, required=False)
     parser.add_argument("--comparison-tag", type=str, default=None, required=False)
+    parser.add_argument(
+        "--comparison-target-version", type=str, default=None, required=False
+    )
     parser.add_argument("--print-regressions-only", type=bool, default=False)
     parser.add_argument("--print-improvements-only", type=bool, default=False)
     parser.add_argument("--skip-unstable", type=bool, default=False)

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.215}/redis_benchmarks_specification/__compare__/compare.py

@@ -248,6 +248,8 @@ def compare_command_logic(args, project_name, project_version):
     testname_regex = args.testname_regex
     auto_approve = args.auto_approve
     running_platform = args.running_platform
+    baseline_target_version = args.baseline_target_version
+    comparison_target_version = args.comparison_target_version
 
     if running_platform is not None:
         logging.info(
@@ -306,6 +308,8 @@ def compare_command_logic(args, project_name, project_version):
         to_ts_ms,
         use_metric_context_path,
         running_platform,
+        baseline_target_version,
+        comparison_target_version,
     )
     prepare_regression_comment(
         auto_approve,
@@ -529,6 +533,8 @@ def compute_regression_table(
     to_ts_ms=None,
     use_metric_context_path=None,
     running_platform=None,
+    baseline_target_version=None,
+    comparison_target_version=None,
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -552,7 +558,11 @@ def compute_regression_table(
         comparison_branch,
         baseline_tag,
         comparison_tag,
+        baseline_target_version,
+        comparison_target_version,
     )
+    logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}")
+    logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}")
     (
         prefix,
         testcases_setname,
@@ -712,6 +722,8 @@ def get_by_strings(
     comparison_branch,
     baseline_tag,
     comparison_tag,
+    baseline_target_version=None,
+    comparison_target_version=None,
 ):
     baseline_covered = False
     comparison_covered = False
@@ -738,6 +750,16 @@ def get_by_strings(
         by_str_baseline = "version"
         baseline_str = baseline_tag
 
+    if baseline_target_version is not None:
+        if comparison_covered:
+            logging.error(
+                "--baseline-branch, --baseline-tag and --baseline-target-version are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        baseline_covered = True
+        by_str_baseline = "target+version"
+        baseline_str = baseline_target_version
+
     if comparison_tag is not None:
         # check if we had already covered comparison
         if comparison_covered:
@@ -748,16 +770,27 @@ def get_by_strings(
         comparison_covered = True
         by_str_comparison = "version"
         comparison_str = comparison_tag
+    if comparison_target_version is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "target+version"
+        comparison_str = comparison_target_version
 
     if baseline_covered is False:
         logging.error(
-            "You need to provider either " + "( --baseline-branch or --baseline-tag ) "
+            "You need to provider either "
+            + "( --baseline-branch, --baseline-tag, or --baseline-target-version ) "
         )
         exit(1)
     if comparison_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --comparison-branch or --comparison-tag ) "
+            + "( --comparison-branch, --comparison-tag, or --comparison-target-version ) "
         )
         exit(1)
     return baseline_str, by_str_baseline, comparison_str, by_str_comparison
@@ -822,6 +855,7 @@ def from_rts_to_regression_table(
         filters_comparison = [
             "{}={}".format(by_str_comparison, comparison_str),
             "metric={}".format(metric_name),
+            "hash==",
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(comparison_deployment_name),
             "triggering_env={}".format(tf_triggering_env),

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.215}/redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -8,7 +8,7 @@ from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import
 
 
 def generate_standalone_redis_server_args(
-    binary, port, dbdir, configuration_parameters=None
+    binary, port, dbdir, configuration_parameters=None, redis_arguments=""
 ):
     added_params = ["port", "protected-mode", "dir"]
     # start redis-server
@@ -30,6 +30,10 @@ def generate_standalone_redis_server_args(
                         parameter_value,
                     ]
                 )
+    if redis_arguments != "":
+        redis_arguments_arr = redis_arguments.split(" ")
+        logging.info(f"adding redis arguments {redis_arguments_arr}")
+        command.extend(redis_arguments_arr)
     return command
 
 

{redis_benchmarks_specification-0.1.214 → redis_benchmarks_specification-0.1.215}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -80,7 +80,6 @@ from redisbench_admin.run.grafana import generate_artifacts_table_grafana_redis
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
 from redisbench_admin.utils.benchmark_config import (
     get_final_benchmark_config,
-    extract_redis_dbconfig_parameters,
     get_defaults,
 )
 from redisbench_admin.utils.local import get_local_run_full_filename
@@ -95,6 +94,8 @@ from redis_benchmarks_specification.__common__.spec import (
     extract_client_cpu_limit,
     extract_client_tool,
     extract_client_container_image,
+    extract_redis_dbconfig_parameters,
+    extract_redis_configuration_from_topology,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import (
     restore_build_artifacts_from_test_details,
@@ -756,6 +757,14 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
                         for topology_spec_name in benchmark_config["redis-topologies"]:
+                            setup_name = topology_spec_name
+                            setup_type = "oss-standalone"
+                            if topology_spec_name in topologies_map:
+                                topology_spec = topologies_map[topology_spec_name]
+                                setup_type = topology_spec["type"]
+                            logging.info(
+                                f"Running topology named {topology_spec_name} of type {setup_type}"
+                            )
                             test_result = False
                             redis_container = None
                             try:
@@ -763,6 +772,11 @@ def process_self_contained_coordinator_stream(
                                 ceil_db_cpu_limit = extract_db_cpu_limit(
                                     topologies_map, topology_spec_name
                                 )
+                                redis_arguments = (
+                                    extract_redis_configuration_from_topology(
+                                        topologies_map, topology_spec_name
+                                    )
+                                )
                                 temporary_dir = tempfile.mkdtemp(dir=home)
                                 temporary_dir_client = tempfile.mkdtemp(dir=home)
                                 logging.info(
@@ -776,8 +790,6 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
 
-                                setup_name = "oss-standalone"
-                                setup_type = "oss-standalone"
                                 tf_triggering_env = "ci"
                                 github_actor = "{}-{}".format(
                                     tf_triggering_env, running_platform
@@ -814,6 +826,7 @@ def process_self_contained_coordinator_stream(
                                     redis_proc_start_port,
                                     mnt_point,
                                     redis_configuration_parameters,
+                                    redis_arguments,
                                 )
                                 command_str = " ".join(command)
                                 db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
@@ -926,7 +939,7 @@ def process_self_contained_coordinator_stream(
                                         start_time_str,
                                         git_hash,
                                         test_name,
-                                        "oss-standalone",
+                                        setup_name,
                                     )
                                 )
                                 logging.info(

redis_benchmarks_specification-0.1.215/redis_benchmarks_specification/setups/topologies/topologies.yml

@@ -0,0 +1,142 @@
+spec:
+  setups:
+  - name: oss-standalone
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    resources:
+      requests:
+        cpus: "1"
+        memory: "10g"
+
+  - name: oss-standalone-02-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 2 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "3"
+        memory: "10g"
+
+  - name: oss-standalone-04-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 4 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "5"
+        memory: "10g"
+
+  - name: oss-standalone-08-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 8 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "9"
+        memory: "10g"
+
+  - name: oss-standalone-16-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 16 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "17"
+        memory: "10g"
+
+  - name: oss-standalone-32-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 32 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "33"
+        memory: "10g"
+
+  - name: oss-standalone-64-io-threads
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 0
+    redis_arguments: --io-threads 64 --io-threads-do-reads yes
+    resources:
+      requests:
+        cpus: "65"
+        memory: "10g"
+  - name: oss-standalone-1replica
+    type: oss-standalone
+    redis_topology:
+      primaries: 1
+      replicas: 1
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "2"
+        memory: "10g"
+
+  - name: oss-cluster-3-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 3
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "3"
+        memory: "30g"
+
+  - name: oss-cluster-5-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 5
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "5"
+        memory: "50g"
+
+  - name: oss-cluster-9-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 9
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "10"
+        memory: "90g"
+
+  - name: oss-cluster-15-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 15
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "15"
+        memory: "150g"
+
+  - name: oss-cluster-30-primaries
+    type: oss-cluster
+    redis_topology:
+      primaries: 30
+      replicas: 0
+      placement: "sparse"
+    resources:
+      requests:
+        cpus: "30"
+        memory: "300g"

redis_benchmarks_specification-0.1.215/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-load-string-with-512B-values
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+- oss-standalone-02-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-08-io-threads
+- oss-standalone-16-io-threads
+- oss-standalone-32-io-threads
+- oss-standalone-64-io-threads
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "512" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram'
+  resources:
+    requests:
+      cpus: '13'
+      memory: 2g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification-0.1.214/redis_benchmarks_specification/__common__/spec.py

@@ -1,30 +0,0 @@
-import math
-
-
-def extract_client_cpu_limit(benchmark_config):
-    db_cpu_limit = benchmark_config["clientconfig"]["resources"]["requests"]["cpus"]
-    ceil_db_cpu_limit = math.ceil(float(db_cpu_limit))
-    return ceil_db_cpu_limit
-
-
-def extract_build_variant_variations(benchmark_config, keyname="build-variants"):
-    result = None
-    if keyname in benchmark_config:
-        result = benchmark_config[keyname]
-    return result
-
-
-def extract_client_container_image(benchmark_config, keyname="clientconfig"):
-    client_container_image = None
-    if keyname in benchmark_config:
-        if "run_image" in benchmark_config[keyname]:
-            client_container_image = benchmark_config[keyname]["run_image"]
-    return client_container_image
-
-
-def extract_client_tool(benchmark_config, keyname="clientconfig"):
-    client_tool = None
-    if keyname in benchmark_config:
-        if "tool" in benchmark_config[keyname]:
-            client_tool = benchmark_config[keyname]["tool"]
-    return client_tool

redis_benchmarks_specification-0.1.214/redis_benchmarks_specification/setups/topologies/topologies.yml

@@ -1,77 +0,0 @@
-spec:
-  setups:
-  - name: oss-standalone
-    type: oss-standalone
-    redis_topology:
-      primaries: 1
-      replicas: 0
-    resources:
-      requests:
-        cpus: "1"
-        memory: "10g"
-
-  - name: oss-standalone-1replica
-    type: oss-standalone
-    redis_topology:
-      primaries: 1
-      replicas: 1
-      placement: "sparse"
-    resources:
-      requests:
-        cpus: "2"
-        memory: "10g"
-
-  - name: oss-cluster-3-primaries
-    type: oss-cluster
-    redis_topology:
-      primaries: 3
-      replicas: 0
-      placement: "sparse"
-    resources:
-      requests:
-        cpus: "3"
-        memory: "30g"
-
-  - name: oss-cluster-5-primaries
-    type: oss-cluster
-    redis_topology:
-      primaries: 5
-      replicas: 0
-      placement: "sparse"
-    resources:
-      requests:
-        cpus: "5"
-        memory: "50g"
-
-  - name: oss-cluster-9-primaries
-    type: oss-cluster
-    redis_topology:
-      primaries: 9
-      replicas: 0
-      placement: "sparse"
-    resources:
-      requests:
-        cpus: "10"
-        memory: "90g"
-
-  - name: oss-cluster-15-primaries
-    type: oss-cluster
-    redis_topology:
-      primaries: 15
-      replicas: 0
-      placement: "sparse"
-    resources:
-      requests:
-        cpus: "15"
-        memory: "150g"
-
-  - name: oss-cluster-30-primaries
-    type: oss-cluster
-    redis_topology:
-      primaries: 30
-      replicas: 0
-      placement: "sparse"
-    resources:
-      requests:
-        cpus: "30"
-        memory: "300g"