redis-benchmarks-specification 0.1.320__tar.gz → 0.1.322__tar.gz

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.320
+Version: 0.1.322
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.320"
+version = "0.1.322"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__builder__/builder.py

@@ -73,18 +73,6 @@ def main():
     parser.add_argument(
         "--arch", type=str, default="amd64", help="arch to build artifacts"
     )
-    parser.add_argument(
-        "--builder-group",
-        type=str,
-        default=STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
-        help="Consumer group name to read from the stream",
-    )
-    parser.add_argument(
-        "--builder-id",
-        type=str,
-        default="1",
-        help="Consumer id to read from the stream",
-    )
     parser.add_argument(
         "--setups-folder",
         type=str,
@@ -161,14 +149,7 @@ def main():
 
     build_spec_image_prefetch(builders_folder, different_build_specs)
 
-    builder_group = args.builder_group
-    builder_id = args.builder_id
-    if builder_group is None:
-        builder_group = STREAM_GH_EVENTS_COMMIT_BUILDERS_CG
-    if builder_id is None:
-        builder_id = "1"
-
-    builder_consumer_group_create(conn, builder_group)
+    builder_consumer_group_create(conn)
     if args.github_token is not None:
         logging.info("detected a github token. will update as much as possible!!! =)")
     previous_id = args.consumer_start_id
@@ -181,26 +162,28 @@ def main():
             args.docker_air_gap,
             arch,
             args.github_token,
-            builder_group,
-            builder_id,
         )
 
 
-def builder_consumer_group_create(
-    conn, builder_group=STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, id="$"
-):
+def builder_consumer_group_create(conn, id="$"):
     try:
         conn.xgroup_create(
             STREAM_KEYNAME_GH_EVENTS_COMMIT,
-            builder_group,
+            STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
             mkstream=True,
             id=id,
         )
         logging.info(
-            "Created consumer group named {} to distribute work.".format(builder_group)
+            "Created consumer group named {} to distribute work.".format(
+                STREAM_GH_EVENTS_COMMIT_BUILDERS_CG
+            )
         )
     except redis.exceptions.ResponseError:
-        logging.info("Consumer group named {} already existed.".format(builder_group))
+        logging.info(
+            "Consumer group named {} already existed.".format(
+                STREAM_GH_EVENTS_COMMIT_BUILDERS_CG
+            )
+        )
 
 
 def check_benchmark_build_comment(comments):
@@ -222,22 +205,14 @@ def builder_process_stream(
     docker_air_gap=False,
     arch="amd64",
     github_token=None,
-    builder_group=None,
-    builder_id=None,
 ):
     new_builds_count = 0
     auto_approve_github_comments = True
     build_stream_fields_arr = []
-    if builder_group is None:
-        builder_group = STREAM_GH_EVENTS_COMMIT_BUILDERS_CG
-    if builder_id is None:
-        builder_id = "1"
-    consumer_name = "{}-proc#{}".format(builder_group, builder_id)
-    logging.info(
-        f"Entering blocking read waiting for work. building for arch: {arch}. Using consumer id {consumer_name}"
-    )
+    logging.info("Entering blocking read waiting for work.")
+    consumer_name = "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1")
     newTestInfo = conn.xreadgroup(
-        builder_group,
+        STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
         consumer_name,
         {STREAM_KEYNAME_GH_EVENTS_COMMIT: previous_id},
         count=1,
@@ -255,21 +230,6 @@ def builder_process_stream(
         docker_client = docker.from_env()
         from pathlib import Path
 
-        build_request_arch = None
-        if b"arch" in testDetails:
-            build_request_arch = testDetails[b"arch"].decode()
-        elif b"build_arch" in testDetails:
-            build_request_arch = testDetails[b"build_arch"].decode()
-        else:
-            logging.info("No arch info found on the stream.")
-        if build_request_arch is not None and build_request_arch != arch:
-            logging.info(
-                "skipping build request given requested build arch {}!={}".format(
-                    build_request_arch, arch
-                )
-            )
-            return previous_id, new_builds_count, build_stream_fields_arr
-
         home = str(Path.home())
         if b"git_hash" in testDetails:
             git_hash = testDetails[b"git_hash"]

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__cli__/args.py

@@ -138,7 +138,7 @@ def spec_cli_args(parser):
     parser.add_argument("--gh_repo", type=str, default="redis")
     parser.add_argument("--server_name", type=str, default=None)
     parser.add_argument("--run_image", type=str, default="redis")
-    parser.add_argument("--build_arch", type=str, default=None)
+    parser.add_argument("--build_arch", type=str, default="amd64")
     parser.add_argument("--id", type=str, default="dockerhub")
     parser.add_argument("--mnt_point", type=str, default="")
     parser.add_argument("--trigger-unstable-commits", type=bool, default=True)

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__cli__/cli.py

@@ -432,9 +432,6 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             commit_dict["tests_groups_regexp"] = tests_groups_regexp
             commit_dict["github_org"] = args.gh_org
             commit_dict["github_repo"] = args.gh_repo
-            if args.build_arch is not None:
-                commit_dict["build_arch"] = args.build_arch
-                commit_dict["arch"] = args.build_arch
             if args.server_name is not None and args.server_name != "":
                 commit_dict["server_name"] = args.server_name
             if args.build_artifacts != "":

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__compare__/args.py

@@ -46,6 +46,12 @@ def create_compare_arguments(parser):
         default="",
         help="specify a test (or a comma separated list of tests) to use for comparison. If none is specified by default will use all of them.",
     )
+    parser.add_argument(
+        "--use-test-suites-folder",
+        action="store_true",
+        default=False,
+        help="Use test names from YAML files in test-suites folder instead of database",
+    )
     parser.add_argument(
         "--defaults_filename",
         type=str,

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__compare__/compare.py

@@ -399,6 +399,8 @@ def compare_command_logic(args, project_name, project_version):
         args.regression_str,
         args.improvement_str,
         tests_with_config,
+        args.use_test_suites_folder,
+        testsuites_folder,
     )
     total_regressions = len(regressions_list)
     total_improvements = len(improvements_list)
@@ -686,6 +688,8 @@ def compute_regression_table(
     regression_str="REGRESSION",
     improvement_str="IMPROVEMENT",
     tests_with_config={},
+    use_test_suites_folder=False,
+    test_suites_folder=None,
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -746,6 +750,10 @@ def compute_regression_table(
     if test != "":
         test_names = test.split(",")
         logging.info("Using test name {}".format(test_names))
+    elif use_test_suites_folder:
+        test_names = get_test_names_from_yaml_files(
+            test_suites_folder, tags_regex_string
+        )
     else:
         test_names = get_test_names_from_db(
             rts, tags_regex_string, test_names, used_key
@@ -1620,6 +1628,31 @@ def get_test_names_from_db(rts, tags_regex_string, test_names, used_key):
     return test_names
 
 
+def get_test_names_from_yaml_files(test_suites_folder, tags_regex_string):
+    """Get test names from YAML files in test-suites folder"""
+    from redis_benchmarks_specification.__common__.runner import get_benchmark_specs
+
+    # Get all YAML files
+    yaml_files = get_benchmark_specs(test_suites_folder, test="", test_regex=".*")
+
+    # Extract test names (remove path and .yml extension)
+    test_names = []
+    for yaml_file in yaml_files:
+        test_name = os.path.basename(yaml_file).replace(".yml", "")
+        # Apply regex filtering like database version
+        match_obj = re.search(tags_regex_string, test_name)
+        if match_obj is not None:
+            test_names.append(test_name)
+
+    test_names.sort()
+    logging.info(
+        "Based on test-suites folder ({}) we have {} comparison points: {}".format(
+            test_suites_folder, len(test_names), test_names
+        )
+    )
+    return test_names
+
+
 def get_line(
     baseline_v_str,
     comparison_v_str,

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__self_contained_coordinator__/docker.py

@@ -15,7 +15,7 @@ def generate_standalone_redis_server_args(
     redis_arguments="",
     password=None,
 ):
-    added_params = ["port", "protected-mode", "dir", "requirepass", "logfile"]
+    added_params = ["port", "protected-mode", "dir", "requirepass"]
     # start redis-server
     command = [
         binary,
@@ -31,7 +31,6 @@ def generate_standalone_redis_server_args(
         logging.info("Redis server will be started with password authentication")
     if dbdir != "":
         command.extend(["--dir", dbdir])
-        command.extend(["--logfile", f"{dbdir}redis.log"])
     if configuration_parameters is not None:
         for parameter, parameter_value in configuration_parameters.items():
             if parameter not in added_params:

{redis_benchmarks_specification-0.1.320 → redis_benchmarks_specification-0.1.322}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -107,43 +107,6 @@ from redis_benchmarks_specification.__self_contained_coordinator__.artifacts imp
 from redis_benchmarks_specification.__self_contained_coordinator__.build_info import (
     extract_build_info_from_streamdata,
 )
-
-
-def print_directory_logs(directory_path, description=""):
-    """Print all .log files in a directory for debugging purposes."""
-    if not os.path.exists(directory_path):
-        logging.warning(f"Directory {directory_path} does not exist")
-        return
-
-    logging.info(
-        f"Printing all .log files in {description} directory: {directory_path}"
-    )
-    try:
-        for root, dirs, files in os.walk(directory_path):
-            for file in files:
-                # Only process .log files
-                if not file.endswith(".log"):
-                    continue
-
-                file_path = os.path.join(root, file)
-                logging.info(f"Found log file: {file_path}")
-                try:
-                    # Try to read and print the log file content
-                    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
-                        content = f.read()
-                        if content.strip():  # Only print non-empty files
-                            logging.info(f"Content of {file_path}:")
-                            logging.info("-" * 40)
-                            logging.info(content)
-                            logging.info("-" * 40)
-                        else:
-                            logging.info(f"Log file {file_path} is empty")
-                except Exception as e:
-                    logging.warning(f"Could not read log file {file_path}: {e}")
-    except Exception as e:
-        logging.error(f"Error walking directory {directory_path}: {e}")
-
-
 from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import (
     extract_db_cpu_limit,
     generate_cpuset_cpus,
@@ -1356,18 +1319,6 @@ def process_self_contained_coordinator_stream(
 
                                     print("-" * 60)
 
-                                # Print all log files in the temporary directories for debugging
-                                logging.critical(
-                                    "Printing all files in temporary directories for debugging..."
-                                )
-                                try:
-                                    print_directory_logs(temporary_dir, "Redis server")
-                                    print_directory_logs(temporary_dir_client, "Client")
-                                except Exception as log_error:
-                                    logging.error(
-                                        f"Failed to print directory logs: {log_error}"
-                                    )
-
                                 test_result = False
                             # tear-down
                             logging.info("Tearing down setup")
@@ -1398,27 +1349,14 @@ def process_self_contained_coordinator_stream(
                                                 )
                                             )
                                             pass
-
-                                # Only remove temporary directories if test passed
-                                if test_result:
-                                    logging.info(
-                                        "Test passed. Removing temporary dirs {} and {}".format(
-                                            temporary_dir, temporary_dir_client
-                                        )
-                                    )
-                                    shutil.rmtree(temporary_dir, ignore_errors=True)
-                                    shutil.rmtree(
-                                        temporary_dir_client, ignore_errors=True
-                                    )
-                                else:
-                                    logging.warning(
-                                        "Test failed. Preserving temporary dirs for debugging: {} and {}".format(
-                                            temporary_dir, temporary_dir_client
-                                        )
+                                logging.info(
+                                    "Removing temporary dirs {} and {}".format(
+                                        temporary_dir, temporary_dir_client
                                     )
-                                    # Print all log files in the temporary directories for debugging
-                                    print_directory_logs(temporary_dir, "Redis server")
-                                    print_directory_logs(temporary_dir_client, "Client")
+                                )
+
+                                shutil.rmtree(temporary_dir, ignore_errors=True)
+                                shutil.rmtree(temporary_dir_client, ignore_errors=True)
 
                             overall_result &= test_result
 

redis_benchmarks_specification-0.1.322/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-1KiB-values-pipeline-10.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-string-with-1KiB-values-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs
+  in which the value has a data size of 1000 Bytes.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:15.2.0-amd64-debian-bookworm-default
+- gcc:15.2.0-arm64-debian-bookworm-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--pipeline 10 --distinct-client-seed --data-size 1000 --ratio 1:0 --key-pattern R:R --key-minimum=1 --key-maximum
+    1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification-0.1.322/redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-caching-string-100k-sessions.yml

@@ -0,0 +1,98 @@
+version: 0.4
+name: memtier_benchmark-session-caching-string-100k-sessions
+description: |
+  Runs memtier_benchmark to simulate a session caching workload for a SaaS application.
+  This benchmark focuses exclusively on **string-based session storage**, where each session
+  is stored in Redis as a serialized JSON string (`session:<id>`) containing fields like
+  user ID, timestamps, device info, and metadata (total ~400–600B).
+
+  The benchmark models a typical read-heavy cache usage pattern, with an approximate
+  **read:write ratio of 80:20**, reflecting session retrievals and infrequent updates.
+
+  Command groups:
+    - Session cache reads (`GET`): ~80%
+    - Session cache writes (`SET`): ~20%
+
+  To better approximate real-world access patterns, the benchmark uses a **Zipfian key distribution**
+  (`--command-key-pattern=Z`). This simulates **skewed access** where a small subset of sessions (hot keys)
+  receives a majority of reads — a common pattern in production workloads.
+
+  While Zipfian is technically a power-law distribution, it effectively mimics **Poisson-like behavior**
+  in large-scale systems, where access frequency is uneven but statistically predictable.
+  This access skew mirrors real-life scenarios such as:
+    - Frequently accessed or "sticky" user sessions
+    - Popular user accounts or active devices
+    - Hot caches for trending or recently used resources
+
+  Using Zipfian distribution allows this benchmark to capture **contention**, **cache pressure**, and
+  **read amplification** effects that occur in real SaaS applications under load.
+
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  resources:
+    requests:
+      memory: 1g
+  init_lua: |
+    local seed = 12345
+    math.randomseed(seed)
+    local now = tonumber(redis.call('TIME')[1])
+    local function rand_str(len)
+      local chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+      local res = ''
+      for i = 1, len do
+        local idx = math.random(#chars)
+        res = res .. chars:sub(idx, idx)
+      end
+      return res
+    end
+    for i = 1, 100000 do
+      local session_id = 'session:' .. i
+      local session_data = string.format(
+        '{"userId":"user-%d","organizationId":"org-%d","role":"member","createdAt":"%d","lastAccessed":"%d","ipAddress":"192.168.1.%d","device":"device-%s","authMethod":"password","status":"active","metadata":"%s"}',
+        i, i, now - math.random(3600), now, (i % 255), rand_str(8), rand_str(200 + (i % 100))
+      )
+      redis.call('SET', session_id, session_data)
+    end
+    return 'OK'
+
+tested-groups:
+- string
+
+tested-commands:
+- get
+- set
+
+redis-topologies:
+- oss-standalone
+
+build-variants:
+- gcc:15.2.0-amd64-debian-bookworm-default
+- gcc:15.2.0-arm64-debian-bookworm-default
+- dockerhub
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: >
+    --key-prefix ""
+    --key-minimum 1
+    --key-maximum 100000
+    --data-size-range=400-600
+    --pipeline=1
+    --print-percentiles=50,90,95,99
+    --run-count=1
+    --test-time=120
+    --command="GET session:__key__"
+    --command-key-pattern=Z
+    --command-ratio=90
+    --command='SET session:__key__ "{\"userId\":\"user-__key__\",\"organizationId\":\"org-__key__\",\"role\":\"member\",\"createdAt\":\"1754905396\",\"lastAccessed\":\"1754906472\",\"ipAddress\":\"192.168.1.36\",\"device\":\"device-2T8YGLbl\",\"authMethod\":\"password\",\"status\":\"active\",\"metadata\":\"wDVmiQsSe2oSEPfhhvYN6jbVxVykSCzQXmnsqCIv5MEmpslD1LMgwJcUe8Wmhvhh56dgTDhH4o3M9vYa6JyAS3Axs2zufVVPWtTsCRVbNGjMmumy7j5vIM0OuclgbHBOxtGDtCU88YBc9IP5oNYiycXXCmq5s7mWAdmRhFrmFxOy3VKlrXJz4ZrtF6KHEflGhoF1FzXtLO8Dckg3B34kdxGsnNiRjNIEtWVxeXekUSDyrxcnwiIy29Zx8SxX2g8ZUrz3sTpEwX7KQZFpIi7XoSKZywvC6I63VMNpZcAO\"}"'
+    --command-key-pattern=Z
+    --command-ratio=10
+    --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 150