redis-benchmarks-specification 0.2.25__tar.gz → 0.2.27__tar.gz

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-benchmarks-specification
-Version: 0.2.25
+Version: 0.2.27
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 License-File: LICENSE
 Author: filipecosta90

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.2.25"
+version = "0.2.27"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/__runner__/runner.py

@@ -1686,7 +1686,7 @@ def process_self_contained_coordinator_stream(
                         logging.info(
                             f"Connected to Redis using individual parameters: {host}:{port}"
                         )
-                    setup_name = "oss-standalone"
+                    setup_name = topology_spec_name
                     r.ping()
 
                     # Auto-detect server information if not explicitly provided
@@ -2440,7 +2440,7 @@ def process_self_contained_coordinator_stream(
                         start_time_str,
                         git_hash,
                         test_name,
-                        setup_type,
+                        setup_name,
                     )
                     logging.info(
                         "Will store benchmark json output to local file {}".format(
@@ -2671,6 +2671,14 @@ def process_self_contained_coordinator_stream(
 
                             if not success:
                                 logging.error(f"Memtier benchmark failed: {stderr}")
+                                # Clean up database after failure (timeout or error)
+                                if args.flushall_on_every_test_end or args.flushall_on_every_test_start:
+                                    logging.warning("Benchmark failed - cleaning up database with FLUSHALL")
+                                    try:
+                                        for r in redis_conns:
+                                            r.flushall()
+                                    except Exception as e:
+                                        logging.error(f"FLUSHALL failed after benchmark failure: {e}")
                                 # Continue with the test but log the failure
                                 client_container_stdout = f"ERROR: {stderr}"
 
@@ -2872,6 +2880,7 @@ def process_self_contained_coordinator_stream(
                             test_name,
                             results_matrix,
                             redis_conns,
+                            setup_name,
                         )
                     else:
                         # Single client - read from file as usual
@@ -2950,6 +2959,7 @@ def process_self_contained_coordinator_stream(
                             test_name,
                             results_matrix,
                             redis_conns,
+                            setup_name,
                         )
 
                     dataset_load_duration_seconds = 0
@@ -3014,6 +3024,15 @@ def process_self_contained_coordinator_stream(
                     print("-" * 60)
                     test_result = False
 
+                    # Clean up database after exception to prevent contamination of next test
+                    if args.flushall_on_every_test_end or args.flushall_on_every_test_start:
+                        logging.warning("Exception caught - cleaning up database with FLUSHALL")
+                        try:
+                            for r in redis_conns:
+                                r.flushall()
+                        except Exception as e:
+                            logging.error(f"FLUSHALL failed after exception: {e}")
+
                     # Check if user requested exit via Ctrl+C
                     if _exit_requested:
                         logging.info(
@@ -3283,7 +3302,13 @@ def used_memory_check(
 ):
     used_memory = 0
     for conn in redis_conns:
-        used_memory = used_memory + conn.info("memory")["used_memory"]
+        info_mem = conn.info("memory")
+        if "used_memory" in info_mem:
+            used_memory = used_memory + info_mem["used_memory"]
+        else:
+            logging.warning(
+                "used_memory not present in Redis memory info. Cannot enforce memory checks."
+            )
     used_memory_gb = int(math.ceil(float(used_memory) / 1024.0 / 1024.0 / 1024.0))
     logging.info("Benchmark used memory at {}: {}g".format(stage, used_memory_gb))
     if used_memory > benchmark_required_memory:
@@ -3519,6 +3544,7 @@ def prepare_overall_total_test_results(
     test_name,
     overall_results_matrix,
     redis_conns=None,
+    topology=None,
 ):
     # check which metrics to extract
     (
@@ -3554,8 +3580,13 @@ def prepare_overall_total_test_results(
 
         return x[0]  # Use original path
 
+    # Include topology in the test name if provided
+    test_name_with_topology = test_name
+    if topology:
+        test_name_with_topology = f"{topology}-{test_name}"
+
     current_test_results_matrix = [
-        [test_name, get_overall_display_name(x), f"{x[3]:.3f}"]
+        [test_name_with_topology, get_overall_display_name(x), f"{x[3]:.3f}"]
         for x in current_test_results_matrix
     ]
     overall_results_matrix.extend(current_test_results_matrix)

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-psetex-expire-use-case.yml

@@ -12,7 +12,7 @@ dbconfig:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
     arguments: '"--data-size" "10" --ratio 1:0 "--key-pattern"
-      "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"  "--key-maximum"
+      "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"  "--key-maximum"
       "1000000" "-n" "allkeys"'
   resources:
     requests:

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-setex-expire-use-case.yml

@@ -12,7 +12,7 @@ dbconfig:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
     arguments: '"--data-size" "10" --ratio 1:0 "--key-pattern"
-      "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"  "--key-maximum"
+      "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"  "--key-maximum"
       "1000000" "-n" "allkeys"'
   resources:
     requests:

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-mixed-50-50-set-get-100B-expire-pipeline-10.yml

@@ -36,7 +36,7 @@ clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
   arguments: '"--data-size" "100" --distinct-client-seed --command "SET __key__ __data__
-    EX 10" --command "GET __key__" --key-pattern R:R --key-minimum=1 --key-maximum
+    EX 10" --command "GET __key__" --command-key-pattern R --key-minimum=1 --key-maximum
     1000000 --test-time 180 -c 25 -t 4 --hide-histogram --pipeline 10'
   resources:
     requests:

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-mixed-50-50-set-get-100B-expire.yml

@@ -36,7 +36,7 @@ clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
   arguments: '"--data-size" "100" --distinct-client-seed --command "SET __key__ __data__
-    EX 10" --command "GET __key__" --key-pattern R:R --key-minimum=1 --key-maximum
+    EX 10" --command "GET __key__" --command-key-pattern R --key-minimum=1 --key-maximum
     1000000 --test-time 180 -c 25 -t 4 --hide-histogram'
   resources:
     requests:

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-leaderboard-top-10.yml

@@ -34,11 +34,11 @@ dbconfig:
     -- Use the same pairs for all keys
     for k = 1, 10000 do
       local key = tostring(k)
-      local args = {'ZADD', key}
+      local args = {key}
       for i = 1, #score_member_pairs do
         table.insert(args, score_member_pairs[i])
       end
-      redis.call(unpack(args))
+      redis.call('ZADD', unpack(args))
     end
     
     return 'OK'

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-leaderboard-top-100.yml

@@ -1,7 +1,7 @@
 version: 0.4
 name: memtier_benchmark-playbook-leaderboard-top-100
 description: Runs memtier_benchmark, for a keyspace length of 10K keys loading/querying ZSETs. Esports/live events with constant score changes, occasional bursts of reads. writes ≈ 60, reads ≈ 40%.
-  with encoding:listpack with 100 elements.
+  with encoding:skiplist with 100 elements.
 
 dbconfig:
   configuration-parameters:
@@ -34,11 +34,11 @@ dbconfig:
     -- Use the same pairs for all keys
     for k = 1, 10000 do
       local key = tostring(k)
-      local args = {'ZADD', key}
+      local args = {key}
       for i = 1, #score_member_pairs do
         table.insert(args, score_member_pairs[i])
       end
-      redis.call(unpack(args))
+      redis.call('ZADD', unpack(args))
     end
     
     return 'OK'

{redis_benchmarks_specification-0.2.25 → redis_benchmarks_specification-0.2.27}/redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-leaderboard-top-1000.yml

@@ -1,7 +1,7 @@
 version: 0.4
 name: memtier_benchmark-playbook-leaderboard-top-1000
 description: Runs memtier_benchmark, for a keyspace length of 10K keys loading/querying ZSETs. Esports/live events with constant score changes, occasional bursts of reads. writes ≈ 60, reads ≈ 40%.
-  with encoding:listpack with 1000 elements.
+  with encoding:skiplist with 1000 elements.
 
 dbconfig:
   configuration-parameters:
@@ -34,11 +34,11 @@ dbconfig:
     -- Use the same pairs for all keys
     for k = 1, 10000 do
       local key = tostring(k)
-      local args = {'ZADD', key}
+      local args = {key}
       for i = 1, #score_member_pairs do
         table.insert(args, score_member_pairs[i])
       end
-      redis.call(unpack(args))
+      redis.call('ZADD', unpack(args))
     end
     
     return 'OK'

redis_benchmarks_specification-0.2.27/redis_benchmarks_specification/test-suites/memtier_benchmark-stream-10M-entries-xread-count-100.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: memtier_benchmark-stream-10M-entries-xread-count-100
+description: 'Runs memtier_benchmark, pre-loading Redis with 10M stream entries using XADD, then testing XREAD performance with COUNT 100.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" --command "XADD stream-key * field __data__" --command-key-pattern="P" -n 50000 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      memory: 4g
+  dataset_name: 1M-stream-entries
+  dataset_description: This dataset contains 1 stream key with 1M entries, each with a field containing 100 bytes of data.
+tested-commands:
+- xread
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:15.2.0-amd64-debian-bookworm-default
+- gcc:15.2.0-arm64-debian-bookworm-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="XREAD COUNT 100 STREAMS stream-key 0" --hide-histogram -n 1000 -c 25 -t 4
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+tested-groups:
+- stream
+priority: 95

redis_benchmarks_specification-0.2.27/redis_benchmarks_specification/test-suites/memtier_benchmark-stream-10M-entries-xreadgroup-count-100.yml

@@ -0,0 +1,38 @@
+version: 0.4
+name: memtier_benchmark-stream-10M-entries-xreadgroup-count-100
+description: 'Runs memtier_benchmark, pre-loading Redis with 10M stream entries using XADD, creating a consumer group, then testing XREADGROUP performance with COUNT 100.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" --command "XADD stream-key * field __data__" --command-key-pattern="P" -n 50000 -c 50 -t 4 --hide-histogram'
+  init_commands:
+  - XGROUP CREATE stream-key test-group 0 MKSTREAM
+  resources:
+    requests:
+      memory: 4g
+  dataset_name: 1k-stream-entries-with-consumer-group
+  dataset_description: This dataset contains 1 stream key with 10M entries, each with a field containing 100 bytes of data, and a consumer group named 'test-group'.
+tested-commands:
+- xreadgroup
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:15.2.0-amd64-debian-bookworm-default
+- gcc:15.2.0-arm64-debian-bookworm-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="XREADGROUP GROUP test-group consumer1 COUNT 100 STREAMS stream-key >" --hide-histogram -n 1000 -c 25 -t 4
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+tested-groups:
+- stream
+priority: 94

redis_benchmarks_specification-0.2.27/redis_benchmarks_specification/test-suites/memtier_benchmark-stream-concurrent-xadd-xreadgroup-70-30.yml

@@ -0,0 +1,50 @@
+version: 0.4
+name: memtier_benchmark-stream-concurrent-xadd-xreadgroup-70-30
+description:
+  Starting with a pre-loaded stream of 500K entries, the benchmark tests concurrent stream operations with a consumer group for distributed processing.  
+  70% of commands produce messages with XADD, while 30% consume with XREADGROUP COUNT 10.
+  500K initial entries plus 200K * 70% minus 200K * 10 * (15% + 15%) ≈ 40K left in the stream at the end.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" --command "XADD stream-key * field __data__" --command-key-pattern="P" -n 2500 -c 50 -t 4 --hide-histogram'
+  init_commands:
+    - XGROUP CREATE stream-key producer-consumer-group 0 MKSTREAM
+  resources:
+    requests:
+      memory: 4g
+  dataset_name: stream-concurrent-producer-consumer-500k-seed
+  dataset_description: This dataset starts with 500K pre-loaded stream entries, then tests concurrent stream operations with a consumer group for distributed processing.
+tested-commands:
+  - xadd
+  - xreadgroup
+redis-topologies:
+  - oss-standalone
+build-variants:
+  - gcc:15.2.0-amd64-debian-bookworm-default
+  - gcc:15.2.0-arm64-debian-bookworm-default
+  - dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: >
+    --data-size 100
+    --command="XADD stream-key * field __data__" --command-key-pattern="P" --command-ratio=70
+    --command="XREADGROUP GROUP producer-consumer-group consumer1 COUNT 10 STREAMS stream-key >" --command-key-pattern="P" --command-ratio=15
+    --command="XREADGROUP GROUP producer-consumer-group consumer2 COUNT 10 STREAMS stream-key >" --command-key-pattern="P" --command-ratio=15
+    --hide-histogram
+    -n 1000
+    -c 50
+    -t 4
+  resources:
+    requests:
+      cpus: "4"
+      memory: 2g
+tested-groups:
+  - stream
+priority: 95