redis-benchmarks-specification 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

redis_benchmarks_specification/__cli__/cli.py

@@ -126,7 +126,9 @@ def trigger_tests_dockerhub_cli_command_logic(args, project_name, project_versio
 
     if result is True:
         # Use architecture-specific stream
-        arch_specific_stream = get_arch_specific_stream_name(args.arch)
+        # Provide default arch if None (defensive programming for tests)
+        arch = args.arch if args.arch is not None else "amd64"
+        arch_specific_stream = get_arch_specific_stream_name(arch)
         logging.info(
             f"CLI adding work to architecture-specific stream: {arch_specific_stream}"
         )

redis_benchmarks_specification/__runner__/runner.py

@@ -2449,21 +2449,6 @@ def process_self_contained_coordinator_stream(
                     )
                     arbitrary_command = False
 
-                    if (
-                        arbitrary_command
-                        and oss_cluster_api_enabled
-                        and "memtier" in benchmark_tool
-                    ):
-                        logging.warning(
-                            "Forcing skip this test given there is an arbitrary commmand and memtier usage. Check https://github.com/RedisLabs/memtier_benchmark/pull/117 ."
-                        )
-                        delete_temporary_files(
-                            temporary_dir_client=temporary_dir_client,
-                            full_result_path=None,
-                            benchmark_tool_global=benchmark_tool_global,
-                        )
-                        continue
-
                     # Check if we have multiple client configurations
                     client_configs = extract_client_configs(benchmark_config)
                     is_multiple_clients = len(client_configs) > 1
@@ -3648,12 +3633,6 @@ def data_prepopulation_step(
             1,
             unix_socket,
         )
-        if arbitrary_command is True and oss_cluster_api_enabled:
-            logging.warning(
-                "Skipping this test given it implies arbitrary command on an cluster setup. Not supported on memtier: https://github.com/RedisLabs/memtier_benchmark/pull/117"
-            )
-            result = False
-            return result
 
         # run the benchmark
         preload_start_time = datetime.datetime.now()

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setget200c-1KiB-pipeline-10.yml

@@ -25,6 +25,9 @@ tested-groups:
 - string
 redis-topologies:
 - oss-standalone
+- oss-standalone-08-io-threads
+- oss-standalone-04-io-threads
+- oss-standalone-02-io-threads
 build-variants:
 - gcc:15.2.0-amd64-debian-bookworm-default
 - gcc:15.2.0-arm64-debian-bookworm-default

redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-rate-limiting-lua-100k-sessions.yml

@@ -1,74 +1,45 @@
 version: 0.4
 name: memtier_benchmark-playbook-rate-limiting-lua-100k-sessions
-description:  |
-  Runs memtier_benchmark to model API rate limiting in Redis using a strings-only Lua script that enforces a sliding window via current/previous time buckets. Each call atomically checks user, org, and global counters with GET/INCRBY/PEXPIRE/TIME, increments only on allow, and returns allow/remaining/reset.
-
-  The benchmark uses a Lua script to atomically enforce quotas:
-
-      local key = KEYS[1]
-      local limit = 100
-      local window = 60
-      local current = redis.call("INCR", key)
-      if current == 1 then
-        redis.call("EXPIRE", key, window)
-      end
-      if current > limit then
-        return 0
-      else
-        return 1
-      end
-
-  This approach ensures that **increments, expiry, and limit checks** are performed without race conditions,
-  accurately modeling production-grade API quota enforcement.
-
-  ### Workload profile
-  - **Command tested**: `EVAL` (rate-limit script)  
-  - **Keyspace size**: 100k user-level rate-limit keys  
-  - **Access pattern**: randomized keys (`--command-key-pattern=R`) to simulate distributed users  
-  - **Response shape**: single integer (1 = allowed, 0 = denied), representing pass/fail on quota checks  
-
-  This benchmark measures Redis performance in **rate limiting scenarios**, where atomic counters and TTLs
-  are critical for API protection and fairness in multi-tenant SaaS workloads.
-
-exporter:
-  redistimeseries:
-    break_by:
-    - version
-    - commit
-    timemetric: $."ALL STATS".Runtime."Start time"
-    metrics:
-    - $."BEST RUN RESULTS".Totals."Ops/sec"
-    - $."BEST RUN RESULTS".Totals."Latency"
-    - $."BEST RUN RESULTS".Totals."Misses/sec"
-    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
-    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
-    - $."ALL STATS".Evals."Ops/sec"
-    - $."ALL STATS".Totals."Ops/sec"
-    - $."ALL STATS".Totals."Latency"
-    - $."ALL STATS".Totals."Misses/sec"
-    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
-    - $."ALL STATS".Totals."Percentile Latencies"."p99.00"
-
+description: |
+  Runs memtier_benchmark to model a **bitmap-based rate limit gate** using Lua: the script performs
+  a `BITCOUNT` on a single bitmap key and **allows** if the count is below a configurable threshold,
+  otherwise **denies** (no other data structures or mutations are performed).
 
 dbconfig:
   configuration-parameters:
     save: '""'
+
+  init_commands:
+  - '"SETBIT" "1" "100000" "1"'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: >
+      --hide-histogram
+      --command "SETBIT __key__ __key__ 1"
+      --key-maximum 100000
+      --key-minimum 1
+      --key-prefix ""
+      -t 1 -c 1
+      --command-key-pattern R
+      --distinct-client-seed
+      -n 100000
+      --pipeline 50
   resources:
     requests:
+      cpus: '2'
       memory: 1g
-tested-groups:
-- hash
-- sorted-set
-- set
-- scripting
 
 tested-commands:
-- incr
-- expire
+- bitcount
 - eval
+tested-groups:
+- bitmap
+- scripting
 
 redis-topologies:
 - oss-standalone
+
 build-variants:
 - gcc:15.2.0-amd64-debian-bookworm-default
 - gcc:15.2.0-arm64-debian-bookworm-default
@@ -78,17 +49,17 @@ clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
   arguments: >
-    --key-prefix ""
-    --key-minimum 1
-    --key-maximum 100000
-    --pipeline=1
     --print-percentiles=50,90,95,99
     --run-count=1
     --test-time=120
-    --command='EVAL "local base=ARGV[1];local limit=tonumber(ARGV[2]);local window=tonumber(ARGV[3]);local slots=tonumber(ARGV[4]);local t=redis.call(\"TIME\");local now=t[1]*1000+math.floor(t[2]/1000);local bucket=math.floor(now/window);local key=base..\":\"..tostring(bucket);local slot=math.floor(((now%window)*slots)/window);local count=redis.call(\"BITCOUNT\",key);local allowed=0;if count<limit then redis.call(\"SETBIT\",key,slot,1);allowed=1;end;redis.call(\"PEXPIRE\",key,window*2);local reset=window-(now%window);local remaining=limit-(count+allowed);if remaining<0 then remaining=0 end;return {allowed,remaining,reset}" 0 ratelimit:{user-__key__}:/api/resource 100 60000 60000'    --hide-histogram
+    --hide-histogram
+    --key-maximum 100000
+    --key-minimum 1
+    --key-prefix ""
+    --command='EVAL "local k=KEYS[1];local limit=tonumber(ARGV[1]);local c=redis.call(\"BITCOUNT\",k);if c>=limit then return {0,0,c} else return {1,limit-c,c} end" 1 __key__ 2000000'
   resources:
     requests:
-      cpus: '4'
+      cpus: '2'
       memory: 2g
 
-priority: 150
+priority: 19

{redis_benchmarks_specification-0.2.10.dist-info → redis_benchmarks_specification-0.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.2.10
+Version: 0.2.12
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.2.10.dist-info → redis_benchmarks_specification-0.2.12.dist-info}/RECORD

@@ -8,7 +8,7 @@ redis_benchmarks_specification/__builder__/builder.py,sha256=vG6Cp0SAAgvZt9zyjTQ
 redis_benchmarks_specification/__builder__/schema.py,sha256=1wcmyVJBcWrBvK58pghN9NCoWLCO3BzPsmdKWYfkVog,584
 redis_benchmarks_specification/__cli__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__cli__/args.py,sha256=y_FHtkjhRKsU532sutlXPeXgtFZu7KGOp7lTSi1C5uc,7427
-redis_benchmarks_specification/__cli__/cli.py,sha256=0S3Lwci-oxpPJMekPIFeG5XJiSF53rpa5sRnS-aAZnA,22416
+redis_benchmarks_specification/__cli__/cli.py,sha256=NesdsV2nS-1cyblvU3A6e0tpz3Gfgw-76dpXFu2u6vU,22547
 redis_benchmarks_specification/__cli__/stats.py,sha256=r9JIfwGCSR3maozYbDZfZrkthNFQSs0xIymS86yZ6Iw,55574
 redis_benchmarks_specification/__common__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb5qj9jzsBhLVNO7K_W2Clumj4pxrkG8,5938
@@ -26,7 +26,7 @@ redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gq
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=K3VGmBC0-9lSv9H6VDp0N-6FGMWvc_4H0pG_TOXN5u8,11312
 redis_benchmarks_specification/__runner__/remote_profiling.py,sha256=R7obNQju8mmY9oKkcndjI4aAuxi84OCLhDSqqaYu1SU,18610
-redis_benchmarks_specification/__runner__/runner.py,sha256=RUY_O56R9TVtsJGHTQb17Sc-wax3-GlRR7z0_3q_a_c,156583
+redis_benchmarks_specification/__runner__/runner.py,sha256=lWIpjThVeYW1hxR2dl2OSIeAwUXrWVHKHfQqsD9mrF8,155505
 redis_benchmarks_specification/__self_contained_coordinator__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__self_contained_coordinator__/args.py,sha256=2nTD4g4V1NjMRjRuDvHaoub5sjcav0GCnxv2HFiXWKc,7329
 redis_benchmarks_specification/__self_contained_coordinator__/artifacts.py,sha256=OVHqJzDgeSSRfUSiKp1ZTAVv14PvSbk-5yJsAAoUfpw,936
@@ -163,7 +163,7 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-mixed
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-mixed-50-50-set-get-with-expiration-240B-400_conns.yml,sha256=1YIVXSLRytR9-QIayu6jCxnFd1KJlY8o0rwJYT28Hx8,1549
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-set-with-ex-100B-pipeline-10.yml,sha256=5C2S9LpQDH-_IpjWwYH9tCnK0jvm9pZdlnyGmJMA9gc,1300
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setex-100B-pipeline-10.yml,sha256=NE5oujnTwuHINrlvHBjMEFSKY_iwKimAvq4twnYfmXI,1297
-redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setget200c-1KiB-pipeline-10.yml,sha256=VWLwsiTixI1WQEF1KJqPvqL0uTUUHOsi8wZ7GONn4HI,1275
+redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setget200c-1KiB-pipeline-10.yml,sha256=TjgswuCZaZrPXdzBUf79pbcMZRSMV3iWoK53mT4fIK0,1368
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B-pipeline-10.yml,sha256=h6haEl469vZSnp9LTB3wds12EwGfyNSEm1iqXxh72s8,1329
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-setrange-100B.yml,sha256=Wg8joxF6TuCOyQbC4CpW8LTx49PpIAKvNhtwqJpX95k,1303
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-100M-bits-bitmap-bitcount.yml,sha256=I4fNkW8P-tVpkkfep8gcPeQ6BgPlZnG4EAytvtxU1hU,1498
@@ -277,7 +277,7 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-mixed
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publish-1K-channels-10B-no-subscribers.yml,sha256=8M9AdpNaVBuSUm78cqKModOF_xITV_RM7RLCTyvtvaQ,825
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml,sha256=zeTHtpbhNGfzTXHfr9P5e62PebxUONRvTsCbXJZhsTs,721
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-leaderboard.yml,sha256=YZTgci8gAfGxTDAM-GHMn_w4hqeNoS4Yi6FYX9JyCt0,6690
-redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-rate-limiting-lua-100k-sessions.yml,sha256=QRG8AGNwfdf3Icwwrdt3Z6GeRl2Aau31zkBYnR2GgIA,3507
+redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-rate-limiting-lua-100k-sessions.yml,sha256=z2APnBAH_leRRYDQXjA-5gFwScNgKT4mVyBh5wLb3SU,1636
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-realtime-analytics-membership-pipeline-10.yml,sha256=uh0J81Y27_YnQfDqzaTpYtJNvIdRFJtYsBUmXGTBrl4,2842
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-realtime-analytics-membership.yml,sha256=y_q6GWbodKpVA48o1a1frqF0bU52utIOnjIjuiQt0qk,2816
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-caching-hash-100k-sessions.yml,sha256=H98DrXlq-lRhoe1M7vehfDBbUdZ7WwWroriTpklx-PI,3646
@@ -287,8 +287,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-st
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage-1k-sessions.yml,sha256=2egtIxPxCze2jlbAfgsk4v9JSQHNMoPLbDWFEW8olDg,7006
 redis_benchmarks_specification/test-suites/template.txt,sha256=ezqGiRPOvuSDO0iG7GEf-AGXNfHbgXI89_G0RUEzL88,481
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
-redis_benchmarks_specification-0.2.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.2.10.dist-info/METADATA,sha256=9qy8SQ5NPYYQq6x6TG3HUApapHjqRU_2wvdIt6lwJow,22767
-redis_benchmarks_specification-0.2.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.2.10.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.2.10.dist-info/RECORD,,
+redis_benchmarks_specification-0.2.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.2.12.dist-info/METADATA,sha256=KSHkm-wxg6iKtiWTYxNEIkftjxxlfAq_SFcevK1NgvQ,22767
+redis_benchmarks_specification-0.2.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.2.12.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.2.12.dist-info/RECORD,,