redis-benchmarks-specification 0.1.301__py3-none-any.whl → 0.1.303__py3-none-any.whl

redis_benchmarks_specification/__common__/runner.py

@@ -15,11 +15,14 @@ from redis_benchmarks_specification.__common__.timeseries import (
 
 def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
     cmds = None
+    lua_scripts = None
     res = 0
     if dbconfig_keyname in benchmark_config:
         for k, v in benchmark_config[dbconfig_keyname].items():
             if "init_commands" in k:
                 cmds = v
+            elif "init_lua" in k:
+                lua_scripts = v
 
     if type(cmds) == str:
         cmds = [cmds]
@@ -37,7 +40,7 @@ def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
                     quoting=csv.QUOTE_ALL,
                     skipinitialspace=True,
                 ):
-                    if lines[0] != " " and len(lines[0]) > 0:
+                    if len(lines) > 0 and lines[0] != " " and len(lines[0]) > 0:
                         cols.append(lines[0])
                 cmd = cols
                 is_array = True
@@ -57,6 +60,21 @@ def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
                     )
                 )
 
+    # Process init_lua scripts
+    if lua_scripts is not None:
+        if type(lua_scripts) == str:
+            lua_scripts = [lua_scripts]
+
+        for lua_script in lua_scripts:
+            try:
+                logging.info("Executing Lua script (length: {} chars)".format(len(lua_script)))
+                # Execute the Lua script using EVAL command with 0 keys
+                stdout = r.execute_command("EVAL", lua_script, 0)
+                logging.info("Lua script result: {}".format(stdout))
+                res = res + 1
+            except Exception as e:
+                logging.error("Lua script execution failed: {}".format(e))
+
     return res
 
 
@@ -452,13 +470,16 @@ def exporter_datasink_common(
     deployment_type_and_name = f"{setup_type}_AND_{setup_name}"
     deployment_type_and_name_and_version = f"{setup_type}_AND_{setup_name}_AND_{git_version}"
 
-    # Add to deployment-specific set
-    deployment_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/{deployment_type_and_name_and_version}:set"
-    datasink_conn.sadd(deployment_set_key, test_name)
+    # Add to deployment-specific set (only if datasink connection is available)
+    if datasink_conn is not None:
+        deployment_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/{deployment_type_and_name_and_version}:set"
+        datasink_conn.sadd(deployment_set_key, test_name)
 
-    # Add to testcases set
-    testcases_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/testcases:set"
-    datasink_conn.sadd(testcases_set_key, test_name)
+        # Add to testcases set
+        testcases_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/testcases:set"
+        datasink_conn.sadd(testcases_set_key, test_name)
+    else:
+        logging.debug("Datasink connection not available, skipping set operations")
 
     # Add metadata fields to timeseries metadata
     metadata["deployment_type_AND_deployment_name"] = deployment_type_and_name

redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage.yml

@@ -0,0 +1,203 @@
+version: 0.4
+name: memtier_benchmark-playbook-session-storage
+description: |
+  Runs memtier_benchmark to simulate a session-based SaaS application.
+  The workload mimics user session CRUD, session tracking, organization-level analytics,
+  and rate limiting. It includes realistic key sizes and command mixes representative
+  of production traffic (e.g., 400–600B session hashes, ZSETs for analytics, and SETs for tracking).
+
+  Rate limiting is modeled using an atomic Lua script, inspired by the Upstash Redis example:
+  https://github.com/upstash/examples/tree/main/examples/ratelimit-with-redis
+
+  Each user has a dedicated key in the form of `ratelimit:user-<id>:/api/resource`, which is
+  used to track usage under a fixed window. The logic is evaluated atomically with the following script:
+
+      local key = KEYS[1]
+      local limit = 100
+      local window = 60
+      local current = redis.call("INCR", key)
+      if current == 1 then
+        redis.call("EXPIRE", key, window)
+      end
+      if current > limit then
+        return 0
+      else
+        return 1
+      end
+
+  This ensures that rate enforcement and usage tracking are done without race conditions, and
+  mirrors a real-world API quota model.
+
+  The workload emphasizes read-heavy patterns to reflect common SaaS access behavior. The overall
+  **read:write ratio is approximately 85:15**, with read operations covering session access, user-session
+  lookups, org analytics, and rate limit enforcement, while writes handle session updates, activity tracking,
+  and quota increments.
+
+  Command groups by use-case (approximate ratio of total operations):
+    - Session CRUD (HGETALL, HSET): ~55%
+    - User session tracking (SMEMBERS, SADD): ~21%
+    - Organization analytics (ZRANGE, ZADD): ~12%
+    - Rate limiting (EVAL-based quota check): ~12%
+
+exporter:
+  redistimeseries:
+    break_by:
+    - version
+    - commit
+    timemetric: $."ALL STATS".Runtime."Start time"
+    metrics:
+    - $."BEST RUN RESULTS".Hgetalls."Ops/sec"
+    - $."BEST RUN RESULTS".Hsets."Ops/sec"
+    - $."BEST RUN RESULTS".Smemberss."Ops/sec"
+    - $."BEST RUN RESULTS".Sadds."Ops/sec"
+    - $."BEST RUN RESULTS".Zranges."Ops/sec"
+    - $."BEST RUN RESULTS".Zadds."Ops/sec"
+    - $."BEST RUN RESULTS".Evals."Ops/sec"
+    - $."BEST RUN RESULTS".Totals."Ops/sec"
+    - $."BEST RUN RESULTS".Totals."Latency"
+    - $."BEST RUN RESULTS".Totals."Misses/sec"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
+    - $."ALL STATS".Hgetalls."Ops/sec"
+    - $."ALL STATS".Hsets."Ops/sec"
+    - $."ALL STATS".Smemberss."Ops/sec"
+    - $."ALL STATS".Sadds."Ops/sec"
+    - $."ALL STATS".Zranges."Ops/sec"
+    - $."ALL STATS".Zadds."Ops/sec"
+    - $."ALL STATS".Evals."Ops/sec"
+    - $."ALL STATS".Totals."Ops/sec"
+    - $."ALL STATS".Totals."Latency"
+    - $."ALL STATS".Totals."Misses/sec"
+    - $."ALL STATS".Hgetalls."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Hsets."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Smemberss."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Sadds."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Zranges."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Zadds."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Evals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p99.00"
+
+
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  resources:
+    requests:
+      memory: 1g
+  init_lua: |
+    -- Use a fixed seed for reproducibility
+    local seed = 12345
+    math.randomseed(seed)
+
+    local now = tonumber(redis.call('TIME')[1])
+
+    local function rand_str(len)
+      local chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+      local res = ''
+      for i = 1, len do
+        local idx = math.random(#chars)
+        res = res .. chars:sub(idx, idx)
+      end
+      return res
+    end
+
+    for i = 1, 100000 do
+      local session_id = 'session:' .. i
+      local user_id = 'user-' .. i
+      local org_id = 'org-' .. i
+
+      redis.call('HSET', session_id,
+        'userId', user_id,
+        'organizationId', org_id,
+        'role', 'member',
+        'createdAt', tostring(now - math.random(3600)),
+        'lastAccessed', tostring(now),
+        'ipAddress', '192.168.1.' .. (i % 255),
+        'device', 'device-' .. rand_str(8),
+        'authMethod', 'password',
+        'status', 'active',
+        'metadata', rand_str(200 + (i % 100))
+      )
+
+      redis.call('SADD', 'user:' .. i .. ':sessions', session_id)
+
+      local org_key = 'org:' .. i .. ':sessions'
+      for j = 1, 10 do
+        local uid = 'user-' .. ((i + j) % 1000 + 1)
+        local sid = 'session:' .. ((i + j) % 1000 + 1)
+        local zmember = uid .. ':' .. sid
+        local zscore = now - math.random(86400)
+        redis.call('ZADD', org_key, zscore, zmember)
+      end
+
+      local endpoint = '/api/resource'
+      local rate_key = 'ratelimit:' .. user_id .. ':' .. endpoint
+      redis.call('INCR', rate_key)
+      redis.call('EXPIRE', rate_key, 60)
+    end
+
+    return 'OK'
+tested-groups:
+- hash
+- zset
+- set
+- string
+
+tested-commands:
+- hgetall
+- hset
+- smembers
+- sadd
+- zrange
+- zadd
+- incr
+- expire
+- get
+
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: >
+    --key-prefix ""
+    --key-minimum 1
+    --key-maximum 100000
+    --data-size-range=400-600
+    --pipeline=1
+    --print-percentiles=50,90,95,99
+    --run-count=1
+    --test-time=120
+    --command="HGETALL session:__key__"
+    --command-key-pattern=R
+    --command-ratio=50
+    --command="HSET session:__key__ userId user-__key__ organizationId org-__key__ role admin email user__key__@example.com name "User __key__" permissions "["read","write"]" lastActivity __timestamp__ ipAddress 192.168.1.__key__ userAgent "Mozilla/5.0" createdAt __timestamp__"
+    --command-key-pattern=R
+    --command-ratio=5
+    --command="SMEMBERS user:__key__:sessions"
+    --command-key-pattern=R
+    --command-ratio=18
+    --command="SADD user:__key__:sessions session-__key__"
+    --command-key-pattern=R
+    --command-ratio=3
+    --command="ZRANGE org:__key__:sessions 0 -1 WITHSCORES"
+    --command-key-pattern=R
+    --command-ratio=10
+    --command="ZADD org:__key__:sessions 1 user-__key__:session-__key__"
+    --command-key-pattern=R
+    --command-ratio=2
+    --command='EVAL "local key=KEYS[1];local limit=10;local window=60;local current=redis.call(\"INCR\",key);if current==1 then redis.call(\"EXPIRE\",key,window) end;if current>limit then return 0 else return 1 end" 1 ratelimit:user-__key__:/api/resource'
+    --command-key-pattern=R
+    --command-ratio=12
+    --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 150

{redis_benchmarks_specification-0.1.301.dist-info → redis_benchmarks_specification-0.1.303.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.301
+Version: 0.1.303
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.301.dist-info → redis_benchmarks_specification-0.1.303.dist-info}/RECORD

@@ -15,7 +15,7 @@ redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb
 redis_benchmarks_specification/__common__/env.py,sha256=kvJ8Ll-fvI_Tc0vynrzUEr22TqnJizzvJ4Lu9RjNr_M,3119
 redis_benchmarks_specification/__common__/github.py,sha256=9TZtnISsSgXTSAN_VQejo5YRPDPhlU0gjxgKGPw_sP8,10699
 redis_benchmarks_specification/__common__/package.py,sha256=4uVt1BAZ999LV2rZkq--Tk6otAVIf9YR3g3KGeUpiW4,834
-redis_benchmarks_specification/__common__/runner.py,sha256=s5IJnxO527SoImhQmOU2UrqR4_h3MnHM2i7dh7wNV1Y,15649
+redis_benchmarks_specification/__common__/runner.py,sha256=7Xc8QjwqYXQH9Vnq7s2YiP052zI3dk-OFhO-8Z5XY3o,16589
 redis_benchmarks_specification/__common__/spec.py,sha256=D_SN48wg6NMthW_-OS1H5bydSDiuZpfd4WPPj7Vfwmc,5760
 redis_benchmarks_specification/__common__/timeseries.py,sha256=w8XQrGPEFuuemDXXz4iny7lYsNbYH0ycQyq3jHIr80g,52916
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
@@ -271,10 +271,11 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-mixed
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-mixed-100-channels-128B-100-publishers-50K-subscribers-5k-conns.yml,sha256=mh_cAomSjsU4WL2l9gt0dhyWdSwfXTrEZgnRz6gzSQE,1144
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publish-1K-channels-10B-no-subscribers.yml,sha256=zwb27Jmg0mwcthbmdUe0KuzqRzAPs7OHcK2gc9-5VBE,779
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml,sha256=7G_J8kUFay7jXhZvsZK5jvVHSLZvhMV0uuDMkZBbeSQ,675
+redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage.yml,sha256=Q5JKvADD1R71hyRx4Qsz5dxHHlYICHqWHD7XTAmX72M,6936
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
-redis_benchmarks_specification-0.1.301.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.301.dist-info/METADATA,sha256=C3ytp_y6zvhWnBn06yKTK1XZgSySycj3i9IJU5Ry9q8,22726
-redis_benchmarks_specification-0.1.301.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.301.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.301.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.303.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.303.dist-info/METADATA,sha256=EhMb2tY2pNg3PZtPh8R40wztMm6GVTR48j5P_fP04nw,22726
+redis_benchmarks_specification-0.1.303.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.303.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.303.dist-info/RECORD,,