redis-benchmarks-specification 0.1.301__tar.gz → 0.1.304__tar.gz

{redis_benchmarks_specification-0.1.301 → redis_benchmarks_specification-0.1.304}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.301
+Version: 0.1.304
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.301 → redis_benchmarks_specification-0.1.304}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.301"
+version = "0.1.304"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.301 → redis_benchmarks_specification-0.1.304}/redis_benchmarks_specification/__common__/runner.py

@@ -15,11 +15,14 @@ from redis_benchmarks_specification.__common__.timeseries import (
 
 def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
     cmds = None
+    lua_scripts = None
     res = 0
     if dbconfig_keyname in benchmark_config:
         for k, v in benchmark_config[dbconfig_keyname].items():
             if "init_commands" in k:
                 cmds = v
+            elif "init_lua" in k:
+                lua_scripts = v
 
     if type(cmds) == str:
         cmds = [cmds]
@@ -37,7 +40,7 @@ def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
                     quoting=csv.QUOTE_ALL,
                     skipinitialspace=True,
                 ):
-                    if lines[0] != " " and len(lines[0]) > 0:
+                    if len(lines) > 0 and lines[0] != " " and len(lines[0]) > 0:
                         cols.append(lines[0])
                 cmd = cols
                 is_array = True
@@ -57,6 +60,21 @@ def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
                     )
                 )
 
+    # Process init_lua scripts
+    if lua_scripts is not None:
+        if type(lua_scripts) == str:
+            lua_scripts = [lua_scripts]
+
+        for lua_script in lua_scripts:
+            try:
+                logging.info("Executing Lua script (length: {} chars)".format(len(lua_script)))
+                # Execute the Lua script using EVAL command with 0 keys
+                stdout = r.execute_command("EVAL", lua_script, 0)
+                logging.info("Lua script result: {}".format(stdout))
+                res = res + 1
+            except Exception as e:
+                logging.error("Lua script execution failed: {}".format(e))
+
     return res
 
 
@@ -452,13 +470,16 @@ def exporter_datasink_common(
     deployment_type_and_name = f"{setup_type}_AND_{setup_name}"
     deployment_type_and_name_and_version = f"{setup_type}_AND_{setup_name}_AND_{git_version}"
 
-    # Add to deployment-specific set
-    deployment_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/{deployment_type_and_name_and_version}:set"
-    datasink_conn.sadd(deployment_set_key, test_name)
+    # Add to deployment-specific set (only if datasink connection is available)
+    if datasink_conn is not None:
+        deployment_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/{deployment_type_and_name_and_version}:set"
+        datasink_conn.sadd(deployment_set_key, test_name)
 
-    # Add to testcases set
-    testcases_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/testcases:set"
-    datasink_conn.sadd(testcases_set_key, test_name)
+        # Add to testcases set
+        testcases_set_key = f"ci.benchmarks.redis/{tf_triggering_env}/testcases:set"
+        datasink_conn.sadd(testcases_set_key, test_name)
+    else:
+        logging.debug("Datasink connection not available, skipping set operations")
 
     # Add metadata fields to timeseries metadata
     metadata["deployment_type_AND_deployment_name"] = deployment_type_and_name

{redis_benchmarks_specification-0.1.301 → redis_benchmarks_specification-0.1.304}/redis_benchmarks_specification/__runner__/runner.py

@@ -1393,17 +1393,27 @@ def process_self_contained_coordinator_stream(
                     ssl_cert_reqs = "required"
                     if tls_skip_verify:
                         ssl_cert_reqs = None
-                    r = redis.StrictRedis(
-                        host=host,
-                        port=port,
-                        password=password,
-                        ssl=tls_enabled,
-                        ssl_cert_reqs=ssl_cert_reqs,
-                        ssl_keyfile=tls_key,
-                        ssl_certfile=tls_cert,
-                        ssl_ca_certs=tls_cacert,
-                        ssl_check_hostname=False,
-                    )
+
+                    # Build Redis connection parameters
+                    redis_params = {
+                        "host": host,
+                        "port": port,
+                        "password": password,
+                        "ssl": tls_enabled,
+                        "ssl_cert_reqs": ssl_cert_reqs,
+                        "ssl_check_hostname": False,
+                    }
+
+                    # Only add SSL certificate parameters if they are provided
+                    if tls_enabled:
+                        if tls_key is not None and tls_key != "":
+                            redis_params["ssl_keyfile"] = tls_key
+                        if tls_cert is not None and tls_cert != "":
+                            redis_params["ssl_certfile"] = tls_cert
+                        if tls_cacert is not None and tls_cacert != "":
+                            redis_params["ssl_ca_certs"] = tls_cacert
+
+                    r = redis.StrictRedis(**redis_params)
                     setup_name = "oss-standalone"
                     r.ping()
 
@@ -1456,17 +1466,26 @@ def process_self_contained_coordinator_stream(
                             slot_network_info = slot[2]
                             prefered_endpoint = slot_network_info[0]
                             prefered_port = slot_network_info[1]
-                            shard_conn = redis.StrictRedis(
-                                host=prefered_endpoint,
-                                port=prefered_port,
-                                password=password,
-                                ssl=tls_enabled,
-                                ssl_cert_reqs=ssl_cert_reqs,
-                                ssl_keyfile=tls_key,
-                                ssl_certfile=tls_cert,
-                                ssl_ca_certs=tls_cacert,
-                                ssl_check_hostname=False,
-                            )
+                            # Build shard connection parameters
+                            shard_params = {
+                                "host": prefered_endpoint,
+                                "port": prefered_port,
+                                "password": password,
+                                "ssl": tls_enabled,
+                                "ssl_cert_reqs": ssl_cert_reqs,
+                                "ssl_check_hostname": False,
+                            }
+
+                            # Only add SSL certificate parameters if they are provided
+                            if tls_enabled:
+                                if tls_key is not None and tls_key != "":
+                                    shard_params["ssl_keyfile"] = tls_key
+                                if tls_cert is not None and tls_cert != "":
+                                    shard_params["ssl_certfile"] = tls_cert
+                                if tls_cacert is not None and tls_cacert != "":
+                                    shard_params["ssl_ca_certs"] = tls_cacert
+
+                            shard_conn = redis.StrictRedis(**shard_params)
                             redis_conns.append(shard_conn)
                         logging.info(
                             "There are a total of {} shards".format(len(redis_conns))

redis_benchmarks_specification-0.1.304/redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage.yml

@@ -0,0 +1,203 @@
+version: 0.4
+name: memtier_benchmark-playbook-session-storage
+description: |
+  Runs memtier_benchmark to simulate a session-based SaaS application.
+  The workload mimics user session CRUD, session tracking, organization-level analytics,
+  and rate limiting. It includes realistic key sizes and command mixes representative
+  of production traffic (e.g., 400–600B session hashes, ZSETs for analytics, and SETs for tracking).
+
+  Rate limiting is modeled using an atomic Lua script, inspired by the Upstash Redis example:
+  https://github.com/upstash/examples/tree/main/examples/ratelimit-with-redis
+
+  Each user has a dedicated key in the form of `ratelimit:user-<id>:/api/resource`, which is
+  used to track usage under a fixed window. The logic is evaluated atomically with the following script:
+
+      local key = KEYS[1]
+      local limit = 100
+      local window = 60
+      local current = redis.call("INCR", key)
+      if current == 1 then
+        redis.call("EXPIRE", key, window)
+      end
+      if current > limit then
+        return 0
+      else
+        return 1
+      end
+
+  This ensures that rate enforcement and usage tracking are done without race conditions, and
+  mirrors a real-world API quota model.
+
+  The workload emphasizes read-heavy patterns to reflect common SaaS access behavior. The overall
+  **read:write ratio is approximately 85:15**, with read operations covering session access, user-session
+  lookups, org analytics, and rate limit enforcement, while writes handle session updates, activity tracking,
+  and quota increments.
+
+  Command groups by use-case (approximate ratio of total operations):
+    - Session CRUD (HGETALL, HSET): ~55%
+    - User session tracking (SMEMBERS, SADD): ~21%
+    - Organization analytics (ZRANGE, ZADD): ~12%
+    - Rate limiting (EVAL-based quota check): ~12%
+
+exporter:
+  redistimeseries:
+    break_by:
+    - version
+    - commit
+    timemetric: $."ALL STATS".Runtime."Start time"
+    metrics:
+    - $."BEST RUN RESULTS".Hgetalls."Ops/sec"
+    - $."BEST RUN RESULTS".Hsets."Ops/sec"
+    - $."BEST RUN RESULTS".Smemberss."Ops/sec"
+    - $."BEST RUN RESULTS".Sadds."Ops/sec"
+    - $."BEST RUN RESULTS".Zranges."Ops/sec"
+    - $."BEST RUN RESULTS".Zadds."Ops/sec"
+    - $."BEST RUN RESULTS".Evals."Ops/sec"
+    - $."BEST RUN RESULTS".Totals."Ops/sec"
+    - $."BEST RUN RESULTS".Totals."Latency"
+    - $."BEST RUN RESULTS".Totals."Misses/sec"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
+    - $."ALL STATS".Hgetalls."Ops/sec"
+    - $."ALL STATS".Hsets."Ops/sec"
+    - $."ALL STATS".Smemberss."Ops/sec"
+    - $."ALL STATS".Sadds."Ops/sec"
+    - $."ALL STATS".Zranges."Ops/sec"
+    - $."ALL STATS".Zadds."Ops/sec"
+    - $."ALL STATS".Evals."Ops/sec"
+    - $."ALL STATS".Totals."Ops/sec"
+    - $."ALL STATS".Totals."Latency"
+    - $."ALL STATS".Totals."Misses/sec"
+    - $."ALL STATS".Hgetalls."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Hsets."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Smemberss."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Sadds."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Zranges."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Zadds."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Evals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p99.00"
+
+
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  resources:
+    requests:
+      memory: 1g
+  init_lua: |
+    -- Use a fixed seed for reproducibility
+    local seed = 12345
+    math.randomseed(seed)
+
+    local now = tonumber(redis.call('TIME')[1])
+
+    local function rand_str(len)
+      local chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+      local res = ''
+      for i = 1, len do
+        local idx = math.random(#chars)
+        res = res .. chars:sub(idx, idx)
+      end
+      return res
+    end
+
+    for i = 1, 100000 do
+      local session_id = 'session:' .. i
+      local user_id = 'user-' .. i
+      local org_id = 'org-' .. i
+
+      redis.call('HSET', session_id,
+        'userId', user_id,
+        'organizationId', org_id,
+        'role', 'member',
+        'createdAt', tostring(now - math.random(3600)),
+        'lastAccessed', tostring(now),
+        'ipAddress', '192.168.1.' .. (i % 255),
+        'device', 'device-' .. rand_str(8),
+        'authMethod', 'password',
+        'status', 'active',
+        'metadata', rand_str(200 + (i % 100))
+      )
+
+      redis.call('SADD', 'user:' .. i .. ':sessions', session_id)
+
+      local org_key = 'org:' .. i .. ':sessions'
+      for j = 1, 10 do
+        local uid = 'user-' .. ((i + j) % 1000 + 1)
+        local sid = 'session:' .. ((i + j) % 1000 + 1)
+        local zmember = uid .. ':' .. sid
+        local zscore = now - math.random(86400)
+        redis.call('ZADD', org_key, zscore, zmember)
+      end
+
+      local endpoint = '/api/resource'
+      local rate_key = 'ratelimit:' .. user_id .. ':' .. endpoint
+      redis.call('INCR', rate_key)
+      redis.call('EXPIRE', rate_key, 60)
+    end
+
+    return 'OK'
+tested-groups:
+- hash
+- zset
+- set
+- string
+
+tested-commands:
+- hgetall
+- hset
+- smembers
+- sadd
+- zrange
+- zadd
+- incr
+- expire
+- get
+
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: >
+    --key-prefix ""
+    --key-minimum 1
+    --key-maximum 100000
+    --data-size-range=400-600
+    --pipeline=1
+    --print-percentiles=50,90,95,99
+    --run-count=1
+    --test-time=120
+    --command="HGETALL session:__key__"
+    --command-key-pattern=R
+    --command-ratio=50
+    --command="HSET session:__key__ userId user-__key__ organizationId org-__key__ role admin email user__key__@example.com name "User __key__" permissions "["read","write"]" lastActivity __timestamp__ ipAddress 192.168.1.__key__ userAgent "Mozilla/5.0" createdAt __timestamp__"
+    --command-key-pattern=R
+    --command-ratio=5
+    --command="SMEMBERS user:__key__:sessions"
+    --command-key-pattern=R
+    --command-ratio=18
+    --command="SADD user:__key__:sessions session-__key__"
+    --command-key-pattern=R
+    --command-ratio=3
+    --command="ZRANGE org:__key__:sessions 0 -1 WITHSCORES"
+    --command-key-pattern=R
+    --command-ratio=10
+    --command="ZADD org:__key__:sessions 1 user-__key__:session-__key__"
+    --command-key-pattern=R
+    --command-ratio=2
+    --command='EVAL "local key=KEYS[1];local limit=10;local window=60;local current=redis.call(\"INCR\",key);if current==1 then redis.call(\"EXPIRE\",key,window) end;if current>limit then return 0 else return 1 end" 1 ratelimit:user-__key__:/api/resource'
+    --command-key-pattern=R
+    --command-ratio=12
+    --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 150