redis-benchmarks-specification 0.2.18__tar.gz → 0.2.19__tar.gz

{redis_benchmarks_specification-0.2.18 → redis_benchmarks_specification-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.2.18
+Version: 0.2.19
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.2.18 → redis_benchmarks_specification-0.2.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.2.18"
+version = "0.2.19"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.2.18 → redis_benchmarks_specification-0.2.19}/redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -207,4 +207,10 @@ def create_self_contained_coordinator_args(project_name):
         action="store_true",
         help="Skip automatically clearing pending messages and resetting consumer group position on startup. By default, pending messages are cleared and consumer group is reset to latest position to skip old work and recover from crashes.",
     )
+    parser.add_argument(
+        "--enable-cpu-distribution",
+        default=False,
+        action="store_true",
+        help="Enable CPU distribution optimization using Docker cpuset_cpus and nano_cpus parameters. This forces better CPU core distribution for Redis and client containers. Disabled by default.",
+    )
     return parser

{redis_benchmarks_specification-0.2.18 → redis_benchmarks_specification-0.2.19}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -772,6 +772,7 @@ def main():
             priority_upper_limit,
             default_baseline_branch,
             default_metrics_str,
+            args=args,
         )
 
 
@@ -808,6 +809,7 @@ def self_contained_coordinator_blocking_read(
     default_metrics_str="ALL_STATS.Totals.Ops/sec",
     docker_keep_env=False,
     restore_build_artifacts_default=True,
+    args=None,
 ):
     num_process_streams = 0
     num_process_test_suites = 0
@@ -1405,6 +1407,8 @@ def process_self_contained_coordinator_stream(
                                     redis_containers,
                                     run_image,
                                     temporary_dir,
+                                    auto_remove=False,
+                                    enable_cpu_distribution=args.enable_cpu_distribution if args else False,
                                 )
 
                                 r = redis.StrictRedis(
@@ -1600,63 +1604,72 @@ def process_self_contained_coordinator_stream(
                                         f"Using default container timeout: {container_timeout}s"
                                     )
 
-                                # Calculate nano_cpus for better CPU distribution
-                                client_cpu_count = len(client_cpuset_cpus.split(','))
-                                client_nano_cpus = int(client_cpu_count * 1e9)  # 1 CPU = 1e9 nano_cpus
-
-                                logging.info(f"Client container will use {client_cpu_count} CPUs (nano_cpus={client_nano_cpus}) on cores {client_cpuset_cpus}")
+                                # Prepare client container arguments
+                                client_container_args = {
+                                    "image": client_container_image,
+                                    "volumes": {
+                                        temporary_dir_client: {
+                                            "bind": client_mnt_point,
+                                            "mode": "rw",
+                                        },
+                                    },
+                                    "auto_remove": False,  # Don't auto-remove so we can get logs if timeout
+                                    "privileged": True,
+                                    "working_dir": benchmark_tool_workdir,
+                                    "command": benchmark_command_str,
+                                    "network_mode": "host",
+                                    "detach": True,  # Detach to enable timeout
+                                }
+
+                                # Add CPU distribution settings if enabled
+                                enable_cpu_distribution = args.enable_cpu_distribution if args else False
+                                if enable_cpu_distribution:
+                                    client_cpu_count = len(client_cpuset_cpus.split(','))
+                                    client_nano_cpus = int(client_cpu_count * 1e9)  # 1 CPU = 1e9 nano_cpus
+
+                                    client_container_args["cpuset_cpus"] = client_cpuset_cpus
+                                    client_container_args["nano_cpus"] = client_nano_cpus
+
+                                    logging.info(f"Client container will use {client_cpu_count} CPUs (nano_cpus={client_nano_cpus}) on cores {client_cpuset_cpus} [CPU distribution enabled]")
+                                else:
+                                    logging.info(f"Client container will use default CPU allocation [CPU distribution disabled]")
 
                                 try:
                                     # Start container with detach=True to enable timeout handling
-                                    container = docker_client.containers.run(
-                                        image=client_container_image,
-                                        volumes={
-                                            temporary_dir_client: {
-                                                "bind": client_mnt_point,
-                                                "mode": "rw",
-                                            },
-                                        },
-                                        auto_remove=False,  # Don't auto-remove so we can get logs if timeout
-                                        privileged=True,
-                                        working_dir=benchmark_tool_workdir,
-                                        command=benchmark_command_str,
-                                        network_mode="host",
-                                        detach=True,  # Detach to enable timeout
-                                        cpuset_cpus=client_cpuset_cpus,
-                                        nano_cpus=client_nano_cpus,  # Force CPU distribution
-                                    )
+                                    container = docker_client.containers.run(**client_container_args)
 
                                     logging.info(
                                         f"Started container {container.name} ({container.id[:12]}) with {container_timeout}s timeout"
                                     )
 
-                                    # Apply CPU affinity using taskset to force client processes to use assigned cores
-                                    try:
-                                        container_info = docker_client.api.inspect_container(container.id)
-                                        container_pid = container_info['State']['Pid']
+                                    # Apply CPU affinity using taskset if CPU distribution is enabled
+                                    if enable_cpu_distribution:
+                                        try:
+                                            container_info = docker_client.api.inspect_container(container.id)
+                                            container_pid = container_info['State']['Pid']
 
-                                        logging.info(f"Setting CPU affinity for client container PID {container_pid} to cores {client_cpuset_cpus}")
+                                            logging.info(f"Setting CPU affinity for client container PID {container_pid} to cores {client_cpuset_cpus}")
 
-                                        # Set CPU affinity for the main process and all its threads
-                                        subprocess.run(f"taskset -cp {client_cpuset_cpus} {container_pid}", shell=True, check=True)
+                                            # Set CPU affinity for the main process and all its threads
+                                            subprocess.run(f"taskset -cp {client_cpuset_cpus} {container_pid}", shell=True, check=True)
 
-                                        # Wait a moment for client to start its threads, then set affinity for all child processes
-                                        time.sleep(1)
-                                        result = subprocess.run(f"pgrep -P {container_pid}", shell=True, capture_output=True, text=True)
-                                        if result.returncode == 0:
-                                            child_pids = result.stdout.strip().split('\n')
-                                            for child_pid in child_pids:
-                                                if child_pid.strip():
-                                                    try:
-                                                        subprocess.run(f"taskset -cp {client_cpuset_cpus} {child_pid.strip()}", shell=True, check=True)
-                                                        logging.info(f"Set CPU affinity for client child process {child_pid.strip()}")
-                                                    except subprocess.CalledProcessError:
-                                                        pass  # Child process may have exited
+                                            # Wait a moment for client to start its threads, then set affinity for all child processes
+                                            time.sleep(1)
+                                            result = subprocess.run(f"pgrep -P {container_pid}", shell=True, capture_output=True, text=True)
+                                            if result.returncode == 0:
+                                                child_pids = result.stdout.strip().split('\n')
+                                                for child_pid in child_pids:
+                                                    if child_pid.strip():
+                                                        try:
+                                                            subprocess.run(f"taskset -cp {client_cpuset_cpus} {child_pid.strip()}", shell=True, check=True)
+                                                            logging.info(f"Set CPU affinity for client child process {child_pid.strip()}")
+                                                        except subprocess.CalledProcessError:
+                                                            pass  # Child process may have exited
 
-                                        logging.info(f"✅ Applied CPU affinity to client container and all child processes")
+                                            logging.info(f"✅ Applied CPU affinity to client container and all child processes")
 
-                                    except Exception as e:
-                                        logging.warning(f"Failed to set CPU affinity for client container: {e}")
+                                        except Exception as e:
+                                            logging.warning(f"Failed to set CPU affinity for client container: {e}")
 
                                     # Wait for container with timeout
                                     try:
@@ -2266,6 +2279,7 @@ def start_redis_container(
     run_image,
     temporary_dir,
     auto_remove=False,
+    enable_cpu_distribution=False,
 ):
     logging.info(
         "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
@@ -2284,55 +2298,62 @@ def start_redis_container(
         logging.info(f"setting volume as follow: {volumes}. working_dir={mnt_point}")
         working_dir = mnt_point
 
-    # Calculate nano_cpus for better CPU distribution
-    redis_cpu_count = len(db_cpuset_cpus.split(','))
-    redis_nano_cpus = int(redis_cpu_count * 1e9)  # 1 CPU = 1e9 nano_cpus
-
-    logging.info(f"Redis container will use {redis_cpu_count} CPUs (nano_cpus={redis_nano_cpus}) on cores {db_cpuset_cpus}")
-
-    redis_container = docker_client.containers.run(
-        image=run_image,
-        volumes=volumes,
-        auto_remove=auto_remove,
-        privileged=True,
-        working_dir=mnt_point,
-        command=command_str,
-        network_mode="host",
-        detach=True,
-        cpuset_cpus=db_cpuset_cpus,
-        nano_cpus=redis_nano_cpus,  # Force CPU distribution
-       # pid_mode="host",
-        publish_all_ports=True,
-    )
+    # Prepare container arguments
+    container_args = {
+        "image": run_image,
+        "volumes": volumes,
+        "auto_remove": auto_remove,
+        "privileged": True,
+        "working_dir": mnt_point,
+        "command": command_str,
+        "network_mode": "host",
+        "detach": True,
+        "publish_all_ports": True,
+    }
+
+    # Add CPU distribution settings if enabled
+    if enable_cpu_distribution:
+        redis_cpu_count = len(db_cpuset_cpus.split(','))
+        redis_nano_cpus = int(redis_cpu_count * 1e9)  # 1 CPU = 1e9 nano_cpus
+
+        container_args["cpuset_cpus"] = db_cpuset_cpus
+        container_args["nano_cpus"] = redis_nano_cpus
+
+        logging.info(f"Redis container will use {redis_cpu_count} CPUs (nano_cpus={redis_nano_cpus}) on cores {db_cpuset_cpus} [CPU distribution enabled]")
+    else:
+        logging.info(f"Redis container will use default CPU allocation [CPU distribution disabled]")
+
+    redis_container = docker_client.containers.run(**container_args)
     time.sleep(5)
 
-    # Apply CPU affinity using taskset to force Redis processes to use assigned cores
-    try:
-        container_info = docker_client.api.inspect_container(redis_container.id)
-        container_pid = container_info['State']['Pid']
+    # Apply CPU affinity using taskset if CPU distribution is enabled
+    if enable_cpu_distribution:
+        try:
+            container_info = docker_client.api.inspect_container(redis_container.id)
+            container_pid = container_info['State']['Pid']
 
-        logging.info(f"Setting CPU affinity for Redis container PID {container_pid} to cores {db_cpuset_cpus}")
+            logging.info(f"Setting CPU affinity for Redis container PID {container_pid} to cores {db_cpuset_cpus}")
 
-        # Set CPU affinity for the main Redis process and all its threads
-        subprocess.run(f"taskset -cp {db_cpuset_cpus} {container_pid}", shell=True, check=True)
+            # Set CPU affinity for the main Redis process and all its threads
+            subprocess.run(f"taskset -cp {db_cpuset_cpus} {container_pid}", shell=True, check=True)
 
-        # Wait a moment for Redis to start its IO threads, then set affinity for all Redis processes
-        time.sleep(2)
-        result = subprocess.run(f"pgrep -P {container_pid}", shell=True, capture_output=True, text=True)
-        if result.returncode == 0:
-            child_pids = result.stdout.strip().split('\n')
-            for child_pid in child_pids:
-                if child_pid.strip():
-                    try:
-                        subprocess.run(f"taskset -cp {db_cpuset_cpus} {child_pid.strip()}", shell=True, check=True)
-                        logging.info(f"Set CPU affinity for Redis child process {child_pid.strip()}")
-                    except subprocess.CalledProcessError:
-                        pass  # Child process may have exited
+            # Wait a moment for Redis to start its IO threads, then set affinity for all Redis processes
+            time.sleep(2)
+            result = subprocess.run(f"pgrep -P {container_pid}", shell=True, capture_output=True, text=True)
+            if result.returncode == 0:
+                child_pids = result.stdout.strip().split('\n')
+                for child_pid in child_pids:
+                    if child_pid.strip():
+                        try:
+                            subprocess.run(f"taskset -cp {db_cpuset_cpus} {child_pid.strip()}", shell=True, check=True)
+                            logging.info(f"Set CPU affinity for Redis child process {child_pid.strip()}")
+                        except subprocess.CalledProcessError:
+                            pass  # Child process may have exited
 
-        logging.info(f"✅ Applied CPU affinity to Redis container and all child processes")
+            logging.info(f"✅ Applied CPU affinity to Redis container and all child processes")
 
-    except Exception as e:
-        logging.warning(f"Failed to set CPU affinity for Redis container: {e}")
+        except Exception as e:
+            logging.warning(f"Failed to set CPU affinity for Redis container: {e}")
 
     redis_containers.append(redis_container)
     return redis_container
@@ -2522,31 +2543,39 @@ def data_prepopulation_step(
         preload_timeout = 1800  # 30 minutes default for data loading
         logging.info(f"Starting preload container with {preload_timeout}s timeout")
 
-        # Calculate nano_cpus for better CPU distribution
-        preload_cpu_count = len(client_cpuset_cpus.split(','))
-        preload_nano_cpus = int(preload_cpu_count * 1e9)  # 1 CPU = 1e9 nano_cpus
+        # Prepare preload container arguments
+        preload_container_args = {
+            "image": preload_image,
+            "volumes": {
+                temporary_dir: {
+                    "bind": client_mnt_point,
+                    "mode": "rw",
+                },
+            },
+            "auto_remove": False,  # Don't auto-remove so we can get logs if timeout
+            "privileged": True,
+            "working_dir": benchmark_tool_workdir,
+            "command": preload_command_str,
+            "network_mode": "host",
+            "detach": True,  # Detach to enable timeout
+        }
+
+        # Add CPU distribution settings if enabled
+        enable_cpu_distribution = args.enable_cpu_distribution if args else False
+        if enable_cpu_distribution:
+            preload_cpu_count = len(client_cpuset_cpus.split(','))
+            preload_nano_cpus = int(preload_cpu_count * 1e9)  # 1 CPU = 1e9 nano_cpus
+
+            preload_container_args["cpuset_cpus"] = client_cpuset_cpus
+            preload_container_args["nano_cpus"] = preload_nano_cpus
 
-        logging.info(f"Preload container will use {preload_cpu_count} CPUs (nano_cpus={preload_nano_cpus}) on cores {client_cpuset_cpus}")
+            logging.info(f"Preload container will use {preload_cpu_count} CPUs (nano_cpus={preload_nano_cpus}) on cores {client_cpuset_cpus} [CPU distribution enabled]")
+        else:
+            logging.info(f"Preload container will use default CPU allocation [CPU distribution disabled]")
 
         try:
             # Start container with detach=True to enable timeout handling
-            container = docker_client.containers.run(
-                image=preload_image,
-                volumes={
-                    temporary_dir: {
-                        "bind": client_mnt_point,
-                        "mode": "rw",
-                    },
-                },
-                auto_remove=False,  # Don't auto-remove so we can get logs if timeout
-                privileged=True,
-                working_dir=benchmark_tool_workdir,
-                command=preload_command_str,
-                network_mode="host",
-                detach=True,  # Detach to enable timeout
-                cpuset_cpus=client_cpuset_cpus,
-                nano_cpus=preload_nano_cpus,  # Force CPU distribution
-            )
+            container = docker_client.containers.run(**preload_container_args)
 
             logging.info(
                 f"Started preload container {container.name} ({container.id[:12]}) with {preload_timeout}s timeout"

{redis_benchmarks_specification-0.2.18 → redis_benchmarks_specification-0.2.19}/redis_benchmarks_specification/setups/topologies/topologies.yml

@@ -26,10 +26,10 @@ spec:
     redis_topology:
       primaries: 1
       replicas: 0
-    redis_arguments: --io-threads 2 --io-threads-do-reads yes --server-cpulist 0,1
+    redis_arguments: --io-threads 2 --io-threads-do-reads yes 
     resources:
       requests:
-        cpus: "3"
+        cpus: "2"
         memory: "10g"
 
   - name: oss-standalone-04-io-threads
@@ -37,10 +37,10 @@ spec:
     redis_topology:
       primaries: 1
       replicas: 0
-    redis_arguments: --io-threads 4 --io-threads-do-reads yes --server-cpulist 0,1,2,3
+    redis_arguments: --io-threads 4 --io-threads-do-reads yes 
     resources:
       requests:
-        cpus: "5"
+        cpus: "4"
         memory: "10g"
 
   - name: oss-standalone-08-io-threads
@@ -48,10 +48,10 @@ spec:
     redis_topology:
       primaries: 1
       replicas: 0
-    redis_arguments: --io-threads 8 --io-threads-do-reads yes --server-cpulist 0,1,2,3,4,5,6,7
+    redis_arguments: --io-threads 8 --io-threads-do-reads yes 
     resources:
       requests:
-        cpus: "9"
+        cpus: "8"
         memory: "10g"
 
   - name: oss-standalone-16-io-threads
@@ -62,7 +62,7 @@ spec:
     redis_arguments: --io-threads 16 --io-threads-do-reads yes
     resources:
       requests:
-        cpus: "17"
+        cpus: "16"
         memory: "10g"
 
   - name: oss-standalone-32-io-threads
@@ -73,7 +73,7 @@ spec:
     redis_arguments: --io-threads 32 --io-threads-do-reads yes
     resources:
       requests:
-        cpus: "33"
+        cpus: "32"
         memory: "10g"
 
   - name: oss-standalone-64-io-threads
@@ -84,7 +84,7 @@ spec:
     redis_arguments: --io-threads 64 --io-threads-do-reads yes
     resources:
       requests:
-        cpus: "65"
+        cpus: "64"
         memory: "10g"
   - name: oss-standalone-1replica
     type: oss-standalone