redis-benchmarks-specification 0.1.274__py3-none-any.whl → 0.1.276__py3-none-any.whl

redis_benchmarks_specification/__runner__/runner.py

@@ -4,6 +4,7 @@ import logging
 import math
 import os
 import shutil
+import subprocess
 import sys
 import tempfile
 import traceback
@@ -62,6 +63,84 @@ from redis_benchmarks_specification.__common__.spec import (
     extract_client_tools,
 )
 from redis_benchmarks_specification.__runner__.args import create_client_runner_args
+from redis_benchmarks_specification.__runner__.remote_profiling import RemoteProfiler
+
+
+def run_local_command_with_timeout(command_str, timeout_seconds, description="command"):
+    """
+    Run a local command with timeout support.
+
+    Args:
+        command_str: The command string to execute
+        timeout_seconds: Timeout in seconds
+        description: Description for logging
+
+    Returns:
+        tuple: (success, stdout, stderr)
+    """
+    try:
+        logging.info(f"Running {description} with {timeout_seconds}s timeout: {command_str}")
+
+        # Use shell=True to support complex command strings with pipes, etc.
+        process = subprocess.Popen(
+            command_str,
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True
+        )
+
+        try:
+            stdout, stderr = process.communicate(timeout=timeout_seconds)
+            return_code = process.returncode
+
+            if return_code == 0:
+                logging.info(f"{description} completed successfully")
+                return True, stdout, stderr
+            else:
+                logging.error(f"{description} failed with return code {return_code}")
+                logging.error(f"stderr: {stderr}")
+                return False, stdout, stderr
+
+        except subprocess.TimeoutExpired:
+            logging.error(f"{description} timed out after {timeout_seconds} seconds")
+            process.kill()
+            try:
+                stdout, stderr = process.communicate(timeout=5)  # Give 5 seconds to cleanup
+            except subprocess.TimeoutExpired:
+                stdout, stderr = "", "Process killed due to timeout"
+            return False, stdout, f"Timeout after {timeout_seconds} seconds. {stderr}"
+
+    except Exception as e:
+        logging.error(f"Error running {description}: {e}")
+        return False, "", str(e)
+
+
+def calculate_process_timeout(command_str, buffer_timeout):
+    """
+    Calculate timeout for a process based on test-time parameter and buffer.
+
+    Args:
+        command_str: The command string to analyze
+        buffer_timeout: Buffer time to add to test-time
+
+    Returns:
+        int: Timeout in seconds
+    """
+    default_timeout = 300  # 5 minutes default
+
+    if "test-time" in command_str:
+        # Try to extract test time and add buffer
+        # Handle both --test-time (memtier) and -test-time (pubsub-sub-bench)
+        test_time_match = re.search(r"--?test-time[=\s]+(\d+)", command_str)
+        if test_time_match:
+            test_time = int(test_time_match.group(1))
+            timeout = test_time + buffer_timeout
+            logging.info(f"Set process timeout to {timeout}s (test-time: {test_time}s + {buffer_timeout}s buffer)")
+            return timeout
+
+    logging.info(f"Using default process timeout: {default_timeout}s")
+    return default_timeout
 
 
 def parse_size(size):
@@ -91,6 +170,31 @@ def parse_size(size):
     return int(number * units[unit])
 
 
+def extract_expected_benchmark_duration(benchmark_command_str, override_memtier_test_time):
+    """
+    Extract expected benchmark duration from command string or override.
+
+    Args:
+        benchmark_command_str: The benchmark command string
+        override_memtier_test_time: Override test time value
+
+    Returns:
+        Expected duration in seconds, or 30 as default
+    """
+    if override_memtier_test_time > 0:
+        return override_memtier_test_time
+
+    # Try to extract test-time from command string
+    if "test-time" in benchmark_command_str:
+        # Handle both --test-time (memtier) and -test-time (pubsub-sub-bench)
+        test_time_match = re.search(r"--?test-time[=\s]+(\d+)", benchmark_command_str)
+        if test_time_match:
+            return int(test_time_match.group(1))
+
+    # Default duration if not found
+    return 30
+
+
 def run_multiple_clients(
     benchmark_config,
     docker_client,
@@ -181,6 +285,30 @@ def run_multiple_clients(
                     unix_socket,
                     None,  # username
                 )
+            elif "vector-db-benchmark" in client_tool:
+                (
+                    _,
+                    benchmark_command_str,
+                    arbitrary_command,
+                    client_env_vars,
+                ) = prepare_vector_db_benchmark_parameters(
+                    client_config,
+                    client_tool,
+                    port,
+                    host,
+                    password,
+                    local_benchmark_output_filename,
+                    oss_cluster_api_enabled,
+                    tls_enabled,
+                    tls_skip_verify,
+                    test_tls_cert,
+                    test_tls_key,
+                    test_tls_cacert,
+                    resp_version,
+                    override_memtier_test_time,
+                    unix_socket,
+                    None,  # username
+                )
             else:
                 # Handle other benchmark tools
                 (
@@ -199,9 +327,8 @@ def run_multiple_clients(
 
             # Calculate container timeout
             container_timeout = 300  # 5 minutes default
-            buffer_timeout = (
-                args.container_timeout_buffer
-            )  # Configurable buffer from command line
+            # Use new timeout_buffer argument, fallback to container_timeout_buffer for backward compatibility
+            buffer_timeout = getattr(args, 'timeout_buffer', getattr(args, 'container_timeout_buffer', 60))
             if "test-time" in benchmark_command_str:
                 # Try to extract test time and add buffer
                 import re
@@ -224,23 +351,51 @@ def run_multiple_clients(
             # Start container (detached)
             import os
 
-            container = docker_client.containers.run(
-                image=client_image,
-                volumes={
-                    temporary_dir_client: {
-                        "bind": client_mnt_point,
-                        "mode": "rw",
-                    },
+            # Set working directory based on tool
+            working_dir = benchmark_tool_workdir
+            if "vector-db-benchmark" in client_tool:
+                working_dir = "/app"  # vector-db-benchmark needs to run from /app
+
+            # Prepare container arguments
+            volumes = {
+                temporary_dir_client: {
+                    "bind": client_mnt_point,
+                    "mode": "rw",
                 },
-                auto_remove=False,
-                privileged=True,
-                working_dir=benchmark_tool_workdir,
-                command=benchmark_command_str,
-                network_mode="host",
-                detach=True,
-                cpuset_cpus=client_cpuset_cpus,
-                user=f"{os.getuid()}:{os.getgid()}",  # Run as current user to fix permissions
-            )
+            }
+
+            # For vector-db-benchmark, also mount the results directory
+            if "vector-db-benchmark" in client_tool:
+                volumes[temporary_dir_client] = {
+                    "bind": "/app/results",
+                    "mode": "rw",
+                }
+
+            container_kwargs = {
+                "image": client_image,
+                "volumes": volumes,
+                "auto_remove": False,
+                "privileged": True,
+                "working_dir": working_dir,
+                "command": benchmark_command_str,
+                "network_mode": "host",
+                "detach": True,
+                "cpuset_cpus": client_cpuset_cpus,
+            }
+
+            # Only add user for non-vector-db-benchmark tools to avoid permission issues
+            if "vector-db-benchmark" not in client_tool:
+                container_kwargs["user"] = f"{os.getuid()}:{os.getgid()}"
+
+            # Add environment variables for vector-db-benchmark
+            if "vector-db-benchmark" in client_tool:
+                try:
+                    container_kwargs["environment"] = client_env_vars
+                except NameError:
+                    # client_env_vars not defined, skip environment variables
+                    pass
+
+            container = docker_client.containers.run(**container_kwargs)
 
             containers.append(
                 {
@@ -334,6 +489,7 @@ def run_multiple_clients(
         aggregated_json = {}
         memtier_json = None
         pubsub_json = None
+        vector_json = None
 
         for result in successful_results:
             client_index = result["client_index"]
@@ -360,6 +516,19 @@ def run_multiple_clients(
                         logging.info(
                             f"Successfully read pubsub-sub-bench JSON output from client {client_index}"
                         )
+                    elif "vector-db-benchmark" in tool:
+                        # For vector-db-benchmark, look for summary JSON file
+                        summary_files = [f for f in os.listdir(temporary_dir_client) if f.endswith("-summary.json")]
+                        if summary_files:
+                            summary_filepath = os.path.join(temporary_dir_client, summary_files[0])
+                            try:
+                                with open(summary_filepath, 'r') as f:
+                                    vector_json = json.load(f)
+                                logging.info(f"Successfully read vector-db-benchmark JSON output from {summary_files[0]}")
+                            except Exception as e:
+                                logging.warning(f"Failed to read vector-db-benchmark JSON from {summary_files[0]}: {e}")
+                        else:
+                            logging.warning(f"No vector-db-benchmark summary JSON file found for client {client_index}")
 
                     logging.info(
                         f"Successfully read JSON output from client {client_index} ({tool})"
@@ -376,16 +545,32 @@ def run_multiple_clients(
                     f"JSON output file not found for client {client_index}: {json_filepath}"
                 )
 
-        # Merge JSON outputs from both tools
-        if memtier_json and pubsub_json:
+        # Merge JSON outputs from all tools
+        if memtier_json and pubsub_json and vector_json:
+            # Use memtier as base and add other metrics
+            aggregated_json = memtier_json.copy()
+            aggregated_json.update(pubsub_json)
+            aggregated_json.update(vector_json)
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using merged JSON results from memtier, pubsub-sub-bench, and vector-db-benchmark clients")
+        elif memtier_json and pubsub_json:
             # Use memtier as base and add pubsub metrics
             aggregated_json = memtier_json.copy()
-            # Add pubsub metrics to the aggregated result
             aggregated_json.update(pubsub_json)
             aggregated_stdout = json.dumps(aggregated_json, indent=2)
-            logging.info(
-                "Using merged JSON results from memtier and pubsub-sub-bench clients"
-            )
+            logging.info("Using merged JSON results from memtier and pubsub-sub-bench clients")
+        elif memtier_json and vector_json:
+            # Use memtier as base and add vector metrics
+            aggregated_json = memtier_json.copy()
+            aggregated_json.update(vector_json)
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using merged JSON results from memtier and vector-db-benchmark clients")
+        elif pubsub_json and vector_json:
+            # Use pubsub as base and add vector metrics
+            aggregated_json = pubsub_json.copy()
+            aggregated_json.update(vector_json)
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using merged JSON results from pubsub-sub-bench and vector-db-benchmark clients")
         elif memtier_json:
             # Only memtier available
             aggregated_json = memtier_json
@@ -396,12 +581,15 @@ def run_multiple_clients(
             aggregated_json = pubsub_json
             aggregated_stdout = json.dumps(aggregated_json, indent=2)
             logging.info("Using JSON results from pubsub-sub-bench client only")
+        elif vector_json:
+            # Only vector-db-benchmark available
+            aggregated_json = vector_json
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using JSON results from vector-db-benchmark client only")
         else:
             # Fall back to concatenated stdout
             aggregated_stdout = "\n".join([r["stdout"] for r in successful_results])
-            logging.warning(
-                "No JSON results found, falling back to concatenated stdout"
-            )
+            logging.warning("No JSON results found, falling back to concatenated stdout")
 
     return aggregated_stdout, results
 
@@ -665,6 +853,71 @@ def prepare_memtier_benchmark_parameters(
     return None, benchmark_command_str, arbitrary_command
 
 
+def prepare_vector_db_benchmark_parameters(
+    clientconfig,
+    full_benchmark_path,
+    port,
+    server,
+    password,
+    local_benchmark_output_filename,
+    oss_cluster_api_enabled=False,
+    tls_enabled=False,
+    tls_skip_verify=False,
+    tls_cert=None,
+    tls_key=None,
+    tls_cacert=None,
+    resp_version=None,
+    override_test_time=0,
+    unix_socket="",
+    username=None,
+):
+    """
+    Prepare vector-db-benchmark command parameters
+    """
+    arbitrary_command = False
+
+    benchmark_command = [
+        "/app/run.py",
+        "--host",
+        f"{server}",
+    ]
+
+    # Add port as environment variable (vector-db-benchmark uses env vars)
+    env_vars = {}
+    if port is not None:
+        env_vars["REDIS_PORT"] = str(port)
+    if password is not None:
+        env_vars["REDIS_AUTH"] = password
+    if username is not None:
+        env_vars["REDIS_USER"] = username
+
+    # Add engines parameter
+    engines = clientconfig.get("engines", "vectorsets-fp32-default")
+    benchmark_command.extend(["--engines", engines])
+
+    # Add datasets parameter
+    datasets = clientconfig.get("datasets", "random-100")
+    benchmark_command.extend(["--datasets", datasets])
+
+    # Add other optional parameters
+    if "parallels" in clientconfig:
+        benchmark_command.extend(["--parallels", str(clientconfig["parallels"])])
+
+    if "queries" in clientconfig:
+        benchmark_command.extend(["--queries", str(clientconfig["queries"])])
+
+    if "timeout" in clientconfig:
+        benchmark_command.extend(["--timeout", str(clientconfig["timeout"])])
+
+    # Add custom arguments if specified
+    if "arguments" in clientconfig:
+        benchmark_command_str = " ".join(benchmark_command) + " " + clientconfig["arguments"]
+    else:
+        benchmark_command_str = " ".join(benchmark_command)
+
+    return benchmark_command, benchmark_command_str, arbitrary_command, env_vars
+
+
 def prepare_pubsub_sub_bench_parameters(
     clientconfig,
     full_benchmark_path,
@@ -899,6 +1152,23 @@ def process_self_contained_coordinator_stream(
                         redis_pid = conn.info()["process_id"]
                         redis_pids.append(redis_pid)
 
+                    # Check if all tested commands are supported by this Redis instance
+                    supported_commands = get_supported_redis_commands(redis_conns)
+                    commands_supported, unsupported_commands = check_test_command_support(
+                        benchmark_config, supported_commands
+                    )
+
+                    if not commands_supported:
+                        logging.warning(
+                            f"Skipping test {test_name} due to unsupported commands: {unsupported_commands}"
+                        )
+                        delete_temporary_files(
+                            temporary_dir_client=temporary_dir_client,
+                            full_result_path=None,
+                            benchmark_tool_global=benchmark_tool_global,
+                        )
+                        continue
+
                     github_actor = f"{tf_triggering_env}-{running_platform}"
                     dso = "redis-server"
                     profilers_artifacts_matrix = []
@@ -1047,6 +1317,9 @@ def process_self_contained_coordinator_stream(
                         continue
                     if "dbconfig" in benchmark_config:
                         if "preload_tool" in benchmark_config["dbconfig"]:
+                            # Get timeout buffer for preload
+                            buffer_timeout = getattr(args, 'timeout_buffer', getattr(args, 'container_timeout_buffer', 60))
+
                             res = data_prepopulation_step(
                                 benchmark_config,
                                 benchmark_tool_workdir,
@@ -1067,6 +1340,7 @@ def process_self_contained_coordinator_stream(
                                 password,
                                 oss_cluster_api_enabled,
                                 unix_socket,
+                                buffer_timeout,
                             )
                             if res is False:
                                 logging.warning(
@@ -1208,6 +1482,30 @@ def process_self_contained_coordinator_stream(
                                 unix_socket,
                                 None,  # username
                             )
+                        elif "vector-db-benchmark" in benchmark_tool:
+                            (
+                                _,
+                                benchmark_command_str,
+                                arbitrary_command,
+                                env_vars,
+                            ) = prepare_vector_db_benchmark_parameters(
+                                benchmark_config["clientconfig"],
+                                full_benchmark_path,
+                                port,
+                                host,
+                                password,
+                                local_benchmark_output_filename,
+                                oss_cluster_api_enabled,
+                                tls_enabled,
+                                tls_skip_verify,
+                                test_tls_cert,
+                                test_tls_key,
+                                test_tls_cacert,
+                                resp_version,
+                                override_memtier_test_time,
+                                unix_socket,
+                                None,  # username
+                            )
                         else:
                             # prepare the benchmark command for other tools
                             (
@@ -1241,6 +1539,40 @@ def process_self_contained_coordinator_stream(
                         profiler_call_graph_mode,
                     )
 
+                    # start remote profiling if enabled
+                    remote_profiler = None
+                    if args.enable_remote_profiling:
+                        try:
+                            remote_profiler = RemoteProfiler(
+                                args.remote_profile_host,
+                                args.remote_profile_port,
+                                args.remote_profile_output_dir,
+                                args.remote_profile_username,
+                                args.remote_profile_password
+                            )
+
+                            # Extract expected benchmark duration
+                            expected_duration = extract_expected_benchmark_duration(
+                                benchmark_command_str, override_memtier_test_time
+                            )
+
+                            # Start remote profiling
+                            profiling_started = remote_profiler.start_profiling(
+                                redis_conns[0] if redis_conns else None,
+                                test_name,
+                                expected_duration
+                            )
+
+                            if profiling_started:
+                                logging.info(f"Started remote profiling for test: {test_name}")
+                            else:
+                                logging.warning(f"Failed to start remote profiling for test: {test_name}")
+                                remote_profiler = None
+
+                        except Exception as e:
+                            logging.error(f"Error starting remote profiling: {e}")
+                            remote_profiler = None
+
                     # run the benchmark
                     benchmark_start_time = datetime.datetime.now()
 
@@ -1284,13 +1616,23 @@ def process_self_contained_coordinator_stream(
                                 + " "
                                 + benchmark_command_str
                             )
-                            logging.info(
-                                "Running memtier benchmark command {}".format(
-                                    benchmark_command_str
-                                )
+
+                            # Calculate timeout for local process
+                            buffer_timeout = getattr(args, 'timeout_buffer', getattr(args, 'container_timeout_buffer', 60))
+                            process_timeout = calculate_process_timeout(benchmark_command_str, buffer_timeout)
+
+                            # Run with timeout
+                            success, client_container_stdout, stderr = run_local_command_with_timeout(
+                                benchmark_command_str,
+                                process_timeout,
+                                "memtier benchmark"
                             )
-                            stream = os.popen(benchmark_command_str)
-                            client_container_stdout = stream.read()
+
+                            if not success:
+                                logging.error(f"Memtier benchmark failed: {stderr}")
+                                # Continue with the test but log the failure
+                                client_container_stdout = f"ERROR: {stderr}"
+
                             move_command = "mv {} {}".format(
                                 local_benchmark_output_filename, temporary_dir_client
                             )
@@ -1305,22 +1647,53 @@ def process_self_contained_coordinator_stream(
                             )
 
                             # Use explicit container management for single client
-                            container = docker_client.containers.run(
-                                image=client_container_image,
-                                volumes={
-                                    temporary_dir_client: {
-                                        "bind": client_mnt_point,
-                                        "mode": "rw",
-                                    },
+                            import os
+
+                            # Set working directory based on tool
+                            working_dir = benchmark_tool_workdir
+                            if "vector-db-benchmark" in benchmark_tool:
+                                working_dir = "/app"  # vector-db-benchmark needs to run from /app
+
+                            # Prepare volumes
+                            volumes = {
+                                temporary_dir_client: {
+                                    "bind": client_mnt_point,
+                                    "mode": "rw",
                                 },
-                                auto_remove=False,
-                                privileged=True,
-                                working_dir=benchmark_tool_workdir,
-                                command=benchmark_command_str,
-                                network_mode="host",
-                                detach=True,
-                                cpuset_cpus=client_cpuset_cpus,
-                            )
+                            }
+
+                            # For vector-db-benchmark, also mount the results directory
+                            if "vector-db-benchmark" in benchmark_tool:
+                                volumes[temporary_dir_client] = {
+                                    "bind": "/app/results",
+                                    "mode": "rw",
+                                }
+
+                            container_kwargs = {
+                                "image": client_container_image,
+                                "volumes": volumes,
+                                "auto_remove": False,
+                                "privileged": True,
+                                "working_dir": working_dir,
+                                "command": benchmark_command_str,
+                                "network_mode": "host",
+                                "detach": True,
+                                "cpuset_cpus": client_cpuset_cpus,
+                            }
+
+                            # Only add user for non-vector-db-benchmark tools to avoid permission issues
+                            if "vector-db-benchmark" not in benchmark_tool:
+                                container_kwargs["user"] = f"{os.getuid()}:{os.getgid()}"
+
+                            # Add environment variables for vector-db-benchmark
+                            if "vector-db-benchmark" in benchmark_tool:
+                                try:
+                                    container_kwargs["environment"] = env_vars
+                                except NameError:
+                                    # env_vars not defined, skip environment variables
+                                    pass
+
+                            container = docker_client.containers.run(**container_kwargs)
 
                             # Wait for container and get output
                             try:
@@ -1370,7 +1743,25 @@ def process_self_contained_coordinator_stream(
                         test_name,
                     )
 
+                    # wait for remote profiling completion
+                    if remote_profiler is not None:
+                        try:
+                            logging.info("Waiting for remote profiling to complete...")
+                            profiling_success = remote_profiler.wait_for_completion(timeout=60)
+                            if profiling_success:
+                                logging.info("Remote profiling completed successfully")
+                            else:
+                                logging.warning("Remote profiling did not complete successfully")
+                        except Exception as e:
+                            logging.error(f"Error waiting for remote profiling completion: {e}")
+
                     logging.info("Printing client tool stdout output")
+                    if client_container_stdout:
+                        print("=== Container Output ===")
+                        print(client_container_stdout)
+                        print("=== End Container Output ===")
+                    else:
+                        logging.warning("No container output captured")
 
                     used_memory_check(
                         test_name,
@@ -1428,13 +1819,30 @@ def process_self_contained_coordinator_stream(
                             full_result_path = "{}/{}".format(
                                 temporary_dir_client, local_benchmark_output_filename
                             )
+                        elif "vector-db-benchmark" in benchmark_tool:
+                            # For vector-db-benchmark, look for summary JSON file
+                            import os
+                            summary_files = [f for f in os.listdir(temporary_dir_client) if f.endswith("-summary.json")]
+                            if summary_files:
+                                full_result_path = os.path.join(temporary_dir_client, summary_files[0])
+                                logging.info(f"Found vector-db-benchmark summary file: {summary_files[0]}")
+                            else:
+                                logging.warning("No vector-db-benchmark summary JSON file found")
+                                # Create empty results dict to avoid crash
+                                results_dict = {}
+
                         logging.info(f"Reading results json from {full_result_path}")
 
-                        with open(
-                            full_result_path,
-                            "r",
-                        ) as json_file:
-                            results_dict = json.load(json_file)
+                        if "vector-db-benchmark" in benchmark_tool and not os.path.exists(full_result_path):
+                            # Handle case where vector-db-benchmark didn't produce results
+                            results_dict = {}
+                            logging.warning("Vector-db-benchmark did not produce results file")
+                        else:
+                            with open(
+                                full_result_path,
+                                "r",
+                            ) as json_file:
+                                results_dict = json.load(json_file)
                         print_results_table_stdout(
                             benchmark_config,
                             default_metrics,
@@ -1654,7 +2062,32 @@ def print_results_table_stdout(
     ]
     results_matrix = extract_results_table(metrics, results_dict)
 
-    results_matrix = [[x[0], f"{x[3]:.3f}"] for x in results_matrix]
+    # Use resolved metric name for precision_summary metrics, otherwise use original path
+    def get_display_name(x):
+        # For precision_summary metrics with wildcards, construct the resolved path
+        if (len(x) > 1 and
+            isinstance(x[0], str) and
+            "precision_summary" in x[0] and
+            "*" in x[0]):
+
+            # Look for the precision level in the cleaned metrics logs
+            # We need to find the corresponding cleaned metric to get the precision level
+            # For now, let's extract it from the time series logs that we know are working
+            # The pattern is: replace "*" with the actual precision level
+
+            # Since we know from logs that the precision level is available,
+            # let's reconstruct it from the metric context path (x[1]) if available
+            if len(x) > 1 and isinstance(x[1], str) and x[1].startswith("'") and x[1].endswith("'"):
+                precision_level = x[1]  # This should be something like "'1.0000'"
+                resolved_path = x[0].replace("*", precision_level)
+                return resolved_path
+
+        return x[0]  # Use original path
+
+    results_matrix = [
+        [get_display_name(x), f"{x[3]:.3f}"]
+        for x in results_matrix
+    ]
     writer = MarkdownTableWriter(
         table_name=table_name,
         headers=results_matrix_headers,
@@ -1668,14 +2101,19 @@ def print_redis_info_section(redis_conns):
     if redis_conns is not None and len(redis_conns) > 0:
         try:
             redis_info = redis_conns[0].info()
+            server_name = "redis"
+            if "server_name" in redis_info:
+                server_name = redis_info['server_name']
 
             print("\n# Redis Server Information")
             redis_info_data = [
-                ["Redis Version", redis_info.get("redis_version", "unknown")],
-                ["Redis Git SHA1", redis_info.get("redis_git_sha1", "unknown")],
-                ["Redis Git Dirty", str(redis_info.get("redis_git_dirty", "unknown"))],
-                ["Redis Build ID", redis_info.get("redis_build_id", "unknown")],
-                ["Redis Mode", redis_info.get("redis_mode", "unknown")],
+                [f"{server_name} version", redis_info.get(f"{server_name}_version", "unknown")],
+                ["redis version", redis_info.get("redis_version", "unknown")],
+                ["io_threads_active", redis_info.get("io_threads_active", "unknown")],
+                [f"{server_name} Git SHA1", redis_info.get("redis_git_sha1", "unknown")],
+                [f"{server_name} Git Dirty", str(redis_info.get("redis_git_dirty", "unknown"))],
+                [f"{server_name} Build ID", redis_info.get("redis_build_id", "unknown")],
+                [f"{server_name} Mode", redis_info.get("redis_mode", "unknown")],
                 ["OS", redis_info.get("os", "unknown")],
                 ["Arch Bits", str(redis_info.get("arch_bits", "unknown"))],
                 ["GCC Version", redis_info.get("gcc_version", "unknown")],
@@ -1703,6 +2141,78 @@ def print_redis_info_section(redis_conns):
             logging.warning(f"Failed to collect Redis server information: {e}")
 
 
+def get_supported_redis_commands(redis_conns):
+    """Get list of supported Redis commands from the server"""
+    if redis_conns is not None and len(redis_conns) > 0:
+        try:
+            # Execute COMMAND to get all supported commands
+            commands_info = redis_conns[0].execute_command("COMMAND")
+            logging.info(f"COMMAND response type: {type(commands_info)}, length: {len(commands_info) if hasattr(commands_info, '__len__') else 'N/A'}")
+
+            # Extract command names
+            supported_commands = set()
+
+            if isinstance(commands_info, dict):
+                # COMMAND response is a dict with command names as keys
+                for cmd_name in commands_info.keys():
+                    if isinstance(cmd_name, bytes):
+                        cmd_name = cmd_name.decode('utf-8')
+                    supported_commands.add(str(cmd_name).upper())
+            elif isinstance(commands_info, (list, tuple)):
+                # Fallback for list format (first element of each command info array)
+                for cmd_info in commands_info:
+                    if isinstance(cmd_info, (list, tuple)) and len(cmd_info) > 0:
+                        cmd_name = cmd_info[0]
+                        if isinstance(cmd_name, bytes):
+                            cmd_name = cmd_name.decode('utf-8')
+                        supported_commands.add(str(cmd_name).upper())
+
+            logging.info(f"Retrieved {len(supported_commands)} supported Redis commands")
+
+            # Log some sample commands for debugging
+            if supported_commands:
+                sample_commands = sorted(list(supported_commands))[:10]
+                logging.info(f"Sample commands: {sample_commands}")
+
+                # Check specifically for vector commands
+                vector_commands = [cmd for cmd in supported_commands if cmd.startswith('V')]
+                if vector_commands:
+                    logging.info(f"Vector commands found: {sorted(vector_commands)}")
+
+            return supported_commands
+        except Exception as e:
+            logging.warning(f"Failed to get supported Redis commands: {e}")
+            logging.warning("Proceeding without command validation")
+            return None
+    return None
+
+
+def check_test_command_support(benchmark_config, supported_commands):
+    """Check if all tested-commands in the benchmark config are supported"""
+    if supported_commands is None:
+        logging.warning("No supported commands list available, skipping command check")
+        return True, []
+
+    if "tested-commands" not in benchmark_config:
+        logging.info("No tested-commands specified in benchmark config")
+        return True, []
+
+    tested_commands = benchmark_config["tested-commands"]
+    unsupported_commands = []
+
+    for cmd in tested_commands:
+        cmd_upper = cmd.upper()
+        if cmd_upper not in supported_commands:
+            unsupported_commands.append(cmd)
+
+    if unsupported_commands:
+        logging.warning(f"Unsupported commands found: {unsupported_commands}")
+        return False, unsupported_commands
+    else:
+        logging.info(f"All tested commands are supported: {tested_commands}")
+        return True, []
+
+
 def prepare_overall_total_test_results(
     benchmark_config,
     default_metrics,
@@ -1721,8 +2231,25 @@ def prepare_overall_total_test_results(
         None,
     )
     current_test_results_matrix = extract_results_table(metrics, results_dict)
+
+    # Use the same display name logic as in the individual test results
+    def get_overall_display_name(x):
+        # For precision_summary metrics with wildcards, construct the resolved path
+        if (len(x) > 1 and
+            isinstance(x[0], str) and
+            "precision_summary" in x[0] and
+            "*" in x[0]):
+
+            # Reconstruct resolved path from metric context path (x[1]) if available
+            if len(x) > 1 and isinstance(x[1], str) and x[1].startswith("'") and x[1].endswith("'"):
+                precision_level = x[1]  # This should be something like "'1.0000'"
+                resolved_path = x[0].replace("*", precision_level)
+                return resolved_path
+
+        return x[0]  # Use original path
+
     current_test_results_matrix = [
-        [test_name, x[0], f"{x[3]:.3f}"] for x in current_test_results_matrix
+        [test_name, get_overall_display_name(x), f"{x[3]:.3f}"] for x in current_test_results_matrix
     ]
     overall_results_matrix.extend(current_test_results_matrix)
 
@@ -1747,6 +2274,7 @@ def data_prepopulation_step(
     password=None,
     oss_cluster_api_enabled=False,
     unix_socket="",
+    timeout_buffer=60,
 ):
     result = True
     # setup the benchmark
@@ -1808,13 +2336,21 @@ def data_prepopulation_step(
             preload_command_str = (
                 "taskset -c " + client_cpuset_cpus + " " + preload_command_str
             )
-            logging.info(
-                "Pre-loading using memtier benchmark command {}".format(
-                    preload_command_str
-                )
+
+            # Calculate timeout for preload process
+            process_timeout = calculate_process_timeout(preload_command_str, timeout_buffer)
+
+            # Run with timeout
+            success, client_container_stdout, stderr = run_local_command_with_timeout(
+                preload_command_str,
+                process_timeout,
+                "memtier preload"
             )
-            stream = os.popen(preload_command_str)
-            client_container_stdout = stream.read()
+
+            if not success:
+                logging.error(f"Memtier preload failed: {stderr}")
+                result = False
+                return result
 
             move_command = "mv {} {}".format(
                 local_benchmark_output_filename, temporary_dir