redis-benchmarks-specification 0.1.273__py3-none-any.whl → 0.1.275__py3-none-any.whl

redis_benchmarks_specification/__common__/timeseries.py

@@ -133,6 +133,9 @@ def extract_results_table(
                 use_metric_context_path = False
                 if len(find_res) > 1:
                     use_metric_context_path = True
+                # Always use context path for precision_summary metrics to show actual precision levels
+                if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath:
+                    use_metric_context_path = True
                 for metric in find_res:
                     metric_name = str(metric.path)
                     metric_value = float(metric.value)
@@ -142,15 +145,34 @@ def extract_results_table(
                     if metric_jsonpath[0] == ".":
                         metric_jsonpath = metric_jsonpath[1:]
 
+                    # For precision_summary metrics, construct the full resolved path for display
+                    display_path = metric_jsonpath
+                    if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath and use_metric_context_path:
+                        # Replace the wildcard with the actual precision level
+                        display_path = metric_jsonpath.replace("*", metric_context_path)
+
                     # retro-compatible naming
                     if use_metric_context_path is False:
                         metric_name = metric_jsonpath
-
-                    metric_name = metric_name.replace("'", "")
-                    metric_name = metric_name.replace('"', "")
-                    metric_name = metric_name.replace("(", "")
-                    metric_name = metric_name.replace(")", "")
-                    metric_name = metric_name.replace(" ", "_")
+                    else:
+                        # For display purposes, use the resolved path for precision_summary
+                        if "precision_summary" in metric_jsonpath and "*" in metric_jsonpath:
+                            metric_name = display_path
+                        else:
+                            # Clean up the metric name for other cases
+                            metric_name = metric_name.replace("'", "")
+                            metric_name = metric_name.replace('"', "")
+                            metric_name = metric_name.replace("(", "")
+                            metric_name = metric_name.replace(")", "")
+                            metric_name = metric_name.replace(" ", "_")
+
+                    # Apply standard cleaning to all metric names
+                    if not ("precision_summary" in metric_jsonpath and "*" in metric_jsonpath and use_metric_context_path):
+                        metric_name = metric_name.replace("'", "")
+                        metric_name = metric_name.replace('"', "")
+                        metric_name = metric_name.replace("(", "")
+                        metric_name = metric_name.replace(")", "")
+                        metric_name = metric_name.replace(" ", "_")
 
                     results_matrix.append(
                         [

redis_benchmarks_specification/__runner__/args.py

@@ -225,4 +225,40 @@ def create_client_runner_args(project_name):
         default="",
         help="UNIX Domain socket name",
     )
+    parser.add_argument(
+        "--enable-remote-profiling",
+        default=False,
+        action="store_true",
+        help="Enable remote profiling of Redis processes via HTTP GET endpoint. Profiles are collected in folded format during benchmark execution.",
+    )
+    parser.add_argument(
+        "--remote-profile-host",
+        type=str,
+        default="localhost",
+        help="Host for remote profiling HTTP endpoint. Default is localhost.",
+    )
+    parser.add_argument(
+        "--remote-profile-port",
+        type=int,
+        default=8080,
+        help="Port for remote profiling HTTP endpoint. Default is 8080.",
+    )
+    parser.add_argument(
+        "--remote-profile-output-dir",
+        type=str,
+        default="profiles",
+        help="Directory to store remote profiling output files. Default is 'profiles/'.",
+    )
+    parser.add_argument(
+        "--remote-profile-username",
+        type=str,
+        default=None,
+        help="Username for HTTP basic authentication to remote profiling endpoint. Optional.",
+    )
+    parser.add_argument(
+        "--remote-profile-password",
+        type=str,
+        default=None,
+        help="Password for HTTP basic authentication to remote profiling endpoint. Optional.",
+    )
     return parser

redis_benchmarks_specification/__runner__/remote_profiling.py

@@ -0,0 +1,329 @@
+"""
+Remote profiling utilities for Redis benchmark runner.
+
+This module provides functionality to trigger remote profiling of Redis processes
+via HTTP GET endpoints during benchmark execution. Profiles are collected in
+folded format for performance analysis.
+"""
+
+import datetime
+import logging
+import os
+import threading
+import time
+from pathlib import Path
+from typing import Optional, Dict, Any
+import requests
+
+
+def extract_redis_pid(redis_conn) -> Optional[int]:
+    """
+    Extract Redis process ID from Redis INFO command.
+    
+    Args:
+        redis_conn: Redis connection object
+        
+    Returns:
+        Redis process ID as integer, or None if not found
+    """
+    try:
+        redis_info = redis_conn.info()
+        pid = redis_info.get("process_id")
+        if pid is not None:
+            logging.info(f"Extracted Redis PID: {pid}")
+            return int(pid)
+        else:
+            logging.warning("Redis process_id not found in INFO command")
+            return None
+    except Exception as e:
+        logging.error(f"Failed to extract Redis PID: {e}")
+        return None
+
+
+def extract_redis_metadata(redis_conn) -> Dict[str, Any]:
+    """
+    Extract Redis metadata for profile comments.
+    
+    Args:
+        redis_conn: Redis connection object
+        
+    Returns:
+        Dictionary containing Redis metadata
+    """
+    try:
+        redis_info = redis_conn.info()
+        metadata = {
+            "redis_version": redis_info.get("redis_version", "unknown"),
+            "redis_git_sha1": redis_info.get("redis_git_sha1", "unknown"),
+            "redis_git_dirty": redis_info.get("redis_git_dirty", "unknown"),
+            "redis_build_id": redis_info.get("redis_build_id", "unknown"),
+            "process_id": redis_info.get("process_id", "unknown"),
+            "tcp_port": redis_info.get("tcp_port", "unknown"),
+        }
+        
+        # Use build_id if git_sha1 is empty or 0
+        if metadata["redis_git_sha1"] in ("", 0, "0"):
+            metadata["redis_git_sha1"] = metadata["redis_build_id"]
+            
+        logging.info(f"Extracted Redis metadata: version={metadata['redis_version']}, sha={metadata['redis_git_sha1']}, pid={metadata['process_id']}")
+        return metadata
+    except Exception as e:
+        logging.error(f"Failed to extract Redis metadata: {e}")
+        return {
+            "redis_version": "unknown",
+            "redis_git_sha1": "unknown", 
+            "redis_git_dirty": "unknown",
+            "redis_build_id": "unknown",
+            "process_id": "unknown",
+            "tcp_port": "unknown",
+        }
+
+
+def calculate_profile_duration(benchmark_duration_seconds: int) -> int:
+    """
+    Calculate profiling duration based on benchmark duration.
+    
+    Args:
+        benchmark_duration_seconds: Expected benchmark duration in seconds
+        
+    Returns:
+        Profiling duration in seconds (minimum: benchmark duration, maximum: 30)
+    """
+    # Minimum duration is the benchmark duration, maximum is 30 seconds
+    duration = min(max(benchmark_duration_seconds, 10), 30)
+    logging.info(f"Calculated profile duration: {duration}s (benchmark: {benchmark_duration_seconds}s)")
+    return duration
+
+
+def trigger_remote_profile(
+    host: str,
+    port: int,
+    pid: int,
+    duration: int,
+    timeout: int = 60,
+    username: Optional[str] = None,
+    password: Optional[str] = None
+) -> Optional[str]:
+    """
+    Trigger remote profiling via HTTP GET request.
+
+    Args:
+        host: Remote host address
+        port: Remote port number
+        pid: Redis process ID
+        duration: Profiling duration in seconds
+        timeout: HTTP request timeout in seconds
+        username: Optional username for HTTP basic authentication
+        password: Optional password for HTTP basic authentication
+
+    Returns:
+        Profile content in folded format, or None if failed
+    """
+    url = f"http://{host}:{port}/debug/folded/profile"
+    params = {
+        "pid": pid,
+        "seconds": duration
+    }
+
+    # Prepare authentication if provided
+    auth = None
+    if username is not None and password is not None:
+        auth = (username, password)
+        logging.info(f"Using HTTP basic authentication with username: {username}")
+
+    try:
+        logging.info(f"Triggering remote profile: {url} with PID={pid}, duration={duration}s")
+        response = requests.get(url, params=params, timeout=timeout, auth=auth)
+        response.raise_for_status()
+
+        profile_content = response.text
+        logging.info(f"Successfully collected profile: {len(profile_content)} characters")
+        return profile_content
+
+    except requests.exceptions.Timeout:
+        logging.error(f"Remote profiling request timed out after {timeout}s")
+        return None
+    except requests.exceptions.ConnectionError:
+        logging.error(f"Failed to connect to remote profiling endpoint: {host}:{port}")
+        return None
+    except requests.exceptions.HTTPError as e:
+        logging.error(f"HTTP error during remote profiling: {e}")
+        if e.response.status_code == 401:
+            logging.error("Authentication failed - check username and password")
+        return None
+    except Exception as e:
+        logging.error(f"Unexpected error during remote profiling: {e}")
+        return None
+
+
+def save_profile_with_metadata(
+    profile_content: str,
+    benchmark_name: str,
+    output_dir: str,
+    redis_metadata: Dict[str, Any],
+    duration: int
+) -> Optional[str]:
+    """
+    Save profile content to file with metadata comments.
+    
+    Args:
+        profile_content: Profile data in folded format
+        benchmark_name: Name of the benchmark
+        output_dir: Output directory path
+        redis_metadata: Redis metadata dictionary
+        duration: Profiling duration in seconds
+        
+    Returns:
+        Path to saved file, or None if failed
+    """
+    try:
+        # Create output directory if it doesn't exist
+        Path(output_dir).mkdir(parents=True, exist_ok=True)
+        
+        # Generate filename
+        filename = f"{benchmark_name}.folded"
+        filepath = os.path.join(output_dir, filename)
+        
+        # Generate timestamp
+        timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        
+        # Create metadata comment
+        metadata_comment = (
+            f"# profile from redis sha = {redis_metadata['redis_git_sha1']} "
+            f"and pid {redis_metadata['process_id']} for duration of {duration}s. "
+            f"collection in date {timestamp}\n"
+            f"# redis_version: {redis_metadata['redis_version']}\n"
+            f"# redis_git_dirty: {redis_metadata['redis_git_dirty']}\n"
+            f"# tcp_port: {redis_metadata['tcp_port']}\n"
+        )
+        
+        # Write file with metadata and profile content
+        with open(filepath, 'w') as f:
+            f.write(metadata_comment)
+            f.write(profile_content)
+            
+        logging.info(f"Saved profile to: {filepath}")
+        return filepath
+        
+    except Exception as e:
+        logging.error(f"Failed to save profile file: {e}")
+        return None
+
+
+class RemoteProfiler:
+    """
+    Remote profiler class to handle threaded profiling execution.
+    """
+
+    def __init__(self, host: str, port: int, output_dir: str, username: Optional[str] = None, password: Optional[str] = None):
+        self.host = host
+        self.port = port
+        self.output_dir = output_dir
+        self.username = username
+        self.password = password
+        self.profile_thread = None
+        self.profile_result = None
+        self.profile_error = None
+        
+    def start_profiling(
+        self,
+        redis_conn,
+        benchmark_name: str,
+        benchmark_duration_seconds: int
+    ) -> bool:
+        """
+        Start profiling in a separate thread.
+        
+        Args:
+            redis_conn: Redis connection object
+            benchmark_name: Name of the benchmark
+            benchmark_duration_seconds: Expected benchmark duration
+            
+        Returns:
+            True if profiling thread started successfully, False otherwise
+        """
+        try:
+            # Extract Redis metadata and PID
+            redis_metadata = extract_redis_metadata(redis_conn)
+            pid = redis_metadata.get("process_id")
+            
+            if pid == "unknown" or pid is None:
+                logging.error("Cannot start remote profiling: Redis PID not available")
+                return False
+                
+            # Calculate profiling duration
+            duration = calculate_profile_duration(benchmark_duration_seconds)
+            
+            # Start profiling thread
+            self.profile_thread = threading.Thread(
+                target=self._profile_worker,
+                args=(pid, duration, benchmark_name, redis_metadata),
+                daemon=True
+            )
+            self.profile_thread.start()
+            
+            logging.info(f"Started remote profiling thread for benchmark: {benchmark_name}")
+            return True
+            
+        except Exception as e:
+            logging.error(f"Failed to start remote profiling: {e}")
+            return False
+    
+    def _profile_worker(self, pid: int, duration: int, benchmark_name: str, redis_metadata: Dict[str, Any]):
+        """
+        Worker function for profiling thread.
+        """
+        try:
+            # Trigger remote profiling
+            profile_content = trigger_remote_profile(
+                self.host, self.port, pid, duration,
+                username=self.username, password=self.password
+            )
+
+            if profile_content:
+                # Save profile with metadata
+                filepath = save_profile_with_metadata(
+                    profile_content, benchmark_name, self.output_dir, redis_metadata, duration
+                )
+                self.profile_result = filepath
+            else:
+                self.profile_error = "Failed to collect profile content"
+
+        except Exception as e:
+            self.profile_error = f"Profile worker error: {e}"
+            logging.error(self.profile_error)
+    
+    def wait_for_completion(self, timeout: int = 60) -> bool:
+        """
+        Wait for profiling thread to complete.
+        
+        Args:
+            timeout: Maximum time to wait in seconds
+            
+        Returns:
+            True if completed successfully, False if timed out or failed
+        """
+        if self.profile_thread is None:
+            return False
+            
+        try:
+            self.profile_thread.join(timeout=timeout)
+            
+            if self.profile_thread.is_alive():
+                logging.warning(f"Remote profiling thread did not complete within {timeout}s")
+                return False
+                
+            if self.profile_error:
+                logging.error(f"Remote profiling failed: {self.profile_error}")
+                return False
+                
+            if self.profile_result:
+                logging.info(f"Remote profiling completed successfully: {self.profile_result}")
+                return True
+            else:
+                logging.warning("Remote profiling completed but no result available")
+                return False
+                
+        except Exception as e:
+            logging.error(f"Error waiting for remote profiling completion: {e}")
+            return False

redis_benchmarks_specification/__runner__/runner.py

@@ -62,6 +62,7 @@ from redis_benchmarks_specification.__common__.spec import (
     extract_client_tools,
 )
 from redis_benchmarks_specification.__runner__.args import create_client_runner_args
+from redis_benchmarks_specification.__runner__.remote_profiling import RemoteProfiler
 
 
 def parse_size(size):
@@ -91,6 +92,31 @@ def parse_size(size):
     return int(number * units[unit])
 
 
+def extract_expected_benchmark_duration(benchmark_command_str, override_memtier_test_time):
+    """
+    Extract expected benchmark duration from command string or override.
+
+    Args:
+        benchmark_command_str: The benchmark command string
+        override_memtier_test_time: Override test time value
+
+    Returns:
+        Expected duration in seconds, or 30 as default
+    """
+    if override_memtier_test_time > 0:
+        return override_memtier_test_time
+
+    # Try to extract test-time from command string
+    if "test-time" in benchmark_command_str:
+        # Handle both --test-time (memtier) and -test-time (pubsub-sub-bench)
+        test_time_match = re.search(r"--?test-time[=\s]+(\d+)", benchmark_command_str)
+        if test_time_match:
+            return int(test_time_match.group(1))
+
+    # Default duration if not found
+    return 30
+
+
 def run_multiple_clients(
     benchmark_config,
     docker_client,
@@ -181,6 +207,30 @@ def run_multiple_clients(
                     unix_socket,
                     None,  # username
                 )
+            elif "vector-db-benchmark" in client_tool:
+                (
+                    _,
+                    benchmark_command_str,
+                    arbitrary_command,
+                    client_env_vars,
+                ) = prepare_vector_db_benchmark_parameters(
+                    client_config,
+                    client_tool,
+                    port,
+                    host,
+                    password,
+                    local_benchmark_output_filename,
+                    oss_cluster_api_enabled,
+                    tls_enabled,
+                    tls_skip_verify,
+                    test_tls_cert,
+                    test_tls_key,
+                    test_tls_cacert,
+                    resp_version,
+                    override_memtier_test_time,
+                    unix_socket,
+                    None,  # username
+                )
             else:
                 # Handle other benchmark tools
                 (
@@ -224,23 +274,51 @@ def run_multiple_clients(
             # Start container (detached)
             import os
 
-            container = docker_client.containers.run(
-                image=client_image,
-                volumes={
-                    temporary_dir_client: {
-                        "bind": client_mnt_point,
-                        "mode": "rw",
-                    },
+            # Set working directory based on tool
+            working_dir = benchmark_tool_workdir
+            if "vector-db-benchmark" in client_tool:
+                working_dir = "/app"  # vector-db-benchmark needs to run from /app
+
+            # Prepare container arguments
+            volumes = {
+                temporary_dir_client: {
+                    "bind": client_mnt_point,
+                    "mode": "rw",
                 },
-                auto_remove=False,
-                privileged=True,
-                working_dir=benchmark_tool_workdir,
-                command=benchmark_command_str,
-                network_mode="host",
-                detach=True,
-                cpuset_cpus=client_cpuset_cpus,
-                user=f"{os.getuid()}:{os.getgid()}",  # Run as current user to fix permissions
-            )
+            }
+
+            # For vector-db-benchmark, also mount the results directory
+            if "vector-db-benchmark" in client_tool:
+                volumes[temporary_dir_client] = {
+                    "bind": "/app/results",
+                    "mode": "rw",
+                }
+
+            container_kwargs = {
+                "image": client_image,
+                "volumes": volumes,
+                "auto_remove": False,
+                "privileged": True,
+                "working_dir": working_dir,
+                "command": benchmark_command_str,
+                "network_mode": "host",
+                "detach": True,
+                "cpuset_cpus": client_cpuset_cpus,
+            }
+
+            # Only add user for non-vector-db-benchmark tools to avoid permission issues
+            if "vector-db-benchmark" not in client_tool:
+                container_kwargs["user"] = f"{os.getuid()}:{os.getgid()}"
+
+            # Add environment variables for vector-db-benchmark
+            if "vector-db-benchmark" in client_tool:
+                try:
+                    container_kwargs["environment"] = client_env_vars
+                except NameError:
+                    # client_env_vars not defined, skip environment variables
+                    pass
+
+            container = docker_client.containers.run(**container_kwargs)
 
             containers.append(
                 {
@@ -334,6 +412,7 @@ def run_multiple_clients(
         aggregated_json = {}
         memtier_json = None
         pubsub_json = None
+        vector_json = None
 
         for result in successful_results:
             client_index = result["client_index"]
@@ -360,6 +439,19 @@ def run_multiple_clients(
                         logging.info(
                             f"Successfully read pubsub-sub-bench JSON output from client {client_index}"
                         )
+                    elif "vector-db-benchmark" in tool:
+                        # For vector-db-benchmark, look for summary JSON file
+                        summary_files = [f for f in os.listdir(temporary_dir_client) if f.endswith("-summary.json")]
+                        if summary_files:
+                            summary_filepath = os.path.join(temporary_dir_client, summary_files[0])
+                            try:
+                                with open(summary_filepath, 'r') as f:
+                                    vector_json = json.load(f)
+                                logging.info(f"Successfully read vector-db-benchmark JSON output from {summary_files[0]}")
+                            except Exception as e:
+                                logging.warning(f"Failed to read vector-db-benchmark JSON from {summary_files[0]}: {e}")
+                        else:
+                            logging.warning(f"No vector-db-benchmark summary JSON file found for client {client_index}")
 
                     logging.info(
                         f"Successfully read JSON output from client {client_index} ({tool})"
@@ -376,16 +468,32 @@ def run_multiple_clients(
                     f"JSON output file not found for client {client_index}: {json_filepath}"
                 )
 
-        # Merge JSON outputs from both tools
-        if memtier_json and pubsub_json:
+        # Merge JSON outputs from all tools
+        if memtier_json and pubsub_json and vector_json:
+            # Use memtier as base and add other metrics
+            aggregated_json = memtier_json.copy()
+            aggregated_json.update(pubsub_json)
+            aggregated_json.update(vector_json)
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using merged JSON results from memtier, pubsub-sub-bench, and vector-db-benchmark clients")
+        elif memtier_json and pubsub_json:
             # Use memtier as base and add pubsub metrics
             aggregated_json = memtier_json.copy()
-            # Add pubsub metrics to the aggregated result
             aggregated_json.update(pubsub_json)
             aggregated_stdout = json.dumps(aggregated_json, indent=2)
-            logging.info(
-                "Using merged JSON results from memtier and pubsub-sub-bench clients"
-            )
+            logging.info("Using merged JSON results from memtier and pubsub-sub-bench clients")
+        elif memtier_json and vector_json:
+            # Use memtier as base and add vector metrics
+            aggregated_json = memtier_json.copy()
+            aggregated_json.update(vector_json)
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using merged JSON results from memtier and vector-db-benchmark clients")
+        elif pubsub_json and vector_json:
+            # Use pubsub as base and add vector metrics
+            aggregated_json = pubsub_json.copy()
+            aggregated_json.update(vector_json)
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using merged JSON results from pubsub-sub-bench and vector-db-benchmark clients")
         elif memtier_json:
             # Only memtier available
             aggregated_json = memtier_json
@@ -396,12 +504,15 @@ def run_multiple_clients(
             aggregated_json = pubsub_json
             aggregated_stdout = json.dumps(aggregated_json, indent=2)
             logging.info("Using JSON results from pubsub-sub-bench client only")
+        elif vector_json:
+            # Only vector-db-benchmark available
+            aggregated_json = vector_json
+            aggregated_stdout = json.dumps(aggregated_json, indent=2)
+            logging.info("Using JSON results from vector-db-benchmark client only")
         else:
             # Fall back to concatenated stdout
             aggregated_stdout = "\n".join([r["stdout"] for r in successful_results])
-            logging.warning(
-                "No JSON results found, falling back to concatenated stdout"
-            )
+            logging.warning("No JSON results found, falling back to concatenated stdout")
 
     return aggregated_stdout, results
 
@@ -665,6 +776,71 @@ def prepare_memtier_benchmark_parameters(
     return None, benchmark_command_str, arbitrary_command
 
 
+def prepare_vector_db_benchmark_parameters(
+    clientconfig,
+    full_benchmark_path,
+    port,
+    server,
+    password,
+    local_benchmark_output_filename,
+    oss_cluster_api_enabled=False,
+    tls_enabled=False,
+    tls_skip_verify=False,
+    tls_cert=None,
+    tls_key=None,
+    tls_cacert=None,
+    resp_version=None,
+    override_test_time=0,
+    unix_socket="",
+    username=None,
+):
+    """
+    Prepare vector-db-benchmark command parameters
+    """
+    arbitrary_command = False
+
+    benchmark_command = [
+        "/app/run.py",
+        "--host",
+        f"{server}",
+    ]
+
+    # Add port as environment variable (vector-db-benchmark uses env vars)
+    env_vars = {}
+    if port is not None:
+        env_vars["REDIS_PORT"] = str(port)
+    if password is not None:
+        env_vars["REDIS_AUTH"] = password
+    if username is not None:
+        env_vars["REDIS_USER"] = username
+
+    # Add engines parameter
+    engines = clientconfig.get("engines", "vectorsets-fp32-default")
+    benchmark_command.extend(["--engines", engines])
+
+    # Add datasets parameter
+    datasets = clientconfig.get("datasets", "random-100")
+    benchmark_command.extend(["--datasets", datasets])
+
+    # Add other optional parameters
+    if "parallels" in clientconfig:
+        benchmark_command.extend(["--parallels", str(clientconfig["parallels"])])
+
+    if "queries" in clientconfig:
+        benchmark_command.extend(["--queries", str(clientconfig["queries"])])
+
+    if "timeout" in clientconfig:
+        benchmark_command.extend(["--timeout", str(clientconfig["timeout"])])
+
+    # Add custom arguments if specified
+    if "arguments" in clientconfig:
+        benchmark_command_str = " ".join(benchmark_command) + " " + clientconfig["arguments"]
+    else:
+        benchmark_command_str = " ".join(benchmark_command)
+
+    return benchmark_command, benchmark_command_str, arbitrary_command, env_vars
+
+
 def prepare_pubsub_sub_bench_parameters(
     clientconfig,
     full_benchmark_path,
@@ -899,6 +1075,23 @@ def process_self_contained_coordinator_stream(
                         redis_pid = conn.info()["process_id"]
                         redis_pids.append(redis_pid)
 
+                    # Check if all tested commands are supported by this Redis instance
+                    supported_commands = get_supported_redis_commands(redis_conns)
+                    commands_supported, unsupported_commands = check_test_command_support(
+                        benchmark_config, supported_commands
+                    )
+
+                    if not commands_supported:
+                        logging.warning(
+                            f"Skipping test {test_name} due to unsupported commands: {unsupported_commands}"
+                        )
+                        delete_temporary_files(
+                            temporary_dir_client=temporary_dir_client,
+                            full_result_path=None,
+                            benchmark_tool_global=benchmark_tool_global,
+                        )
+                        continue
+
                     github_actor = f"{tf_triggering_env}-{running_platform}"
                     dso = "redis-server"
                     profilers_artifacts_matrix = []
@@ -1208,6 +1401,30 @@ def process_self_contained_coordinator_stream(
                                 unix_socket,
                                 None,  # username
                             )
+                        elif "vector-db-benchmark" in benchmark_tool:
+                            (
+                                _,
+                                benchmark_command_str,
+                                arbitrary_command,
+                                env_vars,
+                            ) = prepare_vector_db_benchmark_parameters(
+                                benchmark_config["clientconfig"],
+                                full_benchmark_path,
+                                port,
+                                host,
+                                password,
+                                local_benchmark_output_filename,
+                                oss_cluster_api_enabled,
+                                tls_enabled,
+                                tls_skip_verify,
+                                test_tls_cert,
+                                test_tls_key,
+                                test_tls_cacert,
+                                resp_version,
+                                override_memtier_test_time,
+                                unix_socket,
+                                None,  # username
+                            )
                         else:
                             # prepare the benchmark command for other tools
                             (
@@ -1241,6 +1458,40 @@ def process_self_contained_coordinator_stream(
                         profiler_call_graph_mode,
                     )
 
+                    # start remote profiling if enabled
+                    remote_profiler = None
+                    if args.enable_remote_profiling:
+                        try:
+                            remote_profiler = RemoteProfiler(
+                                args.remote_profile_host,
+                                args.remote_profile_port,
+                                args.remote_profile_output_dir,
+                                args.remote_profile_username,
+                                args.remote_profile_password
+                            )
+
+                            # Extract expected benchmark duration
+                            expected_duration = extract_expected_benchmark_duration(
+                                benchmark_command_str, override_memtier_test_time
+                            )
+
+                            # Start remote profiling
+                            profiling_started = remote_profiler.start_profiling(
+                                redis_conns[0] if redis_conns else None,
+                                test_name,
+                                expected_duration
+                            )
+
+                            if profiling_started:
+                                logging.info(f"Started remote profiling for test: {test_name}")
+                            else:
+                                logging.warning(f"Failed to start remote profiling for test: {test_name}")
+                                remote_profiler = None
+
+                        except Exception as e:
+                            logging.error(f"Error starting remote profiling: {e}")
+                            remote_profiler = None
+
                     # run the benchmark
                     benchmark_start_time = datetime.datetime.now()
 
@@ -1305,22 +1556,53 @@ def process_self_contained_coordinator_stream(
                             )
 
                             # Use explicit container management for single client
-                            container = docker_client.containers.run(
-                                image=client_container_image,
-                                volumes={
-                                    temporary_dir_client: {
-                                        "bind": client_mnt_point,
-                                        "mode": "rw",
-                                    },
+                            import os
+
+                            # Set working directory based on tool
+                            working_dir = benchmark_tool_workdir
+                            if "vector-db-benchmark" in benchmark_tool:
+                                working_dir = "/app"  # vector-db-benchmark needs to run from /app
+
+                            # Prepare volumes
+                            volumes = {
+                                temporary_dir_client: {
+                                    "bind": client_mnt_point,
+                                    "mode": "rw",
                                 },
-                                auto_remove=False,
-                                privileged=True,
-                                working_dir=benchmark_tool_workdir,
-                                command=benchmark_command_str,
-                                network_mode="host",
-                                detach=True,
-                                cpuset_cpus=client_cpuset_cpus,
-                            )
+                            }
+
+                            # For vector-db-benchmark, also mount the results directory
+                            if "vector-db-benchmark" in benchmark_tool:
+                                volumes[temporary_dir_client] = {
+                                    "bind": "/app/results",
+                                    "mode": "rw",
+                                }
+
+                            container_kwargs = {
+                                "image": client_container_image,
+                                "volumes": volumes,
+                                "auto_remove": False,
+                                "privileged": True,
+                                "working_dir": working_dir,
+                                "command": benchmark_command_str,
+                                "network_mode": "host",
+                                "detach": True,
+                                "cpuset_cpus": client_cpuset_cpus,
+                            }
+
+                            # Only add user for non-vector-db-benchmark tools to avoid permission issues
+                            if "vector-db-benchmark" not in benchmark_tool:
+                                container_kwargs["user"] = f"{os.getuid()}:{os.getgid()}"
+
+                            # Add environment variables for vector-db-benchmark
+                            if "vector-db-benchmark" in benchmark_tool:
+                                try:
+                                    container_kwargs["environment"] = env_vars
+                                except NameError:
+                                    # env_vars not defined, skip environment variables
+                                    pass
+
+                            container = docker_client.containers.run(**container_kwargs)
 
                             # Wait for container and get output
                             try:
@@ -1370,7 +1652,25 @@ def process_self_contained_coordinator_stream(
                         test_name,
                     )
 
+                    # wait for remote profiling completion
+                    if remote_profiler is not None:
+                        try:
+                            logging.info("Waiting for remote profiling to complete...")
+                            profiling_success = remote_profiler.wait_for_completion(timeout=60)
+                            if profiling_success:
+                                logging.info("Remote profiling completed successfully")
+                            else:
+                                logging.warning("Remote profiling did not complete successfully")
+                        except Exception as e:
+                            logging.error(f"Error waiting for remote profiling completion: {e}")
+
                     logging.info("Printing client tool stdout output")
+                    if client_container_stdout:
+                        print("=== Container Output ===")
+                        print(client_container_stdout)
+                        print("=== End Container Output ===")
+                    else:
+                        logging.warning("No container output captured")
 
                     used_memory_check(
                         test_name,
@@ -1428,13 +1728,30 @@ def process_self_contained_coordinator_stream(
                             full_result_path = "{}/{}".format(
                                 temporary_dir_client, local_benchmark_output_filename
                             )
+                        elif "vector-db-benchmark" in benchmark_tool:
+                            # For vector-db-benchmark, look for summary JSON file
+                            import os
+                            summary_files = [f for f in os.listdir(temporary_dir_client) if f.endswith("-summary.json")]
+                            if summary_files:
+                                full_result_path = os.path.join(temporary_dir_client, summary_files[0])
+                                logging.info(f"Found vector-db-benchmark summary file: {summary_files[0]}")
+                            else:
+                                logging.warning("No vector-db-benchmark summary JSON file found")
+                                # Create empty results dict to avoid crash
+                                results_dict = {}
+
                         logging.info(f"Reading results json from {full_result_path}")
 
-                        with open(
-                            full_result_path,
-                            "r",
-                        ) as json_file:
-                            results_dict = json.load(json_file)
+                        if "vector-db-benchmark" in benchmark_tool and not os.path.exists(full_result_path):
+                            # Handle case where vector-db-benchmark didn't produce results
+                            results_dict = {}
+                            logging.warning("Vector-db-benchmark did not produce results file")
+                        else:
+                            with open(
+                                full_result_path,
+                                "r",
+                            ) as json_file:
+                                results_dict = json.load(json_file)
                         print_results_table_stdout(
                             benchmark_config,
                             default_metrics,
@@ -1661,7 +1978,32 @@ def print_results_table_stdout(
     ]
     results_matrix = extract_results_table(metrics, results_dict)
 
-    results_matrix = [[x[0], f"{x[3]:.3f}"] for x in results_matrix]
+    # Use resolved metric name for precision_summary metrics, otherwise use original path
+    def get_display_name(x):
+        # For precision_summary metrics with wildcards, construct the resolved path
+        if (len(x) > 1 and
+            isinstance(x[0], str) and
+            "precision_summary" in x[0] and
+            "*" in x[0]):
+
+            # Look for the precision level in the cleaned metrics logs
+            # We need to find the corresponding cleaned metric to get the precision level
+            # For now, let's extract it from the time series logs that we know are working
+            # The pattern is: replace "*" with the actual precision level
+
+            # Since we know from logs that the precision level is available,
+            # let's reconstruct it from the metric context path (x[1]) if available
+            if len(x) > 1 and isinstance(x[1], str) and x[1].startswith("'") and x[1].endswith("'"):
+                precision_level = x[1]  # This should be something like "'1.0000'"
+                resolved_path = x[0].replace("*", precision_level)
+                return resolved_path
+
+        return x[0]  # Use original path
+
+    results_matrix = [
+        [get_display_name(x), f"{x[3]:.3f}"]
+        for x in results_matrix
+    ]
     writer = MarkdownTableWriter(
         table_name=table_name,
         headers=results_matrix_headers,
@@ -1675,14 +2017,19 @@ def print_redis_info_section(redis_conns):
     if redis_conns is not None and len(redis_conns) > 0:
         try:
             redis_info = redis_conns[0].info()
+            server_name = "redis"
+            if "server_name" in redis_info:
+                server_name = redis_info['server_name']
 
             print("\n# Redis Server Information")
             redis_info_data = [
-                ["Redis Version", redis_info.get("redis_version", "unknown")],
-                ["Redis Git SHA1", redis_info.get("redis_git_sha1", "unknown")],
-                ["Redis Git Dirty", str(redis_info.get("redis_git_dirty", "unknown"))],
-                ["Redis Build ID", redis_info.get("redis_build_id", "unknown")],
-                ["Redis Mode", redis_info.get("redis_mode", "unknown")],
+                [f"{server_name} version", redis_info.get(f"{server_name}_version", "unknown")],
+                ["redis version", redis_info.get("redis_version", "unknown")],
+                ["io_threads_active", redis_info.get("io_threads_active", "unknown")],
+                [f"{server_name} Git SHA1", redis_info.get("redis_git_sha1", "unknown")],
+                [f"{server_name} Git Dirty", str(redis_info.get("redis_git_dirty", "unknown"))],
+                [f"{server_name} Build ID", redis_info.get("redis_build_id", "unknown")],
+                [f"{server_name} Mode", redis_info.get("redis_mode", "unknown")],
                 ["OS", redis_info.get("os", "unknown")],
                 ["Arch Bits", str(redis_info.get("arch_bits", "unknown"))],
                 ["GCC Version", redis_info.get("gcc_version", "unknown")],
@@ -1710,6 +2057,78 @@ def print_redis_info_section(redis_conns):
             logging.warning(f"Failed to collect Redis server information: {e}")
 
 
+def get_supported_redis_commands(redis_conns):
+    """Get list of supported Redis commands from the server"""
+    if redis_conns is not None and len(redis_conns) > 0:
+        try:
+            # Execute COMMAND to get all supported commands
+            commands_info = redis_conns[0].execute_command("COMMAND")
+            logging.info(f"COMMAND response type: {type(commands_info)}, length: {len(commands_info) if hasattr(commands_info, '__len__') else 'N/A'}")
+
+            # Extract command names
+            supported_commands = set()
+
+            if isinstance(commands_info, dict):
+                # COMMAND response is a dict with command names as keys
+                for cmd_name in commands_info.keys():
+                    if isinstance(cmd_name, bytes):
+                        cmd_name = cmd_name.decode('utf-8')
+                    supported_commands.add(str(cmd_name).upper())
+            elif isinstance(commands_info, (list, tuple)):
+                # Fallback for list format (first element of each command info array)
+                for cmd_info in commands_info:
+                    if isinstance(cmd_info, (list, tuple)) and len(cmd_info) > 0:
+                        cmd_name = cmd_info[0]
+                        if isinstance(cmd_name, bytes):
+                            cmd_name = cmd_name.decode('utf-8')
+                        supported_commands.add(str(cmd_name).upper())
+
+            logging.info(f"Retrieved {len(supported_commands)} supported Redis commands")
+
+            # Log some sample commands for debugging
+            if supported_commands:
+                sample_commands = sorted(list(supported_commands))[:10]
+                logging.info(f"Sample commands: {sample_commands}")
+
+                # Check specifically for vector commands
+                vector_commands = [cmd for cmd in supported_commands if cmd.startswith('V')]
+                if vector_commands:
+                    logging.info(f"Vector commands found: {sorted(vector_commands)}")
+
+            return supported_commands
+        except Exception as e:
+            logging.warning(f"Failed to get supported Redis commands: {e}")
+            logging.warning("Proceeding without command validation")
+            return None
+    return None
+
+
+def check_test_command_support(benchmark_config, supported_commands):
+    """Check if all tested-commands in the benchmark config are supported"""
+    if supported_commands is None:
+        logging.warning("No supported commands list available, skipping command check")
+        return True, []
+
+    if "tested-commands" not in benchmark_config:
+        logging.info("No tested-commands specified in benchmark config")
+        return True, []
+
+    tested_commands = benchmark_config["tested-commands"]
+    unsupported_commands = []
+
+    for cmd in tested_commands:
+        cmd_upper = cmd.upper()
+        if cmd_upper not in supported_commands:
+            unsupported_commands.append(cmd)
+
+    if unsupported_commands:
+        logging.warning(f"Unsupported commands found: {unsupported_commands}")
+        return False, unsupported_commands
+    else:
+        logging.info(f"All tested commands are supported: {tested_commands}")
+        return True, []
+
+
 def prepare_overall_total_test_results(
     benchmark_config,
     default_metrics,
@@ -1728,8 +2147,25 @@ def prepare_overall_total_test_results(
         None,
     )
     current_test_results_matrix = extract_results_table(metrics, results_dict)
+
+    # Use the same display name logic as in the individual test results
+    def get_overall_display_name(x):
+        # For precision_summary metrics with wildcards, construct the resolved path
+        if (len(x) > 1 and
+            isinstance(x[0], str) and
+            "precision_summary" in x[0] and
+            "*" in x[0]):
+
+            # Reconstruct resolved path from metric context path (x[1]) if available
+            if len(x) > 1 and isinstance(x[1], str) and x[1].startswith("'") and x[1].endswith("'"):
+                precision_level = x[1]  # This should be something like "'1.0000'"
+                resolved_path = x[0].replace("*", precision_level)
+                return resolved_path
+
+        return x[0]  # Use original path
+
     current_test_results_matrix = [
-        [test_name, x[0], f"{x[3]:.3f}"] for x in current_test_results_matrix
+        [test_name, get_overall_display_name(x), f"{x[3]:.3f}"] for x in current_test_results_matrix
     ]
     overall_results_matrix.extend(current_test_results_matrix)
 

redis_benchmarks_specification/test-suites/defaults.yml

@@ -27,3 +27,6 @@ exporter:
     - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
     - $."ALL STATS".Totals."Percentile Latencies"."p99.00"
     - $."MessageRate"
+    - $."precision_summary".*.qps
+    - $."precision_summary".*.p50
+    - $."precision_summary".*.p95

{redis_benchmarks_specification-0.1.273.dist-info → redis_benchmarks_specification-0.1.275.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.273
+Version: 0.1.275
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.273.dist-info → redis_benchmarks_specification-0.1.275.dist-info}/RECORD

@@ -17,14 +17,15 @@ redis_benchmarks_specification/__common__/github.py,sha256=9TZtnISsSgXTSAN_VQejo
 redis_benchmarks_specification/__common__/package.py,sha256=4uVt1BAZ999LV2rZkq--Tk6otAVIf9YR3g3KGeUpiW4,834
 redis_benchmarks_specification/__common__/runner.py,sha256=2IpMl0IEHi2IZvfLc4_h0e-E3ZfnlB8EkCA_SE8VDCY,7033
 redis_benchmarks_specification/__common__/spec.py,sha256=D_SN48wg6NMthW_-OS1H5bydSDiuZpfd4WPPj7Vfwmc,5760
-redis_benchmarks_specification/__common__/timeseries.py,sha256=mwKlHP_sV5sK5mcAoHdxlzBqmW-HKWgUwSWjsEn3Tyk,51032
+redis_benchmarks_specification/__common__/timeseries.py,sha256=Jd8kGrLiuIs_1508F0MfiM4qlN_6gZWazG8arHxz9xA,52643
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
 redis_benchmarks_specification/__compare__/args.py,sha256=FlKD1wutBoKxeahpXw1gY2H_1FOPH5y-o5QsIPfFsT0,6802
 redis_benchmarks_specification/__compare__/compare.py,sha256=O6ZuB6Ln5xkTX5jRaizpj1PTPhmoETcf-_PY-A_CGr8,57179
 redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gqDKgzvNJnkGNKY,491
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
-redis_benchmarks_specification/__runner__/args.py,sha256=lrLc890bzN7_r1MSlVQpXVlqoU2HpcgJsHa-_5S6DlU,7741
-redis_benchmarks_specification/__runner__/runner.py,sha256=tUVhm5ZX-5DxCTkDrkkb8xb7bc9bPWg8kR_j8sStHIU,74011
+redis_benchmarks_specification/__runner__/args.py,sha256=HAqOfkjSU4Y6odJRvhm0q8G9v46_g4uKTj0rDTaCgQg,8991
+redis_benchmarks_specification/__runner__/remote_profiling.py,sha256=338UHq6gDH4dy6q8nj2ewipaHVj5fPmsNiHbFfaELfk,11378
+redis_benchmarks_specification/__runner__/runner.py,sha256=x3rIb5zJd0BK6hsWHgTCT6JcdFMAksZMRlWuUqdaNWg,94332
 redis_benchmarks_specification/__self_contained_coordinator__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__self_contained_coordinator__/args.py,sha256=uxBjdQ78klvsVi6lOfGYQVaWIxc8OI-DwYKY16SgvCY,5952
 redis_benchmarks_specification/__self_contained_coordinator__/artifacts.py,sha256=OVHqJzDgeSSRfUSiKp1ZTAVv14PvSbk-5yJsAAoUfpw,936
@@ -50,7 +51,7 @@ redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-def
 redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml,sha256=I6qEO7MZKduVx6xbBrRniE1i6NK9R8-uQXdQJT9o5G4,511
 redis_benchmarks_specification/setups/platforms/aws-ec2-1node-c5.4xlarge.yml,sha256=l7HsjccpebwZXeutnt3SHSETw4iiRwQ9dCDXLOySSRQ,622
 redis_benchmarks_specification/setups/topologies/topologies.yml,sha256=N2UOKA8tG_pLpaSFtn7WdUmDNYwxRyTv9Ln_PCOPTco,3261
-redis_benchmarks_specification/test-suites/defaults.yml,sha256=nlL_HBhiCt9qenj0H8nQi-vSMP6e7HSn_xF-RYqYugs,1290
+redis_benchmarks_specification/test-suites/defaults.yml,sha256=EJHv9INdjoNVMOgHY8qo4IVCHfvXVz5sv7Vxtr3DAIE,1392
 redis_benchmarks_specification/test-suites/generate.py,sha256=1QJBuWiouJ5OLil_r4OMG_UtZkmA8TLcyPlQAUuxCUw,4175
 redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml,sha256=BR5hCMZtC_rrbWtGuGs96yZXIxeJobB59MY1hqR0m0E,2009
 redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-load-hash-20-fields-with-1B-values-pipeline-30.yml,sha256=46e6GJWv9pDzd4YZXM62HWuhAX0kWtq5WH4F7d2Iqb0,1758
@@ -274,8 +275,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publi
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml,sha256=rJuWWXubUeRKQ2GSfHlbPMLeOyM9Eu_MzvN2vgKcAhA,672
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=uhaSP6YUVmPvZU-qMtPPGdvNEUgUBqOfveUbeJ9WsbI,972
-redis_benchmarks_specification-0.1.273.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.273.dist-info/METADATA,sha256=dd5weai6mqaPSfhQh7oonXbrOCNC-54FK7-qgeg4mDM,22726
-redis_benchmarks_specification-0.1.273.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.273.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.273.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.275.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.275.dist-info/METADATA,sha256=2VD-N9KbazAqJWh-P0qpG0QDliIZWFsmYogwkrk_TL8,22726
+redis_benchmarks_specification-0.1.275.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.275.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.275.dist-info/RECORD,,