redis-benchmarks-specification 0.1.329__tar.gz → 0.1.331__tar.gz

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.329
+Version: 0.1.331
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.329"
+version = "0.1.331"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/redis_benchmarks_specification/__common__/runner.py

@@ -398,6 +398,7 @@ def exporter_datasink_common(
         running_platform,
         None,
         git_hash,
+        disable_target_tables=True,
     )
     if collect_memory_metrics:
         logging.info("Collecting memory metrics")

redis_benchmarks_specification-0.1.331/redis_benchmarks_specification/__common__/suppress_warnings.py

@@ -0,0 +1,20 @@
+"""
+Warning suppression module that should be imported first to suppress known warnings.
+"""
+
+import warnings
+
+# Suppress cryptography deprecation warnings from paramiko
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="paramiko")
+warnings.filterwarnings("ignore", message=".*TripleDES.*", category=DeprecationWarning)
+warnings.filterwarnings(
+    "ignore", message=".*cryptography.*", category=DeprecationWarning
+)
+
+# Also suppress the specific CryptographyDeprecationWarning if it exists
+try:
+    from cryptography.utils import CryptographyDeprecationWarning
+
+    warnings.filterwarnings("ignore", category=CryptographyDeprecationWarning)
+except ImportError:
+    pass

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/redis_benchmarks_specification/__common__/timeseries.py

@@ -1183,6 +1183,7 @@ def timeseries_test_sucess_flow(
     running_platform=None,
     timeseries_dict=None,
     git_hash=None,
+    disable_target_tables=False,
 ):
     testcase_metric_context_paths = []
     version_target_tables = None
@@ -1219,7 +1220,7 @@ def timeseries_test_sucess_flow(
             )
         )
         push_data_to_redistimeseries(rts, timeseries_dict)
-        if version_target_tables is not None:
+        if not disable_target_tables and version_target_tables is not None:
             logging.info(
                 "There are a total of {} distinct target tables by version".format(
                     len(version_target_tables.keys())
@@ -1239,7 +1240,12 @@ def timeseries_test_sucess_flow(
                 rts.hset(
                     version_target_table_keyname, None, None, version_target_table_dict
                 )
-        if branch_target_tables is not None:
+        elif disable_target_tables:
+            logging.info(
+                "Target tables disabled - skipping version target table creation"
+            )
+
+        if not disable_target_tables and branch_target_tables is not None:
             logging.info(
                 "There are a total of {} distinct target tables by branch".format(
                     len(branch_target_tables.keys())
@@ -1260,6 +1266,10 @@ def timeseries_test_sucess_flow(
                 rts.hset(
                     branch_target_table_keyname, None, None, branch_target_table_dict
                 )
+        elif disable_target_tables:
+            logging.info(
+                "Target tables disabled - skipping branch target table creation"
+            )
         if test_name is not None:
             if type(test_name) is str:
                 update_secondary_result_keys(

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/redis_benchmarks_specification/__runner__/runner.py

@@ -1,3 +1,6 @@
+# Import warning suppression first
+from redis_benchmarks_specification.__common__.suppress_warnings import *
+
 import datetime
 import json
 import logging
@@ -19,15 +22,11 @@ from docker.models.containers import Container
 from pytablewriter import CsvTableWriter, MarkdownTableWriter
 from redisbench_admin.profilers.profilers_local import (
     check_compatible_system_and_kernel_and_prepare_profile,
-    local_profilers_platform_checks,
-    profilers_start_if_required,
-    profilers_stop_if_required,
 )
 from redisbench_admin.run.common import (
     get_start_time_vars,
     merge_default_and_config_metrics,
     prepare_benchmark_parameters,
-    dbconfig_keyspacelen_check,
 )
 
 from redis_benchmarks_specification.__common__.runner import (
@@ -158,6 +157,83 @@ def parse_redis_uri(uri):
         return {}
 
 
+def validate_benchmark_metrics(
+    results_dict, test_name, benchmark_config=None, default_metrics=None
+):
+    """
+    Validate benchmark metrics to ensure they contain reasonable values.
+    Fails the test if critical metrics indicate something is wrong.
+
+    Args:
+        results_dict: Dictionary containing benchmark results
+        test_name: Name of the test being validated
+        benchmark_config: Benchmark configuration (unused, for compatibility)
+        default_metrics: Default metrics configuration (unused, for compatibility)
+
+    Returns:
+        tuple: (is_valid, error_message)
+    """
+    try:
+        # Define validation rules
+        throughput_patterns = [
+            "ops/sec",
+            "qps",
+            "totals.ops/sec",
+            "all_stats.totals.ops/sec",
+        ]
+
+        latency_patterns = ["latency", "p50", "p95", "p99", "p999", "usec", "msec"]
+
+        validation_errors = []
+
+        def check_nested_dict(data, path=""):
+            """Recursively check nested dictionary for metrics"""
+            if isinstance(data, dict):
+                for key, value in data.items():
+                    current_path = f"{path}.{key}" if path else key
+                    check_nested_dict(value, current_path)
+            elif isinstance(data, (int, float)):
+                metric_path_lower = path.lower()
+
+                # Check throughput metrics
+                for pattern in throughput_patterns:
+                    if pattern in metric_path_lower:
+                        if data <= 10:  # Below 10 QPS threshold
+                            validation_errors.append(
+                                f"Throughput metric '{path}' has invalid value: {data} "
+                                f"(below 10 QPS threshold)"
+                            )
+                        break
+
+                # Check latency metrics
+                for pattern in latency_patterns:
+                    if pattern in metric_path_lower:
+                        if data <= 0.0:  # Invalid latency
+                            validation_errors.append(
+                                f"Latency metric '{path}' has invalid value: {data} "
+                                f"(should be > 0.0)"
+                            )
+                        break
+
+        # Validate the results dictionary
+        check_nested_dict(results_dict)
+
+        if validation_errors:
+            error_msg = f"Test {test_name} failed metric validation:\n" + "\n".join(
+                validation_errors
+            )
+            logging.error(error_msg)
+            return False, error_msg
+
+        logging.info(f"Test {test_name} passed metric validation")
+        return True, None
+
+    except Exception as e:
+        logging.warning(f"Error during metric validation for test {test_name}: {e}")
+        # Don't fail the test if validation itself fails
+        return True, None
+
+
 def run_local_command_with_timeout(command_str, timeout_seconds, description="command"):
     """
     Run a local command with timeout support.
@@ -2753,6 +2829,23 @@ def process_self_contained_coordinator_stream(
                             "Using aggregated JSON results from multi-client execution"
                         )
                         results_dict = json.loads(client_container_stdout)
+
+                        # Validate benchmark metrics
+                        is_valid, validation_error = validate_benchmark_metrics(
+                            results_dict, test_name, benchmark_config, default_metrics
+                        )
+                        if not is_valid:
+                            logging.error(
+                                f"Test {test_name} failed metric validation: {validation_error}"
+                            )
+                            test_result = False
+                            delete_temporary_files(
+                                temporary_dir_client=temporary_dir_client,
+                                full_result_path=full_result_path,
+                                benchmark_tool_global=benchmark_tool_global,
+                            )
+                            continue
+
                         # Print results table for multi-client
                         print_results_table_stdout(
                             benchmark_config,
@@ -2815,6 +2908,23 @@ def process_self_contained_coordinator_stream(
                                 "r",
                             ) as json_file:
                                 results_dict = json.load(json_file)
+
+                        # Validate benchmark metrics
+                        is_valid, validation_error = validate_benchmark_metrics(
+                            results_dict, test_name, benchmark_config, default_metrics
+                        )
+                        if not is_valid:
+                            logging.error(
+                                f"Test {test_name} failed metric validation: {validation_error}"
+                            )
+                            test_result = False
+                            delete_temporary_files(
+                                temporary_dir_client=temporary_dir_client,
+                                full_result_path=full_result_path,
+                                benchmark_tool_global=benchmark_tool_global,
+                            )
+                            continue
+
                         print_results_table_stdout(
                             benchmark_config,
                             default_metrics,

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -28,6 +28,12 @@ def create_self_contained_coordinator_args(project_name):
         description=project_name,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=project_name,
+        help="Show version information and exit",
+    )
     parser.add_argument("--event_stream_host", type=str, default=GH_REDIS_SERVER_HOST)
     parser.add_argument("--event_stream_port", type=int, default=GH_REDIS_SERVER_PORT)
     parser.add_argument("--event_stream_pass", type=str, default=GH_REDIS_SERVER_AUTH)
@@ -171,4 +177,28 @@ def create_self_contained_coordinator_args(project_name):
         default="",
         help="Filter tests to run only with the specified topology (e.g. oss-standalone)",
     )
+    parser.add_argument(
+        "--exclusive-hardware",
+        default=False,
+        action="store_true",
+        help="Enable exclusive hardware mode. Kills all memtier processes and stops all docker containers before and after each test.",
+    )
+    parser.add_argument(
+        "--http-port",
+        type=int,
+        default=8080,
+        help="Port for HTTP server endpoints (/ping health check and /reset-queue POST endpoint).",
+    )
+    parser.add_argument(
+        "--http-auth-username",
+        type=str,
+        default=None,
+        help="Username for HTTP endpoint authentication. HTTP server is disabled if not provided.",
+    )
+    parser.add_argument(
+        "--http-auth-password",
+        type=str,
+        default=None,
+        help="Password for HTTP endpoint authentication. HTTP server is disabled if not provided.",
+    )
     return parser

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -615,6 +615,9 @@ def process_self_contained_coordinator_stream(
                                 metadata,
                                 build_variant_name,
                                 running_platform,
+                                None,
+                                None,
+                                disable_target_tables=True,
                             )
                             test_result = True
                             total_test_suite_runs = total_test_suite_runs + 1

{redis_benchmarks_specification-0.1.329 → redis_benchmarks_specification-0.1.331}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -1,9 +1,14 @@
+# Import warning suppression first
+from redis_benchmarks_specification.__common__.suppress_warnings import *
+
 import datetime
 import json
 import logging
 import pathlib
 import shutil
+import subprocess
 import tempfile
+import threading
 import traceback
 import re
 import docker
@@ -13,6 +18,9 @@ import os
 from pathlib import Path
 import sys
 import time
+import base64
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from urllib.parse import urlparse, parse_qs
 
 from docker.models.containers import Container
 from redis_benchmarks_specification.__self_contained_coordinator__.post_processing import (
@@ -59,6 +67,7 @@ from redis_benchmarks_specification.__compare__.compare import (
 from redis_benchmarks_specification.__runner__.runner import (
     print_results_table_stdout,
     prepare_memtier_benchmark_parameters,
+    validate_benchmark_metrics,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.args import (
     create_self_contained_coordinator_args,
@@ -108,6 +117,192 @@ from redis_benchmarks_specification.__self_contained_coordinator__.build_info im
     extract_build_info_from_streamdata,
 )
 
+# Global variables for HTTP server control
+_reset_queue_requested = False
+_exclusive_hardware = False
+_http_auth_username = None
+_http_auth_password = None
+
+
+class CoordinatorHTTPHandler(BaseHTTPRequestHandler):
+    """HTTP request handler for coordinator endpoints"""
+
+    def log_message(self, format, *args):
+        """Override to use our logging system"""
+        logging.info(f"HTTP {format % args}")
+
+    def _authenticate(self):
+        """Check if the request is authenticated"""
+        global _http_auth_username, _http_auth_password
+
+        # Check for Authorization header
+        auth_header = self.headers.get("Authorization")
+        if not auth_header:
+            return False
+
+        # Parse Basic auth
+        try:
+            if not auth_header.startswith("Basic "):
+                return False
+
+            # Decode base64 credentials
+            encoded_credentials = auth_header[6:]  # Remove 'Basic ' prefix
+            decoded_credentials = base64.b64decode(encoded_credentials).decode("utf-8")
+            username, password = decoded_credentials.split(":", 1)
+
+            # Verify credentials
+            return username == _http_auth_username and password == _http_auth_password
+
+        except Exception as e:
+            logging.warning(f"Authentication error: {e}")
+            return False
+
+    def _send_auth_required(self):
+        """Send 401 Unauthorized response"""
+        self.send_response(401)
+        self.send_header(
+            "WWW-Authenticate", 'Basic realm="Redis Benchmarks Coordinator"'
+        )
+        self.send_header("Content-type", "application/json")
+        self.end_headers()
+        response = {
+            "error": "Authentication required",
+            "message": "Please provide valid credentials using Basic authentication",
+        }
+        self.wfile.write(json.dumps(response).encode())
+
+    def do_GET(self):
+        """Handle GET requests"""
+        # Check authentication
+        if not self._authenticate():
+            self._send_auth_required()
+            return
+
+        parsed_path = urlparse(self.path)
+
+        if parsed_path.path == "/ping":
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            response = {
+                "status": "healthy",
+                "timestamp": datetime.datetime.utcnow().isoformat(),
+                "service": "redis-benchmarks-self-contained-coordinator",
+            }
+            self.wfile.write(json.dumps(response).encode())
+        else:
+            self.send_response(404)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Not found"}).encode())
+
+    def do_POST(self):
+        """Handle POST requests"""
+        # Check authentication
+        if not self._authenticate():
+            self._send_auth_required()
+            return
+
+        global _reset_queue_requested
+
+        parsed_path = urlparse(self.path)
+
+        if parsed_path.path == "/reset-queue":
+            try:
+                # Read request body
+                content_length = int(self.headers.get("Content-Length", 0))
+                if content_length > 0:
+                    post_data = self.rfile.read(content_length)
+                    try:
+                        request_data = json.loads(post_data.decode())
+                    except json.JSONDecodeError:
+                        request_data = {}
+                else:
+                    request_data = {}
+
+                # Set the reset flag
+                _reset_queue_requested = True
+                logging.info("Queue reset requested via HTTP endpoint")
+
+                self.send_response(200)
+                self.send_header("Content-type", "application/json")
+                self.end_headers()
+                response = {
+                    "status": "success",
+                    "message": "Queue reset requested",
+                    "timestamp": datetime.datetime.utcnow().isoformat(),
+                }
+                self.wfile.write(json.dumps(response).encode())
+
+            except Exception as e:
+                logging.error(f"Error handling reset-queue request: {e}")
+                self.send_response(500)
+                self.send_header("Content-type", "application/json")
+                self.end_headers()
+                self.wfile.write(json.dumps({"error": str(e)}).encode())
+        else:
+            self.send_response(404)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            self.wfile.write(json.dumps({"error": "Not found"}).encode())
+
+
+def start_http_server(port=8080):
+    """Start the HTTP server in a separate thread"""
+
+    def run_server():
+        try:
+            server = HTTPServer(("0.0.0.0", port), CoordinatorHTTPHandler)
+            logging.info(f"Starting HTTP server on port {port}")
+            logging.info(f"Available endpoints:")
+            logging.info(f"  GET  /ping - Health check")
+            logging.info(
+                f"  POST /reset-queue - Reset pending streams and skip running tests"
+            )
+            server.serve_forever()
+        except Exception as e:
+            logging.error(f"HTTP server error: {e}")
+
+    server_thread = threading.Thread(target=run_server, daemon=True)
+    server_thread.start()
+    return server_thread
+
+
+def cleanup_system_processes():
+    """Clean up memtier processes and docker containers for exclusive hardware mode"""
+    global _exclusive_hardware
+
+    if not _exclusive_hardware:
+        return
+
+    logging.info("Exclusive hardware mode: Cleaning up system processes")
+
+    try:
+        # Kill all memtier_benchmark processes
+        logging.info("Killing all memtier_benchmark processes")
+        subprocess.run(["pkill", "-f", "memtier_benchmark"], check=False)
+
+        # Stop all docker containers
+        logging.info("Stopping all docker containers")
+        docker_client = docker.from_env()
+        containers = docker_client.containers.list()
+        for container in containers:
+            try:
+                logging.info(
+                    f"Stopping container: {container.name} ({container.id[:12]})"
+                )
+                container.stop(timeout=10)
+                container.remove(force=True)
+            except Exception as e:
+                logging.warning(f"Error stopping container {container.id[:12]}: {e}")
+
+        # Wait a moment for cleanup to complete
+        time.sleep(2)
+        logging.info("System cleanup completed")
+
+    except Exception as e:
+        logging.error(f"Error during system cleanup: {e}")
+
 
 def print_directory_logs(directory_path, description=""):
     """Print all .log files in a directory for debugging purposes."""
@@ -157,6 +352,8 @@ from redis_benchmarks_specification.__self_contained_coordinator__.docker import
 
 
 def main():
+    global _exclusive_hardware, _http_auth_username, _http_auth_password
+
     _, _, project_version = populate_with_poetry_data()
     project_name = "redis-benchmarks-spec runner(self-contained)"
     parser = create_self_contained_coordinator_args(
@@ -164,6 +361,7 @@ def main():
     )
     args = parser.parse_args()
 
+    # Configure logging first, before any logging calls
     if args.logname is not None:
         print("Writting log to {}".format(args.logname))
         logging.basicConfig(
@@ -180,6 +378,23 @@ def main():
             level=LOG_LEVEL,
             datefmt=LOG_DATEFMT,
         )
+
+    # Set global exclusive hardware flag
+    _exclusive_hardware = args.exclusive_hardware
+    if _exclusive_hardware:
+        logging.info("Exclusive hardware mode enabled")
+
+    # Set HTTP authentication credentials and start server only if credentials are provided
+    _http_auth_username = args.http_auth_username
+    _http_auth_password = args.http_auth_password
+
+    if _http_auth_username and _http_auth_password:
+        logging.info(
+            "Starting HTTP server with authentication on port {}".format(args.http_port)
+        )
+        start_http_server(args.http_port)
+    else:
+        logging.info("HTTP server disabled - no authentication credentials provided")
     logging.info(get_version_string(project_name, project_version))
     topologies_folder = os.path.abspath(args.setups_folder + "/topologies")
     logging.info("Using topologies folder dir {}".format(topologies_folder))
@@ -750,7 +965,7 @@ def process_self_contained_coordinator_stream(
                     github_event_conn.expire(
                         stream_test_list_pending, REDIS_BINS_EXPIRE_SECS
                     )
-                    logging.info(
+                    logging.debug(
                         f"Added test named {test_name} to the pending test list in key {stream_test_list_pending}"
                     )
 
@@ -781,6 +996,22 @@ def process_self_contained_coordinator_stream(
                         )
 
                 for test_file in filtered_test_files:
+                    # Check if queue reset was requested
+                    global _reset_queue_requested
+                    if _reset_queue_requested:
+                        logging.info(
+                            "Queue reset requested. Skipping remaining tests and clearing queues."
+                        )
+                        # Clear all pending tests from the queue
+                        github_event_conn.delete(stream_test_list_pending)
+                        github_event_conn.delete(stream_test_list_running)
+                        logging.info("Cleared pending and running test queues")
+                        _reset_queue_requested = False
+                        break
+
+                    # Clean up system processes if in exclusive hardware mode
+                    cleanup_system_processes()
+
                     redis_containers = []
                     client_containers = []
                     with open(test_file, "r") as stream:
@@ -794,8 +1025,8 @@ def process_self_contained_coordinator_stream(
                         github_event_conn.expire(
                             stream_test_list_running, REDIS_BINS_EXPIRE_SECS
                         )
-                        logging.info(
-                            f"Added test named {test_name} to the pending test list in key {stream_test_list_running}"
+                        logging.debug(
+                            f"Added test named {test_name} to the running test list in key {stream_test_list_running}"
                         )
                         (
                             _,
@@ -1250,6 +1481,23 @@ def process_self_contained_coordinator_stream(
                                     results_dict = post_process_vector_db(
                                         temporary_dir_client
                                     )
+
+                                    # Validate benchmark metrics for vector-db-benchmark
+                                    is_valid, validation_error = (
+                                        validate_benchmark_metrics(
+                                            results_dict,
+                                            test_name,
+                                            benchmark_config,
+                                            default_metrics,
+                                        )
+                                    )
+                                    if not is_valid:
+                                        logging.error(
+                                            f"Test {test_name} failed metric validation: {validation_error}"
+                                        )
+                                        test_result = False
+                                        failed_tests += 1
+                                        continue
                                 else:
                                     post_process_benchmark_results(
                                         benchmark_tool,
@@ -1276,6 +1524,24 @@ def process_self_contained_coordinator_stream(
                                         "r",
                                     ) as json_file:
                                         results_dict = json.load(json_file)
+
+                                        # Validate benchmark metrics
+                                        is_valid, validation_error = (
+                                            validate_benchmark_metrics(
+                                                results_dict,
+                                                test_name,
+                                                benchmark_config,
+                                                default_metrics,
+                                            )
+                                        )
+                                        if not is_valid:
+                                            logging.error(
+                                                f"Test {test_name} failed metric validation: {validation_error}"
+                                            )
+                                            test_result = False
+                                            failed_tests += 1
+                                            continue
+
                                         print_results_table_stdout(
                                             benchmark_config,
                                             default_metrics,
@@ -1430,6 +1696,9 @@ def process_self_contained_coordinator_stream(
 
                             overall_result &= test_result
 
+                    # Clean up system processes after test completion if in exclusive hardware mode
+                    cleanup_system_processes()
+
                     github_event_conn.lrem(stream_test_list_running, 1, test_name)
                     github_event_conn.lpush(stream_test_list_completed, test_name)
                     github_event_conn.expire(
@@ -1592,7 +1861,7 @@ def process_self_contained_coordinator_stream(
                                     e.__str__()
                                 )
                             )
-                    logging.info(
+                    logging.debug(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"
                     )
         else: