redis-benchmarks-specification 0.1.328__tar.gz → 0.1.330__tar.gz

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.328
+Version: 0.1.330
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.328"
+version = "0.1.330"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/redis_benchmarks_specification/__common__/runner.py

@@ -291,13 +291,7 @@ def export_redis_metrics(
             metric_name,
             metric_value,
         ) in overall_end_time_metrics.items():
-            tsname_metric = "{}/{}/{}/benchmark_end/{}/{}".format(
-                sprefix,
-                test_name,
-                by_variant,
-                setup_name,
-                metric_name,
-            )
+            tsname_metric = f"{sprefix}/{test_name}/{by_variant}/benchmark_end/{running_platform}/{setup_name}/{metric_name}"
 
             logging.debug(
                 "Adding a redis server side metric collected at the end of benchmark."
@@ -404,6 +398,7 @@ def exporter_datasink_common(
         running_platform,
         None,
         git_hash,
+        disable_target_tables=True,
     )
     if collect_memory_metrics:
         logging.info("Collecting memory metrics")

redis_benchmarks_specification-0.1.330/redis_benchmarks_specification/__common__/suppress_warnings.py

@@ -0,0 +1,20 @@
+"""
+Warning suppression module that should be imported first to suppress known warnings.
+"""
+
+import warnings
+
+# Suppress cryptography deprecation warnings from paramiko
+warnings.filterwarnings("ignore", category=DeprecationWarning, module="paramiko")
+warnings.filterwarnings("ignore", message=".*TripleDES.*", category=DeprecationWarning)
+warnings.filterwarnings(
+    "ignore", message=".*cryptography.*", category=DeprecationWarning
+)
+
+# Also suppress the specific CryptographyDeprecationWarning if it exists
+try:
+    from cryptography.utils import CryptographyDeprecationWarning
+
+    warnings.filterwarnings("ignore", category=CryptographyDeprecationWarning)
+except ImportError:
+    pass

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/redis_benchmarks_specification/__common__/timeseries.py

@@ -838,34 +838,47 @@ def common_exporter_logic(
         and artifact_version != ""
         and artifact_version != "N/A"
     ):
-        # extract per-version datapoints
-        total_hs_ts = len(per_hash_time_series_dict.keys())
-        logging.info(
-            f"Extending the by.hash {git_hash} timeseries ({total_hs_ts}) with version info {artifact_version}"
-        )
-        for hash_timeserie in per_hash_time_series_dict.values():
-            hash_timeserie["labels"]["version"] = artifact_version
-        (
-            _,
-            per_version_time_series_dict,
-            version_target_tables,
-        ) = extract_perversion_timeseries_from_results(
-            used_ts,
-            metrics,
-            results_dict,
-            artifact_version,
-            tf_github_org,
-            tf_github_repo,
-            deployment_name,
-            deployment_type,
-            test_name,
-            tf_triggering_env,
-            metadata_tags,
-            build_variant_name,
-            running_platform,
-            testcase_metric_context_paths,
-        )
-        total_break_by_added += 1
+        # Check if version 255.255.255 should only be pushed for unstable branch
+        should_push_version = True
+        if artifact_version == "255.255.255":
+            if tf_github_branch != "unstable":
+                logging.info(
+                    f"Skipping version 255.255.255 data push for branch '{tf_github_branch}' "
+                    f"(only pushing for 'unstable' branch)"
+                )
+                should_push_version = False
+            else:
+                logging.info(f"Pushing version 255.255.255 data for unstable branch")
+
+        if should_push_version:
+            # extract per-version datapoints
+            total_hs_ts = len(per_hash_time_series_dict.keys())
+            logging.info(
+                f"Extending the by.hash {git_hash} timeseries ({total_hs_ts}) with version info {artifact_version}"
+            )
+            for hash_timeserie in per_hash_time_series_dict.values():
+                hash_timeserie["labels"]["version"] = artifact_version
+            (
+                _,
+                per_version_time_series_dict,
+                version_target_tables,
+            ) = extract_perversion_timeseries_from_results(
+                used_ts,
+                metrics,
+                results_dict,
+                artifact_version,
+                tf_github_org,
+                tf_github_repo,
+                deployment_name,
+                deployment_type,
+                test_name,
+                tf_triggering_env,
+                metadata_tags,
+                build_variant_name,
+                running_platform,
+                testcase_metric_context_paths,
+            )
+            total_break_by_added += 1
     else:
         logging.warning(
             "there was no git VERSION information to push data brokedown by VERSION"
@@ -1118,6 +1131,7 @@ def add_standardized_metric_byversion(
             tf_triggering_env,
             metadata_tags,
             build_variant_name,
+            running_platform,
         )
         labels["version"] = artifact_version
         labels["deployment_name+version"] = "{} {}".format(
@@ -1169,6 +1183,7 @@ def timeseries_test_sucess_flow(
     running_platform=None,
     timeseries_dict=None,
     git_hash=None,
+    disable_target_tables=False,
 ):
     testcase_metric_context_paths = []
     version_target_tables = None
@@ -1205,7 +1220,7 @@ def timeseries_test_sucess_flow(
             )
         )
         push_data_to_redistimeseries(rts, timeseries_dict)
-        if version_target_tables is not None:
+        if not disable_target_tables and version_target_tables is not None:
             logging.info(
                 "There are a total of {} distinct target tables by version".format(
                     len(version_target_tables.keys())
@@ -1225,7 +1240,12 @@ def timeseries_test_sucess_flow(
                 rts.hset(
                     version_target_table_keyname, None, None, version_target_table_dict
                 )
-        if branch_target_tables is not None:
+        elif disable_target_tables:
+            logging.info(
+                "Target tables disabled - skipping version target table creation"
+            )
+
+        if not disable_target_tables and branch_target_tables is not None:
             logging.info(
                 "There are a total of {} distinct target tables by branch".format(
                     len(branch_target_tables.keys())
@@ -1246,6 +1266,10 @@ def timeseries_test_sucess_flow(
                 rts.hset(
                     branch_target_table_keyname, None, None, branch_target_table_dict
                 )
+        elif disable_target_tables:
+            logging.info(
+                "Target tables disabled - skipping branch target table creation"
+            )
         if test_name is not None:
             if type(test_name) is str:
                 update_secondary_result_keys(

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/redis_benchmarks_specification/__runner__/runner.py

@@ -1,3 +1,6 @@
+# Import warning suppression first
+from redis_benchmarks_specification.__common__.suppress_warnings import *
+
 import datetime
 import json
 import logging
@@ -19,15 +22,11 @@ from docker.models.containers import Container
 from pytablewriter import CsvTableWriter, MarkdownTableWriter
 from redisbench_admin.profilers.profilers_local import (
     check_compatible_system_and_kernel_and_prepare_profile,
-    local_profilers_platform_checks,
-    profilers_start_if_required,
-    profilers_stop_if_required,
 )
 from redisbench_admin.run.common import (
     get_start_time_vars,
     merge_default_and_config_metrics,
     prepare_benchmark_parameters,
-    dbconfig_keyspacelen_check,
 )
 
 from redis_benchmarks_specification.__common__.runner import (
@@ -158,6 +157,83 @@ def parse_redis_uri(uri):
         return {}
 
 
+def validate_benchmark_metrics(
+    results_dict, test_name, benchmark_config=None, default_metrics=None
+):
+    """
+    Validate benchmark metrics to ensure they contain reasonable values.
+    Fails the test if critical metrics indicate something is wrong.
+
+    Args:
+        results_dict: Dictionary containing benchmark results
+        test_name: Name of the test being validated
+        benchmark_config: Benchmark configuration (unused, for compatibility)
+        default_metrics: Default metrics configuration (unused, for compatibility)
+
+    Returns:
+        tuple: (is_valid, error_message)
+    """
+    try:
+        # Define validation rules
+        throughput_patterns = [
+            "ops/sec",
+            "qps",
+            "totals.ops/sec",
+            "all_stats.totals.ops/sec",
+        ]
+
+        latency_patterns = ["latency", "p50", "p95", "p99", "p999", "usec", "msec"]
+
+        validation_errors = []
+
+        def check_nested_dict(data, path=""):
+            """Recursively check nested dictionary for metrics"""
+            if isinstance(data, dict):
+                for key, value in data.items():
+                    current_path = f"{path}.{key}" if path else key
+                    check_nested_dict(value, current_path)
+            elif isinstance(data, (int, float)):
+                metric_path_lower = path.lower()
+
+                # Check throughput metrics
+                for pattern in throughput_patterns:
+                    if pattern in metric_path_lower:
+                        if data <= 10:  # Below 10 QPS threshold
+                            validation_errors.append(
+                                f"Throughput metric '{path}' has invalid value: {data} "
+                                f"(below 10 QPS threshold)"
+                            )
+                        break
+
+                # Check latency metrics
+                for pattern in latency_patterns:
+                    if pattern in metric_path_lower:
+                        if data <= 0.0:  # Invalid latency
+                            validation_errors.append(
+                                f"Latency metric '{path}' has invalid value: {data} "
+                                f"(should be > 0.0)"
+                            )
+                        break
+
+        # Validate the results dictionary
+        check_nested_dict(results_dict)
+
+        if validation_errors:
+            error_msg = f"Test {test_name} failed metric validation:\n" + "\n".join(
+                validation_errors
+            )
+            logging.error(error_msg)
+            return False, error_msg
+
+        logging.info(f"Test {test_name} passed metric validation")
+        return True, None
+
+    except Exception as e:
+        logging.warning(f"Error during metric validation for test {test_name}: {e}")
+        # Don't fail the test if validation itself fails
+        return True, None
+
+
 def run_local_command_with_timeout(command_str, timeout_seconds, description="command"):
     """
     Run a local command with timeout support.
@@ -2753,6 +2829,23 @@ def process_self_contained_coordinator_stream(
                             "Using aggregated JSON results from multi-client execution"
                         )
                         results_dict = json.loads(client_container_stdout)
+
+                        # Validate benchmark metrics
+                        is_valid, validation_error = validate_benchmark_metrics(
+                            results_dict, test_name, benchmark_config, default_metrics
+                        )
+                        if not is_valid:
+                            logging.error(
+                                f"Test {test_name} failed metric validation: {validation_error}"
+                            )
+                            test_result = False
+                            delete_temporary_files(
+                                temporary_dir_client=temporary_dir_client,
+                                full_result_path=full_result_path,
+                                benchmark_tool_global=benchmark_tool_global,
+                            )
+                            continue
+
                         # Print results table for multi-client
                         print_results_table_stdout(
                             benchmark_config,
@@ -2815,6 +2908,23 @@ def process_self_contained_coordinator_stream(
                                 "r",
                             ) as json_file:
                                 results_dict = json.load(json_file)
+
+                        # Validate benchmark metrics
+                        is_valid, validation_error = validate_benchmark_metrics(
+                            results_dict, test_name, benchmark_config, default_metrics
+                        )
+                        if not is_valid:
+                            logging.error(
+                                f"Test {test_name} failed metric validation: {validation_error}"
+                            )
+                            test_result = False
+                            delete_temporary_files(
+                                temporary_dir_client=temporary_dir_client,
+                                full_result_path=full_result_path,
+                                benchmark_tool_global=benchmark_tool_global,
+                            )
+                            continue
+
                         print_results_table_stdout(
                             benchmark_config,
                             default_metrics,

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/redis_benchmarks_specification/__self_contained_coordinator__/args.py

@@ -28,6 +28,12 @@ def create_self_contained_coordinator_args(project_name):
         description=project_name,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=project_name,
+        help="Show version information and exit",
+    )
     parser.add_argument("--event_stream_host", type=str, default=GH_REDIS_SERVER_HOST)
     parser.add_argument("--event_stream_port", type=int, default=GH_REDIS_SERVER_PORT)
     parser.add_argument("--event_stream_pass", type=str, default=GH_REDIS_SERVER_AUTH)
@@ -171,4 +177,28 @@ def create_self_contained_coordinator_args(project_name):
         default="",
         help="Filter tests to run only with the specified topology (e.g. oss-standalone)",
     )
+    parser.add_argument(
+        "--exclusive-hardware",
+        default=False,
+        action="store_true",
+        help="Enable exclusive hardware mode. Kills all memtier processes and stops all docker containers before and after each test.",
+    )
+    parser.add_argument(
+        "--http-port",
+        type=int,
+        default=8080,
+        help="Port for HTTP server endpoints (/ping health check and /reset-queue POST endpoint).",
+    )
+    parser.add_argument(
+        "--http-auth-username",
+        type=str,
+        default=None,
+        help="Username for HTTP endpoint authentication. HTTP server is disabled if not provided.",
+    )
+    parser.add_argument(
+        "--http-auth-password",
+        type=str,
+        default=None,
+        help="Password for HTTP endpoint authentication. HTTP server is disabled if not provided.",
+    )
     return parser

{redis_benchmarks_specification-0.1.328 → redis_benchmarks_specification-0.1.330}/redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -615,6 +615,9 @@ def process_self_contained_coordinator_stream(
                                 metadata,
                                 build_variant_name,
                                 running_platform,
+                                None,
+                                None,
+                                disable_target_tables=True,
                             )
                             test_result = True
                             total_test_suite_runs = total_test_suite_runs + 1