redis-benchmarks-specification 0.1.308__tar.gz → 0.1.310__tar.gz

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: redis-benchmarks-specification
-Version: 0.1.308
+Version: 0.1.310
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: Flask (>=2.0.3,<3.0.0)
 Requires-Dist: Flask-HTTPAuth (>=4.4.0,<5.0.0)
 Requires-Dist: GitPython (>=3.1.20,<4.0.0)

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.308"
+version = "0.1.310"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__cli__/stats.py

@@ -223,6 +223,9 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         command = command.replace("'", "")
                         if "-key-pattern" in command:
                             continue
+                        # Skip command-ratio and other memtier arguments that start with -
+                        if command.startswith("-"):
+                            continue
                         command = command.lower()
                         if command not in tested_commands:
                             tested_commands.append(command)
@@ -238,14 +241,16 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         if command not in tracked_commands_json:
                             tracked_commands_json[command] = command_json
 
-                        group = command_json["group"]
-                        if group not in tested_groups:
+                        # Only process if command_json has group information
+                        if "group" in command_json:
+                            group = command_json["group"]
+                            if group not in tested_groups:
 
-                            tested_groups.append(group)
-                        if group not in tracked_groups:
-                            tracked_groups.append(group)
-                            tracked_groups_hist[group] = 0
-                        tracked_groups_hist[group] = tracked_groups_hist[group] + 1
+                                tested_groups.append(group)
+                            if group not in tracked_groups:
+                                tracked_groups.append(group)
+                                tracked_groups_hist[group] = 0
+                            tracked_groups_hist[group] = tracked_groups_hist[group] + 1
 
                 # Calculate total connections
                 total_connections = clients * threads
@@ -262,12 +267,14 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                     data_sizes[data_size] = 0
                 data_sizes[data_size] = data_sizes[data_size] + 1
 
-                if tested_commands != origin_tested_commands:
+                if sorted(tested_commands) != sorted(origin_tested_commands):
                     requires_override = True
                     benchmark_config["tested-commands"] = tested_commands
                     logging.warn(
                         "there is a difference between specified test-commands in the yaml (name={}) and the ones we've detected {}!={}".format(
-                            test_name, origin_tested_commands, tested_commands
+                            test_name,
+                            sorted(origin_tested_commands),
+                            sorted(tested_commands),
                         )
                     )
 
@@ -323,12 +330,14 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         )
                     )
 
-                if tested_groups != origin_tested_groups:
+                if sorted(tested_groups) != sorted(origin_tested_groups):
                     tested_groups_match_origin = False
                     benchmark_config["tested-groups"] = tested_groups
                     logging.warn(
                         "there is a difference between specified test-groups in the yaml (name={}) and the ones we've detected {}!={}".format(
-                            test_name, origin_tested_groups, tested_groups
+                            test_name,
+                            sorted(origin_tested_groups),
+                            sorted(tested_groups),
                         )
                     )
 

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__common__/runner.py

@@ -67,7 +67,9 @@ def execute_init_commands(benchmark_config, r, dbconfig_keyname="dbconfig"):
 
         for lua_script in lua_scripts:
             try:
-                logging.info("Executing Lua script (length: {} chars)".format(len(lua_script)))
+                logging.info(
+                    "Executing Lua script (length: {} chars)".format(len(lua_script))
+                )
                 # Execute the Lua script using EVAL command with 0 keys
                 stdout = r.execute_command("EVAL", lua_script, 0)
                 logging.info("Lua script result: {}".format(stdout))
@@ -137,7 +139,6 @@ def extract_testsuites(args):
     return testsuite_spec_files
 
 
-
 def commandstats_latencystats_process_name(
     metric_name, prefix, setup_name, variant_labels_dict
 ):
@@ -235,6 +236,7 @@ def collect_redis_metrics(
 
     return start_time_ms, res, kv_overall
 
+
 def export_redis_metrics(
     artifact_version,
     end_time_ms,
@@ -274,9 +276,9 @@ def export_redis_metrics(
             "branch": tf_github_branch
         }
     if git_hash is not None and git_hash != "":
-        by_variants["by.hash/{}".format(git_hash)] = {
-            "hash": git_hash
-        }
+        by_variants["by.hash/{}".format(git_hash)] = {"hash": git_hash}
+        if artifact_version is not None and artifact_version != "":
+            by_variants["by.hash/{}".format(git_hash)]["version"] = artifact_version
     if artifact_version is not None and artifact_version != "":
         by_variants["by.version/{}".format(artifact_version)] = {
             "version": artifact_version
@@ -339,8 +341,6 @@ def export_redis_metrics(
     return datapoint_errors, datapoint_inserts
 
 
-
-
 def reset_commandstats(redis_conns):
     for pos, redis_conn in enumerate(redis_conns):
         logging.info("Resetting commmandstats for shard {}".format(pos))
@@ -468,7 +468,9 @@ def exporter_datasink_common(
 
     # Update deployment tracking sets
     deployment_type_and_name = f"{setup_type}_AND_{setup_name}"
-    deployment_type_and_name_and_version = f"{setup_type}_AND_{setup_name}_AND_{git_version}"
+    deployment_type_and_name_and_version = (
+        f"{setup_type}_AND_{setup_name}_AND_{git_version}"
+    )
 
     # Add to deployment-specific set (only if datasink connection is available)
     if datasink_conn is not None:
@@ -483,4 +485,6 @@ def exporter_datasink_common(
 
     # Add metadata fields to timeseries metadata
     metadata["deployment_type_AND_deployment_name"] = deployment_type_and_name
-    metadata["deployment_type_AND_deployment_name_AND_version"] = deployment_type_and_name_and_version
+    metadata["deployment_type_AND_deployment_name_AND_version"] = (
+        deployment_type_and_name_and_version
+    )

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__common__/timeseries.py

@@ -76,7 +76,7 @@ def get_ts_metric_name(
     else:
         deployment_name = ""
     ts_name = (
-        "ci.benchmarks.redislabs/{by}/"
+        "ci.benchmarks.redis/{by}/"
         "{triggering_env}/{github_org}/{github_repo}/"
         "{test_name}/{build_variant_str}{running_platform_str}{deployment_type}{deployment_name}/{by_value}/{metric}".format(
             by=by,
@@ -323,7 +323,7 @@ def from_metric_kv_to_timeserie(
     }
 
     original_ts_name = ts_name
-    target_table_keyname = "target_tables:{triggering_env}:ci.benchmarks.redislabs/{break_by_key}/{break_by_str}/{tf_github_org}/{tf_github_repo}/{deployment_type}/{deployment_name}/{test_name}/{metric_name}".format(
+    target_table_keyname = "target_tables:{triggering_env}:ci.benchmarks.redis/{break_by_key}/{break_by_str}/{tf_github_org}/{tf_github_repo}/{deployment_type}/{deployment_name}/{test_name}/{metric_name}".format(
         triggering_env=tf_triggering_env,
         break_by_key=break_by_key,
         break_by_str=break_by_str,
@@ -679,7 +679,7 @@ def get_overall_dashboard_keynames(
     if running_platform is not None:
         running_platform_str = "/{}".format(running_platform)
     sprefix = (
-        "ci.benchmarks.redislabs/"
+        "ci.benchmarks.redis/"
         + "{triggering_env}/{github_org}/{github_repo}".format(
             triggering_env=tf_triggering_env,
             github_org=tf_github_org,

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__compare__/args.py

@@ -76,9 +76,7 @@ def create_compare_arguments(parser):
     parser.add_argument("--baseline_deployment_name", type=str, default="")
     parser.add_argument("--comparison_deployment_name", type=str, default="")
     parser.add_argument("--metric_name", type=str, default="ALL_STATS.Totals.Ops/sec")
-    parser.add_argument(
-        "--running_platform", type=str, default="intel64-ubuntu22.04-redis-icx1"
-    )
+    parser.add_argument("--running_platform", type=str, default=None)
     parser.add_argument(
         "--running_platform_baseline",
         type=str,

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__compare__/compare.py

@@ -59,7 +59,7 @@ def get_overall_dashboard_keynames(
     if running_platform is not None:
         running_platform_str = "/{}".format(running_platform)
     sprefix = (
-        "ci.benchmarks.redislabs/"
+        "ci.benchmarks.redis/"
         + "{triggering_env}/{github_org}/{github_repo}".format(
             triggering_env=tf_triggering_env,
             github_org=tf_github_org,
@@ -128,7 +128,7 @@ def get_start_time_vars(start_time=None):
 
 
 def get_project_compare_zsets(triggering_env, org, repo):
-    return "ci.benchmarks.redislabs/{}/{}/{}:compare:pull_requests:zset".format(
+    return "ci.benchmarks.redis/{}/{}/{}:compare:pull_requests:zset".format(
         triggering_env, org, repo
     )
 
@@ -1179,10 +1179,12 @@ def from_rts_to_regression_table(
         if baseline_str != "":
             filters_baseline.append("{}={}".format(by_str_baseline, baseline_str))
         if baseline_deployment_name != "":
-            filters_baseline.append("deployment_name={}".format(baseline_deployment_name))
+            filters_baseline.append(
+                "deployment_name={}".format(baseline_deployment_name)
+            )
         if baseline_github_org != "":
             filters_baseline.append(f"github_org={baseline_github_org}")
-        if running_platform_baseline is not None:
+        if running_platform_baseline is not None and running_platform_baseline != "":
             filters_baseline.append(
                 "running_platform={}".format(running_platform_baseline)
             )
@@ -1195,14 +1197,19 @@ def from_rts_to_regression_table(
         if comparison_str != "":
             filters_comparison.append("{}={}".format(by_str_comparison, comparison_str))
         if comparison_deployment_name != "":
-            filters_comparison.append("deployment_name={}".format(comparison_deployment_name))
+            filters_comparison.append(
+                "deployment_name={}".format(comparison_deployment_name)
+            )
         if comparison_github_org != "":
             filters_comparison.append(f"github_org={comparison_github_org}")
         if "hash" not in by_str_baseline:
             filters_baseline.append("hash==")
         if "hash" not in by_str_comparison:
             filters_comparison.append("hash==")
-        if running_platform_comparison is not None:
+        if (
+            running_platform_comparison is not None
+            and running_platform_comparison != ""
+        ):
             filters_comparison.append(
                 "running_platform={}".format(running_platform_comparison)
             )
@@ -1538,12 +1545,22 @@ def from_rts_to_regression_table(
 
 def get_only_Totals(baseline_timeseries):
     logging.warning("\t\tTime-series: {}".format(", ".join(baseline_timeseries)))
-    logging.info("Checking if Totals will reduce timeseries.")
+    logging.info(
+        f"Checking if Totals will reduce timeseries. initial len={len(baseline_timeseries)}"
+    )
     new_base = []
     for ts_name in baseline_timeseries:
+        if "io-threads" in ts_name:
+            continue
+        if "oss-cluster" in ts_name:
+            continue
         if "Totals" in ts_name:
             new_base.append(ts_name)
     baseline_timeseries = new_base
+    logging.info(
+        f"                                          final len={len(baseline_timeseries)}"
+    )
+
     return baseline_timeseries
 
 

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__runner__/args.py

@@ -44,19 +44,19 @@ def create_client_runner_args(project_name):
         "--deployment_type",
         type=str,
         default="oss-standalone",
-        help="Deployment type for the Redis instance (e.g., oss-standalone, oss-cluster, enterprise)"
+        help="Deployment type for the Redis instance (e.g., oss-standalone, oss-cluster, enterprise)",
     )
     parser.add_argument(
         "--deployment_name",
         type=str,
         default="redis",
-        help="Deployment name identifier for the Redis instance"
+        help="Deployment name identifier for the Redis instance",
     )
     parser.add_argument(
         "--core_count",
         type=int,
         default=None,
-        help="Number of CPU cores available to the Redis instance"
+        help="Number of CPU cores available to the Redis instance",
     )
     parser.add_argument("--github_repo", type=str, default="redis")
     parser.add_argument("--github_org", type=str, default="redis")

{redis_benchmarks_specification-0.1.308 → redis_benchmarks_specification-0.1.310}/redis_benchmarks_specification/__runner__/remote_profiling.py

@@ -238,7 +238,9 @@ def extract_server_metadata_for_timeseries(redis_conn) -> Dict[str, str]:
         else:
             metadata["config_file"] = "none"
 
-        logging.info(f"Extracted {len(metadata)} server metadata fields for timeseries: {list(metadata.keys())}")
+        logging.info(
+            f"Extracted {len(metadata)} server metadata fields for timeseries: {list(metadata.keys())}"
+        )
 
         return metadata