redisbench-admin 0.11.72__tar.gz → 0.11.74__tar.gz

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redisbench-admin
-Version: 0.11.72
+Version: 0.11.74
 Summary: Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... ).
 License-File: LICENSE
 Author: filipecosta90

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.11.72"
+version = "0.11.74"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/redisbench_admin/run/common.py

@@ -38,6 +38,7 @@ from redisbench_admin.run.ycsb.ycsb import (
     prepare_ycsb_benchmark_command,
     prepare_go_ycsb_benchmark_command,
 )
+from redisbench_admin.utils.utils import get_remote_input_file_from_url
 from redisbench_admin.run_remote.args import OVERRIDE_MODULES
 from redisbench_admin.run_remote.remote_helpers import (
     extract_module_semver_from_info_modules_cmd,
@@ -66,6 +67,38 @@ PERFORMANCE_GH_TOKEN = os.getenv("PERFORMANCE_GH_TOKEN", None)
 REDIS_BINARY = os.getenv("REDIS_BINARY", "redis-server")
 
 
+def extract_input_file_url_from_parameters(entry, benchmark_tool):
+    """
+    Extract the input file URL from the parameters entry.
+    Different tools use different parameter names:
+    - ftsb_*: uses "input" parameter
+    - tsbs_*, aibench_*: uses "file" parameter
+
+    Args:
+        entry: The benchmark config entry containing "parameters"
+        benchmark_tool: The benchmark tool name
+
+    Returns:
+        The file URL if found, None otherwise
+    """
+    if "parameters" not in entry:
+        return None
+
+    # Determine which parameter name to look for based on the tool
+    if "ftsb_" in benchmark_tool:
+        param_name = "input"
+    elif "tsbs_" in benchmark_tool or "aibench_" in benchmark_tool:
+        param_name = "file"
+    else:
+        return None
+
+    for param in entry["parameters"]:
+        if param_name in param:
+            return param[param_name]
+
+    return None
+
+
 def prepare_benchmark_parameters(
     benchmark_config,
     benchmark_tool,
@@ -88,6 +121,10 @@ def prepare_benchmark_parameters(
     if type(benchmark_config[config_key]) == list:
         for entry in benchmark_config[config_key]:
             if "parameters" in entry:
+                # Extract input file URL from parameters
+                input_file_url = extract_input_file_url_from_parameters(
+                    entry, benchmark_tool
+                )
                 command_arr, command_str = prepare_benchmark_parameters_specif_tooling(
                     benchmark_tool,
                     cluster_api_enabled,
@@ -104,10 +141,13 @@ def prepare_benchmark_parameters(
                     private_key,
                     client_ssh_port,
                     redis_password,
+                    input_file_url,
                 )
     # v0.4 spec
     elif type(benchmark_config[config_key]) == dict:
         entry = benchmark_config[config_key]
+        # Extract input file URL from parameters
+        input_file_url = extract_input_file_url_from_parameters(entry, benchmark_tool)
         command_arr, command_str = prepare_benchmark_parameters_specif_tooling(
             benchmark_tool,
             cluster_api_enabled,
@@ -124,6 +164,7 @@ def prepare_benchmark_parameters(
             private_key,
             client_ssh_port,
             redis_password,
+            input_file_url,
         )
     printed_command_str = command_str
     printed_command_arr = command_arr
@@ -154,6 +195,7 @@ def prepare_benchmark_parameters_specif_tooling(
     private_key,
     client_ssh_port,
     redis_password=None,
+    input_file_url=None,
 ):
     if "redis-benchmark" in benchmark_tool:
         command_arr, command_str = prepare_redis_benchmark_command(
@@ -207,7 +249,7 @@ def prepare_benchmark_parameters_specif_tooling(
         input_data_file = None
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
-            input_data_file = "/tmp/input.data"
+            input_data_file = get_remote_input_file_from_url(input_file_url)
         (
             command_arr,
             command_str,
@@ -260,7 +302,7 @@ def prepare_benchmark_parameters_specif_tooling(
         input_data_file = None
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
-            input_data_file = "/tmp/input.data"
+            input_data_file = get_remote_input_file_from_url(input_file_url)
         (
             command_arr,
             command_str,
@@ -280,7 +322,7 @@ def prepare_benchmark_parameters_specif_tooling(
         input_data_file = None
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
-            input_data_file = "/tmp/input.data"
+            input_data_file = get_remote_input_file_from_url(input_file_url)
         (
             command_arr,
             command_str,

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/redisbench_admin/run_local/local_db.py

@@ -4,6 +4,7 @@
 #  All rights reserved.
 #
 import logging
+import os
 import tempfile
 import datetime
 
@@ -32,6 +33,7 @@ from redisbench_admin.utils.local import (
     check_dataset_local_requirements,
     is_process_alive,
 )
+from redisbench_admin.utils.utils import setup_search_clusterset
 
 
 def local_db_spin(
@@ -163,6 +165,9 @@ def local_db_spin(
         r.client_setname("redisbench-admin-standalone")
         redis_conns.append(r)
 
+    # Setup search CLUSTERSET if enabled (after all servers are started and cluster is set up)
+    setup_search_clusterset(redis_conns, args.host, args.port, args.password)
+
     if dataset is None:
         if flushall_on_every_test_start:
             logging.info("Will flush all data at test start...")
@@ -213,8 +218,13 @@ def local_db_spin(
         )
     dbconfig_keyspacelen_check(benchmark_config, redis_conns, ignore_keyspace_errors)
 
-    artifact_version = run_redis_pre_steps(
-        benchmark_config, redis_conns[0], required_modules
-    )
+    if 'SEARCH_CLUSTERSET' in os.environ:
+        logging.info("SEARCH_CLUSTERSET is set. Running run_redis_pre_steps for each shard")
+        for conn in redis_conns:
+            artifact_version = run_redis_pre_steps(benchmark_config, conn, required_modules)
+    else:
+        artifact_version = run_redis_pre_steps(
+            benchmark_config, redis_conns[0], required_modules
+        )
 
     return result, artifact_version, cluster_api_enabled, redis_conns, redis_processes

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/redisbench_admin/run_remote/remote_db.py

@@ -5,7 +5,7 @@
 #
 import datetime
 import logging
-
+import os
 import redis
 
 from redisbench_admin.environments.oss_cluster import setup_redis_cluster_from_conns
@@ -37,6 +37,7 @@ from redisbench_admin.utils.remote import (
     check_dataset_remote_requirements,
     get_run_full_filename,
 )
+from redisbench_admin.utils.utils import setup_search_clusterset
 
 
 def remote_tmpdir_prune(
@@ -277,6 +278,14 @@ def remote_db_spin(
         )
         server_plaintext_port = cluster_start_port
 
+    # Setup BigRedis CLUSTERSET if enabled (after all servers are started and cluster is set up)
+    setup_search_clusterset(
+        redis_conns,
+        server_private_ip,
+        cluster_start_port if cluster_enabled else server_plaintext_port,
+        redis_password,
+    )
+
     topology_setup_end_time = datetime.datetime.now()
     topology_setup_duration_seconds = (
         topology_setup_end_time - topology_setup_start_time
@@ -406,9 +415,12 @@ def remote_db_spin(
         ignore_keyspace_errors,
         keyspace_check_timeout,
     )
-    artifact_version = run_redis_pre_steps(
-        benchmark_config, redis_conns[0], required_modules
-    )
+    if 'SEARCH_CLUSTERSET' in os.environ:
+        logging.info("SEARCH_CLUSTERSET is set. Running run_redis_pre_steps for each shard")
+        for conn in redis_conns:
+            artifact_version = run_redis_pre_steps(benchmark_config, conn, required_modules)
+    else:
+        artifact_version = run_redis_pre_steps(benchmark_config, redis_conns[0], required_modules)
     return (
         artifact_version,
         cluster_enabled,

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/redisbench_admin/run_remote/remote_helpers.py

@@ -28,6 +28,7 @@ from redisbench_admin.utils.remote import (
     extract_redisgraph_version_from_resultdict,
 )
 from redisbench_admin.utils.results import post_process_benchmark_results
+from redisbench_admin.utils.utils import get_remote_input_file_from_url
 
 
 def absoluteFilePaths(directory):
@@ -166,7 +167,7 @@ def remote_tool_pre_bench_step(
                 queries_file_link, remote_tool_link, tool_link
             )
         )
-        remote_input_file = "/tmp/input.data"
+        remote_input_file = get_remote_input_file_from_url(queries_file_link)
         setup_remote_benchmark_tool_requirements_ftsb(
             client_public_ip,
             username,
@@ -191,7 +192,7 @@ def remote_tool_pre_bench_step(
             "redisearch",
             "go-ycsb",
         )
-        remote_input_file = "/tmp/input.data"
+        remote_input_file = get_remote_input_file_from_url(queries_file_link)
         setup_remote_benchmark_tool_requirements_tsbs(
             client_public_ip,
             username,
@@ -211,7 +212,7 @@ def remote_tool_pre_bench_step(
         ) = extract_remote_tool_extra_links(
             benchmark_config, benchmark_tool, config_key, os_str, arch_str
         )
-        remote_input_file = "/tmp/input.data"
+        remote_input_file = get_remote_input_file_from_url(queries_file_link)
         setup_remote_benchmark_tool_requirements_tsbs(
             client_public_ip,
             username,
@@ -228,7 +229,7 @@ def remote_tool_pre_bench_step(
             remote_tool_link,
             tool_link,
         ) = extract_aibench_extra_links(benchmark_config, benchmark_tool)
-        remote_input_file = "/tmp/input.data"
+        remote_input_file = get_remote_input_file_from_url(queries_file_link)
         setup_remote_benchmark_tool_requirements_tsbs(
             client_public_ip,
             username,

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/redisbench_admin/run_remote/standalone.py

@@ -583,6 +583,17 @@ def generate_remote_standalone_redis_cmd(
                 )
     if remote_module_files is not None:
         initial_redis_cmd += " " + " ".join(command)
+
+    # Add BigRedis configuration if enabled via environment variable
+    if os.getenv("BIGREDIS_ENABLED") is not None:
+        bigredis_path = os.getenv("BIGREDIS_PATH", f"{temporary_dir}/redis.big")
+        bigredis_use_async = os.getenv("BIGREDIS_USE_ASYNC", "no")
+        logging.info(f"BigRedis enabled. Using bigredis-path: {bigredis_path}")
+        initial_redis_cmd += " --bigredis-enabled yes"
+        initial_redis_cmd += " --bigredis-driver speedb"
+        initial_redis_cmd += f" --bigredis-use-async {bigredis_use_async}"
+        initial_redis_cmd += f" --bigredis-path {bigredis_path}"
+
     return full_logfile, initial_redis_cmd
 
 

{redisbench_admin-0.11.72 → redisbench_admin-0.11.74}/redisbench_admin/utils/utils.py

@@ -11,10 +11,12 @@ import logging
 import operator
 import os
 import os.path
+import re
 import tarfile
 import time
 from functools import reduce
-from urllib.parse import quote_plus
+from pathlib import PurePosixPath
+from urllib.parse import quote_plus, urlparse, unquote
 from zipfile import ZipFile
 
 import boto3
@@ -135,6 +137,75 @@ def generate_common_server_args(
     return command
 
 
+def setup_search_clusterset(redis_conns, host="127.0.0.1", start_port=6379, password=None):
+    """
+    Configure  SEARCH.CLUSTERSET for Redis instances.
+    This is required when SEARCH_CLUSTERSET is set to configure the search module
+    to work with BigRedis. Supports both standalone (single conn) and cluster (multiple conns).
+
+    Args:
+        redis_conns: List of Redis connection objects (or single connection)
+        host: Redis host address (default: 127.0.0.1)
+        start_port: Starting port number (default: 6379)
+        password: Redis password (optional)
+    """
+    if os.getenv("SEARCH_CLUSTERSET") is None:
+        return
+
+    # Handle single connection case
+    if not isinstance(redis_conns, list):
+        redis_conns = [redis_conns]
+
+    shard_count = len(redis_conns)
+    logging.info(f"Search Clusterset enabled - configuring SEARCH.CLUSTERSET for {shard_count} shard(s)")
+
+    # Calculate slots per shard
+    total_slots = 16384
+    slots_per_shard = total_slots // shard_count
+
+    for shard_id, conn in enumerate(redis_conns, start=1):
+        port = start_port + shard_id - 1
+
+        # Calculate slot range for this shard
+        slot_start = (shard_id - 1) * slots_per_shard
+        if shard_id == shard_count:
+            # Last shard gets remaining slots
+            slot_end = total_slots - 1
+        else:
+            slot_end = shard_id * slots_per_shard - 1
+
+        # Build the address string with optional password
+        if password:
+            addr = f"{password}@{host}:{port}"
+        else:
+            addr = f"{host}:{port}"
+
+        logging.info(
+            f"Sending SEARCH.CLUSTERSET to shard {shard_id}: "
+            f"slots {slot_start}-{slot_end}, addr {host}:{port}"
+        )
+
+        try:
+            conn.execute_command(
+                "SEARCH.CLUSTERSET",
+                "MYID",
+                str(shard_id),
+                "RANGES",
+                "1",
+                "SHARD",
+                str(shard_id),
+                "SLOTRANGE",
+                str(slot_start),
+                str(slot_end),
+                "ADDR",
+                addr,
+                "MASTER",
+            )
+            logging.info(f"SEARCH.CLUSTERSET command executed successfully for shard {shard_id}")
+        except Exception as e:
+            logging.warning(f"Failed to execute SEARCH.CLUSTERSET for shard {shard_id}: {e}")
+
+
 def upload_artifacts_to_s3(
     artifacts,
     s3_bucket_name,
@@ -484,6 +555,60 @@ def make_dashboard_callback(
     )
 
 
+def get_remote_input_file_from_url(url):
+    """
+    Generate a unique remote input file path based on the file name from the URL.
+    This ensures each input file has its own path and avoids reuse issues between benchmark runs.
+
+    Example:
+        url = "https://s3.amazonaws.com/.../AND_QUERY.CSV"
+        returns "/tmp/input-AND_QUERY.data"
+
+    Args:
+        url: The URL or path to the input file
+
+    Returns:
+        A unique remote file path like /tmp/input-{file_name}.data,
+        or "/tmp/input.data" as fallback if url is None or empty
+    """
+    default_path = PurePosixPath("/tmp/input.data")
+
+    if url is None or url == "":
+        return str(default_path)
+
+    # Parse the URL to extract the path component
+    # This handles both http(s):// URLs and s3:// URIs
+    parsed = urlparse(url)
+    url_path = parsed.path if parsed.path else url
+
+    # Use PurePosixPath to extract the file name (works for both URLs and paths)
+    path = PurePosixPath(url_path)
+    file_name = path.name
+
+    # If no file name could be extracted, return default
+    if not file_name:
+        return str(default_path)
+
+    # Decode URL-encoded characters (e.g., %3A -> :)
+    file_name = unquote(file_name)
+
+    # Remove extension using pathlib's stem property
+    base_name = PurePosixPath(file_name).stem
+
+    # If stem is empty (e.g., hidden file like ".data"), use the full name
+    if not base_name:
+        base_name = file_name
+
+    # Sanitize the base name: replace characters that might cause issues in file paths
+    # Keep only alphanumeric, underscore, hyphen, and dot
+    base_name = re.sub(r"[^a-zA-Z0-9_\-.]", "_", base_name)
+
+    # Construct the final path using PurePosixPath for proper path joining
+    result_path = PurePosixPath("/tmp") / f"input-{base_name}.data"
+
+    return str(result_path)
+
+
 EC2_REGION = os.getenv("AWS_DEFAULT_REGION", None)
 EC2_SECRET_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", None)
 EC2_PRIVATE_PEM = os.getenv("EC2_PRIVATE_PEM", None)