redisbench-admin 0.11.57__tar.gz → 0.11.58__tar.gz

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redisbench-admin
-Version: 0.11.57
+Version: 0.11.58
 Summary: Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... ).
 License-File: LICENSE
 Author: filipecosta90

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.11.57"
+version = "0.11.58"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/redisbench_admin/run/ftsb/ftsb.py

@@ -44,6 +44,8 @@ def prepare_ftsb_benchmark_command(
                     input_file, "/tmp", None, remote_queries_file, is_remote
                 )
                 command_arr.extend(["--input", input_file])
+            elif "max-token-size-mb" in k:
+                command_arr.extend(["--max-token-size-mb", str(k["max-token-size-mb"])])
             else:
                 for kk in k.keys():
                     command_arr.extend(["--{}".format(kk), str(k[kk])])

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/redisbench_admin/run/ssh.py

@@ -35,6 +35,9 @@ def ssh_tunnel_redisconn(
         ssh_address_or_host=(server_public_ip, ssh_port),
         ssh_username=username,
         ssh_pkey=ssh_pkey,
+        ssh_config_file=None,
+        allow_agent=False,
+        host_pkey_directories=[],
         logger=logging.getLogger(),
         remote_bind_address=(
             server_private_ip,

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/redisbench_admin/run_remote/remote_helpers.py

@@ -51,6 +51,50 @@ def benchmark_tools_sanity_check(allowed_tools, benchmark_tool):
         )
 
 
+def ensure_aws_cli_available(client_public_ip, username, private_key, client_ssh_port):
+    """Check if AWS CLI is available, install if not"""
+    logging.info("Checking if AWS CLI is available on remote client...")
+
+    # Check if aws command exists
+    check_result = execute_remote_commands(
+        client_public_ip, username, private_key, ["which aws"], client_ssh_port
+    )
+
+    # Check the result
+    if len(check_result) > 0:
+        [recv_exit_status, stdout, stderr] = check_result[0]
+        if recv_exit_status != 0:
+            logging.info("AWS CLI not found, installing...")
+
+            # Install AWS CLI v2
+            install_commands = [
+                "curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o 'awscliv2.zip'",
+                "unzip -q awscliv2.zip",
+                "sudo ./aws/install",
+                "rm -rf awscliv2.zip aws/"
+            ]
+
+            install_result = execute_remote_commands(
+                client_public_ip, username, private_key, install_commands, client_ssh_port
+            )
+
+            # Check if installation was successful
+            for pos, res_pos in enumerate(install_result):
+                [recv_exit_status, stdout, stderr] = res_pos
+                if recv_exit_status != 0:
+                    logging.warning(
+                        "AWS CLI installation command {} returned exit code {}. stdout: {}. stderr: {}".format(
+                            pos, recv_exit_status, stdout, stderr
+                        )
+                    )
+
+            logging.info("AWS CLI installation completed")
+        else:
+            logging.info("AWS CLI is already available")
+    else:
+        logging.error("Failed to check AWS CLI availability")
+
+
 def remote_tool_pre_bench_step(
     benchmark_config,
     benchmark_min_tool_version,
@@ -221,12 +265,16 @@ def _setup_remote_benchmark_tool_requirements(
     ]
 
     # detect if queries_file_link is a s3 URI or http one and act accordingly (use aws cli or wget)
-    if queries_file_link.startswith("s3://"):
-        commands.append(
-            "aws s3 cp {} {} --no-sign-request".format(queries_file_link, remote_input_file)
-        )
+    if queries_file_link is not None:
+      if queries_file_link.startswith("s3://"):
+          ensure_aws_cli_available(client_public_ip, username, private_key, client_ssh_port)
+          commands.append(
+              "timeout 600 aws s3 cp {} {} --no-sign-request --cli-read-timeout 300".format(queries_file_link, remote_input_file)
+          )
+      else:
+          commands.append("wget {} -q -O {}".format(queries_file_link, remote_input_file))
     else:
-        commands.append("wget {} -q -O {}".format(queries_file_link, remote_input_file))
+        logging.info("No queries file link provided. Skipping download.")
     execute_remote_commands(
         client_public_ip, username, private_key, commands, client_ssh_port
     )

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/redisbench_admin/utils/benchmark_config.py

@@ -195,36 +195,59 @@ def merge_default_and_specific_properties_dict_type(
             )
         )
     else:
-        usecase_kpi = None
-        use_case_specific_properties = benchmark_config[propertygroup_keyname]
-        for default_property in default_properties:
-            default_rule, default_details = list(default_property.items())[0]
-            default_condition = list(default_details.values())[0]
-            comparison_key = "{}{}".format(default_rule, default_condition)
-            found = False
-            for usecase_kpi in use_case_specific_properties:
-                usecase_rule, usecase_details = list(usecase_kpi.items())[0]
-                usecase_condition = list(usecase_details.values())[0]
-                usecase_comparison_key = "{}{}".format(usecase_rule, usecase_condition)
-                if comparison_key == usecase_comparison_key:
-                    found = True
-            if found:
-                logging.info(
-                    "Skipping to add default '{}' property ({}) given the file {}"
-                    " had the same specific property ({})".format(
-                        propertygroup_keyname,
-                        default_property,
-                        usecase_filename,
-                        usecase_kpi,
+        # Special handling for remote configs
+        if propertygroup_keyname == "remote":
+            # For remote, merge by adding missing keys from defaults
+            use_case_specific_properties = benchmark_config[propertygroup_keyname]
+            existing_keys = set()
+            for item in use_case_specific_properties:
+                existing_keys.update(item.keys())
+
+            for default_property in default_properties:
+                for key in default_property.keys():
+                    if key not in existing_keys:
+                        use_case_specific_properties.append(default_property)
+                        logging.info(
+                            "Adding a default '{}' property ({}) given the file {} did not have the specific property".format(
+                                propertygroup_keyname, default_property, usecase_filename
+                            )
+                        )
+                        break
+        else:
+            # Original logic for kpis and other properties
+            usecase_kpi = None
+            use_case_specific_properties = benchmark_config[propertygroup_keyname]
+            for default_property in default_properties:
+                default_rule, default_details = list(default_property.items())[0]
+                if isinstance(default_details, dict):
+                    default_condition = list(default_details.values())[0]
+                else:
+                    default_condition = default_details
+                comparison_key = "{}{}".format(default_rule, default_condition)
+                found = False
+                for usecase_kpi in use_case_specific_properties:
+                    usecase_rule, usecase_details = list(usecase_kpi.items())[0]
+                    usecase_condition = list(usecase_details.values())[0]
+                    usecase_comparison_key = "{}{}".format(usecase_rule, usecase_condition)
+                    if comparison_key == usecase_comparison_key:
+                        found = True
+                if found:
+                    logging.info(
+                        "Skipping to add default '{}' property ({}) given the file {}"
+                        " had the same specific property ({})".format(
+                            propertygroup_keyname,
+                            default_property,
+                            usecase_filename,
+                            usecase_kpi,
+                        )
                     )
-                )
-            else:
-                use_case_specific_properties.append(default_property)
-                logging.info(
-                    "Adding a default '{}' property ({}) given the file {} did not had the specific property".format(
-                        propertygroup_keyname, default_property, usecase_filename
+                else:
+                    use_case_specific_properties.append(default_property)
+                    logging.info(
+                        "Adding a default '{}' property ({}) given the file {} did not have the specific property".format(
+                            propertygroup_keyname, default_property, usecase_filename
+                        )
                     )
-                )
 
 
 def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):

{redisbench_admin-0.11.57 → redisbench_admin-0.11.58}/redisbench_admin/utils/local.py

@@ -70,7 +70,7 @@ def check_dataset_local_requirements(
 def check_if_needs_remote_fetch(
     property, localtemp_dir, dirname, full_path=None, is_remote=False
 ):
-    if property.startswith("http"):
+    if property.startswith("http") or property.startswith("s3"):
         if not os.path.isdir(localtemp_dir):
             os.mkdir(localtemp_dir)
         if full_path is None:
@@ -82,7 +82,10 @@ def check_if_needs_remote_fetch(
                     property, full_path, localtemp_dir
                 )
             )
-            wget.download(property, full_path)
+            if property.startswith("s3://"):
+              logging.error("Unexpected s3 URL. It should have already been downloaded. Skipping...")
+            else:
+              wget.download(property, full_path)
         else:
             logging.info(
                 "Reusing cached remote file (located at {} ).".format(full_path)