redisbench-admin 0.11.56__py3-none-any.whl → 0.11.58__py3-none-any.whl

redisbench_admin/run/ftsb/ftsb.py

@@ -44,6 +44,8 @@ def prepare_ftsb_benchmark_command(
                     input_file, "/tmp", None, remote_queries_file, is_remote
                 )
                 command_arr.extend(["--input", input_file])
+            elif "max-token-size-mb" in k:
+                command_arr.extend(["--max-token-size-mb", str(k["max-token-size-mb"])])
             else:
                 for kk in k.keys():
                     command_arr.extend(["--{}".format(kk), str(k[kk])])

redisbench_admin/run/ssh.py

@@ -35,6 +35,9 @@ def ssh_tunnel_redisconn(
         ssh_address_or_host=(server_public_ip, ssh_port),
         ssh_username=username,
         ssh_pkey=ssh_pkey,
+        ssh_config_file=None,
+        allow_agent=False,
+        host_pkey_directories=[],
         logger=logging.getLogger(),
         remote_bind_address=(
             server_private_ip,

redisbench_admin/run_remote/remote_helpers.py

@@ -51,6 +51,50 @@ def benchmark_tools_sanity_check(allowed_tools, benchmark_tool):
         )
 
 
+def ensure_aws_cli_available(client_public_ip, username, private_key, client_ssh_port):
+    """Check if AWS CLI is available, install if not"""
+    logging.info("Checking if AWS CLI is available on remote client...")
+
+    # Check if aws command exists
+    check_result = execute_remote_commands(
+        client_public_ip, username, private_key, ["which aws"], client_ssh_port
+    )
+
+    # Check the result
+    if len(check_result) > 0:
+        [recv_exit_status, stdout, stderr] = check_result[0]
+        if recv_exit_status != 0:
+            logging.info("AWS CLI not found, installing...")
+
+            # Install AWS CLI v2
+            install_commands = [
+                "curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o 'awscliv2.zip'",
+                "unzip -q awscliv2.zip",
+                "sudo ./aws/install",
+                "rm -rf awscliv2.zip aws/"
+            ]
+
+            install_result = execute_remote_commands(
+                client_public_ip, username, private_key, install_commands, client_ssh_port
+            )
+
+            # Check if installation was successful
+            for pos, res_pos in enumerate(install_result):
+                [recv_exit_status, stdout, stderr] = res_pos
+                if recv_exit_status != 0:
+                    logging.warning(
+                        "AWS CLI installation command {} returned exit code {}. stdout: {}. stderr: {}".format(
+                            pos, recv_exit_status, stdout, stderr
+                        )
+                    )
+
+            logging.info("AWS CLI installation completed")
+        else:
+            logging.info("AWS CLI is already available")
+    else:
+        logging.error("Failed to check AWS CLI availability")
+
+
 def remote_tool_pre_bench_step(
     benchmark_config,
     benchmark_min_tool_version,
@@ -205,8 +249,7 @@ def remote_tool_pre_bench_step(
         )
     logging.info("Finished up remote tool {} requirements".format(benchmark_tool))
 
-
-def setup_remote_benchmark_tool_requirements_ftsb(
+def _setup_remote_benchmark_tool_requirements(
     client_public_ip,
     username,
     private_key,
@@ -218,14 +261,47 @@ def setup_remote_benchmark_tool_requirements_ftsb(
 ):
     commands = [
         "wget {} -q -O {}".format(tool_link, remote_tool_link),
-        "wget {} -q -O {}".format(queries_file_link, remote_input_file),
-        "chmod 755 {}".format(remote_tool_link),
+        "chmod 755 {}".format(remote_tool_link)
     ]
+
+    # detect if queries_file_link is a s3 URI or http one and act accordingly (use aws cli or wget)
+    if queries_file_link is not None:
+      if queries_file_link.startswith("s3://"):
+          ensure_aws_cli_available(client_public_ip, username, private_key, client_ssh_port)
+          commands.append(
+              "timeout 600 aws s3 cp {} {} --no-sign-request --cli-read-timeout 300".format(queries_file_link, remote_input_file)
+          )
+      else:
+          commands.append("wget {} -q -O {}".format(queries_file_link, remote_input_file))
+    else:
+        logging.info("No queries file link provided. Skipping download.")
     execute_remote_commands(
         client_public_ip, username, private_key, commands, client_ssh_port
     )
 
 
+def setup_remote_benchmark_tool_requirements_ftsb(
+    client_public_ip,
+    username,
+    private_key,
+    tool_link,
+    queries_file_link,
+    remote_tool_link,
+    remote_input_file,
+    client_ssh_port,
+):
+    _setup_remote_benchmark_tool_requirements(
+        client_public_ip,
+        username,
+        private_key,
+        tool_link,
+        queries_file_link,
+        remote_tool_link,
+        remote_input_file,
+        client_ssh_port,
+    )
+
+
 def setup_remote_benchmark_tool_requirements_tsbs(
     client_public_ip,
     username,
@@ -236,13 +312,15 @@ def setup_remote_benchmark_tool_requirements_tsbs(
     remote_input_file,
     client_ssh_port,
 ):
-    commands = [
-        "wget {} -q -O {}".format(tool_link, remote_tool_link),
-        "wget {} -q -O {}".format(queries_file_link, remote_input_file),
-        "chmod 755 {}".format(remote_tool_link),
-    ]
-    execute_remote_commands(
-        client_public_ip, username, private_key, commands, client_ssh_port
+    _setup_remote_benchmark_tool_requirements(
+        client_public_ip,
+        username,
+        private_key,
+        tool_link,
+        queries_file_link,
+        remote_tool_link,
+        remote_input_file,
+        client_ssh_port,
     )
 
 

redisbench_admin/utils/benchmark_config.py

@@ -195,36 +195,59 @@ def merge_default_and_specific_properties_dict_type(
             )
         )
     else:
-        usecase_kpi = None
-        use_case_specific_properties = benchmark_config[propertygroup_keyname]
-        for default_property in default_properties:
-            default_rule, default_details = list(default_property.items())[0]
-            default_condition = list(default_details.values())[0]
-            comparison_key = "{}{}".format(default_rule, default_condition)
-            found = False
-            for usecase_kpi in use_case_specific_properties:
-                usecase_rule, usecase_details = list(usecase_kpi.items())[0]
-                usecase_condition = list(usecase_details.values())[0]
-                usecase_comparison_key = "{}{}".format(usecase_rule, usecase_condition)
-                if comparison_key == usecase_comparison_key:
-                    found = True
-            if found:
-                logging.info(
-                    "Skipping to add default '{}' property ({}) given the file {}"
-                    " had the same specific property ({})".format(
-                        propertygroup_keyname,
-                        default_property,
-                        usecase_filename,
-                        usecase_kpi,
+        # Special handling for remote configs
+        if propertygroup_keyname == "remote":
+            # For remote, merge by adding missing keys from defaults
+            use_case_specific_properties = benchmark_config[propertygroup_keyname]
+            existing_keys = set()
+            for item in use_case_specific_properties:
+                existing_keys.update(item.keys())
+
+            for default_property in default_properties:
+                for key in default_property.keys():
+                    if key not in existing_keys:
+                        use_case_specific_properties.append(default_property)
+                        logging.info(
+                            "Adding a default '{}' property ({}) given the file {} did not have the specific property".format(
+                                propertygroup_keyname, default_property, usecase_filename
+                            )
+                        )
+                        break
+        else:
+            # Original logic for kpis and other properties
+            usecase_kpi = None
+            use_case_specific_properties = benchmark_config[propertygroup_keyname]
+            for default_property in default_properties:
+                default_rule, default_details = list(default_property.items())[0]
+                if isinstance(default_details, dict):
+                    default_condition = list(default_details.values())[0]
+                else:
+                    default_condition = default_details
+                comparison_key = "{}{}".format(default_rule, default_condition)
+                found = False
+                for usecase_kpi in use_case_specific_properties:
+                    usecase_rule, usecase_details = list(usecase_kpi.items())[0]
+                    usecase_condition = list(usecase_details.values())[0]
+                    usecase_comparison_key = "{}{}".format(usecase_rule, usecase_condition)
+                    if comparison_key == usecase_comparison_key:
+                        found = True
+                if found:
+                    logging.info(
+                        "Skipping to add default '{}' property ({}) given the file {}"
+                        " had the same specific property ({})".format(
+                            propertygroup_keyname,
+                            default_property,
+                            usecase_filename,
+                            usecase_kpi,
+                        )
                     )
-                )
-            else:
-                use_case_specific_properties.append(default_property)
-                logging.info(
-                    "Adding a default '{}' property ({}) given the file {} did not had the specific property".format(
-                        propertygroup_keyname, default_property, usecase_filename
+                else:
+                    use_case_specific_properties.append(default_property)
+                    logging.info(
+                        "Adding a default '{}' property ({}) given the file {} did not have the specific property".format(
+                            propertygroup_keyname, default_property, usecase_filename
+                        )
                     )
-                )
 
 
 def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname):

redisbench_admin/utils/local.py

@@ -70,7 +70,7 @@ def check_dataset_local_requirements(
 def check_if_needs_remote_fetch(
     property, localtemp_dir, dirname, full_path=None, is_remote=False
 ):
-    if property.startswith("http"):
+    if property.startswith("http") or property.startswith("s3"):
         if not os.path.isdir(localtemp_dir):
             os.mkdir(localtemp_dir)
         if full_path is None:
@@ -82,7 +82,10 @@ def check_if_needs_remote_fetch(
                     property, full_path, localtemp_dir
                 )
             )
-            wget.download(property, full_path)
+            if property.startswith("s3://"):
+              logging.error("Unexpected s3 URL. It should have already been downloaded. Skipping...")
+            else:
+              wget.download(property, full_path)
         else:
             logging.info(
                 "Reusing cached remote file (located at {} ).".format(full_path)

redisbench_admin/utils/remote.py

@@ -178,6 +178,8 @@ def connect_remote_ssh(port, private_key, server_public_ip, username):
     c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
     logging.info("Connecting to remote server {}".format(server_public_ip))
     c.connect(hostname=server_public_ip, port=port, username=username, pkey=k)
+    transport = c.get_transport()
+    transport.set_keepalive(10)  # Send keepalive every 10 seconds
     logging.info("Connected to remote server {}".format(server_public_ip))
     return c
 

{redisbench_admin-0.11.56.dist-info → redisbench_admin-0.11.58.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redisbench-admin
-Version: 0.11.56
+Version: 0.11.58
 Summary: Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... ).
 License-File: LICENSE
 Author: filipecosta90

{redisbench_admin-0.11.56.dist-info → redisbench_admin-0.11.58.dist-info}/RECORD

@@ -53,7 +53,7 @@ redisbench_admin/run/asm.py,sha256=Hybna8Xs7SnsP2IBwg6kMsiGPoCK0TlvhyKX94XmOEs,1
 redisbench_admin/run/cluster.py,sha256=_Y6a8Dbu1cJ7OxhgymKQSZcCmV8cZ3UpGEWL6b6O84Y,6363
 redisbench_admin/run/common.py,sha256=1zJPQtIk3tjvLTjNUgxvDdcviN73psiaDBQ8cRJMIl8,29341
 redisbench_admin/run/ftsb/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
-redisbench_admin/run/ftsb/ftsb.py,sha256=MnviWzIxMPxAw__R1lQT6tTsH7nP7t5ejCXrALm2MUw,2580
+redisbench_admin/run/ftsb/ftsb.py,sha256=0SS4InU7e9x-yZ0u-D1yOYJ0u0G37IwVWwiK78LRXE8,2712
 redisbench_admin/run/git.py,sha256=6UYGcTN0MPzf4QDVoJnFkou0yZasLF6jLG7f0zoySq8,3064
 redisbench_admin/run/grafana.py,sha256=iMDgMyJKinpZMTD43rZ1IcRGkadjFjCxaB48mYWkvG4,9421
 redisbench_admin/run/memtier_benchmark/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
@@ -67,7 +67,7 @@ redisbench_admin/run/redisgraph_benchmark_go/redisgraph_benchmark_go.py,sha256=D
 redisbench_admin/run/redistimeseries.py,sha256=x3PA7QoHXu53zs5v0ekK2sVmUnA9_ZF2JxgCDf1Mui4,21331
 redisbench_admin/run/run.py,sha256=AwE4_qkejAXOcDVFmwHjVqYjqi1BFZKdGNrCKBHkAHI,6030
 redisbench_admin/run/s3.py,sha256=pXQXZ1rrwDCWeBegGR4aKzbKqWWwMrmqvIjFxEB3bh4,442
-redisbench_admin/run/ssh.py,sha256=gRW6ROoTKlaxLKhS5tM-Ejjd6zk2iO1KN9dzBzw7GOk,2835
+redisbench_admin/run/ssh.py,sha256=WinLylTbLsS8Bgqtiko1IDuD1OqHMU81nVAt8mSwQUI,2926
 redisbench_admin/run/tsbs_run_queries_redistimeseries/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
 redisbench_admin/run/tsbs_run_queries_redistimeseries/tsbs_run_queries_redistimeseries.py,sha256=CfXcoLS25pEkhGtka82JwgDZVPV4w2lU_VhI-reH5h4,2514
 redisbench_admin/run/ycsb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -94,24 +94,24 @@ redisbench_admin/run_remote/remote_client.py,sha256=rRmDro1weto01wzqYpId8NMPoizE
 redisbench_admin/run_remote/remote_db.py,sha256=EEDeiOZk-godr5EINscEkOJLGWUN3gFfH6RaBzAKbak,14566
 redisbench_admin/run_remote/remote_env.py,sha256=Ux_0QT1unNRlKl3cakzjG5Px1uuxOOfBoF_pnalx_T8,4936
 redisbench_admin/run_remote/remote_failures.py,sha256=IOo6DyxarcwwMPCeN4gWB2JrhuC9iBLwq0nCROqr5ak,1567
-redisbench_admin/run_remote/remote_helpers.py,sha256=skWeGyDJBmyx_UwUekT3N3_nOJvF2-Hvu-E7vKlO9gg,10598
+redisbench_admin/run_remote/remote_helpers.py,sha256=6rbENEUs62ZAqXDVgLMfh9o8gttcG0bG0lXE0HNdFeo,13271
 redisbench_admin/run_remote/run_remote.py,sha256=tZqCu1fTfB5gWooVIEsSDoaVfnVRfxeCpn-RLmYI3IM,75476
 redisbench_admin/run_remote/standalone.py,sha256=WRgiY8oMJwDUPht1LAPG9tnOLvhJmhRvVOi3GBnMoX4,27952
 redisbench_admin/run_remote/terraform.py,sha256=vV3eWXNwj7vsnFNqUgCir5ueZS4VYopEyzWiTtoSq0Q,4018
 redisbench_admin/utils/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
-redisbench_admin/utils/benchmark_config.py,sha256=bC2C6rnj89wkkSlOXyyfe0N15unn_M1t1zfskfVkb98,21387
-redisbench_admin/utils/local.py,sha256=zUvyVI9LZMT3qyxs1pO3mXL6Bt_1z9EZUGppaRcWNRA,3890
+redisbench_admin/utils/benchmark_config.py,sha256=SREplEGIfPtR9NKIjJXb5pug-ikbJDsDMpZ9UM5sg0M,22676
+redisbench_admin/utils/local.py,sha256=uVU91Bvk3FAaDGmTIbI0zXYd1K24A9-Xwm15xI_SYC4,4086
 redisbench_admin/utils/redisearch.py,sha256=lchUEzpt0zB1rHwlDlw9LLifAnxFWcLP-PePw7TjL-0,1602
 redisbench_admin/utils/redisgraph_benchmark_go.py,sha256=os7EJt6kBxsFJLKkSoANbjMT7-cEq4-Ns-49alk2Tf8,2048
-redisbench_admin/utils/remote.py,sha256=RAQ2VxfmlK7swN7ujCuwSI2soGSycjnxbQw_IrLxIFE,42205
+redisbench_admin/utils/remote.py,sha256=MsRL6x-eekzJOVE6Ho05igZzUX54m-4fRVCQlSYxyw0,42306
 redisbench_admin/utils/results.py,sha256=uKk3uNJ--bSXlUj_HGQ2OaV6MVqmXJVM8xTzFV6EOw4,3267
 redisbench_admin/utils/ssh.py,sha256=QW4AwlocMHJt05QMdN_4f8WeDmxiEwR80ny8VBThq6k,6533
 redisbench_admin/utils/utils.py,sha256=XVSvo1_DdcYwk2jOxL3VPVPbnDnhGYt8ieYfANo6rTo,15085
 redisbench_admin/watchdog/__init__.py,sha256=cD7zfXt0VEmy0b7452HvcAxX_9kVj6Vm213yNdUHP20,95
 redisbench_admin/watchdog/args.py,sha256=nKsG1G6ATOZlAMHMtT9u3kXxduKCbejSZ5x8oB_ynZ8,1312
 redisbench_admin/watchdog/watchdog.py,sha256=0wWYge3x_OMxWrzazNhJif2NK4tKsI963HVZqjczRag,6189
-redisbench_admin-0.11.56.dist-info/METADATA,sha256=wxAuRSDGZqqFHiamBoQhtv8Zh3sX3yHa1XzCdEACoSo,5724
-redisbench_admin-0.11.56.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-redisbench_admin-0.11.56.dist-info/entry_points.txt,sha256=UUawXk_AS-PlieKJ1QxPQXGsRLb6OW_F0MtmA1W0KE8,113
-redisbench_admin-0.11.56.dist-info/licenses/LICENSE,sha256=AAMtfs82zOOvmG68vILivm6lxi2rcOlGObmA8jzxQvw,10768
-redisbench_admin-0.11.56.dist-info/RECORD,,
+redisbench_admin-0.11.58.dist-info/METADATA,sha256=c-SW0SwVQ3NXU7l69rv4zHC-J9DpCbePfKteRNPVYvU,5724
+redisbench_admin-0.11.58.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+redisbench_admin-0.11.58.dist-info/entry_points.txt,sha256=UUawXk_AS-PlieKJ1QxPQXGsRLb6OW_F0MtmA1W0KE8,113
+redisbench_admin-0.11.58.dist-info/licenses/LICENSE,sha256=AAMtfs82zOOvmG68vILivm6lxi2rcOlGObmA8jzxQvw,10768
+redisbench_admin-0.11.58.dist-info/RECORD,,