redisbench-admin 0.11.37__py3-none-any.whl → 0.11.39__py3-none-any.whl

redisbench_admin/deploy/deploy.py

@@ -73,15 +73,7 @@ def deploy_command_logic(args, project_name, project_version):
     tf_triggering_env = "redisbench-admin-deploy"
     logging.info("Setting an infra timeout of {} secs".format(infra_timeout_secs))
     if args.destroy is False:
-        (
-            tf_return_code,
-            _,
-            _,
-            _,
-            _,
-            _,
-            _,
-        ) = setup_remote_environment(
+        (tf_return_code, _, _, _, _, _, _,) = setup_remote_environment(
             tf,
             tf_github_sha,
             tf_github_actor,

redisbench_admin/export/export.py

@@ -42,13 +42,7 @@ def export_command_logic(args, project_name, project_version):
     deployment_name = args.deployment_name
     deployment_type = args.deployment_type
     results_format = args.results_format
-    (
-        _,
-        github_branch,
-        github_org,
-        github_repo,
-        _,
-    ) = git_vars_crosscheck(
+    (_, github_branch, github_org, github_repo, _,) = git_vars_crosscheck(
         None, args.github_branch, args.github_org, args.github_repo, None
     )
     exporter_timemetric_path = None

redisbench_admin/profilers/perf.py

@@ -400,9 +400,9 @@ class Perf:
             "Main THREAD Flame Graph: " + use_case, details
         )
         if artifact_result is True:
-            outputs["Main THREAD Flame Graph {}".format(identifier)] = (
-                flame_graph_output
-            )
+            outputs[
+                "Main THREAD Flame Graph {}".format(identifier)
+            ] = flame_graph_output
         result &= artifact_result
 
         tid = self.pid
@@ -440,9 +440,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report per dso,sym
@@ -460,9 +460,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym with callgraph {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym with callgraph {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report per dso,sym,srcline
@@ -487,9 +487,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report per dso,sym,srcline {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report per dso,sym,srcline {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         self.logger.info(
@@ -527,9 +527,9 @@ class Perf:
         )
 
         if artifact_result is True:
-            outputs["perf report top self-cpu {}".format(identifier)] = (
-                perf_report_artifact
-            )
+            outputs[
+                "perf report top self-cpu {}".format(identifier)
+            ] = perf_report_artifact
         result &= artifact_result
 
         # generate perf report --stdio report
@@ -546,9 +546,9 @@ class Perf:
             )
 
             if artifact_result is True:
-                outputs["perf report top self-cpu (dso={})".format(binary)] = (
-                    perf_report_artifact
-                )
+                outputs[
+                    "perf report top self-cpu (dso={})".format(binary)
+                ] = perf_report_artifact
         result &= artifact_result
 
         if self.callgraph_mode == "dwarf":
@@ -590,9 +590,9 @@ class Perf:
                 )
                 result &= artifact_result
                 if artifact_result is True:
-                    outputs["Top entries in text form by LOC"] = (
-                        pprof_artifact_text_output
-                    )
+                    outputs[
+                        "Top entries in text form by LOC"
+                    ] = pprof_artifact_text_output
                     tabular_data_map["text-lines"] = tabular_data
                 self.logger.info("Generating pprof png output")
                 pprof_png_output = self.output + ".pprof.png"
@@ -604,9 +604,9 @@ class Perf:
                     self.output,
                 )
                 if artifact_result is True:
-                    outputs["Output graph image in PNG format"] = (
-                        pprof_artifact_png_output
-                    )
+                    outputs[
+                        "Output graph image in PNG format"
+                    ] = pprof_artifact_png_output
                 result &= artifact_result
 
         # save stack collapsed

redisbench_admin/run/cluster.py

@@ -115,6 +115,12 @@ def spin_up_redis_cluster_remote_redis(
     logname,
     redis_7=True,
 ):
+    # Import the function from standalone module
+    from redisbench_admin.run_remote.standalone import ensure_redis_server_available
+
+    # Ensure redis-server is available before trying to start cluster
+    ensure_redis_server_available(server_public_ip, username, private_key, ssh_port)
+
     logging.info("Generating the remote redis-server command arguments")
     redis_process_commands = []
     logfiles = []

redisbench_admin/run/common.py

@@ -206,10 +206,7 @@ def prepare_benchmark_parameters_specif_tooling(
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
             input_data_file = "/tmp/input.data"
-        (
-            command_arr,
-            command_str,
-        ) = prepare_tsbs_benchmark_command(
+        (command_arr, command_str,) = prepare_tsbs_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -221,10 +218,7 @@ def prepare_benchmark_parameters_specif_tooling(
             cluster_api_enabled,
         )
     if "memtier_benchmark" in benchmark_tool:
-        (
-            command_arr,
-            command_str,
-        ) = prepare_memtier_benchmark_command(
+        (command_arr, command_str,) = prepare_memtier_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -242,10 +236,7 @@ def prepare_benchmark_parameters_specif_tooling(
             ann_path = stdout[0].strip() + "/run/ann/pkg/multirun.py"
             logging.info("Remote ann-benchmark path: {}".format(ann_path))
 
-        (
-            command_arr,
-            command_str,
-        ) = prepare_ann_benchmark_command(
+        (command_arr, command_str,) = prepare_ann_benchmark_command(
             server_private_ip,
             server_plaintext_port,
             cluster_api_enabled,
@@ -259,10 +250,7 @@ def prepare_benchmark_parameters_specif_tooling(
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
             input_data_file = "/tmp/input.data"
-        (
-            command_arr,
-            command_str,
-        ) = prepare_ftsb_benchmark_command(
+        (command_arr, command_str,) = prepare_ftsb_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -279,10 +267,7 @@ def prepare_benchmark_parameters_specif_tooling(
         if isremote is True:
             benchmark_tool = "/tmp/{}".format(benchmark_tool)
             input_data_file = "/tmp/input.data"
-        (
-            command_arr,
-            command_str,
-        ) = prepare_aibench_benchmark_command(
+        (command_arr, command_str,) = prepare_aibench_benchmark_command(
             benchmark_tool,
             server_private_ip,
             server_plaintext_port,
@@ -787,10 +772,7 @@ def print_results_table_stdout(
     metric_names=[],
 ):
     # check which metrics to extract
-    (
-        _,
-        metrics,
-    ) = merge_default_and_config_metrics(
+    (_, metrics,) = merge_default_and_config_metrics(
         benchmark_config,
         default_metrics,
         None,

redisbench_admin/run_async/async_terraform.py

@@ -114,11 +114,7 @@ class TerraformClass:
     def async_runner_setup(
         self,
     ):
-        (
-            remote_setup,
-            deployment_type,
-            remote_id,
-        ) = fetch_remote_setup_from_config(
+        (remote_setup, deployment_type, remote_id,) = fetch_remote_setup_from_config(
             [{"type": "async", "setup": "runner"}],
             "https://github.com/RedisLabsModules/testing-infrastructure.git",
             "master",
@@ -233,11 +229,7 @@ def terraform_spin_or_reuse_env(
     tf_override_name=None,
     tf_folder_path=None,
 ):
-    (
-        remote_setup,
-        deployment_type,
-        remote_id,
-    ) = fetch_remote_setup_from_config(
+    (remote_setup, deployment_type, remote_id,) = fetch_remote_setup_from_config(
         benchmark_config["remote"],
         "https://github.com/RedisLabsModules/testing-infrastructure.git",
         "master",

redisbench_admin/run_async/render_files.py

@@ -28,9 +28,9 @@ WantedBy=multi-user.target
         argv.append("--private_key")
         argv.append("/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem")
     else:
-        argv[argv.index(args.private_key)] = (
-            "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
-        )
+        argv[
+            argv.index(args.private_key)
+        ] = "/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
     if len(args.module_path) != 0:
         argv[argv.index(args.module_path[0])] = (
             "/home/ubuntu/work_dir/tests/benchmarks/"

redisbench_admin/run_local/run_local.py

@@ -687,17 +687,17 @@ def commandstats_latencystats_process_name(
             branch = variant_labels_dict["branch"]
 
         if version is not None:
-            variant_labels_dict["command_and_metric_and_version"] = (
-                "{} - {} - {}".format(command, metric, version)
-            )
-            variant_labels_dict["command_and_metric_and_setup_and_version"] = (
-                "{} - {} - {} - {}".format(command, metric, setup_name, version)
-            )
+            variant_labels_dict[
+                "command_and_metric_and_version"
+            ] = "{} - {} - {}".format(command, metric, version)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_version"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, version)
 
         if branch is not None:
-            variant_labels_dict["command_and_metric_and_branch"] = (
-                "{} - {} - {}".format(command, metric, branch)
-            )
-            variant_labels_dict["command_and_metric_and_setup_and_branch"] = (
-                "{} - {} - {} - {}".format(command, metric, setup_name, branch)
-            )
+            variant_labels_dict[
+                "command_and_metric_and_branch"
+            ] = "{} - {} - {}".format(command, metric, branch)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_branch"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, branch)

redisbench_admin/run_remote/remote_db.py

@@ -406,9 +406,17 @@ def db_error_artifacts(
     upload_s3,
     username,
 ):
+    # Import the zip check function
+    from redisbench_admin.run_remote.standalone import ensure_zip_available
+
+    # Ensure zip is available before trying to use it
+    ensure_zip_available(server_public_ip, username, private_key, db_ssh_port)
+
     local_zipfile = "{}.zip".format(logname)
     remote_zipfile = "/home/{}/{}".format(username, local_zipfile)
-    execute_remote_commands(
+
+    # Create zip file
+    zip_result = execute_remote_commands(
         server_public_ip,
         username,
         private_key,
@@ -417,26 +425,57 @@ def db_error_artifacts(
         ],
         db_ssh_port,
     )
-    failed_remote_run_artifact_store(
-        upload_s3,
-        server_public_ip,
-        dirname,
-        remote_zipfile,
-        local_zipfile,
-        s3_bucket_name,
-        s3_bucket_path,
-        username,
-        private_key,
-    )
+
+    # Check if zip creation was successful
+    zip_success = True
+    for pos, res_pos in enumerate(zip_result):
+        [recv_exit_status, stdout, stderr] = res_pos
+        if recv_exit_status != 0:
+            logging.warning(
+                "Zip creation failed with exit code {}. stdout: {}. stderr: {}".format(
+                    recv_exit_status, stdout, stderr
+                )
+            )
+            zip_success = False
+
+    # Only try to upload if zip was created successfully
+    if zip_success:
+        try:
+            failed_remote_run_artifact_store(
+                upload_s3,
+                server_public_ip,
+                dirname,
+                remote_zipfile,
+                local_zipfile,
+                s3_bucket_name,
+                s3_bucket_path,
+                username,
+                private_key,
+            )
+        except Exception as e:
+            logging.warning(
+                "Failed to upload zip file to S3: {}. Continuing without upload.".format(
+                    e
+                )
+            )
+    else:
+        logging.warning("Skipping S3 upload due to zip creation failure")
     if len(full_logfiles) > 0:
-        failed_remote_run_artifact_store(
-            upload_s3,
-            server_public_ip,
-            dirname,
-            full_logfiles[0],
-            logname,
-            s3_bucket_name,
-            s3_bucket_path,
-            username,
-            private_key,
-        )
+        try:
+            failed_remote_run_artifact_store(
+                upload_s3,
+                server_public_ip,
+                dirname,
+                full_logfiles[0],
+                logname,
+                s3_bucket_name,
+                s3_bucket_path,
+                username,
+                private_key,
+            )
+        except Exception as e:
+            logging.warning(
+                "Failed to upload logfile to S3: {}. Continuing without upload.".format(
+                    e
+                )
+            )

redisbench_admin/run_remote/remote_helpers.py

@@ -71,6 +71,23 @@ def remote_tool_pre_bench_step(
     logging.info(
         f"Settting up remote tool {benchmark_tool} requirements. architecture ={architecture}"
     )
+
+    # Check and install benchmark tools if needed
+    if benchmark_tool == "memtier_benchmark":
+        from redisbench_admin.run_remote.standalone import (
+            ensure_memtier_benchmark_available,
+        )
+
+        ensure_memtier_benchmark_available(
+            client_public_ip, username, private_key, client_ssh_port
+        )
+    elif benchmark_tool == "redis-benchmark":
+        from redisbench_admin.run_remote.standalone import ensure_redis_server_available
+
+        # redis-benchmark comes with redis-server, so ensure redis-server is installed
+        ensure_redis_server_available(
+            client_public_ip, username, private_key, client_ssh_port
+        )
     if benchmark_tool == "redisgraph-benchmark-go":
         setup_remote_benchmark_tool_redisgraph_benchmark_go(
             client_public_ip,
@@ -89,11 +106,7 @@ def remote_tool_pre_bench_step(
         )
 
     if "ftsb_" in benchmark_tool:
-        (
-            queries_file_link,
-            remote_tool_link,
-            tool_link,
-        ) = extract_ftsb_extra_links(
+        (queries_file_link, remote_tool_link, tool_link,) = extract_ftsb_extra_links(
             benchmark_config, benchmark_tool, config_key, architecture
         )
         logging.info(

redisbench_admin/run_remote/run_remote.py

@@ -161,7 +161,24 @@ def run_remote_command_logic(args, project_name, project_version):
         )
         webhook_client_slack = WebhookClient(webhook_url)
 
-    if args.skip_env_vars_verify is False:
+    # Only check AWS credentials when actually needed
+    needs_aws_for_infrastructure = (
+        args.inventory is None
+    )  # No inventory = need to deploy with Terraform
+    needs_aws_for_s3 = args.upload_results_s3  # S3 upload enabled
+
+    if args.skip_env_vars_verify is False and (
+        needs_aws_for_infrastructure or needs_aws_for_s3
+    ):
+        # Log why AWS credentials are being checked
+        aws_reasons = []
+        if needs_aws_for_infrastructure:
+            aws_reasons.append("infrastructure deployment (no --inventory provided)")
+        if needs_aws_for_s3:
+            aws_reasons.append("S3 upload (--upload_results_s3 enabled)")
+
+        logging.info("AWS credentials required for: {}".format(", ".join(aws_reasons)))
+
         env_check_status, failure_reason = check_ec2_env()
         if env_check_status is False:
             if webhook_notifications_active:
@@ -177,6 +194,10 @@ def run_remote_command_logic(args, project_name, project_version):
                 )
             logging.critical("{}. Exiting right away!".format(failure_reason))
             exit(1)
+    elif args.skip_env_vars_verify is False:
+        logging.info(
+            "AWS credentials check skipped (using --inventory and S3 upload disabled)"
+        )
 
     continue_on_module_check_error = args.continue_on_module_check_error
     module_check_status, error_message = redis_modules_check(local_module_files)
@@ -1408,20 +1429,20 @@ def commandstats_latencystats_process_name(
             branch = variant_labels_dict["branch"]
 
         if version is not None:
-            variant_labels_dict["command_and_metric_and_version"] = (
-                "{} - {} - {}".format(command, metric, version)
-            )
-            variant_labels_dict["command_and_metric_and_setup_and_version"] = (
-                "{} - {} - {} - {}".format(command, metric, setup_name, version)
-            )
+            variant_labels_dict[
+                "command_and_metric_and_version"
+            ] = "{} - {} - {}".format(command, metric, version)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_version"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, version)
 
         if branch is not None:
-            variant_labels_dict["command_and_metric_and_branch"] = (
-                "{} - {} - {}".format(command, metric, branch)
-            )
-            variant_labels_dict["command_and_metric_and_setup_and_branch"] = (
-                "{} - {} - {} - {}".format(command, metric, setup_name, branch)
-            )
+            variant_labels_dict[
+                "command_and_metric_and_branch"
+            ] = "{} - {} - {}".format(command, metric, branch)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_branch"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, branch)
 
 
 def shutdown_remote_redis(redis_conns, ssh_tunnel):

redisbench_admin/run_remote/standalone.py

@@ -15,6 +15,139 @@ from redisbench_admin.utils.ssh import SSHSession
 from redisbench_admin.utils.utils import redis_server_config_module_part
 
 
+def ensure_redis_server_available(server_public_ip, username, private_key, port=22):
+    """Check if redis-server is available, install if not"""
+    logging.info("Checking if redis-server is available on remote server...")
+
+    # Check if redis-server exists
+    check_result = execute_remote_commands(
+        server_public_ip, username, private_key, ["which redis-server"], port
+    )
+
+    # Check the result
+    if len(check_result) > 0:
+        [recv_exit_status, stdout, stderr] = check_result[0]
+        if recv_exit_status != 0:
+            logging.info("redis-server not found, installing Redis...")
+
+            # Install Redis using the provided commands
+            install_commands = [
+                "sudo apt-get install lsb-release curl gpg -y",
+                "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg",
+                "sudo chmod 644 /usr/share/keyrings/redis-archive-keyring.gpg",
+                'echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list',
+                "sudo apt-get update",
+                "sudo apt-get install redis -y",
+                "sudo systemctl disable redis-server",
+            ]
+
+            install_result = execute_remote_commands(
+                server_public_ip, username, private_key, install_commands, port
+            )
+
+            # Check if installation was successful
+            for pos, res_pos in enumerate(install_result):
+                [recv_exit_status, stdout, stderr] = res_pos
+                if recv_exit_status != 0:
+                    logging.warning(
+                        "Redis installation command {} returned exit code {}. stdout: {}. stderr: {}".format(
+                            pos, recv_exit_status, stdout, stderr
+                        )
+                    )
+
+            logging.info("Redis installation completed and auto-start disabled")
+        else:
+            logging.info("redis-server is already available")
+    else:
+        logging.error("Failed to check redis-server availability")
+
+
+def ensure_zip_available(server_public_ip, username, private_key, port=22):
+    """Check if zip is available, install if not"""
+    logging.info("Checking if zip is available on remote server...")
+
+    # Check if zip exists
+    check_result = execute_remote_commands(
+        server_public_ip, username, private_key, ["which zip"], port
+    )
+
+    # Check the result
+    if len(check_result) > 0:
+        [recv_exit_status, stdout, stderr] = check_result[0]
+        if recv_exit_status != 0:
+            logging.info("zip not found, installing...")
+
+            # Install zip
+            install_commands = ["sudo apt-get install zip -y"]
+
+            install_result = execute_remote_commands(
+                server_public_ip, username, private_key, install_commands, port
+            )
+
+            # Check if installation was successful
+            for pos, res_pos in enumerate(install_result):
+                [recv_exit_status, stdout, stderr] = res_pos
+                if recv_exit_status != 0:
+                    logging.warning(
+                        "Zip installation command {} returned exit code {}. stdout: {}. stderr: {}".format(
+                            pos, recv_exit_status, stdout, stderr
+                        )
+                    )
+
+            logging.info("Zip installation completed")
+        else:
+            logging.info("zip is already available")
+    else:
+        logging.error("Failed to check zip availability")
+
+
+def ensure_memtier_benchmark_available(
+    client_public_ip, username, private_key, port=22
+):
+    """Check if memtier_benchmark is available, install if not"""
+    logging.info("Checking if memtier_benchmark is available on remote client...")
+
+    # Check if memtier_benchmark exists
+    check_result = execute_remote_commands(
+        client_public_ip, username, private_key, ["which memtier_benchmark"], port
+    )
+
+    # Check the result
+    if len(check_result) > 0:
+        [recv_exit_status, stdout, stderr] = check_result[0]
+        if recv_exit_status != 0:
+            logging.info("memtier_benchmark not found, installing...")
+
+            # Install memtier_benchmark using the provided commands
+            install_commands = [
+                "sudo apt install lsb-release curl gpg -y",
+                "curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg",
+                'echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list',
+                "sudo apt-get update",
+                "sudo apt-get install memtier-benchmark -y",
+            ]
+
+            install_result = execute_remote_commands(
+                client_public_ip, username, private_key, install_commands, port
+            )
+
+            # Check if installation was successful
+            for pos, res_pos in enumerate(install_result):
+                [recv_exit_status, stdout, stderr] = res_pos
+                if recv_exit_status != 0:
+                    logging.warning(
+                        "memtier_benchmark installation command {} returned exit code {}. stdout: {}. stderr: {}".format(
+                            pos, recv_exit_status, stdout, stderr
+                        )
+                    )
+
+            logging.info("memtier_benchmark installation completed")
+        else:
+            logging.info("memtier_benchmark is already available")
+    else:
+        logging.error("Failed to check memtier_benchmark availability")
+
+
 def spin_up_standalone_remote_redis(
     temporary_dir,
     server_public_ip,
@@ -27,6 +160,9 @@ def spin_up_standalone_remote_redis(
     modules_configuration_parameters_map={},
     redis_7=True,
 ):
+    # Ensure redis-server is available before trying to start it
+    ensure_redis_server_available(server_public_ip, username, private_key, port)
+
     full_logfile, initial_redis_cmd = generate_remote_standalone_redis_cmd(
         logfile,
         redis_configuration_parameters,

redisbench_admin/run_remote/terraform.py

@@ -32,11 +32,7 @@ def terraform_spin_or_reuse_env(
     tf_folder_path=None,
     architecture="x86_64",
 ):
-    (
-        remote_setup,
-        deployment_type,
-        remote_id,
-    ) = fetch_remote_setup_from_config(
+    (remote_setup, deployment_type, remote_id,) = fetch_remote_setup_from_config(
         benchmark_config["remote"],
         "https://github.com/redis-performance/testing-infrastructure.git",
         "master",