redis-benchmarks-specification 0.1.101__py3-none-any.whl → 0.1.150__py3-none-any.whl

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -11,6 +11,7 @@ import redis
 import os
 from pathlib import Path
 import sys
+import time
 
 from docker.models.containers import Container
 from redisbench_admin.profilers.profilers_local import (
@@ -43,6 +44,9 @@ from redis_benchmarks_specification.__common__.runner import (
     exporter_datasink_common,
     execute_init_commands,
 )
+from redis_benchmarks_specification.__common__.timeseries import (
+    datasink_profile_tabular_data,
+)
 from redis_benchmarks_specification.__compare__.compare import (
     compute_regression_table,
     prepare_regression_comment,
@@ -72,13 +76,10 @@ from redisbench_admin.run.common import (
     prepare_benchmark_parameters,
 )
 from redisbench_admin.run.grafana import generate_artifacts_table_grafana_redis
-from redisbench_admin.run.redistimeseries import (
-    datasink_profile_tabular_data,
-)
+
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
 from redisbench_admin.utils.benchmark_config import (
     get_final_benchmark_config,
-    extract_redis_dbconfig_parameters,
     get_defaults,
 )
 from redisbench_admin.utils.local import get_local_run_full_filename
@@ -93,6 +94,8 @@ from redis_benchmarks_specification.__common__.spec import (
     extract_client_cpu_limit,
     extract_client_tool,
     extract_client_container_image,
+    extract_redis_dbconfig_parameters,
+    extract_redis_configuration_from_topology,
 )
 from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import (
     restore_build_artifacts_from_test_details,
@@ -229,6 +232,7 @@ def main():
 
     defaults_filename = args.defaults_filename
     (
+        _,
         _,
         default_metrics,
         _,
@@ -308,6 +312,12 @@ def main():
         )
 
 
+def check_health(container):
+    logging.info(container.attrs["State"])
+    health_status = container.attrs["State"].get("Health", {}).get("Status")
+    return health_status
+
+
 def self_contained_coordinator_blocking_read(
     github_event_conn,
     datasink_push_results_redistimeseries,
@@ -333,6 +343,8 @@ def self_contained_coordinator_blocking_read(
     priority_upper_limit=10000,
     default_baseline_branch="unstable",
     default_metrics_str="ALL_STATS.Totals.Ops/sec",
+    docker_keep_env=False,
+    restore_build_artifacts_default=True,
 ):
     num_process_streams = 0
     num_process_test_suites = 0
@@ -384,6 +396,8 @@ def self_contained_coordinator_blocking_read(
             priority_upper_limit,
             default_baseline_branch,
             default_metrics_str,
+            docker_keep_env,
+            restore_build_artifacts_default,
         )
         num_process_streams = num_process_streams + 1
         num_process_test_suites = num_process_test_suites + total_test_suite_runs
@@ -397,7 +411,7 @@ def self_contained_coordinator_blocking_read(
                 ack_reply = ack_reply.decode()
             if ack_reply == "1" or ack_reply == 1:
                 logging.info(
-                    "Sucessfully acknowledge build variation stream with id {}.".format(
+                    "Sucessfully acknowledge BENCHMARK variation stream with id {}.".format(
                         stream_id
                     )
                 )
@@ -452,7 +466,7 @@ def process_self_contained_coordinator_stream(
     grafana_profile_dashboard="",
     cpuset_start_pos=0,
     redis_proc_start_port=6379,
-    docker_air_gap=False,
+    default_docker_air_gap=False,
     defaults_filename="defaults.yml",
     override_test_time=0,
     default_metrics=[],
@@ -462,6 +476,8 @@ def process_self_contained_coordinator_stream(
     priority_upper_limit=10000,
     default_baseline_branch="unstable",
     default_metrics_str="ALL_STATS.Totals.Ops/sec",
+    docker_keep_env=False,
+    restore_build_artifacts_default=True,
 ):
     stream_id = "n/a"
     overall_result = False
@@ -475,12 +491,17 @@ def process_self_contained_coordinator_stream(
     regression_comment = None
     pull_request = None
     auto_approve_github = True
+    # defaults
+    default_github_org = "redis"
+    default_github_repo = "redis"
+    restore_build_artifacts = restore_build_artifacts_default
+
     try:
         stream_id, testDetails = newTestInfo[0][1][0]
         stream_id = stream_id.decode()
         logging.info("Received work . Stream id {}.".format(stream_id))
 
-        if b"git_hash" in testDetails:
+        if b"run_image" in testDetails:
             (
                 build_variant_name,
                 metadata,
@@ -493,6 +514,60 @@ def process_self_contained_coordinator_stream(
                 git_timestamp_ms,
                 run_arch,
             ) = extract_build_info_from_streamdata(testDetails)
+            tf_github_org = default_github_org
+            if b"github_org" in testDetails:
+                tf_github_org = testDetails[b"github_org"].decode()
+                logging.info(
+                    f"detected a github_org definition on the streamdata: {tf_github_org}. Overriding the default one: {default_github_org}"
+                )
+            tf_github_repo = default_github_repo
+            if b"github_repo" in testDetails:
+                tf_github_repo = testDetails[b"github_repo"].decode()
+                logging.info(
+                    f"detected a github_org definition on the streamdata: {tf_github_repo}. Overriding the default one: {default_github_repo}"
+                )
+
+            mnt_point = "/mnt/redis/"
+            if b"mnt_point" in testDetails:
+                mnt_point = testDetails[b"mnt_point"].decode()
+                logging.info(
+                    f"detected a mnt_point definition on the streamdata: {mnt_point}."
+                )
+
+            executable = f"{mnt_point}redis-server"
+            if b"executable" in testDetails:
+                executable = testDetails[b"executable"].decode()
+                logging.info(
+                    f"detected a executable definition on the streamdata: {executable}."
+                )
+
+            server_name = "redis"
+            if b"server_name" in testDetails:
+                server_name = testDetails[b"server_name"].decode()
+                logging.info(
+                    f"detected a server_name definition on the streamdata: {server_name}."
+                )
+                new_executable = f"{mnt_point}{server_name}-server"
+                logging.info(
+                    "changing executable from {executable} to {new_executable}"
+                )
+                executable = new_executable
+
+            if b"restore_build_artifacts" in testDetails:
+                restore_build_artifacts = bool(
+                    testDetails[b"restore_build_artifacts"].decode()
+                )
+                logging.info(
+                    f"detected a restore_build_artifacts config {restore_build_artifacts} overriding the default just for this test"
+                )
+
+            test_docker_air_gap = default_docker_air_gap
+            # check if we override the docker air gap on this test details
+            if b"docker_air_gap" in testDetails:
+                test_docker_air_gap = bool(testDetails[b"docker_air_gap"].decode())
+                logging.info(
+                    f"detected a docker air gap config {test_docker_air_gap} overriding the default of {default_docker_air_gap} just for this test"
+                )
 
             if b"priority_upper_limit" in testDetails:
                 stream_priority_upper_limit = int(
@@ -530,11 +605,18 @@ def process_self_contained_coordinator_stream(
                     fn, github_token, pull_request, "redis", "redis", verbose
                 )
 
-            test_regexp = ".*"
-            if b"test_regexp" in testDetails:
-                test_regexp = testDetails[b"test_regexp"]
+            tests_regexp = ".*"
+            if b"tests_regexp" in testDetails:
+                tests_regexp = testDetails[b"tests_regexp"].decode()
                 logging.info(
-                    f"detected a regexp definition on the streamdata {test_regexp}"
+                    f"detected a regexp definition on the streamdata {tests_regexp}"
+                )
+
+            command_groups_regexp = None
+            if b"tests_groups_regexp" in testDetails:
+                command_groups_regexp = testDetails[b"tests_groups_regexp"].decode()
+                logging.info(
+                    f"detected a command groups regexp definition on the streamdata {command_groups_regexp}"
                 )
 
             skip_test = False
@@ -559,7 +641,7 @@ def process_self_contained_coordinator_stream(
             if skip_test is False:
                 overall_result = True
                 profiler_dashboard_links = []
-                if docker_air_gap:
+                if test_docker_air_gap:
                     airgap_key = "docker:air-gap:{}".format(run_image)
                     logging.info(
                         "Restoring docker image: {} from {}".format(
@@ -577,7 +659,7 @@ def process_self_contained_coordinator_stream(
                     {stream_id: stream_time_ms},
                 )
                 logging.info(
-                    f"Added stream with id {stream_id} to zset {zset_running_platform_benchmarks}"
+                    f"Added stream with id {stream_id} to zset {zset_running_platform_benchmarks}. res={res}"
                 )
 
                 stream_test_list_pending = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_pending"
@@ -589,8 +671,9 @@ def process_self_contained_coordinator_stream(
                     defaults_filename,
                     priority_lower_limit,
                     priority_upper_limit,
-                    test_regexp,
+                    tests_regexp,
                     testsuite_spec_files,
+                    command_groups_regexp,
                 )
 
                 for test_file in filtered_test_files:
@@ -599,7 +682,7 @@ def process_self_contained_coordinator_stream(
                             _,
                             benchmark_config,
                             test_name,
-                        ) = get_final_benchmark_config(None, stream, "")
+                        ) = get_final_benchmark_config(None, None, stream, "")
                     github_event_conn.lpush(stream_test_list_pending, test_name)
                     github_event_conn.expire(
                         stream_test_list_pending, REDIS_BINS_EXPIRE_SECS
@@ -611,16 +694,16 @@ def process_self_contained_coordinator_stream(
                 pending_tests = len(filtered_test_files)
                 failed_tests = 0
                 benchmark_suite_start_datetime = datetime.datetime.utcnow()
-                comment_body = generate_benchmark_started_pr_comment(
-                    stream_id,
-                    pending_tests,
-                    len(filtered_test_files),
-                    failed_tests,
-                    benchmark_suite_start_datetime,
-                    0,
-                )
                 # update on github if needed
                 if is_actionable_pr:
+                    comment_body = generate_benchmark_started_pr_comment(
+                        stream_id,
+                        pending_tests,
+                        len(filtered_test_files),
+                        failed_tests,
+                        benchmark_suite_start_datetime,
+                        0,
+                    )
                     if contains_benchmark_run_comment:
                         update_comment_if_needed(
                             auto_approve_github,
@@ -642,7 +725,7 @@ def process_self_contained_coordinator_stream(
                             _,
                             benchmark_config,
                             test_name,
-                        ) = get_final_benchmark_config(None, stream, "")
+                        ) = get_final_benchmark_config(None, None, stream, "")
                         github_event_conn.lrem(stream_test_list_pending, 1, test_name)
                         github_event_conn.lpush(stream_test_list_running, test_name)
                         github_event_conn.expire(
@@ -679,6 +762,14 @@ def process_self_contained_coordinator_stream(
                                     )
                                 )
                         for topology_spec_name in benchmark_config["redis-topologies"]:
+                            setup_name = topology_spec_name
+                            setup_type = "oss-standalone"
+                            if topology_spec_name in topologies_map:
+                                topology_spec = topologies_map[topology_spec_name]
+                                setup_type = topology_spec["type"]
+                            logging.info(
+                                f"Running topology named {topology_spec_name} of type {setup_type}"
+                            )
                             test_result = False
                             redis_container = None
                             try:
@@ -686,6 +777,11 @@ def process_self_contained_coordinator_stream(
                                 ceil_db_cpu_limit = extract_db_cpu_limit(
                                     topologies_map, topology_spec_name
                                 )
+                                redis_arguments = (
+                                    extract_redis_configuration_from_topology(
+                                        topologies_map, topology_spec_name
+                                    )
+                                )
                                 temporary_dir = tempfile.mkdtemp(dir=home)
                                 temporary_dir_client = tempfile.mkdtemp(dir=home)
                                 logging.info(
@@ -698,15 +794,12 @@ def process_self_contained_coordinator_stream(
                                         temporary_dir_client
                                     )
                                 )
-                                tf_github_org = "redis"
-                                tf_github_repo = "redis"
-                                setup_name = "oss-standalone"
-                                setup_type = "oss-standalone"
+
                                 tf_triggering_env = "ci"
                                 github_actor = "{}-{}".format(
                                     tf_triggering_env, running_platform
                                 )
-                                dso = "redis-server"
+                                dso = server_name
                                 profilers_artifacts_matrix = []
 
                                 collection_summary_str = ""
@@ -725,54 +818,61 @@ def process_self_contained_coordinator_stream(
                                             collection_summary_str
                                         )
                                     )
+                                if restore_build_artifacts:
+                                    restore_build_artifacts_from_test_details(
+                                        build_artifacts,
+                                        github_event_conn,
+                                        temporary_dir,
+                                        testDetails,
+                                    )
 
-                                restore_build_artifacts_from_test_details(
-                                    build_artifacts,
-                                    github_event_conn,
-                                    temporary_dir,
-                                    testDetails,
-                                )
-                                mnt_point = "/mnt/redis/"
                                 command = generate_standalone_redis_server_args(
-                                    "{}redis-server".format(mnt_point),
+                                    executable,
                                     redis_proc_start_port,
                                     mnt_point,
                                     redis_configuration_parameters,
+                                    redis_arguments,
                                 )
                                 command_str = " ".join(command)
                                 db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
                                     ceil_db_cpu_limit, current_cpu_pos
                                 )
-                                logging.info(
-                                    "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
-                                        run_image, db_cpuset_cpus, command_str
-                                    )
-                                )
-                                redis_container = docker_client.containers.run(
-                                    image=run_image,
-                                    volumes={
-                                        temporary_dir: {
-                                            "bind": mnt_point,
-                                            "mode": "rw",
-                                        },
-                                    },
-                                    auto_remove=True,
-                                    privileged=True,
-                                    working_dir=mnt_point,
-                                    command=command_str,
-                                    network_mode="host",
-                                    detach=True,
-                                    cpuset_cpus=db_cpuset_cpus,
-                                    pid_mode="host",
+                                redis_container = start_redis_container(
+                                    command_str,
+                                    db_cpuset_cpus,
+                                    docker_client,
+                                    mnt_point,
+                                    redis_containers,
+                                    run_image,
+                                    temporary_dir,
                                 )
-                                redis_containers.append(redis_container)
 
                                 r = redis.StrictRedis(port=redis_proc_start_port)
                                 r.ping()
                                 redis_conns = [r]
                                 reset_commandstats(redis_conns)
                                 redis_pids = []
-                                first_redis_pid = r.info()["process_id"]
+                                redis_info = r.info()
+                                first_redis_pid = redis_info["process_id"]
+                                if git_hash is None and "redis_git_sha1" in redis_info:
+                                    git_hash = redis_info["redis_git_sha1"]
+                                    if (
+                                        git_hash == "" or git_hash == 0
+                                    ) and "redis_build_id" in redis_info:
+                                        git_hash = redis_info["redis_build_id"]
+                                    logging.info(
+                                        f"Given git_hash was None, we've collected that info from the server reply. git_hash={git_hash}"
+                                    )
+
+                                server_version_keyname = f"{server_name}_version"
+                                if (
+                                    git_version is None
+                                    and server_version_keyname in redis_info
+                                ):
+                                    git_version = redis_info[server_version_keyname]
+                                    logging.info(
+                                        f"Given git_version was None, we've collected that info from the server reply key named {server_version_keyname}. git_version={git_version}"
+                                    )
                                 redis_pids.append(first_redis_pid)
                                 ceil_client_cpu_limit = extract_client_cpu_limit(
                                     benchmark_config
@@ -821,7 +921,7 @@ def process_self_contained_coordinator_stream(
                                         start_time_str,
                                         git_hash,
                                         test_name,
-                                        "oss-standalone",
+                                        setup_name,
                                     )
                                 )
                                 logging.info(
@@ -1091,6 +1191,7 @@ def process_self_contained_coordinator_stream(
                                         tf_triggering_env,
                                         topology_spec_name,
                                         default_metrics,
+                                        git_hash,
                                     )
                                     r.shutdown(save=False)
 
@@ -1132,9 +1233,10 @@ def process_self_contained_coordinator_stream(
                                                 stdout=True, stderr=True
                                             )
                                         )
+                                        redis_container.remove()
                                     except docker.errors.NotFound:
                                         logging.info(
-                                            "When trying to stop DB container with id {} and image {} it was already stopped".format(
+                                            "When trying to fetch logs from DB container with id {} and image {} it was already stopped".format(
                                                 redis_container.id,
                                                 redis_container.image,
                                             )
@@ -1146,36 +1248,41 @@ def process_self_contained_coordinator_stream(
                                 test_result = False
                             # tear-down
                             logging.info("Tearing down setup")
-                            for redis_container in redis_containers:
-                                try:
-                                    redis_container.stop()
-                                except docker.errors.NotFound:
-                                    logging.info(
-                                        "When trying to stop DB container with id {} and image {} it was already stopped".format(
-                                            redis_container.id, redis_container.image
-                                        )
-                                    )
-                                    pass
-
-                            for redis_container in client_containers:
-                                if type(redis_container) == Container:
+                            if docker_keep_env is False:
+                                for redis_container in redis_containers:
                                     try:
                                         redis_container.stop()
+                                        redis_container.remove()
                                     except docker.errors.NotFound:
                                         logging.info(
-                                            "When trying to stop Client container with id {} and image {} it was already stopped".format(
+                                            "When trying to stop DB container with id {} and image {} it was already stopped".format(
                                                 redis_container.id,
                                                 redis_container.image,
                                             )
                                         )
                                         pass
-                            logging.info(
-                                "Removing temporary dirs {} and {}".format(
-                                    temporary_dir, temporary_dir_client
+
+                                for redis_container in client_containers:
+                                    if type(redis_container) == Container:
+                                        try:
+                                            redis_container.stop()
+                                            redis_container.remove()
+                                        except docker.errors.NotFound:
+                                            logging.info(
+                                                "When trying to stop Client container with id {} and image {} it was already stopped".format(
+                                                    redis_container.id,
+                                                    redis_container.image,
+                                                )
+                                            )
+                                            pass
+                                logging.info(
+                                    "Removing temporary dirs {} and {}".format(
+                                        temporary_dir, temporary_dir_client
+                                    )
                                 )
-                            )
-                            shutil.rmtree(temporary_dir, ignore_errors=True)
-                            shutil.rmtree(temporary_dir_client, ignore_errors=True)
+
+                                shutil.rmtree(temporary_dir, ignore_errors=True)
+                                shutil.rmtree(temporary_dir_client, ignore_errors=True)
 
                             overall_result &= test_result
 
@@ -1240,9 +1347,7 @@ def process_self_contained_coordinator_stream(
                             tf_github_repo,
                             verbose,
                         )
-                        logging.info(
-                            f"Preparing regression info for the data available"
-                        )
+                        logging.info("Preparing regression info for the data available")
                         print_improvements_only = False
                         print_regressions_only = False
                         skip_unstable = False
@@ -1261,7 +1366,7 @@ def process_self_contained_coordinator_stream(
                         to_date = datetime.datetime.utcnow()
                         from_date = to_date - datetime.timedelta(days=180)
                         baseline_branch = default_baseline_branch
-                        comparison_tag = git_tag
+                        comparison_tag = git_version
                         comparison_branch = git_branch
                         to_ts_ms = None
                         from_ts_ms = None
@@ -1269,8 +1374,8 @@ def process_self_contained_coordinator_stream(
                         (
                             detected_regressions,
                             table_output,
-                            total_improvements,
-                            total_regressions,
+                            improvement_list,
+                            regressions_list,
                             total_stable,
                             total_unstable,
                             total_comparison_points,
@@ -1282,8 +1387,8 @@ def process_self_contained_coordinator_stream(
                             metric_name,
                             comparison_branch,
                             baseline_branch,
-                            baseline_tag,
-                            comparison_tag,
+                            None,  # we only compare by branch on CI automation
+                            None,  # we only compare by branch on CI automation
                             baseline_deployment_name,
                             comparison_deployment_name,
                             print_improvements_only,
@@ -1304,41 +1409,50 @@ def process_self_contained_coordinator_stream(
                             use_metric_context_path,
                             running_platform,
                         )
+                        total_regressions = len(regressions_list)
+                        total_improvements = len(improvement_list)
                         auto_approve = True
                         grafana_link_base = "https://benchmarksredisio.grafana.net/d/1fWbtb7nz/experimental-oss-spec-benchmarks"
-
-                        prepare_regression_comment(
-                            auto_approve,
-                            baseline_branch,
-                            baseline_tag,
-                            comparison_branch,
-                            comparison_tag,
-                            contains_regression_comment,
-                            github_pr,
-                            grafana_link_base,
-                            is_actionable_pr,
-                            old_regression_comment_body,
-                            pr_link,
-                            regression_comment,
-                            datasink_conn,
-                            running_platform,
-                            table_output,
-                            tf_github_org,
-                            tf_github_repo,
-                            tf_triggering_env,
-                            total_comparison_points,
-                            total_improvements,
-                            total_regressions,
-                            total_stable,
-                            total_unstable,
-                            verbose,
-                            regressions_percent_lower_limit,
-                        )
+                        try:
+                            prepare_regression_comment(
+                                auto_approve,
+                                baseline_branch,
+                                baseline_tag,
+                                comparison_branch,
+                                comparison_tag,
+                                contains_regression_comment,
+                                github_pr,
+                                grafana_link_base,
+                                is_actionable_pr,
+                                old_regression_comment_body,
+                                pr_link,
+                                regression_comment,
+                                datasink_conn,
+                                running_platform,
+                                table_output,
+                                tf_github_org,
+                                tf_github_repo,
+                                tf_triggering_env,
+                                total_comparison_points,
+                                total_improvements,
+                                total_regressions,
+                                total_stable,
+                                total_unstable,
+                                verbose,
+                                regressions_percent_lower_limit,
+                                regressions_list,
+                            )
+                        except Exception as e:
+                            logging.error(
+                                "Failed to produce regression comment but continuing... Error: {}".format(
+                                    e.__str__()
+                                )
+                            )
                     logging.info(
                         f"Added test named {test_name} to the completed test list in key {stream_test_list_completed}"
                     )
         else:
-            logging.error("Missing commit information within received message.")
+            logging.error("Missing run image information within received message.")
 
     except Exception as e:
         logging.critical(
@@ -1356,29 +1470,74 @@ def process_self_contained_coordinator_stream(
     return stream_id, overall_result, total_test_suite_runs
 
 
+def start_redis_container(
+    command_str,
+    db_cpuset_cpus,
+    docker_client,
+    mnt_point,
+    redis_containers,
+    run_image,
+    temporary_dir,
+    auto_remove=False,
+):
+    logging.info(
+        "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
+            run_image, db_cpuset_cpus, command_str
+        )
+    )
+    volumes = {}
+    working_dir = "/"
+    if mnt_point != "":
+        volumes = {
+            temporary_dir: {
+                "bind": mnt_point,
+                "mode": "rw",
+            },
+        }
+        logging.info(f"setting volume as follow: {volumes}. working_dir={mnt_point}")
+        working_dir = mnt_point
+    redis_container = docker_client.containers.run(
+        image=run_image,
+        volumes=volumes,
+        auto_remove=auto_remove,
+        privileged=True,
+        working_dir=mnt_point,
+        command=command_str,
+        network_mode="host",
+        detach=True,
+        cpuset_cpus=db_cpuset_cpus,
+        pid_mode="host",
+        publish_all_ports=True,
+    )
+    time.sleep(5)
+    redis_containers.append(redis_container)
+    return redis_container
+
+
 def filter_test_files(
     defaults_filename,
     priority_lower_limit,
     priority_upper_limit,
-    test_regexp,
+    tests_regexp,
     testsuite_spec_files,
+    command_groups_regexp=None,
 ):
     filtered_test_files = []
     for test_file in testsuite_spec_files:
         if defaults_filename in test_file:
             continue
 
-        if test_regexp != ".*":
+        if tests_regexp != ".*":
             logging.info(
-                "Filtering all tests via a regular expression: {}".format(test_regexp)
+                "Filtering all tests via a regular expression: {}".format(tests_regexp)
             )
-            tags_regex_string = re.compile(test_regexp)
+            tags_regex_string = re.compile(tests_regexp)
 
             match_obj = re.search(tags_regex_string, test_file)
             if match_obj is None:
                 logging.info(
                     "Skipping {} given it does not match regex {}".format(
-                        test_file, test_regexp
+                        test_file, tests_regexp
                     )
                 )
                 continue
@@ -1388,7 +1547,7 @@ def filter_test_files(
                 result,
                 benchmark_config,
                 test_name,
-            ) = get_final_benchmark_config(None, stream, "")
+            ) = get_final_benchmark_config(None, None, stream, "")
             if result is False:
                 logging.error(
                     "Skipping {} given there were errors while calling get_final_benchmark_config()".format(
@@ -1397,6 +1556,34 @@ def filter_test_files(
                 )
                 continue
 
+            if command_groups_regexp is not None:
+                logging.info(
+                    "Filtering all test command groups via a regular expression: {}".format(
+                        command_groups_regexp
+                    )
+                )
+                if "tested-groups" in benchmark_config:
+                    command_groups = benchmark_config["tested-groups"]
+                    logging.info(
+                        f"The file {test_file} (test name = {test_name}) contains the following groups: {command_groups}"
+                    )
+                    groups_regex_string = re.compile(command_groups_regexp)
+                    found = False
+                    for command_group in command_groups:
+                        match_obj = re.search(groups_regex_string, command_group)
+                        if match_obj is not None:
+                            found = True
+                            logging.info(f"found the command group {command_group}")
+                    if found is False:
+                        logging.info(
+                            f"Skipping {test_file} given the following groups: {command_groups} does not match command group regex {command_groups_regexp}"
+                        )
+                        continue
+                else:
+                    logging.warning(
+                        f"The file {test_file} (test name = {test_name}) does not contain the property 'tested-groups'. Cannot filter based uppon groups..."
+                    )
+
             if "priority" in benchmark_config:
                 priority = benchmark_config["priority"]
 
@@ -1458,11 +1645,16 @@ def data_prepopulation_step(
     full_benchmark_path = "/usr/local/bin/{}".format(preload_tool)
     client_mnt_point = "/mnt/client/"
     if "memtier_benchmark" in preload_tool:
-        (_, preload_command_str,) = prepare_memtier_benchmark_parameters(
+        (
+            _,
+            preload_command_str,
+            _,
+        ) = prepare_memtier_benchmark_parameters(
             benchmark_config["dbconfig"]["preload_tool"],
             full_benchmark_path,
             port,
             "localhost",
+            None,
             local_benchmark_output_filename,
             False,
         )