redis-benchmarks-specification 0.1.72__py3-none-any.whl → 0.1.74__py3-none-any.whl

redis_benchmarks_specification/__runner__/runner.py

@@ -358,7 +358,21 @@ def process_self_contained_coordinator_stream(
     preserve_temporary_client_dirs=False,
     resp_version=None,
     override_memtier_test_time=0,
+    used_memory_check_fail=False,
 ):
+    def delete_temporary_files(
+        temporary_dir_client, full_result_path, benchmark_tool_global
+    ):
+        if preserve_temporary_client_dirs is True:
+            logging.info(f"Preserving temporary client dir {temporary_dir_client}")
+        else:
+            if "redis-benchmark" in benchmark_tool_global:
+                if full_result_path is not None:
+                    os.remove(full_result_path)
+                    logging.info("Removing temporary JSON file")
+            shutil.rmtree(temporary_dir_client, ignore_errors=True)
+            logging.info(f"Removing temporary client dir {temporary_dir_client}")
+
     overall_result = True
     results_matrix = []
     total_test_suite_runs = 0
@@ -507,6 +521,11 @@ def process_self_contained_coordinator_stream(
                                 test_name, maxmemory, benchmark_required_memory
                             )
                         )
+                        delete_temporary_files(
+                            temporary_dir_client=temporary_dir_client,
+                            full_result_path=None,
+                            benchmark_tool_global=benchmark_tool_global,
+                        )
                         continue
 
                     reset_commandstats(redis_conns)
@@ -543,6 +562,11 @@ def process_self_contained_coordinator_stream(
                                         test_name, priority_upper_limit, priority
                                     )
                                 )
+                                delete_temporary_files(
+                                    temporary_dir_client=temporary_dir_client,
+                                    full_result_path=None,
+                                    benchmark_tool_global=benchmark_tool_global,
+                                )
                                 continue
                             if priority < priority_lower_limit:
                                 logging.warning(
@@ -550,6 +574,11 @@ def process_self_contained_coordinator_stream(
                                         test_name, priority_lower_limit, priority
                                     )
                                 )
+                                delete_temporary_files(
+                                    temporary_dir_client=temporary_dir_client,
+                                    full_result_path=None,
+                                    benchmark_tool_global=benchmark_tool_global,
+                                )
                                 continue
                             logging.info(
                                 "Test {} priority ({}) is within the priority limit [{},{}]".format(
@@ -559,46 +588,61 @@ def process_self_contained_coordinator_stream(
                                     priority_upper_limit,
                                 )
                             )
-
-                    if "dataset" in benchmark_config["dbconfig"]:
-                        if args.run_tests_with_dataset is False:
-                            logging.warning(
-                                "Skipping test {} giving it implies dataset preload".format(
-                                    test_name
+                    if "dbconfig" in benchmark_config:
+                        if "dataset" in benchmark_config["dbconfig"]:
+                            if args.run_tests_with_dataset is False:
+                                logging.warning(
+                                    "Skipping test {} giving it implies dataset preload".format(
+                                        test_name
+                                    )
                                 )
-                            )
-                            continue
+                                delete_temporary_files(
+                                    temporary_dir_client=temporary_dir_client,
+                                    full_result_path=None,
+                                    benchmark_tool_global=benchmark_tool_global,
+                                )
+                                continue
 
                     if dry_run is True:
                         dry_run_count = dry_run_count + 1
-                        continue
-
-                    if "preload_tool" in benchmark_config["dbconfig"]:
-                        res = data_prepopulation_step(
-                            benchmark_config,
-                            benchmark_tool_workdir,
-                            client_cpuset_cpus,
-                            docker_client,
-                            git_hash,
-                            port,
-                            temporary_dir_client,
-                            test_name,
-                            host,
-                            tls_enabled,
-                            tls_skip_verify,
-                            test_tls_cert,
-                            test_tls_key,
-                            test_tls_cacert,
-                            resp_version,
-                            args.benchmark_local_install,
-                            password,
-                            oss_cluster_api_enabled,
+                        delete_temporary_files(
+                            temporary_dir_client=temporary_dir_client,
+                            full_result_path=None,
+                            benchmark_tool_global=benchmark_tool_global,
                         )
-                        if res is False:
-                            logging.warning(
-                                "Skipping this test given preload result was false"
+                        continue
+                    if "dbconfig" in benchmark_config:
+                        if "preload_tool" in benchmark_config["dbconfig"]:
+                            res = data_prepopulation_step(
+                                benchmark_config,
+                                benchmark_tool_workdir,
+                                client_cpuset_cpus,
+                                docker_client,
+                                git_hash,
+                                port,
+                                temporary_dir_client,
+                                test_name,
+                                host,
+                                tls_enabled,
+                                tls_skip_verify,
+                                test_tls_cert,
+                                test_tls_key,
+                                test_tls_cacert,
+                                resp_version,
+                                args.benchmark_local_install,
+                                password,
+                                oss_cluster_api_enabled,
                             )
-                            continue
+                            if res is False:
+                                logging.warning(
+                                    "Skipping this test given preload result was false"
+                                )
+                                delete_temporary_files(
+                                    temporary_dir_client=temporary_dir_client,
+                                    full_result_path=None,
+                                    benchmark_tool_global=benchmark_tool_global,
+                                )
+                                continue
                     execute_init_commands(
                         benchmark_config, r, dbconfig_keyname="dbconfig"
                     )
@@ -608,6 +652,7 @@ def process_self_contained_coordinator_stream(
                         benchmark_required_memory,
                         redis_conns,
                         "start of benchmark",
+                        used_memory_check_fail,
                     )
 
                     logging.info("Checking if there is a keyspace check being enforced")
@@ -618,6 +663,11 @@ def process_self_contained_coordinator_stream(
 
                     if dry_run_include_preload is True:
                         dry_run_count = dry_run_count + 1
+                        delete_temporary_files(
+                            temporary_dir_client=temporary_dir_client,
+                            full_result_path=None,
+                            benchmark_tool_global=benchmark_tool_global,
+                        )
                         continue
 
                     benchmark_tool = extract_client_tool(benchmark_config)
@@ -692,6 +742,11 @@ def process_self_contained_coordinator_stream(
                         logging.warning(
                             "Forcing skip this test given there is an arbitrary commmand and memtier usage. Check https://github.com/RedisLabs/memtier_benchmark/pull/117 ."
                         )
+                        delete_temporary_files(
+                            temporary_dir_client=temporary_dir_client,
+                            full_result_path=None,
+                            benchmark_tool_global=benchmark_tool_global,
+                        )
                         continue
 
                     client_container_image = extract_client_container_image(
@@ -789,6 +844,7 @@ def process_self_contained_coordinator_stream(
                         benchmark_required_memory,
                         redis_conns,
                         "end of benchmark",
+                        used_memory_check_fail,
                     )
 
                     if args.flushall_on_every_test_end:
@@ -899,18 +955,11 @@ def process_self_contained_coordinator_stream(
                     shutil.copy(full_result_path, dest_fpath)
                 overall_result &= test_result
 
-                if preserve_temporary_client_dirs is True:
-                    logging.info(
-                        f"Preserving temporary client dir {temporary_dir_client}"
-                    )
-                else:
-                    if "redis-benchmark" in benchmark_tool_global:
-                        os.remove(full_result_path)
-                        logging.info("Removing temporary JSON file")
-                    shutil.rmtree(temporary_dir_client, ignore_errors=True)
-                    logging.info(
-                        f"Removing temporary client dir {temporary_dir_client}"
-                    )
+                delete_temporary_files(
+                    temporary_dir_client=temporary_dir_client,
+                    full_result_path=full_result_path,
+                    benchmark_tool_global=benchmark_tool_global,
+                )
 
     table_name = "Results for entire test-suite"
     results_matrix_headers = [
@@ -957,22 +1006,33 @@ def get_maxmemory(r):
 
 def get_benchmark_required_memory(benchmark_config):
     benchmark_required_memory = 0
-    if "resources" in benchmark_config["dbconfig"]:
-        resources = benchmark_config["dbconfig"]["resources"]
-        if "requests" in resources:
-            resources_requests = benchmark_config["dbconfig"]["resources"]["requests"]
-            if "memory" in resources_requests:
-                benchmark_required_memory = resources_requests["memory"]
-                benchmark_required_memory = int(parse_size(benchmark_required_memory))
-                logging.info(
-                    "Benchmark required memory: {} Bytes".format(
-                        benchmark_required_memory
+    if "dbconfig" in benchmark_config:
+        if "resources" in benchmark_config["dbconfig"]:
+            resources = benchmark_config["dbconfig"]["resources"]
+            if "requests" in resources:
+                resources_requests = benchmark_config["dbconfig"]["resources"][
+                    "requests"
+                ]
+                if "memory" in resources_requests:
+                    benchmark_required_memory = resources_requests["memory"]
+                    benchmark_required_memory = int(
+                        parse_size(benchmark_required_memory)
+                    )
+                    logging.info(
+                        "Benchmark required memory: {} Bytes".format(
+                            benchmark_required_memory
+                        )
                     )
-                )
     return benchmark_required_memory
 
 
-def used_memory_check(test_name, benchmark_required_memory, redis_conns, stage):
+def used_memory_check(
+    test_name,
+    benchmark_required_memory,
+    redis_conns,
+    stage,
+    used_memory_check_fail=False,
+):
     used_memory = 0
     for conn in redis_conns:
         used_memory = used_memory + conn.info("memory")["used_memory"]
@@ -984,7 +1044,8 @@ def used_memory_check(test_name, benchmark_required_memory, redis_conns, stage):
                 test_name, benchmark_required_memory, used_memory_gb
             )
         )
-        exit(1)
+        if used_memory_check_fail:
+            exit(1)
 
 
 def cp_to_workdir(benchmark_tool_workdir, srcfile):

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -433,7 +433,26 @@ def process_self_contained_coordinator_stream(
                 run_arch,
             ) = extract_build_info_from_streamdata(testDetails)
 
-            if run_arch == arch:
+            skip_test = False
+            if b"platform" in testDetails:
+                platform = testDetails[b"platform"]
+                if running_platform != platform:
+                    skip_test = True
+                    logging.info(
+                        "skipping stream_id {} given plaform {}!={}".format(
+                            stream_id, running_platform, platform
+                        )
+                    )
+
+            if run_arch != arch:
+                skip_test = True
+                logging.info(
+                    "skipping stream_id {} given arch {}!={}".format(
+                        stream_id, run_arch, arch
+                    )
+                )
+
+            if skip_test is False:
                 overall_result = True
                 profiler_dashboard_links = []
                 if docker_air_gap:
@@ -912,9 +931,7 @@ def process_self_contained_coordinator_stream(
                                     logging.critical("Printing redis container log....")
                                     print("-" * 60)
                                     print(
-                                        redis_container.logs(
-                                            stdout=True, stderr=True, logs=True
-                                        )
+                                        redis_container.logs(stdout=True, stderr=True)
                                     )
                                     print("-" * 60)
                                 test_result = False
@@ -953,12 +970,6 @@ def process_self_contained_coordinator_stream(
 
                             overall_result &= test_result
 
-            else:
-                logging.info(
-                    "skipping stream_id {} given arch {}!={}".format(
-                        stream_id, run_arch, arch
-                    )
-                )
         else:
             logging.error("Missing commit information within received message.")
     except:

redis_benchmarks_specification/test-suites/create-re-string.py

@@ -0,0 +1,286 @@
+tests = [
+    {
+        "name": "memtier_benchmark-100Kkeys-string-setget50c-20KiB",
+        "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys",
+        "check": {"keyspacelen": 100000},
+        "command": "--data-size 20000  --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=100000  --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-string-setget50c-20KiB-pipeline-10",
+        "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys",
+        "check": {"keyspacelen": 100000},
+        "command": "--pipeline 10 --data-size 20000 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --key-minimum=1 --key-maximum=100000  --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-string-setget200c-20KiB",
+        "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys",
+        "check": {"keyspacelen": 100000},
+        "command": "--data-size 20000  --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=100000 --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-string-setget200c-20KiB-pipeline-10",
+        "precommand": "--data-size 20000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=100000 -n allkeys",
+        "check": {"keyspacelen": 100000},
+        "command": "--pipeline 10 --data-size 20000 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --key-minimum=1 --key-maximum=100000 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-load-string50c-with-20KiB-values",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 25 -t 2 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-load-string50c-with-20KiB-values-pipeline-10",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--pipeline 10 --data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 25 -t 2 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-load-string200c-with-20KiB-values",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 50 -t 4 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-100Kkeys-load-string200c-with-20KiB-values-pipeline-10",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--pipeline 10 --data-size 20000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum=100000 --test-time 180 -c 50 -t 4 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set-20k"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget50c-100B",
+        "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--data-size 100 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget50c-1KiB",
+        "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--data-size 1000  --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget50c-100B-pipeline-10",
+        "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--pipeline 10 --data-size 100 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget50c-1KiB-pipeline-10",
+        "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 25 -t 2 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--pipeline 10 --data-size 1000 --ratio 1:10 --key-pattern R:R -c 25 -t 2 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget200c-100B",
+        "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--data-size 100 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget200c-1KiB",
+        "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--data-size 1000  --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget200c-100B-pipeline-10",
+        "precommand": "--data-size 100 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--pipeline 10 --data-size 100 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-setget200c-1KiB-pipeline-10",
+        "precommand": "--data-size 1000 --ratio 1:0 --key-pattern P:P -c 50 -t 4 --hide-histogram --key-minimum=1 --key-maximum=1000000 -n allkeys",
+        "check": {"keyspacelen": 1000000},
+        "command": "--pipeline 10 --data-size 1000 --ratio 1:10 --key-pattern R:R -c 50 -t 4 --hide-histogram --test-time 180",
+        "kpis": {},
+        "tested-commands": ["setget"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string50c-with-100B-values",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string50c-with-1KiB-values",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string50c-with-100B-values-pipeline-10",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--pipeline 10 --data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string50c-with-1KiB-values-pipeline-10",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--pipeline 10 --data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 25 -t 2 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string200c-with-100B-values",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string200c-with-1KiB-values",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string200c-with-100B-values-pipeline-10",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--pipeline 10 --data-size 100 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-load-string200c-with-1KiB-values-pipeline-10",
+        "precommand": "",
+        "check": {"keyspacelen": 0},
+        "command": "--pipeline 10 --data-size 1000 --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram",
+        "kpis": {},
+        "tested-commands": ["set"],
+        "tested-groups": ["string"],
+    },
+    {
+        "name": "memtier_benchmark-1Mkeys-string-mget-1KiB",
+        "precommand": "--data-size 1000 --key-minimum=1 --key-maximum 1000000 -n allkeys --ratio=1:0  --key-pattern P:P --hide-histogram -t 2 -c 100",
+        "check": {"keyspacelen": 1000000},
+        "command": ' --command="MGET __key__ memtier-1 memtier-2 memtier-3 memtier-4 memtier-5 memtier-6 memtier-7 memtier-8 memtier-9" --command-key-pattern=R -c 50 -t 2 --hide-histogram --test-time 180',
+        "kpis": {},
+        "tested-commands": ["mget"],
+        "tested-groups": ["string"],
+    },
+]
+
+print(len(tests))
+re_filenames = [x["name"] for x in tests]
+re_test_specs = {}
+for x in tests:
+    re_test_specs[x["name"]] = x
+
+import yaml
+import json
+import pathlib
+
+
+defaults_filename = "default.yml"
+prefix = "memtier_benchmark-"
+test_glob = "memtier_*.yml"
+files = pathlib.Path().glob(test_glob)
+files = [str(x) for x in files]
+
+base_yaml = yaml.safe_load(open("memtier_benchmark-1Mkeys-string-get-1KiB.yml"))
+del base_yaml["description"]
+# del base_yaml["clientconfig"]["resources"]
+# del base_yaml["build-variants"]
+# del base_yaml["priority"]
+# del base_yaml["redis-topologies"]
+# del base_yaml["tested-commands"]
+# del base_yaml["version"]
+# del base_yaml["tested-groups"]
+#
+
+
+#
+# for file in files:
+#     if defaults_filename in file:
+#         files.remove(file)
+
+for re_file in re_filenames:
+    re_spec = re_test_specs[re_file]
+    precommand = ""
+    if "precommand" in re_spec:
+        precommand = re_spec["precommand"]
+
+    command = ""
+    if "command" in re_spec:
+        command = re_spec["command"]
+    if "dbconfig" in base_yaml:
+        del base_yaml["dbconfig"]
+    if precommand != "":
+        base_yaml["dbconfig"] = {}
+        base_yaml["dbconfig"]["preload_tool"] = {}
+        base_yaml["dbconfig"]["preload_tool"][
+            "run_image"
+        ] = "redislabs/memtier_benchmark:edge"
+        base_yaml["dbconfig"]["preload_tool"]["tool"] = "memtier_benchmark"
+        base_yaml["dbconfig"]["preload_tool"]["arguments"] = f"{precommand}"
+
+    base_yaml["clientconfig"]["arguments"] = command
+    base_yaml["name"] = re_file
+    with open(f"re-string/{re_file}.yml", "w") as outfile:
+        yaml.dump(base_yaml, outfile)