redis-benchmarks-specification 0.1.217__py3-none-any.whl → 0.1.219__py3-none-any.whl

redis_benchmarks_specification/__cli__/cli.py

@@ -351,16 +351,22 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
                 hash_regexp
             )
         )
+    enable_hash_filtering = False
+    hash_filters = []
+    if args.git_hash != "":
+        enable_hash_filtering = True
+        hash_filters = args.git_hash.split(",")
+        logging.info(
+            f"There is a total of {len(hash_filters)} commit hash fitlers: {hash_filters}"
+        )
     hash_regexp_string = re.compile(hash_regexp)
     filtered_hash_commits = []
     for cdict in commits:
         commit_hash = cdict["git_hash"]
-        if args.git_hash != "":
-            if args.git_hash != commit_hash:
+        if enable_hash_filtering:
+            if commit_hash not in hash_filters:
                 logging.info(
-                    "Skipping {} given it does not match commit hash {}".format(
-                        commit_hash, args.git_hash
-                    )
+                    f"Skipping {commit_hash} given it does not match any commit hash in {hash_filters}"
                 )
                 continue
         commit_summary = cdict["commit_summary"]

redis_benchmarks_specification/__common__/builder_schema.py

@@ -174,6 +174,10 @@ def get_branch_version_from_test_details(testDetails):
             git_branch = git_branch.decode()
         if git_branch.startswith("/refs/heads/"):
             git_branch = git_branch.replace("/refs/heads/", "")
+        if git_branch.startswith("refs/heads/"):
+            git_branch = git_branch.replace("refs/heads/", "")
+        if git_branch.startswith("/"):
+            git_branch = git_branch[1:]
     if git_version is not None:
         if type(git_version) == bytes:
             git_version = git_version.decode()

redis_benchmarks_specification/__compare__/args.py

@@ -92,14 +92,28 @@ def create_compare_arguments(parser):
     )
     parser.add_argument("--baseline-branch", type=str, default=None, required=False)
     parser.add_argument("--baseline-tag", type=str, default=None, required=False)
+    parser.add_argument("--baseline-hash", type=str, default=None, required=False)
     parser.add_argument(
         "--baseline-target-version", type=str, default=None, required=False
     )
+    parser.add_argument(
+        "--baseline-target-branch", type=str, default=None, required=False
+    )
     parser.add_argument("--comparison-branch", type=str, default=None, required=False)
+    parser.add_argument(
+        "--baseline-github-repo", type=str, default="redis", required=False
+    )
+    parser.add_argument(
+        "--comparison-github-repo", type=str, default="redis", required=False
+    )
     parser.add_argument("--comparison-tag", type=str, default=None, required=False)
+    parser.add_argument("--comparison-hash", type=str, default=None, required=False)
     parser.add_argument(
         "--comparison-target-version", type=str, default=None, required=False
     )
+    parser.add_argument(
+        "--comparison-target-branch", type=str, default=None, required=False
+    )
     parser.add_argument("--print-regressions-only", type=bool, default=False)
     parser.add_argument("--print-improvements-only", type=bool, default=False)
     parser.add_argument("--skip-unstable", type=bool, default=False)

redis_benchmarks_specification/__compare__/compare.py

@@ -203,6 +203,8 @@ def compare_command_logic(args, project_name, project_version):
             )
         )
         baseline_branch = default_baseline_branch
+    if baseline_branch == "":
+        baseline_branch = None
     comparison_branch = args.comparison_branch
     simplify_table = args.simple_table
     print_regressions_only = args.print_regressions_only
@@ -250,6 +252,12 @@ def compare_command_logic(args, project_name, project_version):
     running_platform = args.running_platform
     baseline_target_version = args.baseline_target_version
     comparison_target_version = args.comparison_target_version
+    baseline_target_branch = args.baseline_target_branch
+    comparison_target_branch = args.comparison_target_branch
+    baseline_github_repo = args.baseline_github_repo
+    comparison_github_repo = args.comparison_github_repo
+    baseline_hash = args.baseline_hash
+    comparison_hash = args.comparison_hash
 
     if running_platform is not None:
         logging.info(
@@ -310,6 +318,12 @@ def compare_command_logic(args, project_name, project_version):
         running_platform,
         baseline_target_version,
         comparison_target_version,
+        baseline_hash,
+        comparison_hash,
+        baseline_github_repo,
+        comparison_github_repo,
+        baseline_target_branch,
+        comparison_target_branch,
     )
     prepare_regression_comment(
         auto_approve,
@@ -535,6 +549,12 @@ def compute_regression_table(
     running_platform=None,
     baseline_target_version=None,
     comparison_target_version=None,
+    comparison_hash=None,
+    baseline_hash=None,
+    baseline_github_repo="redis",
+    comparison_github_repo="redis",
+    baseline_target_branch=None,
+    comparison_target_branch=None,
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -560,6 +580,10 @@ def compute_regression_table(
         comparison_tag,
         baseline_target_version,
         comparison_target_version,
+        comparison_hash,
+        baseline_hash,
+        baseline_target_branch,
+        comparison_target_branch,
     )
     logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}")
     logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}")
@@ -605,6 +629,8 @@ def compute_regression_table(
         total_stable,
         total_unstable,
         total_comparison_points,
+        regressions_list,
+        improvements_list,
     ) = from_rts_to_regression_table(
         baseline_deployment_name,
         comparison_deployment_name,
@@ -629,6 +655,8 @@ def compute_regression_table(
         tf_triggering_env,
         verbose,
         running_platform,
+        baseline_github_repo,
+        comparison_github_repo,
     )
     logging.info(
         "Printing differential analysis between {} and {}".format(
@@ -661,6 +689,8 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
+        test_names_str = "|".join(regressions_list)
+        table_output += f"Regressions test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
 
@@ -682,6 +712,8 @@ def compute_regression_table(
         writer_regressions.dump(mystdout, False)
         table_output += mystdout.getvalue()
         table_output += "\n\n"
+        test_names_str = "|".join(improvements_list)
+        table_output += f"Improvements test regexp names: {test_names_str}\n\n"
         mystdout.close()
         sys.stdout = old_stdout
 
@@ -717,6 +749,11 @@ def compute_regression_table(
     )
 
 
+def get_by_error(name, by_str_arr):
+    by_string = ",".join(by_str_arr)
+    return f"--{name}-branch, --{name}-tag, --{name}-target-branch, --{name}-hash, and --{name}-target-version are mutually exclusive. You selected a total of {len(by_str_arr)}: {by_string}. Pick one..."
+
+
 def get_by_strings(
     baseline_branch,
     comparison_branch,
@@ -724,6 +761,10 @@ def get_by_strings(
     comparison_tag,
     baseline_target_version=None,
     comparison_target_version=None,
+    baseline_hash=None,
+    comparison_hash=None,
+    baseline_target_branch=None,
+    comparison_target_branch=None,
 ):
     baseline_covered = False
     comparison_covered = False
@@ -731,40 +772,64 @@ def get_by_strings(
     by_str_comparison = ""
     baseline_str = ""
     comparison_str = ""
+    baseline_by_arr = []
+    comparison_by_arr = []
+
+    ################# BASELINE BY ....
+
     if baseline_branch is not None:
-        baseline_covered = True
         by_str_baseline = "branch"
+        baseline_covered = True
         baseline_str = baseline_branch
-    if comparison_branch is not None:
-        comparison_covered = True
-        by_str_comparison = "branch"
-        comparison_str = comparison_branch
+        baseline_by_arr.append(by_str_baseline)
 
     if baseline_tag is not None:
-        if comparison_covered:
-            logging.error(
-                "--baseline-branch and --baseline-tag are mutually exclusive. Pick one..."
-            )
+        by_str_baseline = "version"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
             exit(1)
         baseline_covered = True
-        by_str_baseline = "version"
         baseline_str = baseline_tag
 
     if baseline_target_version is not None:
-        if comparison_covered:
-            logging.error(
-                "--baseline-branch, --baseline-tag and --baseline-target-version are mutually exclusive. Pick one..."
-            )
+        by_str_baseline = "target+version"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
             exit(1)
         baseline_covered = True
-        by_str_baseline = "target+version"
         baseline_str = baseline_target_version
 
+    if baseline_hash is not None:
+        by_str_baseline = "hash"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
+            exit(1)
+        baseline_covered = True
+        baseline_str = baseline_hash
+    if baseline_target_branch is not None:
+        by_str_baseline = "target+branch"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
+            exit(1)
+        baseline_covered = True
+        baseline_str = baseline_target_branch
+
+    ################# COMPARISON BY ....
+
+    if comparison_branch is not None:
+        by_str_comparison = "branch"
+        comparison_covered = True
+        comparison_str = comparison_branch
+
     if comparison_tag is not None:
         # check if we had already covered comparison
         if comparison_covered:
             logging.error(
-                "--comparison-branch and --comparison-tag are mutually exclusive. Pick one..."
+                "--comparison-branch and --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
             )
             exit(1)
         comparison_covered = True
@@ -774,23 +839,45 @@ def get_by_strings(
         # check if we had already covered comparison
         if comparison_covered:
             logging.error(
-                "--comparison-branch, --comparison-tag, and --comparison-target-table are mutually exclusive. Pick one..."
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
             )
             exit(1)
         comparison_covered = True
         by_str_comparison = "target+version"
         comparison_str = comparison_target_version
 
+    if comparison_target_branch is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "target+branch"
+        comparison_str = comparison_target_branch
+
+    if comparison_hash is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "hash"
+        comparison_str = comparison_hash
+
     if baseline_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --baseline-branch, --baseline-tag, or --baseline-target-version ) "
+            + "( --baseline-branch, --baseline-tag, --baseline-hash, --baseline-target-branch or --baseline-target-version ) "
         )
         exit(1)
     if comparison_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --comparison-branch, --comparison-tag, or --comparison-target-version ) "
+            + "( --comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch or --comparison-target-version ) "
         )
         exit(1)
     return baseline_str, by_str_baseline, comparison_str, by_str_comparison
@@ -820,6 +907,8 @@ def from_rts_to_regression_table(
     tf_triggering_env,
     verbose,
     running_platform=None,
+    baseline_github_repo="redis",
+    comparison_github_repo="redis",
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
     table_full = []
@@ -835,6 +924,8 @@ def from_rts_to_regression_table(
     total_comparison_points = 0
     noise_waterline = 3
     progress = tqdm(unit="benchmark time-series", total=len(test_names))
+    regressions_list = []
+    improvements_list = []
     for test_name in test_names:
         compare_version = "v0.1.208"
         github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
@@ -848,6 +939,7 @@ def from_rts_to_regression_table(
             "metric={}".format(metric_name),
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(baseline_deployment_name),
+            "github_repo={}".format(baseline_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
         if running_platform is not None:
@@ -855,11 +947,15 @@ def from_rts_to_regression_table(
         filters_comparison = [
             "{}={}".format(by_str_comparison, comparison_str),
             "metric={}".format(metric_name),
-            "hash==",
             "{}={}".format(test_filter, test_name),
             "deployment_name={}".format(comparison_deployment_name),
+            "github_repo={}".format(comparison_github_repo),
             "triggering_env={}".format(tf_triggering_env),
         ]
+        if "hash" not in by_str_baseline:
+            filters_baseline.append("hash==")
+        if "hash" not in by_str_comparison:
+            filters_comparison.append("hash==")
         if running_platform is not None:
             filters_comparison.append("running_platform={}".format(running_platform))
         baseline_timeseries = rts.ts().queryindex(filters_baseline)
@@ -1037,9 +1133,11 @@ def from_rts_to_regression_table(
                 test_link,
             )
             if detected_regression:
+                regressions_list.append(test_name)
                 table_regressions.append(line)
 
             if detected_improvement:
+                improvements_list.append(test_name)
                 table_improvements.append(line)
 
             if unstable:
@@ -1072,6 +1170,8 @@ def from_rts_to_regression_table(
         total_stable,
         total_unstable,
         total_comparison_points,
+        regressions_list,
+        improvements_list,
     )
 
 
@@ -1194,9 +1294,9 @@ def get_v_pct_change_and_largest_var(
             if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n):
                 comparison_values.append(tuple[1])
         comparison_df = pd.DataFrame(comparison_values)
-        comparison_median = float(comparison_df.median())
+        comparison_median = float(comparison_df.median().iloc[0])
         comparison_v = comparison_median
-        comparison_std = float(comparison_df.std())
+        comparison_std = float(comparison_df.std().iloc[0])
         if verbose:
             logging.info(
                 "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-geo-2-elements-geopos
+description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geopos, and we query it using GEOPOS command.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"'
+tested-groups:
+- geo
+tested-commands:
+- geopos
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: -c 50 -t 4 --command="GEOPOS Sicily Palermo Catania"  --hide-histogram --test-time 120
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 138

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml

@@ -0,0 +1,33 @@
+version: 0.4
+name: memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord
+description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geosearch, and we query it using GEOSEARCH command.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"'
+   - '"GEOADD" "Sicily" "12.758489" "38.788135" "edge1" "17.241510" "38.788135" "edge2"'
+tested-groups:
+- geo
+tested-commands:
+- geosearch
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: -c 50 -t 4 --command="GEOSEARCH Sicily FROMLONLAT 15 37 BYBOX 400 400 km ASC WITHCOORD WITHDIST"  --hide-histogram --test-time 120
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 138

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-set-100-elements-sismember-is-a-member
+description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is a member. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"'
+tested-groups:
+- set
+tested-commands:
+- sismember
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="SISMEMBER set:100 lysbgqqfqw"  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-set-100-elements-sismember-not-a-member
+description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is not a member. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"'
+tested-groups:
+- set
+tested-commands:
+- sismember
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="SISMEMBER set:100 not-a-member"  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance
+description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 10M elements. We query it using SISMEMBER in which the value has 50% change of being member. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --command="SADD set:10M:elements __key__" --command-key-pattern=P --key-maximum 10000000 --key-prefix "" -n 10000000 --hide-histogram -t 1 -c 1
+tested-groups:
+- set
+tested-commands:
+- sadd
+- sismember
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="SISMEMBER set:10M:elements __key__"  --key-maximum 20000000 --key-prefix ""  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance
+description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key with 1M elements. We query it using SISMEMBER in which the value has 50% change of being member. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --command="SADD set:1M:elements __key__" --command-key-pattern=P --key-maximum 1000000 --key-prefix "" -n 1000000 --hide-histogram -t 1 -c 1
+tested-groups:
+- set
+tested-commands:
+- sadd
+- sismember
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="SISMEMBER set:1M:elements __key__"  --key-maximum 2000000 --key-prefix ""  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml

@@ -0,0 +1,22 @@
+version: 0.4
+name: memtier_benchmark-connection-hello
+description: 'Runs memtier_benchmark, for no keyspace, benchmarking the connection setup scenario using HELLO command.'
+tested-groups:
+- connection
+tested-commands:
+- hello
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: -c 50 -t 4 --command="HELLO 2 SETNAME __key__"  --hide-histogram --test-time 120
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 1

{redis_benchmarks_specification-0.1.217.dist-info → redis_benchmarks_specification-0.1.219.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.217
+Version: 0.1.219
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.217.dist-info → redis_benchmarks_specification-0.1.219.dist-info}/RECORD

@@ -8,10 +8,10 @@ redis_benchmarks_specification/__builder__/builder.py,sha256=T5dTB_qgX1FdPA_qQfF
 redis_benchmarks_specification/__builder__/schema.py,sha256=1wcmyVJBcWrBvK58pghN9NCoWLCO3BzPsmdKWYfkVog,584
 redis_benchmarks_specification/__cli__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__cli__/args.py,sha256=9uP9p2hqxr60k0XjpDl0TS22d3bHYdxsVZdgrgf5dg0,7013
-redis_benchmarks_specification/__cli__/cli.py,sha256=vswKHtFCvWoxSW9bgehqjMTSfhtC8D5rwZ9w_OhtgU4,21116
+redis_benchmarks_specification/__cli__/cli.py,sha256=X5kmZoZfm85AtKNK5UoM8TeNDXWti3OIWnSguRgsylw,21363
 redis_benchmarks_specification/__cli__/stats.py,sha256=wahzZRbpfokv8dQU8O4BH5JFrOZk-l6k8LWdKfue9_0,20204
 redis_benchmarks_specification/__common__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-redis_benchmarks_specification/__common__/builder_schema.py,sha256=oU6zFrBsmPRgCr2c1zf7bM0o9bCyuqUpooJo0-nIHrE,5747
+redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb5qj9jzsBhLVNO7K_W2Clumj4pxrkG8,5938
 redis_benchmarks_specification/__common__/env.py,sha256=kvJ8Ll-fvI_Tc0vynrzUEr22TqnJizzvJ4Lu9RjNr_M,3119
 redis_benchmarks_specification/__common__/github.py,sha256=QEHG05DgvJXkJPMMprLa7E1fwn3PrizfR-_N3_Ff93Q,10398
 redis_benchmarks_specification/__common__/package.py,sha256=4uVt1BAZ999LV2rZkq--Tk6otAVIf9YR3g3KGeUpiW4,834
@@ -19,8 +19,8 @@ redis_benchmarks_specification/__common__/runner.py,sha256=7DBI09eu_4RibK6MwcYyI
 redis_benchmarks_specification/__common__/spec.py,sha256=eTF5559epBB0FrJPx-jRDQVeP_ZVOgyC7Vjxr2xk6fo,3262
 redis_benchmarks_specification/__common__/timeseries.py,sha256=_LJFtC5sVP7DTaLZaIzv5g7wRxPTQZRwFIYvWX4p4N8,50533
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
-redis_benchmarks_specification/__compare__/args.py,sha256=HKp94Tpb4iH0J7I4DK8d63D_sgYNu8f5jI-F1Vadh8o,5557
-redis_benchmarks_specification/__compare__/compare.py,sha256=GUsDWPHMXjAR682HKv3lamelIbJ-TMF_VdQOpfJvdtU,41180
+redis_benchmarks_specification/__compare__/args.py,sha256=k1e2L67IQKEBPvNBfY-jC_cR1x4ueSg-7u-1e8tJ9aQ,6155
+redis_benchmarks_specification/__compare__/compare.py,sha256=E-O5NfByFWWVTktrhwq-aRtBdboHf9eZEVFFMf0dwSQ,45313
 redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gqDKgzvNJnkGNKY,491
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=lYvbPd_3ppHZv4f2sRwXcF-fcBrwRSn3H2RMmNVkojY,7221
@@ -108,6 +108,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-20KiB.yml,sha256=nZ_c5DV8lKWMmgW9Scgh4WGNA9tD0nkCYssp0syHJfs,950
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2MB.yml,sha256=JslW9y5ItM3I5yoZY2yNiDSD0XQrW9m1hKLDtC8ImVQ,1011
 redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-mget-1KiB.yml,sha256=s1PLNH__vexG6bE-GL_xPAqVelupxY73AWWE5siYIYE,999
+redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml,sha256=Mni3vvF4iSy1OJ_9USsYhh6RPd0x5Sss37E1UcL2J3Q,931
+redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml,sha256=qo26OHdRFwmtH9ZGzj-bsasjzYoa4DGL2ODuDbOdgAE,1097
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-60M-elements-geodist-pipeline-10.yml,sha256=fnLeuvhBxNit7iO2D7P4Ybxahw0e_U6wXy0IudNhQVM,890
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-60M-elements-geodist.yml,sha256=l88SV31p15gOTtRycS3ZLHsV3b4A5BpQ-T6bLXOfVnI,864
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-60M-elements-geohash-pipeline-10.yml,sha256=-GZ3nPHpJn50JIgadeW7YolmxYvSNFuOD4zTpsstFNw,869
@@ -124,9 +126,13 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-1key-pfadd-4KB-valu
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10-elements-smembers-pipeline-10.yml,sha256=GGTJjBl03pAUrDqpSB60uPZQn8CFH8nT-5ec1bdPmzA,944
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10-elements-smembers.yml,sha256=yNIpCPO7slCQbv8H6gRCZyHu5ARteVZSkZw4jPl6CBU,918
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10-elements-smismember.yml,sha256=MkSsnFsMhSfoqV13dHa2joAkGwk98rB1vHLG1ZcJ_0Q,992
+redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml,sha256=T28C1dlaLqErpmeJjcCFSXGmYwK8i6x5xgp9bK2KdLc,2123
+redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml,sha256=Fu-fiYNxvy7KfOl4aY5GH8yq3omnesPKO8Uto2c6bI8,2130
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-smembers.yml,sha256=v_YPymKgk-TAAax1Z-WLPb9FlGDxR3YZzHTu3TOLdMY,2090
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-smismember.yml,sha256=yIpS_oixm5c9XjWrXXbnD7HpkdS-9qcoqbpQNjvT2kQ,2171
+redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-10M-elements-sismember-50pct-chance.yml,sha256=2dn2klO6QnFfttaZi2OXlv_lHggL7_bxYBL3PZJFQw4,1083
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1K-elements-smembers.yml,sha256=obobyni8avSLoTOTEEWfT_YFuVgpTma9fGszc_JX2lI,13792
+redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-1M-elements-sismember-50pct-chance.yml,sha256=o47aHfpCJ78VX2SOg8TKYkEHTyRGmQj-MRgtkiX4I0I,1076
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-200K-elements-sadd-constant.yml,sha256=spiUdTx0kSZ0xcilXjOzN5UtmFm08dekzR2YQ0QPJUE,1060
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-2M-elements-sadd-increasing.yml,sha256=cmAyCOBnDSbtoxj1AJKAfAUwGk4zmqv2whlDB7z0QJw,766
 redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zincrby-1M-elements-pipeline-1.yml,sha256=_gp943gEk2E-hYtwbHLhgJ0G4v4-nRkHJyKgNLXVPJM,1098
@@ -151,9 +157,10 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-set-10-100-el
 redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-stream-5-entries-xread-all-entries-pipeline-10.yml,sha256=7yWCmQTtiypmF1tHGV61z9Zh5EH8CVuETu7bDNC4V6g,1138
 redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-stream-5-entries-xread-all-entries.yml,sha256=Z6T75dIbjRb4YO1tFIV9K4S_KFzRHfAa4q3kOg0vcHw,1112
 redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml,sha256=XAIFlbR6VJnmQRwedLGBGenbIsMC_I3uA35Mz_bkTTc,1028
+redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml,sha256=8JX9InqlMPW7Irn94gOsrGJtq6Q6pMlCGx901-oUkmA,583
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
-redis_benchmarks_specification-0.1.217.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.217.dist-info/METADATA,sha256=Xl-aBoQ5ko_6URB4Keum16X-QCAd18fRcytsWRoaM9k,22726
-redis_benchmarks_specification-0.1.217.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.217.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.217.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.219.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.219.dist-info/METADATA,sha256=b23LCfzcZJCRP6nQ-F9SeeKpZUrxX1W92d3YoPfxOWo,22726
+redis_benchmarks_specification-0.1.219.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.219.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.219.dist-info/RECORD,,