redis-benchmarks-specification 0.1.245__py3-none-any.whl → 0.1.247__py3-none-any.whl

redis_benchmarks_specification/__compare__/compare.py

@@ -569,7 +569,7 @@ def compute_regression_table(
     verbose=False,
     last_n_baseline=-1,
     last_n_comparison=-1,
-    metric_mode="higher-better",
+    metric_mode="lower-better",
     from_date=None,
     from_ts_ms=None,
     to_date=None,
@@ -1019,8 +1019,13 @@ def from_rts_to_regression_table(
     baseline_only_list = []
     comparison_only_list = []
     no_datapoints_list = []
+    no_datapoints_baseline_list = []
+    no_datapoints_comparison_list = []
+    original_metric_mode = metric_mode
     for test_name in test_names:
+        metric_mode = original_metric_mode
         compare_version = "main"
+        # GE
         github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
         test_path = f"redis_benchmarks_specification/test-suites/{test_name}.yml"
         test_link = f"[{test_name}]({github_link}/{compare_version}/{test_path})"
@@ -1076,6 +1081,22 @@ def from_rts_to_regression_table(
         if len(baseline_timeseries) > 1 and multi_value_baseline is False:
             baseline_timeseries = get_only_Totals(baseline_timeseries)
 
+        if len(baseline_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for baseline timeseries {baseline_timeseries}"
+            )
+            no_datapoints_baseline_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
+        if len(comparison_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for comparison timeseries {comparison_timeseries}"
+            )
+            no_datapoints_comparison_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
         if len(baseline_timeseries) != 1 and multi_value_baseline is False:
             if verbose:
                 logging.warning(
@@ -1152,11 +1173,14 @@ def from_rts_to_regression_table(
             )
 
             waterline = regressions_percent_lower_limit
-            if regressions_percent_lower_limit < largest_variance:
-                note = "waterline={:.1f}%.".format(largest_variance)
-                waterline = largest_variance
+            # if regressions_percent_lower_limit < largest_variance:
+            #     note = "waterline={:.1f}%.".format(largest_variance)
+            #     waterline = largest_variance
 
-        except redis.exceptions.ResponseError:
+        except redis.exceptions.ResponseError as e:
+            logging.error(
+                "Detected a redis.exceptions.ResponseError. {}".format(e.__str__())
+            )
             pass
         except ZeroDivisionError as e:
             logging.error("Detected a ZeroDivisionError. {}".format(e.__str__()))
@@ -1198,7 +1222,7 @@ def from_rts_to_regression_table(
             else:
                 # lower-better
                 percentage_change = (
-                    float(baseline_v) / float(comparison_v) - 1
+                    -(float(baseline_v) - float(comparison_v)) / float(baseline_v)
                 ) * 100.0
         else:
             logging.warn(
@@ -1280,16 +1304,27 @@ def from_rts_to_regression_table(
             logging.warning(
                 "There were no datapoints both for baseline and comparison for test: {test_name}"
             )
-            no_datapoints_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
     logging.warning(
         f"There is a total of {len(no_datapoints_list)} tests without datapoints for baseline AND comparison"
     )
     logging.info(
         f"There is a total of {len(comparison_only_list)} tests without datapoints for baseline"
     )
+    print(
+        "No datapoint baseline regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_baseline_list)
+        )
+    )
     logging.info(
         f"There is a total of {len(baseline_only_list)} tests without datapoints for comparison"
     )
+    print(
+        "No datapoint comparison regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_comparison_list)
+        )
+    )
     logging.info(f"There is a total of {len(unstable_list)} UNSTABLE tests")
     return (
         detected_regressions,

redis_benchmarks_specification/test-suites/defaults.yml

@@ -10,15 +10,19 @@ exporter:
     - $."BEST RUN RESULTS".Totals."Latency"
     - $."BEST RUN RESULTS".Totals."Misses/sec"
     - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
     - $."WORST RUN RESULTS".Totals."Ops/sec"
     - $."WORST RUN RESULTS".Totals."Latency"
     - $."WORST RUN RESULTS".Totals."Misses/sec"
     - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Ops/sec"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Latency"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Misses/sec"
-    - $."AGGREGATED AVERAGE RESULTS (5 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Percentile Latencies"."p99.00"
     - $."ALL STATS".Totals."Ops/sec"
     - $."ALL STATS".Totals."Latency"
     - $."ALL STATS".Totals."Misses/sec"
     - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p99.00"

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-2000_conns.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-2000_conns
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 1000 Bytes, with 400 clients running random GET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 3000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --data-size 1000 --ratio 1:0 -n allkeys --pipeline 10 --key-maximum 3000000 --key-pattern P:P --key-minimum 1 --hide-histogram -t 4 -c 10
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "1000" --distinct-client-seed --ratio 0:1 --key-pattern R:R --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 200 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 3g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-400_conns.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-400_conns
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 1000 Bytes, with 400 clients running random GET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 3000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --data-size 1000 --ratio 1:0 -n allkeys --pipeline 10 --key-maximum 3000000 --key-pattern P:P --key-minimum 1 --hide-histogram -t 4 -c 10
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "1000" --distinct-client-seed --ratio 0:1 --key-pattern R:R --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 40 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 3g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-40_conns.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-40_conns
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 1000 Bytes, with 40 clients running random GET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 3000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --data-size 1000 --ratio 1:0 -n allkeys --pipeline 10 --key-maximum 3000000 --key-pattern P:P --key-minimum 1 --hide-histogram -t 4 -c 10
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "1000" --distinct-client-seed --ratio 0:1 --key-pattern R:R --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 4 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 3g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-2000_conns.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-2000_conns
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 1000 Bytes, with 400 clients running random GET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 3000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --data-size 512 --ratio 1:0 -n allkeys --pipeline 10 --key-maximum 3000000 --key-pattern P:P --key-minimum 1 --hide-histogram -t 4 -c 10
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "512" --distinct-client-seed --ratio 1:4 --key-pattern R:R --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 200 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 3g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-400_conns.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-400_conns
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 1000 Bytes, with 400 clients running random GET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 3000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --data-size 512 --ratio 1:0 -n allkeys --pipeline 10 --key-maximum 3000000 --key-pattern P:P --key-minimum 1 --hide-histogram -t 4 -c 10
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "512" --distinct-client-seed --ratio 1:4 --key-pattern R:R --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 40 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 3g
+
+tested-groups:
+- string
+priority: 17

redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-5200_conns.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-5200_conns
+description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 1000 Bytes, with 400 clients running random GET commands.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 3000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --data-size 512 --ratio 1:0 -n allkeys --pipeline 10 --key-maximum 3000000 --key-pattern P:P --key-minimum 1 --hide-histogram -t 4 -c 10
+  resources:
+    requests:
+      memory: 3g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "512" --distinct-client-seed --ratio 1:4 --key-pattern R:R --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 520 -t 10 --hide-histogram'
+  resources:
+    requests:
+      cpus: '10'
+      memory: 3g
+
+tested-groups:
+- string
+priority: 17

{redis_benchmarks_specification-0.1.245.dist-info → redis_benchmarks_specification-0.1.247.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.245
+Version: 0.1.247
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.245.dist-info → redis_benchmarks_specification-0.1.247.dist-info}/RECORD

@@ -20,7 +20,7 @@ redis_benchmarks_specification/__common__/spec.py,sha256=eTF5559epBB0FrJPx-jRDQV
 redis_benchmarks_specification/__common__/timeseries.py,sha256=_LJFtC5sVP7DTaLZaIzv5g7wRxPTQZRwFIYvWX4p4N8,50533
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
 redis_benchmarks_specification/__compare__/args.py,sha256=-vQco6WG73DJtUFA7s-5bNk05vwHjOc94puVKZMZUYE,6513
-redis_benchmarks_specification/__compare__/compare.py,sha256=MXQuTITR-84Lh235jrPGwy8Le-gVNPeryHRJVRG2PZU,52078
+redis_benchmarks_specification/__compare__/compare.py,sha256=0uv61Xlvlb7cUd6ZvIf1P3icj8xEe9FkPco7kuIGitI,53479
 redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gqDKgzvNJnkGNKY,491
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=lYvbPd_3ppHZv4f2sRwXcF-fcBrwRSn3H2RMmNVkojY,7221
@@ -49,7 +49,7 @@ redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-def
 redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml,sha256=I6qEO7MZKduVx6xbBrRniE1i6NK9R8-uQXdQJT9o5G4,511
 redis_benchmarks_specification/setups/platforms/aws-ec2-1node-c5.4xlarge.yml,sha256=l7HsjccpebwZXeutnt3SHSETw4iiRwQ9dCDXLOySSRQ,622
 redis_benchmarks_specification/setups/topologies/topologies.yml,sha256=4h8e-zXA9-_iuJ2MsgldUnPKpHTUUD3BgLdZ9FsLPdI,3007
-redis_benchmarks_specification/test-suites/defaults.yml,sha256=Ckte0Bv8b5L7vxWYqochiVGZnlqqw49-vGjFRsppXdg,991
+redis_benchmarks_specification/test-suites/defaults.yml,sha256=CfNfn_nKTtVvbbTwmksJgLKzCzzBAm5SO7UAdrcqncg,1268
 redis_benchmarks_specification/test-suites/generate.py,sha256=FApiWulMw-_2jhl1zqrLUnTnOk3Vm7YxC3oGdauhv3A,3835
 redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml,sha256=HSFrXsVcX1c69IsOaRgrcnJrKs-qaNrtuFLpKuykbWI,2182
 redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml,sha256=HbE7SDAJbF7t6J4ue58D3U6BT36q2RZKwRHunQn7AhA,1904
@@ -321,10 +321,16 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-stream-5-entr
 redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunion.yml,sha256=_UVlSaeCFNvJHitG1wWs7S07M6o05e8uDYIGz4EgseE,39486
 redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-zset-300-elements-skiplist-encoded-zunionstore.yml,sha256=xkD4HHGvfHZF9YEsp1eErWOnC_WFdgFXyo-RW0_s7Fo,39526
 redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml,sha256=XAIFlbR6VJnmQRwedLGBGenbIsMC_I3uA35Mz_bkTTc,1028
+redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-2000_conns.yml,sha256=OUj2JnDs6Peax-mOA6Mb5rJJaSkEu1SzChWifJxP--Q,1137
+redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-400_conns.yml,sha256=947oH59ss6ZPDSSNpgqaOueltPusY7UMCD04lH_sP9k,1135
+redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-get-with-1KiB-values-pipeline-10-40_conns.yml,sha256=yQZr0SPEdqG-H2nUuAtlfQ7BAGo36axPiiHsR2-IFAY,1132
+redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-2000_conns.yml,sha256=dxlhwxliYODCNUmCCXElwSUusSlBdh5vu8EwuwXYgeA,1143
+redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-400_conns.yml,sha256=veXypnds7M6Hfv4wSXllFzHyWTnLefZdr7awC7xFTMk,1141
+redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-5200_conns.yml,sha256=4NHvtQR0UY-4iH4mPSUhJKrk-b5vSYtqZc2J7pm4vMA,1143
 redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml,sha256=VIFNaaAqRQMeDvtXmKGDpJTnu658Lv5i_oAju5uSi_c,708
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
-redis_benchmarks_specification-0.1.245.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.245.dist-info/METADATA,sha256=2ZL-D-d2sPzYrtIERWohS1d5pRotpSjcJN8atyTxKsI,22726
-redis_benchmarks_specification-0.1.245.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.245.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.245.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.247.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.247.dist-info/METADATA,sha256=4-472_bg8tn_XhVJ8dyc7FdTI9jd0SLirFkEOEbK8bA,22726
+redis_benchmarks_specification-0.1.247.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.247.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.247.dist-info/RECORD,,