redis-benchmarks-specification 0.1.246__py3-none-any.whl → 0.1.247__py3-none-any.whl

redis_benchmarks_specification/__compare__/compare.py

@@ -569,7 +569,7 @@ def compute_regression_table(
     verbose=False,
     last_n_baseline=-1,
     last_n_comparison=-1,
-    metric_mode="higher-better",
+    metric_mode="lower-better",
     from_date=None,
     from_ts_ms=None,
     to_date=None,
@@ -1019,6 +1019,8 @@ def from_rts_to_regression_table(
     baseline_only_list = []
     comparison_only_list = []
     no_datapoints_list = []
+    no_datapoints_baseline_list = []
+    no_datapoints_comparison_list = []
     original_metric_mode = metric_mode
     for test_name in test_names:
         metric_mode = original_metric_mode
@@ -1079,6 +1081,22 @@ def from_rts_to_regression_table(
         if len(baseline_timeseries) > 1 and multi_value_baseline is False:
             baseline_timeseries = get_only_Totals(baseline_timeseries)
 
+        if len(baseline_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for baseline timeseries {baseline_timeseries}"
+            )
+            no_datapoints_baseline_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
+        if len(comparison_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for comparison timeseries {comparison_timeseries}"
+            )
+            no_datapoints_comparison_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
         if len(baseline_timeseries) != 1 and multi_value_baseline is False:
             if verbose:
                 logging.warning(
@@ -1159,7 +1177,10 @@ def from_rts_to_regression_table(
             #     note = "waterline={:.1f}%.".format(largest_variance)
             #     waterline = largest_variance
 
-        except redis.exceptions.ResponseError:
+        except redis.exceptions.ResponseError as e:
+            logging.error(
+                "Detected a redis.exceptions.ResponseError. {}".format(e.__str__())
+            )
             pass
         except ZeroDivisionError as e:
             logging.error("Detected a ZeroDivisionError. {}".format(e.__str__()))
@@ -1201,7 +1222,7 @@ def from_rts_to_regression_table(
             else:
                 # lower-better
                 percentage_change = (
-                    float(baseline_v) / float(comparison_v) - 1
+                    -(float(baseline_v) - float(comparison_v)) / float(baseline_v)
                 ) * 100.0
         else:
             logging.warn(
@@ -1283,16 +1304,27 @@ def from_rts_to_regression_table(
             logging.warning(
                 "There were no datapoints both for baseline and comparison for test: {test_name}"
             )
-            no_datapoints_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
     logging.warning(
         f"There is a total of {len(no_datapoints_list)} tests without datapoints for baseline AND comparison"
     )
     logging.info(
         f"There is a total of {len(comparison_only_list)} tests without datapoints for baseline"
     )
+    print(
+        "No datapoint baseline regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_baseline_list)
+        )
+    )
     logging.info(
         f"There is a total of {len(baseline_only_list)} tests without datapoints for comparison"
     )
+    print(
+        "No datapoint comparison regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_comparison_list)
+        )
+    )
     logging.info(f"There is a total of {len(unstable_list)} UNSTABLE tests")
     return (
         detected_regressions,

redis_benchmarks_specification/test-suites/defaults.yml

@@ -10,15 +10,19 @@ exporter:
     - $."BEST RUN RESULTS".Totals."Latency"
     - $."BEST RUN RESULTS".Totals."Misses/sec"
     - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
     - $."WORST RUN RESULTS".Totals."Ops/sec"
     - $."WORST RUN RESULTS".Totals."Latency"
     - $."WORST RUN RESULTS".Totals."Misses/sec"
     - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Ops/sec"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Latency"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Misses/sec"
-    - $."AGGREGATED AVERAGE RESULTS (5 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Percentile Latencies"."p99.00"
     - $."ALL STATS".Totals."Ops/sec"
     - $."ALL STATS".Totals."Latency"
     - $."ALL STATS".Totals."Misses/sec"
     - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p99.00"

{redis_benchmarks_specification-0.1.246.dist-info → redis_benchmarks_specification-0.1.247.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.246
+Version: 0.1.247
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.246.dist-info → redis_benchmarks_specification-0.1.247.dist-info}/RECORD

@@ -20,7 +20,7 @@ redis_benchmarks_specification/__common__/spec.py,sha256=eTF5559epBB0FrJPx-jRDQV
 redis_benchmarks_specification/__common__/timeseries.py,sha256=_LJFtC5sVP7DTaLZaIzv5g7wRxPTQZRwFIYvWX4p4N8,50533
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
 redis_benchmarks_specification/__compare__/args.py,sha256=-vQco6WG73DJtUFA7s-5bNk05vwHjOc94puVKZMZUYE,6513
-redis_benchmarks_specification/__compare__/compare.py,sha256=YWFgwedCkqnp4FMjzwZt50cJI75qabxYAdtPkJRqerk,52179
+redis_benchmarks_specification/__compare__/compare.py,sha256=0uv61Xlvlb7cUd6ZvIf1P3icj8xEe9FkPco7kuIGitI,53479
 redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gqDKgzvNJnkGNKY,491
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=lYvbPd_3ppHZv4f2sRwXcF-fcBrwRSn3H2RMmNVkojY,7221
@@ -49,7 +49,7 @@ redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-def
 redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml,sha256=I6qEO7MZKduVx6xbBrRniE1i6NK9R8-uQXdQJT9o5G4,511
 redis_benchmarks_specification/setups/platforms/aws-ec2-1node-c5.4xlarge.yml,sha256=l7HsjccpebwZXeutnt3SHSETw4iiRwQ9dCDXLOySSRQ,622
 redis_benchmarks_specification/setups/topologies/topologies.yml,sha256=4h8e-zXA9-_iuJ2MsgldUnPKpHTUUD3BgLdZ9FsLPdI,3007
-redis_benchmarks_specification/test-suites/defaults.yml,sha256=Ckte0Bv8b5L7vxWYqochiVGZnlqqw49-vGjFRsppXdg,991
+redis_benchmarks_specification/test-suites/defaults.yml,sha256=CfNfn_nKTtVvbbTwmksJgLKzCzzBAm5SO7UAdrcqncg,1268
 redis_benchmarks_specification/test-suites/generate.py,sha256=FApiWulMw-_2jhl1zqrLUnTnOk3Vm7YxC3oGdauhv3A,3835
 redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml,sha256=HSFrXsVcX1c69IsOaRgrcnJrKs-qaNrtuFLpKuykbWI,2182
 redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml,sha256=HbE7SDAJbF7t6J4ue58D3U6BT36q2RZKwRHunQn7AhA,1904
@@ -329,8 +329,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed
 redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-string-mixed-20-80-with-512B-values-pipeline-10-5200_conns.yml,sha256=4NHvtQR0UY-4iH4mPSUhJKrk-b5vSYtqZc2J7pm4vMA,1143
 redis_benchmarks_specification/test-suites/memtier_benchmark-connection-hello.yml,sha256=VIFNaaAqRQMeDvtXmKGDpJTnu658Lv5i_oAju5uSi_c,708
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
-redis_benchmarks_specification-0.1.246.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.246.dist-info/METADATA,sha256=DjL5FxymUfsHtWSvmkx7eKojIHRtOFKz2dVeoKYmXdw,22726
-redis_benchmarks_specification-0.1.246.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.246.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.246.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.247.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.247.dist-info/METADATA,sha256=4-472_bg8tn_XhVJ8dyc7FdTI9jd0SLirFkEOEbK8bA,22726
+redis_benchmarks_specification-0.1.247.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.247.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.247.dist-info/RECORD,,