redis-benchmarks-specification 0.1.246__tar.gz → 0.1.247__tar.gz

{redis_benchmarks_specification-0.1.246 → redis_benchmarks_specification-0.1.247}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.246
+Version: 0.1.247
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.246 → redis_benchmarks_specification-0.1.247}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.246"
+version = "0.1.247"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.246 → redis_benchmarks_specification-0.1.247}/redis_benchmarks_specification/__compare__/compare.py

@@ -569,7 +569,7 @@ def compute_regression_table(
     verbose=False,
     last_n_baseline=-1,
     last_n_comparison=-1,
-    metric_mode="higher-better",
+    metric_mode="lower-better",
     from_date=None,
     from_ts_ms=None,
     to_date=None,
@@ -1019,6 +1019,8 @@ def from_rts_to_regression_table(
     baseline_only_list = []
     comparison_only_list = []
     no_datapoints_list = []
+    no_datapoints_baseline_list = []
+    no_datapoints_comparison_list = []
     original_metric_mode = metric_mode
     for test_name in test_names:
         metric_mode = original_metric_mode
@@ -1079,6 +1081,22 @@ def from_rts_to_regression_table(
         if len(baseline_timeseries) > 1 and multi_value_baseline is False:
             baseline_timeseries = get_only_Totals(baseline_timeseries)
 
+        if len(baseline_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for baseline timeseries {baseline_timeseries}"
+            )
+            no_datapoints_baseline_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
+        if len(comparison_timeseries) == 0:
+            logging.warning(
+                f"No datapoints for test={test_name} for comparison timeseries {comparison_timeseries}"
+            )
+            no_datapoints_comparison_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
+
         if len(baseline_timeseries) != 1 and multi_value_baseline is False:
             if verbose:
                 logging.warning(
@@ -1159,7 +1177,10 @@ def from_rts_to_regression_table(
             #     note = "waterline={:.1f}%.".format(largest_variance)
             #     waterline = largest_variance
 
-        except redis.exceptions.ResponseError:
+        except redis.exceptions.ResponseError as e:
+            logging.error(
+                "Detected a redis.exceptions.ResponseError. {}".format(e.__str__())
+            )
             pass
         except ZeroDivisionError as e:
             logging.error("Detected a ZeroDivisionError. {}".format(e.__str__()))
@@ -1201,7 +1222,7 @@ def from_rts_to_regression_table(
             else:
                 # lower-better
                 percentage_change = (
-                    float(baseline_v) / float(comparison_v) - 1
+                    -(float(baseline_v) - float(comparison_v)) / float(baseline_v)
                 ) * 100.0
         else:
             logging.warn(
@@ -1283,16 +1304,27 @@ def from_rts_to_regression_table(
             logging.warning(
                 "There were no datapoints both for baseline and comparison for test: {test_name}"
             )
-            no_datapoints_list.append(test_name)
+            if test_name not in no_datapoints_list:
+                no_datapoints_list.append(test_name)
     logging.warning(
         f"There is a total of {len(no_datapoints_list)} tests without datapoints for baseline AND comparison"
     )
     logging.info(
         f"There is a total of {len(comparison_only_list)} tests without datapoints for baseline"
     )
+    print(
+        "No datapoint baseline regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_baseline_list)
+        )
+    )
     logging.info(
         f"There is a total of {len(baseline_only_list)} tests without datapoints for comparison"
     )
+    print(
+        "No datapoint comparison regex={test_names_str}".format(
+            test_names_str="|".join(no_datapoints_comparison_list)
+        )
+    )
     logging.info(f"There is a total of {len(unstable_list)} UNSTABLE tests")
     return (
         detected_regressions,

{redis_benchmarks_specification-0.1.246 → redis_benchmarks_specification-0.1.247}/redis_benchmarks_specification/test-suites/defaults.yml

@@ -10,15 +10,19 @@ exporter:
     - $."BEST RUN RESULTS".Totals."Latency"
     - $."BEST RUN RESULTS".Totals."Misses/sec"
     - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."BEST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
     - $."WORST RUN RESULTS".Totals."Ops/sec"
     - $."WORST RUN RESULTS".Totals."Latency"
     - $."WORST RUN RESULTS".Totals."Misses/sec"
     - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p50.00"
+    - $."WORST RUN RESULTS".Totals."Percentile Latencies"."p99.00"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Ops/sec"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Latency"
     - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Misses/sec"
-    - $."AGGREGATED AVERAGE RESULTS (5 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Percentile Latencies"."p50.00"
+    - $."AGGREGATED AVERAGE RESULTS (3 runs)".Totals."Percentile Latencies"."p99.00"
     - $."ALL STATS".Totals."Ops/sec"
     - $."ALL STATS".Totals."Latency"
     - $."ALL STATS".Totals."Misses/sec"
     - $."ALL STATS".Totals."Percentile Latencies"."p50.00"
+    - $."ALL STATS".Totals."Percentile Latencies"."p99.00"