mite 2.4.3__tar.gz → 2.4.4__tar.gz

{mite-2.4.3 → mite-2.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mite
-Version: 2.4.3
+Version: 2.4.4
 Summary: A Python Performance Testing Framework
 Home-page: https://github.com/sky-uk/mite/
 Classifier: License :: OSI Approved :: MIT License

{mite-2.4.3 → mite-2.4.4}/mite/__main__.py

@@ -97,6 +97,7 @@ Options:
     --mean-response-time-threshold=THRESHOLD                Set the mean response time accepted before setting exit status to 1 [default: 0]
     --standard-deviation-response-time-threshold=THRESHOLD  Set the response time standard deviation accepted before setting exit status to 1 [default: 0]
     --standard-deviation-req-sec-threshold=THRESHOLD        Set the request per second standard deviation accepted before setting exit status to 1 [default: 0]
+    --benchmark-percentiles=NEW_VALUE                       Percentiles(int):threshold(int in milliseconds) pairs separated by commas [default: 50:0,90:0,98:0,99:0]
 """
 import asyncio
 import logging

{mite-2.4.3 → mite-2.4.4}/mite/cli/test.py

@@ -128,6 +128,10 @@ async def controller_report(controller, receiver):
         controller.report(receiver.recieve)
 
 
+def time_unit_format(number):
+    return f"{number * 1000:.2f}ms" if number < 1 else f"{number:.3f}s"
+
+
 def test_scenarios(test_name, opts, scenarios, config_manager):
     scenario_manager = _create_scenario_manager(opts)
     for journey_spec, datapool, volumemodel in scenarios:
@@ -191,6 +195,16 @@ def test_scenarios(test_name, opts, scenarios, config_manager):
 
     if opts.get("--report"):
         benchmark_report(http_stats_output)
+        for (
+            percentile,
+            threshold,
+            result,
+        ) in http_stats_output.percentiles_list_resp_time_store:
+            if threshold != 0 and result * 1000 >= threshold:
+                has_error = True
+                logging.error(
+                    f"Response time at {percentile}th percentile exceeded {threshold}ms: {time_unit_format(result)}"
+                )
 
     # Ensure any open files get closed
     del receiver._raw_listeners
@@ -207,25 +221,62 @@ def human_readable_bytes(size):
     return size, "PB"
 
 
+def colorise_result(result, threshold):
+    formatted_result = time_unit_format(result)
+    if threshold != 0:
+        result_ms = result * 1000
+        if result_ms >= threshold:
+            return f"\033[91m{formatted_result}\033[0m"  # red
+        else:
+            return f"\033[92m{formatted_result}\033[0m"  # green
+    return formatted_result
+
+
 def benchmark_report(http_stats_output):
-    table = PrettyTable()
-    table.field_names = ["Metric", "Avg", "Min", "Max", "Std Dev", "+/- Std Dev"]
-    number_format = (
-        lambda number: f"{number / 1000:.2f}K" if number >= 1000 else f"{number:.2f}"
-    )
+    perctile_thresh_results_list = http_stats_output.percentiles_list_resp_time_store
 
-    table.add_row(
+    percentiles_headers = [
+        f"{int(percentile[0])} %tile" for percentile in perctile_thresh_results_list
+    ]
+    standard_headers = ["", "Avg", "Min", "Max", "Std Dev", "+/- Std Dev"]
+
+    latency_table = PrettyTable()
+    latency_table.field_names = standard_headers + percentiles_headers
+
+    latency_table.add_row(
         [
             "Latency",
-            f"{http_stats_output.mean_resp_time * 1000:.2f}ms",
-            f"{http_stats_output._resp_time_min * 1000:.2f}ms",
-            f"{http_stats_output._resp_time_max * 1000:.2f}ms",
-            f"{http_stats_output.resp_time_standard_deviation * 1000:.2f}ms",
+            time_unit_format(http_stats_output.mean_resp_time),
+            time_unit_format(http_stats_output._resp_time_min),
+            time_unit_format(http_stats_output._resp_time_max),
+            time_unit_format(http_stats_output.resp_time_standard_deviation),
             f"{http_stats_output.resp_time_within_standard_deviation:.2f}%",
-        ]
+            *[
+                colorise_result(result, threshold)
+                for _, threshold, result in perctile_thresh_results_list
+            ],
+        ],
+        divider=True,
     )
 
-    table.add_row(
+    if any(threshold[1] != 0 for threshold in perctile_thresh_results_list):
+        latency_table.add_row(
+            [
+                "Test Thresholds",
+                *["-" for _ in range(len(standard_headers) - 1)],
+                *[
+                    f"{threshold[1]}ms" if threshold[1] != 0 else "-"
+                    for threshold in perctile_thresh_results_list
+                ],
+            ],
+        )
+
+    requests_table = PrettyTable()
+    requests_table.field_names = standard_headers
+    number_format = (
+        lambda number: f"{number / 1000:.2f}K" if number >= 1000 else f"{number:.2f}"
+    )
+    requests_table.add_row(
         [
             "Req/Sec",
             number_format(http_stats_output.req_sec_mean),
@@ -236,11 +287,14 @@ def benchmark_report(http_stats_output):
         ]
     )
 
-    table.align["Metric"] = "l"
     data_transfer, data_unit = human_readable_bytes(http_stats_output._data_transferred)
-    print(table)
+    print("\n" + "=" * 20 + " MITE Benchmark Report " + "=" * 20, end="\n\n")
+    print(latency_table, end="\n\n")
+    print(requests_table, end="\n\n")
+    print("=" * 27 + " Summary " + "=" * 27)
     print(
-        f"{http_stats_output._req_total} requests in {http_stats_output._scenarios_completed_time - http_stats_output._init_time:.2f}s, {data_transfer:.2f} {data_unit} data transferred"
+        f"{http_stats_output._req_total} requests in {http_stats_output._scenarios_completed_time - http_stats_output._init_time:.2f}s, {data_transfer:.2f} {data_unit} data transferred",
+        end="\n\n",
     )
 
 

{mite-2.4.3 → mite-2.4.4}/mite/config.py

@@ -46,10 +46,10 @@ def default_config_loader():
     for name, value in os.environ.items():
         if name.startswith("MITE_CONF_"):
             key_name = name[10:]
-            logger.info(f'Setting config "{key_name}" from environment variable')
+            logger.debug(f'Setting config "{key_name}" from environment variable')
             result[key_name] = value
         if name.startswith("MITE_EVAL_CONF_"):
             key_name = name[15:]
-            logger.info(f'Setting config "{key_name}" by eval\'ing environment variable')
+            logger.debug(f'Setting config "{key_name}" by eval\'ing environment variable')
             result[key_name] = eval(value)
     return result

{mite-2.4.3 → mite-2.4.4}/mite/logoutput.py

@@ -64,6 +64,7 @@ class GenericStatsOutput:
         self._data_transferred = 0
         self._standard_deviation = 0
         self._error_journeys = defaultdict(int)
+        self._benchmark_percentiles = opts.get("--benchmark-percentiles")
 
     def _pct(self, percentile):
         """Percentile calculation with linear interpolation.
@@ -157,6 +158,17 @@ class GenericStatsOutput:
                 journey_name = message.get("journey")
                 self._error_journeys[journey_name] += 1
 
+    @property
+    def percentiles_list_resp_time_store(self):
+        percentiles = self._benchmark_percentiles.split(",")
+        percentiles_thresholds_list = [
+            (int(p.split(":")[0]), int(p.split(":")[1])) for p in percentiles
+        ]
+        if not self._resp_time_store:
+            return []
+        quantiles = statistics.quantiles(self._resp_time_store, n=100)
+        return [(p[0], p[1], quantiles[p[0] - 1]) for p in percentiles_thresholds_list]
+
     @property
     def mean_resp_time(self):
         if not self._resp_time_store:
@@ -165,15 +177,15 @@ class GenericStatsOutput:
 
     @property
     def max_resp_time_recent(self):
-        if not self._resp_time_recent:
+        if not self._resp_time_store:
             return 0
-        return max(self._resp_time_recent)
+        return max(self._resp_time_store)
 
     @property
     def min_resp_time_recent(self):
-        if not self._resp_time_recent:
+        if not self._resp_time_store:
             return 0
-        return min(self._resp_time_recent)
+        return min(self._resp_time_store)
 
     @property
     def req_sec_mean(self):

{mite-2.4.3 → mite-2.4.4}/mite.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mite
-Version: 2.4.3
+Version: 2.4.4
 Summary: A Python Performance Testing Framework
 Home-page: https://github.com/sky-uk/mite/
 Classifier: License :: OSI Approved :: MIT License

{mite-2.4.3 → mite-2.4.4}/test-requirements.txt

@@ -13,5 +13,5 @@ pytest-cov
 pytest-httpbin
 pytest-httpserver
 sanic
-werkzeug==3.0.3
+werkzeug==3.0.6
 uvloop==0.19.0