rosotacom-dev 2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rosotacom/__init__.py +32 -0
- rosotacom/__main__.py +8 -0
- rosotacom/anonymize.py +360 -0
- rosotacom/bag_ground_truth.py +395 -0
- rosotacom/benched_set.py +492 -0
- rosotacom/benchmark.py +1842 -0
- rosotacom/bundle_check.py +517 -0
- rosotacom/cli.py +8637 -0
- rosotacom/cli_benchmark.py +8211 -0
- rosotacom/deployment.py +196 -0
- rosotacom/ffmpeg_packet.py +123 -0
- rosotacom/forensics.py +1172 -0
- rosotacom/geomap.py +848 -0
- rosotacom/network_profiles.py +563 -0
- rosotacom/network_shaper.py +127 -0
- rosotacom/plots.py +609 -0
- rosotacom/py.typed +0 -0
- rosotacom/resources/benched-set.yaml +317 -0
- rosotacom/resources/examples/.gitignore +1 -0
- rosotacom/resources/examples/README.md +163 -0
- rosotacom/resources/examples/__init__.py +1 -0
- rosotacom/resources/examples/bags/8_drop_reference/metadata.yaml +17 -0
- rosotacom/resources/examples/deployment.example.yaml +14 -0
- rosotacom/resources/examples/profiles.yaml +85 -0
- rosotacom/resources/examples/ros2docker.json +33 -0
- rosotacom/resources/examples/rosotacom.yaml +12 -0
- rosotacom/resources/examples/scenarios/2_native_chatter/scenario-definition.yaml +10 -0
- rosotacom/resources/examples/scripts/1_heartbeat/run_machine_a.sh +9 -0
- rosotacom/resources/examples/scripts/1_heartbeat/run_machine_b.sh +9 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_a/external.ros2docker.json +21 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_a/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_a/run_external.py +15 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_a/wait_for_echo_topic.sh +10 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_b/external.ros2docker.json +19 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_b/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/2_native_chatter/machine_b/run_external.py +15 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/external.ros2docker.json +12 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/external_session.yaml +9 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/foxglove.json +89 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/run_external.py +15 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/costmap_1second/costmap_1second_0.mcap +0 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/costmap_1second/metadata.yaml +44 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/external.ros2docker.json +13 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/run_external.py +15 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/external.ros2docker.json +12 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/external_session.yaml +9 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/foxglove.json +89 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/run_external.py +15 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/costmap_1second/costmap_1second_0.mcap +0 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/costmap_1second/metadata.yaml +44 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/external.ros2docker.json +13 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/run_external.py +15 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_a/external.ros2docker.json +20 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_a/external_session.yaml +22 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_a/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_a/run_external.py +15 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_b/external.ros2docker.json +20 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_b/external_session.yaml +22 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_b/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/5_sized_payload/machine_b/run_external.py +15 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/external.ros2docker.json +18 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/external_session.yaml +22 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/run_external.py +15 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/external.ros2docker.json +18 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/external_session.yaml +22 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/run_communication.sh +9 -0
- rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/run_external.py +15 -0
- rosotacom/resources/examples/sessions/10_restamp/session-definition.yaml +39 -0
- rosotacom/resources/examples/sessions/11_trickle/session-definition.yaml +42 -0
- rosotacom/resources/examples/sessions/12_content_integrity/session-definition.yaml +33 -0
- rosotacom/resources/examples/sessions/13_link_latency/session-definition.yaml +42 -0
- rosotacom/resources/examples/sessions/14_remote_assist_anonymized/session-definition.yaml +211 -0
- rosotacom/resources/examples/sessions/15_remote_assist_anonymized_costmap/session-definition.yaml +59 -0
- rosotacom/resources/examples/sessions/16_remote_assist_anonymized_camera/session-definition.yaml +61 -0
- rosotacom/resources/examples/sessions/17_synthetic_camera_quality/session-definition.yaml +57 -0
- rosotacom/resources/examples/sessions/1_heartbeat/session-definition.yaml +26 -0
- rosotacom/resources/examples/sessions/1_heartbeat_status/session-definition.yaml +37 -0
- rosotacom/resources/examples/sessions/2_native_chatter/session-definition.yaml +34 -0
- rosotacom/resources/examples/sessions/3_comp_occ_grid/session-definition.yaml +43 -0
- rosotacom/resources/examples/sessions/4_comp_occ_grid_zen/session-definition.yaml +43 -0
- rosotacom/resources/examples/sessions/5_sized_payload/session-definition.yaml +39 -0
- rosotacom/resources/examples/sessions/6_sized_payload_zen/session-definition.yaml +50 -0
- rosotacom/resources/examples/sessions/7_latched_static/session-definition.yaml +49 -0
- rosotacom/resources/examples/sessions/8_drop/session-definition.yaml +37 -0
- rosotacom/resources/examples/sessions/9_throttle/session-definition.yaml +37 -0
- rosotacom/resources/examples/sessions/bench_1_1_capacity/session-definition.yaml +42 -0
- rosotacom/resources/examples/sessions/bench_1_2_load_sweep/session-definition.yaml +51 -0
- rosotacom/resources/examples/sessions/bench_1_3_ramp/session-definition.yaml +41 -0
- rosotacom/resources/examples/sessions/bench_1_4_recovery/session-definition.yaml +47 -0
- rosotacom/resources/examples/sessions/bench_1_5_camera/session-definition.yaml +45 -0
- rosotacom/resources/ros2docker.json.example +33 -0
- rosotacom/resources/ws/.vscode/c_cpp_properties.json +20 -0
- rosotacom/resources/ws/.vscode/settings.json +8 -0
- rosotacom/resources/ws/ota_configs/archive/fastdds_easy_mode.xml.template +32 -0
- rosotacom/resources/ws/ota_configs/archive/fastdds_initial_peer_list.xml.template +21 -0
- rosotacom/resources/ws/ota_configs/archive/fastdds_v2.xml.template +17 -0
- rosotacom/resources/ws/ota_configs/archive/fastdds_v3.xml.template +48 -0
- rosotacom/resources/ws/ota_configs/archive/fastdds_v4.xml.template +95 -0
- rosotacom/resources/ws/ota_configs/archive/fastdds_v5.xml.template +64 -0
- rosotacom/resources/ws/ota_configs/create_zenoh_json5.py +56 -0
- rosotacom/resources/ws/ota_configs/cyclonedds_minimal.xml.template +9 -0
- rosotacom/resources/ws/ota_configs/cyclonedds_tuned.xml.template +33 -0
- rosotacom/resources/ws/ota_configs/fastdds_unicast.xml.template +71 -0
- rosotacom/resources/ws/ota_configs/get_ota_xml.py +214 -0
- rosotacom/resources/ws/ota_configs/utils.py +74 -0
- rosotacom/resources/ws/ota_configs/zenoh.json5.template +32 -0
- rosotacom/resources/ws/ros2src/.gitkeep +0 -0
- rosotacom/resources/ws/ros2src/com_cpp/CMakeLists.txt +67 -0
- rosotacom/resources/ws/ros2src/com_cpp/COLCON_IGNORE +0 -0
- rosotacom/resources/ws/ros2src/com_cpp/LICENSE.txt +25 -0
- rosotacom/resources/ws/ros2src/com_cpp/package.xml +37 -0
- rosotacom/resources/ws/ros2src/com_cpp/src/hz_monitor.cpp +192 -0
- rosotacom/resources/ws/ros2src/com_cpp/src/universal_compressor.cpp +283 -0
- rosotacom/resources/ws/ros2src/com_cpp/src/universal_decompressor.cpp +306 -0
- rosotacom/resources/ws/ros2src/com_msgs/CMakeLists.txt +22 -0
- rosotacom/resources/ws/ros2src/com_msgs/LICENSE.txt +25 -0
- rosotacom/resources/ws/ros2src/com_msgs/msg/CompressedData.msg +3 -0
- rosotacom/resources/ws/ros2src/com_msgs/msg/EchoHeartbeat.msg +10 -0
- rosotacom/resources/ws/ros2src/com_msgs/msg/OtaStamped.msg +4 -0
- rosotacom/resources/ws/ros2src/com_msgs/msg/SizedPayload.msg +4 -0
- rosotacom/resources/ws/ros2src/com_msgs/package.xml +25 -0
- rosotacom/resources/ws/ros2src/com_py/LICENSE.txt +25 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/__init__.py +0 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/base_bridge_relay.py +106 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/bridge_in.py +76 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/bridge_out.py +70 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/heartbeat_echo.py +221 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/hz_monitor.py +178 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/image_bandwidth_reducer.py +218 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/image_pixel_capper.py +370 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/lanelet_saver.py +80 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/latch_relay.py +205 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/legacy/costmap_remapper.py +73 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/legacy/marker_remapper.py +93 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/legacy/speed_test.py +107 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/legacy/tf_remapper.py +156 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/legacy/traffic_monitor.py +91 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/legacy/wifi_network_analysis_1.py +213 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/link_bytes.py +177 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/link_trace.py +186 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/local_global_frame_bridge.py +465 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/localization_map_split_reduce_saver.py +258 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/msg_transform.py +277 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/pair_management.py +87 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/pub_sub_pair.py +145 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/qos.py +254 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/relay_in.py +78 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/relay_out.py +75 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/restamp.py +191 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/sized_publisher.py +244 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/stage_latency.py +64 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/stage_latency_core.py +51 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/status_overview.py +476 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/status_overview_core.py +989 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/tf_map_to_cart_saver.py +99 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/topic_monitor.py +327 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/topic_resolution.py +171 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/trickle.py +183 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/universal_compressor.py +653 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/universal_decompressor.py +372 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/universal_ota_unwrapper.py +191 -0
- rosotacom/resources/ws/ros2src/com_py/com_py/universal_ota_wrapper.py +154 -0
- rosotacom/resources/ws/ros2src/com_py/package.xml +34 -0
- rosotacom/resources/ws/ros2src/com_py/resource/com_py +0 -0
- rosotacom/resources/ws/ros2src/com_py/setup.cfg +6 -0
- rosotacom/resources/ws/ros2src/com_py/setup.py +55 -0
- rosotacom/resources/ws/ros2src/com_py/test/test_copyright.py +25 -0
- rosotacom/resources/ws/ros2src/com_py/test/test_flake8.py +25 -0
- rosotacom/resources/ws/ros2src/com_py/test/test_pep257.py +23 -0
- rosotacom/resources/ws/session/content/base/session_plugin_base.yaml +633 -0
- rosotacom/resources/ws/session/content/network/up_connectivity.py +364 -0
- rosotacom/resources/ws/session/content/network/wifi_network_analysis_2.py +215 -0
- rosotacom/resources/ws/session/creation/anonymize_bag.py +156 -0
- rosotacom/resources/ws/session/creation/catmux_log_setup.sh +65 -0
- rosotacom/resources/ws/session/creation/create_session_yaml.py +145 -0
- rosotacom/resources/ws/session/creation/generate_session_files.py +3217 -0
- rosotacom/resources/ws/session/creation/run_session.py +447 -0
- rosotacom/resources/ws/session/creation/run_session_in_external_terminal.py +54 -0
- rosotacom/resources/ws/session/creation/strip_ansi.py +42 -0
- rosotacom/status_eval.py +475 -0
- rosotacom/stream_stats.py +650 -0
- rosotacom/trace_profiles.py +561 -0
- rosotacom/transit.py +249 -0
- rosotacom/video_quality.py +906 -0
- rosotacom_dev-2.2.dist-info/METADATA +874 -0
- rosotacom_dev-2.2.dist-info/RECORD +195 -0
- rosotacom_dev-2.2.dist-info/WHEEL +5 -0
- rosotacom_dev-2.2.dist-info/entry_points.txt +4 -0
- rosotacom_dev-2.2.dist-info/licenses/LICENSE.txt +25 -0
- rosotacom_dev-2.2.dist-info/top_level.txt +1 -0
rosotacom/benchmark.py
ADDED
|
@@ -0,0 +1,1842 @@
|
|
|
1
|
+
"""RFC 0005 — benchmark genres (sweep/capacity & perturbation/recovery).
|
|
2
|
+
|
|
3
|
+
Pure, host-testable driver and verdict logic. The *runs* themselves (live ROS
|
|
4
|
+
graphs, emulated profiles, the nightly runner topology) are non-deterministic and
|
|
5
|
+
FZI-private — they live in the harness, not here (RFC 0005, "CI distribution").
|
|
6
|
+
What lives here is everything that can be exercised by a deterministic host test:
|
|
7
|
+
|
|
8
|
+
* size-pattern expansion (the a/b irregular-size load, mirrors ``sized_publisher``);
|
|
9
|
+
* the capacity binary-search driver and its oracle (``loss < p`` and ``latency < L``);
|
|
10
|
+
* sweep bounds + the shared-link guard (an unshaped run never saturates the LAN);
|
|
11
|
+
* the committed band store, the two-sided compare, and the ratchet (RFC 0007);
|
|
12
|
+
* recovery-metric extraction from a timeline of RFC 0003 transit records;
|
|
13
|
+
* fixed-probe time-bin characterization for latency/loss/Hz/bandwidth plots;
|
|
14
|
+
* the coarse linear-ramp curve (monitor-only trend).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import math
|
|
21
|
+
import os
|
|
22
|
+
import platform
|
|
23
|
+
import re
|
|
24
|
+
import socket
|
|
25
|
+
from collections.abc import Callable, Iterable, Mapping, Sequence
|
|
26
|
+
from dataclasses import asdict, dataclass, field, replace
|
|
27
|
+
from enum import Enum
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
# --------------------------------------------------------------------------- #
|
|
32
|
+
# Genre 1 — load driver: a/b size pattern
|
|
33
|
+
# --------------------------------------------------------------------------- #
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def parse_size_pattern(pattern: str) -> list[str]:
|
|
37
|
+
"""Parse ``"a*4,b*1"`` / ``"ax4,bx1"`` into a cyclic list of ``a``/``b`` tokens.
|
|
38
|
+
|
|
39
|
+
Mirrors ``sized_publisher._parse_pattern`` so the host test validates the same
|
|
40
|
+
generation logic the ROS node uses. An empty pattern means a single ``a`` size.
|
|
41
|
+
"""
|
|
42
|
+
if not pattern.strip():
|
|
43
|
+
return ["a"]
|
|
44
|
+
|
|
45
|
+
tokens: list[str] = []
|
|
46
|
+
for raw_token in pattern.split(","):
|
|
47
|
+
token = raw_token.strip().lower()
|
|
48
|
+
match = re.fullmatch(r"([a-z])(?:[*x](\d+))?", token)
|
|
49
|
+
if not match:
|
|
50
|
+
raise ValueError(f"Invalid pattern token '{raw_token}'. Use tokens like 'a', 'b', 'ax4', or 'b*2'.")
|
|
51
|
+
count = int(match.group(2) or "1")
|
|
52
|
+
if count < 1:
|
|
53
|
+
raise ValueError(f"Pattern token '{raw_token}' must repeat at least once.")
|
|
54
|
+
tokens.extend([match.group(1)] * count)
|
|
55
|
+
return tokens
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse_payload_size_bytes(raw: str) -> int:
|
|
59
|
+
"""Parse a payload byte size with optional decimal/binary units."""
|
|
60
|
+
text = raw.strip()
|
|
61
|
+
match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([A-Za-z]*)", text)
|
|
62
|
+
if not match:
|
|
63
|
+
raise ValueError(f"Invalid payload size {raw!r}. Use values like 20000, 20KB, or 20KiB.")
|
|
64
|
+
|
|
65
|
+
value = float(match.group(1))
|
|
66
|
+
unit = match.group(2).lower()
|
|
67
|
+
units = {
|
|
68
|
+
"": 1,
|
|
69
|
+
"b": 1,
|
|
70
|
+
"k": 1_000,
|
|
71
|
+
"kb": 1_000,
|
|
72
|
+
"m": 1_000_000,
|
|
73
|
+
"mb": 1_000_000,
|
|
74
|
+
"g": 1_000_000_000,
|
|
75
|
+
"gb": 1_000_000_000,
|
|
76
|
+
"ki": 1024,
|
|
77
|
+
"kib": 1024,
|
|
78
|
+
"mi": 1024 * 1024,
|
|
79
|
+
"mib": 1024 * 1024,
|
|
80
|
+
"gi": 1024 * 1024 * 1024,
|
|
81
|
+
"gib": 1024 * 1024 * 1024,
|
|
82
|
+
}
|
|
83
|
+
if unit not in units:
|
|
84
|
+
raise ValueError(f"Invalid payload size unit {match.group(2)!r}. Use B, KB, MB, KiB, or MiB.")
|
|
85
|
+
bytes_value = value * units[unit]
|
|
86
|
+
if bytes_value < 0 or not bytes_value.is_integer():
|
|
87
|
+
raise ValueError(f"Payload size {raw!r} does not resolve to a whole non-negative byte count.")
|
|
88
|
+
return int(bytes_value)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _compress_size_pattern(tokens: Sequence[str]) -> str:
|
|
92
|
+
compressed: list[str] = []
|
|
93
|
+
current = tokens[0]
|
|
94
|
+
count = 1
|
|
95
|
+
for token in tokens[1:]:
|
|
96
|
+
if token == current:
|
|
97
|
+
count += 1
|
|
98
|
+
continue
|
|
99
|
+
compressed.append(f"{current}*{count}")
|
|
100
|
+
current = token
|
|
101
|
+
count = 1
|
|
102
|
+
compressed.append(f"{current}*{count}")
|
|
103
|
+
return ",".join(compressed)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def parse_size_pattern_load(pattern: str) -> dict[str, Any]:
|
|
107
|
+
"""Parse ``1x20KB+1x0KB`` into ``sized_publisher`` load parameters.
|
|
108
|
+
|
|
109
|
+
Pattern terms are ``COUNTxSIZE`` or just ``SIZE``, separated by ``+`` or
|
|
110
|
+
``,``. Decimal units (KB/MB/GB) use powers of 1000; binary units
|
|
111
|
+
(KiB/MiB/GiB) use powers of 1024.
|
|
112
|
+
"""
|
|
113
|
+
if not pattern.strip():
|
|
114
|
+
raise ValueError("size pattern must not be empty.")
|
|
115
|
+
|
|
116
|
+
label_by_size: dict[int, str] = {}
|
|
117
|
+
size_by_label: dict[str, int] = {}
|
|
118
|
+
tokens: list[str] = []
|
|
119
|
+
|
|
120
|
+
for raw_term in re.split(r"[+,]", pattern):
|
|
121
|
+
term = raw_term.strip()
|
|
122
|
+
if not term:
|
|
123
|
+
raise ValueError(f"Invalid size pattern {pattern!r}: empty term.")
|
|
124
|
+
match = re.fullmatch(r"(?:(\d+)\s*[x*]\s*)?(.+)", term)
|
|
125
|
+
if not match:
|
|
126
|
+
raise ValueError(f"Invalid size pattern term {raw_term!r}. Use terms like 1x20KB or 0KB.")
|
|
127
|
+
count = int(match.group(1) or "1")
|
|
128
|
+
if count < 1:
|
|
129
|
+
raise ValueError(f"Size pattern term {raw_term!r} must repeat at least once.")
|
|
130
|
+
size = parse_payload_size_bytes(match.group(2))
|
|
131
|
+
label = label_by_size.get(size)
|
|
132
|
+
if label is None:
|
|
133
|
+
label = chr(ord("a") + len(label_by_size))
|
|
134
|
+
label_by_size[size] = label
|
|
135
|
+
size_by_label[label] = size
|
|
136
|
+
tokens.extend([label] * count)
|
|
137
|
+
|
|
138
|
+
sizes_list = [size_by_label[t] for t in tokens]
|
|
139
|
+
|
|
140
|
+
load: dict[str, Any] = {
|
|
141
|
+
"sizes": sizes_list,
|
|
142
|
+
"pattern": _compress_size_pattern(tokens),
|
|
143
|
+
"size_pattern": pattern,
|
|
144
|
+
}
|
|
145
|
+
for label, size in size_by_label.items():
|
|
146
|
+
load[f"size_{label}"] = size
|
|
147
|
+
|
|
148
|
+
return load
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def expand_size_pattern(pattern: str, size_a: int, size_b: int | None = None, **kwargs: Any) -> list[int]:
|
|
152
|
+
"""Expand a pattern into the concrete cyclic byte-size sequence it publishes."""
|
|
153
|
+
if size_a < 0:
|
|
154
|
+
raise ValueError("size_a must be >= 0.")
|
|
155
|
+
tokens = parse_size_pattern(pattern)
|
|
156
|
+
sizes = {"a": size_a, "b": size_b}
|
|
157
|
+
sizes.update(kwargs)
|
|
158
|
+
for token in tokens:
|
|
159
|
+
if token not in sizes or sizes[token] is None:
|
|
160
|
+
raise ValueError(f"Pattern references size {token!r} but it was not provided.")
|
|
161
|
+
return [int(sizes[token]) for token in tokens] # type: ignore[arg-type]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def pattern_mean_bytes(pattern: str, size_a: int, size_b: int | None = None, **kwargs: Any) -> float:
|
|
165
|
+
"""Mean payload of one pattern cycle — the basis for the offered-bandwidth bound."""
|
|
166
|
+
sizes = expand_size_pattern(pattern, size_a, size_b, **kwargs)
|
|
167
|
+
return sum(sizes) / len(sizes)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# --------------------------------------------------------------------------- #
|
|
171
|
+
# Genre 1 — oracle: loss < p AND latency < L over a window
|
|
172
|
+
# --------------------------------------------------------------------------- #
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@dataclass(frozen=True)
|
|
176
|
+
class OracleThresholds:
|
|
177
|
+
"""Pass/fail bound for a capacity probe, read off the RFC 0003 backbone."""
|
|
178
|
+
|
|
179
|
+
max_loss_pct: float
|
|
180
|
+
max_latency_ms: float
|
|
181
|
+
latency_quantile: str = "p95"
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def oracle_passes(loss_pct: float, latency_ms: float | None, thresholds: OracleThresholds) -> bool:
|
|
185
|
+
"""A point passes iff loss and the chosen latency quantile are both under bound.
|
|
186
|
+
|
|
187
|
+
``latency_ms is None`` means nothing was delivered in the window — a failure.
|
|
188
|
+
"""
|
|
189
|
+
if latency_ms is None:
|
|
190
|
+
return False
|
|
191
|
+
return loss_pct <= thresholds.max_loss_pct and latency_ms <= thresholds.max_latency_ms
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def oracle_passes_topic(topic_summary: dict[str, Any], thresholds: OracleThresholds) -> bool:
|
|
195
|
+
"""Apply the oracle to one topic of ``transit.summarize_transit_records`` output."""
|
|
196
|
+
loss_pct = float(topic_summary.get("loss_pct", 100.0))
|
|
197
|
+
latency = (topic_summary.get("ota_hop_ms") or {}).get(thresholds.latency_quantile)
|
|
198
|
+
return oracle_passes(loss_pct, None if latency is None else float(latency), thresholds)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# --------------------------------------------------------------------------- #
|
|
202
|
+
# Bag-as-load oracle: a whole replay contract, judged per topic
|
|
203
|
+
# --------------------------------------------------------------------------- #
|
|
204
|
+
#
|
|
205
|
+
# When the benchmark *load* is a bag replay (a real session/scenario) rather than
|
|
206
|
+
# a single synthetic stream, one probe point delivers many topics at once. The
|
|
207
|
+
# verdict is then per topic — each contract topic must clear the network-loss and
|
|
208
|
+
# latency bound *and* deliver a high-enough share of the bag's known messages
|
|
209
|
+
# (completeness ground truth from ``bag_ground_truth``) — aggregated to a run
|
|
210
|
+
# verdict: the run passes iff every required topic passes, and the failing topics
|
|
211
|
+
# are named. This mirrors ``oracle_passes_topic`` for the loss/latency part and
|
|
212
|
+
# adds the completeness gate the known-source replay makes possible (RFC 0002,
|
|
213
|
+
# "Live vs replay"). It is pure so the search over replay verdicts is host-tested.
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@dataclass(frozen=True)
|
|
217
|
+
class TopicVerdict:
|
|
218
|
+
"""Per-topic verdict of one replay run against the contract + bag ground truth."""
|
|
219
|
+
|
|
220
|
+
topic: str
|
|
221
|
+
passes: bool
|
|
222
|
+
expected: int | None # bag ground-truth count when known, else the run's own expected
|
|
223
|
+
delivered: int | None
|
|
224
|
+
lost: int | None
|
|
225
|
+
reordered: int | None
|
|
226
|
+
loss_pct: float
|
|
227
|
+
completeness: float | None # delivered / bag-count; None when no ground truth
|
|
228
|
+
latency_p95_ms: float | None
|
|
229
|
+
jitter_p95_ms: float | None
|
|
230
|
+
loss_free: bool # zero network loss AND (when known) complete against the bag
|
|
231
|
+
latency_ok: bool
|
|
232
|
+
reason: str # "ok" or the first failing check, for the failing-topics list
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@dataclass(frozen=True)
|
|
236
|
+
class BagRunVerdict:
|
|
237
|
+
"""Run verdict aggregated over every required contract topic of one replay."""
|
|
238
|
+
|
|
239
|
+
passes: bool
|
|
240
|
+
loss_free: bool
|
|
241
|
+
latency_ok: bool
|
|
242
|
+
topics: tuple[TopicVerdict, ...]
|
|
243
|
+
failing_topics: tuple[str, ...]
|
|
244
|
+
representative_topic: str
|
|
245
|
+
# Row aggregates (summed counts; worst-case rate metrics across topics):
|
|
246
|
+
expected: int
|
|
247
|
+
delivered: int
|
|
248
|
+
lost: int
|
|
249
|
+
reordered: int
|
|
250
|
+
loss_pct: float
|
|
251
|
+
latency_p95_ms: float | None
|
|
252
|
+
jitter_p95_ms: float | None
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def summary_topic_name(label: str) -> str:
|
|
256
|
+
"""The bare ROS topic of a ``summarize_transit_records`` label.
|
|
257
|
+
|
|
258
|
+
Labels are ``"<source>-><target>:<topic>"`` when directions are known (peer
|
|
259
|
+
identities carry no ``:``) and just ``"<topic>"`` otherwise; ROS topics never
|
|
260
|
+
contain ``:``, so the topic is everything after the first ``:``.
|
|
261
|
+
"""
|
|
262
|
+
if "->" in label and ":" in label:
|
|
263
|
+
return label.split(":", 1)[1]
|
|
264
|
+
return label
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _topic_data_by_name(summary: dict[str, Any]) -> dict[str, tuple[str, dict[str, Any]]]:
|
|
268
|
+
"""Index a run summary by bare topic name, keeping the richest per-topic row.
|
|
269
|
+
|
|
270
|
+
A contract topic flows one direction, but if a name appears under several
|
|
271
|
+
labels the entry with the most delivered messages is the real stream.
|
|
272
|
+
"""
|
|
273
|
+
topics = summary.get("topics")
|
|
274
|
+
if not isinstance(topics, dict):
|
|
275
|
+
return {}
|
|
276
|
+
indexed: dict[str, tuple[str, dict[str, Any]]] = {}
|
|
277
|
+
for label, data in topics.items():
|
|
278
|
+
if not isinstance(data, dict):
|
|
279
|
+
continue
|
|
280
|
+
name = summary_topic_name(str(label))
|
|
281
|
+
current = indexed.get(name)
|
|
282
|
+
if current is None or int(data.get("delivered") or 0) > int(current[1].get("delivered") or 0):
|
|
283
|
+
indexed[name] = (str(label), data)
|
|
284
|
+
return indexed
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def evaluate_bag_run(
|
|
288
|
+
summary: dict[str, Any],
|
|
289
|
+
*,
|
|
290
|
+
thresholds: OracleThresholds,
|
|
291
|
+
ground_truth: dict[str, dict[str, Any]] | None = None,
|
|
292
|
+
min_completeness: float = 0.95,
|
|
293
|
+
required_topics: Sequence[str] | None = None,
|
|
294
|
+
) -> BagRunVerdict:
|
|
295
|
+
"""Judge one replay run's transit summary against the whole contract.
|
|
296
|
+
|
|
297
|
+
``required_topics`` is the set the run must deliver (the session's carried
|
|
298
|
+
topics); when omitted it is taken from ``ground_truth`` if given, else from
|
|
299
|
+
every topic in the summary. ``ground_truth`` maps ROS topic -> bag facts
|
|
300
|
+
(``bag_ground_truth``); when a topic has a bag ``count`` the completeness gate
|
|
301
|
+
(``delivered / count >= min_completeness``) applies on top of the loss/latency
|
|
302
|
+
oracle. A required topic missing from the summary counts as fully lost.
|
|
303
|
+
"""
|
|
304
|
+
if not 0.0 < min_completeness <= 1.0:
|
|
305
|
+
raise ValueError("min_completeness must be within (0, 1].")
|
|
306
|
+
indexed = _topic_data_by_name(summary)
|
|
307
|
+
if required_topics is not None:
|
|
308
|
+
names = [str(name) for name in required_topics]
|
|
309
|
+
elif ground_truth is not None:
|
|
310
|
+
names = sorted(name for name in ground_truth if name in indexed) or sorted(indexed)
|
|
311
|
+
else:
|
|
312
|
+
names = sorted(indexed)
|
|
313
|
+
|
|
314
|
+
quantile = thresholds.latency_quantile
|
|
315
|
+
verdicts: list[TopicVerdict] = []
|
|
316
|
+
for name in names:
|
|
317
|
+
label, data = indexed.get(name, (name, {}))
|
|
318
|
+
gt = (ground_truth or {}).get(name) or {}
|
|
319
|
+
gt_count = int(gt.get("count") or 0) if gt else 0
|
|
320
|
+
|
|
321
|
+
delivered = data.get("delivered")
|
|
322
|
+
delivered_n = int(delivered) if delivered is not None else 0
|
|
323
|
+
lost = data.get("lost")
|
|
324
|
+
reordered = data.get("reordered")
|
|
325
|
+
run_expected = data.get("expected")
|
|
326
|
+
loss_pct = float(data.get("loss_pct", 100.0)) if data else 100.0
|
|
327
|
+
latency_p95 = (data.get("ota_hop_ms") or {}).get(quantile) if data else None
|
|
328
|
+
jitter_p95 = (data.get("jitter_ms") or {}).get("p95") if data else None
|
|
329
|
+
latency_val = None if latency_p95 is None else float(latency_p95)
|
|
330
|
+
|
|
331
|
+
expected_out = gt_count if gt_count > 0 else (int(run_expected) if run_expected is not None else None)
|
|
332
|
+
completeness = (delivered_n / gt_count) if gt_count > 0 else None
|
|
333
|
+
|
|
334
|
+
latency_ok = latency_val is not None and latency_val <= thresholds.max_latency_ms
|
|
335
|
+
loss_ok = loss_pct <= thresholds.max_loss_pct
|
|
336
|
+
no_network_loss = loss_pct <= 0.0 and int(lost or 0) == 0
|
|
337
|
+
complete = completeness is None or completeness >= min_completeness
|
|
338
|
+
passes = bool(data) and loss_ok and latency_ok and complete
|
|
339
|
+
loss_free = bool(data) and no_network_loss and complete
|
|
340
|
+
|
|
341
|
+
if not data:
|
|
342
|
+
reason = "absent"
|
|
343
|
+
elif not complete:
|
|
344
|
+
reason = f"incomplete({completeness:.3f}<{min_completeness:g})"
|
|
345
|
+
elif not loss_ok:
|
|
346
|
+
reason = f"loss({loss_pct:.3g}%>{thresholds.max_loss_pct:g}%)"
|
|
347
|
+
elif not latency_ok:
|
|
348
|
+
if latency_val is None:
|
|
349
|
+
reason = "latency(none)"
|
|
350
|
+
else:
|
|
351
|
+
reason = f"latency({latency_val:.3g}>{thresholds.max_latency_ms:g}ms)"
|
|
352
|
+
else:
|
|
353
|
+
reason = "ok"
|
|
354
|
+
|
|
355
|
+
verdicts.append(
|
|
356
|
+
TopicVerdict(
|
|
357
|
+
topic=name,
|
|
358
|
+
passes=passes,
|
|
359
|
+
expected=expected_out,
|
|
360
|
+
delivered=delivered_n if data else 0,
|
|
361
|
+
lost=int(lost) if lost is not None else (expected_out if not data and expected_out else None),
|
|
362
|
+
reordered=int(reordered) if reordered is not None else None,
|
|
363
|
+
loss_pct=loss_pct,
|
|
364
|
+
completeness=round(completeness, 6) if completeness is not None else None,
|
|
365
|
+
latency_p95_ms=latency_val,
|
|
366
|
+
jitter_p95_ms=None if jitter_p95 is None else float(jitter_p95),
|
|
367
|
+
loss_free=loss_free,
|
|
368
|
+
latency_ok=latency_ok,
|
|
369
|
+
reason=reason,
|
|
370
|
+
)
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
failing = tuple(v.topic for v in verdicts if not v.passes)
|
|
374
|
+
passes = bool(verdicts) and not failing
|
|
375
|
+
loss_free = bool(verdicts) and all(v.loss_free for v in verdicts)
|
|
376
|
+
latency_ok = bool(verdicts) and all(v.latency_ok for v in verdicts)
|
|
377
|
+
|
|
378
|
+
expected_total = sum(int(v.expected) for v in verdicts if v.expected is not None)
|
|
379
|
+
delivered_total = sum(int(v.delivered) for v in verdicts if v.delivered is not None)
|
|
380
|
+
lost_total = sum(int(v.lost) for v in verdicts if v.lost is not None)
|
|
381
|
+
reordered_total = sum(int(v.reordered) for v in verdicts if v.reordered is not None)
|
|
382
|
+
loss_values = [v.loss_pct for v in verdicts]
|
|
383
|
+
latency_values = [v.latency_p95_ms for v in verdicts if v.latency_p95_ms is not None]
|
|
384
|
+
jitter_values = [v.jitter_p95_ms for v in verdicts if v.jitter_p95_ms is not None]
|
|
385
|
+
representative = failing[0] if failing else (verdicts[0].topic if verdicts else "")
|
|
386
|
+
|
|
387
|
+
return BagRunVerdict(
|
|
388
|
+
passes=passes,
|
|
389
|
+
loss_free=loss_free,
|
|
390
|
+
latency_ok=latency_ok,
|
|
391
|
+
topics=tuple(verdicts),
|
|
392
|
+
failing_topics=failing,
|
|
393
|
+
representative_topic=representative,
|
|
394
|
+
expected=expected_total,
|
|
395
|
+
delivered=delivered_total,
|
|
396
|
+
lost=lost_total,
|
|
397
|
+
reordered=reordered_total,
|
|
398
|
+
loss_pct=max(loss_values) if loss_values else 100.0,
|
|
399
|
+
latency_p95_ms=max(latency_values) if latency_values else None,
|
|
400
|
+
jitter_p95_ms=max(jitter_values) if jitter_values else None,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
# --------------------------------------------------------------------------- #
|
|
405
|
+
# Genre 1 — sweep bounds + shared-link guard
|
|
406
|
+
# --------------------------------------------------------------------------- #
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
@dataclass(frozen=True)
|
|
410
|
+
class SweepBounds:
|
|
411
|
+
"""Operationally-relevant ceilings; a sweep stops here, not "until it breaks".
|
|
412
|
+
|
|
413
|
+
``max_bandwidth_bps`` is the offered-load ceiling that keeps an unshaped / LAN
|
|
414
|
+
run from saturating the shared network (RFC 0005, "Bounded, not unbounded").
|
|
415
|
+
"""
|
|
416
|
+
|
|
417
|
+
max_size: int | None = None
|
|
418
|
+
max_rate: float | None = None
|
|
419
|
+
max_bandwidth_bps: float | None = None
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def offered_bandwidth_bps(size_bytes: float, rate_hz: float) -> float:
|
|
423
|
+
return size_bytes * 8.0 * rate_hz
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def within_shared_link_budget(size_bytes: float, rate_hz: float, bounds: SweepBounds) -> bool:
|
|
427
|
+
if bounds.max_bandwidth_bps is None:
|
|
428
|
+
return True
|
|
429
|
+
return offered_bandwidth_bps(size_bytes, rate_hz) <= bounds.max_bandwidth_bps
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def guard_shared_link(size_bytes: float, rate_hz: float, bounds: SweepBounds, *, shared_link: bool) -> None:
|
|
433
|
+
"""Raise if a *shared* (unshaped/LAN) run would exceed the offered-load budget."""
|
|
434
|
+
if shared_link and not within_shared_link_budget(size_bytes, rate_hz, bounds):
|
|
435
|
+
raise ValueError(
|
|
436
|
+
f"offered load {offered_bandwidth_bps(size_bytes, rate_hz):.0f} bps at "
|
|
437
|
+
f"size={size_bytes} B, rate={rate_hz} Hz would saturate the shared link "
|
|
438
|
+
f"(budget {bounds.max_bandwidth_bps:.0f} bps)"
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def size_ceiling(bounds: SweepBounds, *, rate_hz: float) -> int | None:
|
|
443
|
+
"""Largest payload the sweep may probe at ``rate_hz`` under the configured bounds."""
|
|
444
|
+
ceilings: list[int] = []
|
|
445
|
+
if bounds.max_size is not None:
|
|
446
|
+
ceilings.append(int(bounds.max_size))
|
|
447
|
+
if bounds.max_bandwidth_bps is not None and rate_hz > 0:
|
|
448
|
+
ceilings.append(int(bounds.max_bandwidth_bps // (8.0 * rate_hz)))
|
|
449
|
+
return min(ceilings) if ceilings else None
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
# --------------------------------------------------------------------------- #
|
|
453
|
+
# Genre 1 — capacity binary search
|
|
454
|
+
# --------------------------------------------------------------------------- #
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@dataclass(frozen=True)
|
|
458
|
+
class CapacitySlice:
|
|
459
|
+
"""A capacity is a slice of (size × rate × profile); always state the slice."""
|
|
460
|
+
|
|
461
|
+
profile: str
|
|
462
|
+
knob: str # the swept parameter, e.g. "size" or "rate"
|
|
463
|
+
fixed: dict[str, float] = field(default_factory=dict)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
@dataclass(frozen=True)
|
|
467
|
+
class CapacityResult:
|
|
468
|
+
slice: CapacitySlice
|
|
469
|
+
capacity: int | None # highest passing value, or None if even ``low`` fails
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def capacity_binary_search(low: int, high: int, probe: Callable[[int], bool]) -> int | None:
|
|
473
|
+
"""Highest integer value in ``[low, high]`` for which ``probe`` passes.
|
|
474
|
+
|
|
475
|
+
Assumes monotonicity (passing at smaller values) — see the RFC's "honest
|
|
476
|
+
limits": real cellular's non-monotonic loss can fool a single search, so the
|
|
477
|
+
live driver confirms with repeats/medians. Returns ``None`` if ``low`` fails.
|
|
478
|
+
"""
|
|
479
|
+
if low > high:
|
|
480
|
+
raise ValueError("low must be <= high")
|
|
481
|
+
if not probe(low):
|
|
482
|
+
return None
|
|
483
|
+
best = low
|
|
484
|
+
lo, hi = low + 1, high
|
|
485
|
+
while lo <= hi:
|
|
486
|
+
mid = (lo + hi) // 2
|
|
487
|
+
if probe(mid):
|
|
488
|
+
best = mid
|
|
489
|
+
lo = mid + 1
|
|
490
|
+
else:
|
|
491
|
+
hi = mid - 1
|
|
492
|
+
return best
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def find_capacity(
|
|
496
|
+
slice_: CapacitySlice,
|
|
497
|
+
low: int,
|
|
498
|
+
high: int,
|
|
499
|
+
probe: Callable[[int], bool],
|
|
500
|
+
*,
|
|
501
|
+
bounds: SweepBounds | None = None,
|
|
502
|
+
rate_hz: float | None = None,
|
|
503
|
+
) -> CapacityResult:
|
|
504
|
+
"""Bounded capacity search: clamp ``high`` to the sweep ceiling, then search.
|
|
505
|
+
|
|
506
|
+
When ``bounds`` and ``rate_hz`` are given the swept ceiling never exceeds the
|
|
507
|
+
offered-load budget, so a sweep on a shared link cannot be asked to saturate it.
|
|
508
|
+
"""
|
|
509
|
+
if bounds is not None and rate_hz is not None:
|
|
510
|
+
ceiling = size_ceiling(bounds, rate_hz=rate_hz)
|
|
511
|
+
if ceiling is not None:
|
|
512
|
+
high = min(high, ceiling)
|
|
513
|
+
return CapacityResult(slice=slice_, capacity=capacity_binary_search(low, high, probe))
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
# --------------------------------------------------------------------------- #
|
|
517
|
+
# Genre 1/2 — committed bands + the ratchet (RFC 0007, the analogue of ``expect``)
|
|
518
|
+
# --------------------------------------------------------------------------- #
|
|
519
|
+
|
|
520
|
+
BAND_SCHEMA = 2
|
|
521
|
+
|
|
522
|
+
# Half-width = max(k·σ, floor). The defaults are starting points; the runner-class
|
|
523
|
+
# calibration (RFC 0007 checklist) settles them empirically per metric.
|
|
524
|
+
DEFAULT_WIDTH_K = 3.0
|
|
525
|
+
DEFAULT_FLOOR_FRAC = 0.02
|
|
526
|
+
|
|
527
|
+
# Fingerprint for bands rewritten mechanically without measured runs (e.g. the v1
|
|
528
|
+
# → v2 schema rewrite). It never matches a real runner class, so ``compare``
|
|
529
|
+
# refuses these bands until a real calibration replaces them.
|
|
530
|
+
UNCALIBRATED_FINGERPRINT = "uncalibrated"
|
|
531
|
+
|
|
532
|
+
RECALIBRATE_HINT = "recalibrate on the target runner class: rosotacom benchmark ratchet <result.json ...> --recalibrate"
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
class Better(str, Enum):
|
|
536
|
+
"""A metric's better-direction: which way out of the band is an improvement."""
|
|
537
|
+
|
|
538
|
+
HIGHER = "higher" # capacity numbers, completeness
|
|
539
|
+
LOWER = "lower" # latency, loss, recovery times/bursts
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
class Verdict(str, Enum):
|
|
543
|
+
WITHIN = "WITHIN"
|
|
544
|
+
REGRESSED = "REGRESSED" # out of band on the worse side
|
|
545
|
+
IMPROVED = "IMPROVED" # out of band on the better side — gate-red too: ratchet it
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
class BandError(ValueError):
|
|
549
|
+
"""A band refusal. Every path that declines to compare or ratchet raises this."""
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
class FingerprintMismatch(BandError):
|
|
553
|
+
"""Bands never transfer across runner classes (RFC 0007 §3)."""
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
class WideningRefused(BandError):
|
|
557
|
+
"""A plain ratchet only turns one way; moving toward worse needs ``--recalibrate``."""
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
@dataclass(frozen=True)
|
|
561
|
+
class BandProvenance:
|
|
562
|
+
"""Where a band's width comes from — a band without provenance is a guess."""
|
|
563
|
+
|
|
564
|
+
fingerprint: str # runner class whose variance calibrated the width
|
|
565
|
+
window_s: float # measurement window of one calibration run
|
|
566
|
+
repeats: int # K calibration runs behind sigma
|
|
567
|
+
sigma: float # run-to-run standard deviation across the K runs
|
|
568
|
+
floor: float # minimum half-width; keeps one lucky calibration from minting an impossibly tight band
|
|
569
|
+
k: float # half-width = max(k·sigma, floor)
|
|
570
|
+
source_sha: str # commit whose run(s) last moved the band
|
|
571
|
+
ratcheted_at: str # ISO timestamp of the last ratchet
|
|
572
|
+
note: str = "" # one-line cause note for the last move ("gop default changed → smaller keyframes")
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
@dataclass(frozen=True)
|
|
576
|
+
class Band:
|
|
577
|
+
"""Committed two-sided envelope for one ``(row, profile, metric)``.
|
|
578
|
+
|
|
579
|
+
Leaving ``[lo, hi]`` toward worse is ``REGRESSED``; toward better is
|
|
580
|
+
``IMPROVED`` — and in a gate lane both are red, the latter with the exact
|
|
581
|
+
ratchet command. Bands are never hand-edited: they change only through
|
|
582
|
+
:func:`ratchet_band`.
|
|
583
|
+
"""
|
|
584
|
+
|
|
585
|
+
row: str
|
|
586
|
+
profile: str
|
|
587
|
+
metric: str
|
|
588
|
+
lo: float
|
|
589
|
+
hi: float
|
|
590
|
+
better: Better
|
|
591
|
+
provenance: BandProvenance
|
|
592
|
+
|
|
593
|
+
def __post_init__(self) -> None:
|
|
594
|
+
if self.lo > self.hi:
|
|
595
|
+
raise BandError(f"band [{self.lo}, {self.hi}] for {self.metric!r} is inverted (lo > hi)")
|
|
596
|
+
|
|
597
|
+
@property
|
|
598
|
+
def center(self) -> float:
|
|
599
|
+
return 0.5 * (self.lo + self.hi)
|
|
600
|
+
|
|
601
|
+
@property
|
|
602
|
+
def half_width(self) -> float:
|
|
603
|
+
return 0.5 * (self.hi - self.lo)
|
|
604
|
+
|
|
605
|
+
@property
|
|
606
|
+
def key(self) -> tuple[str, str, str]:
|
|
607
|
+
return (self.row, self.profile, self.metric)
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
@dataclass(frozen=True)
|
|
611
|
+
class BandComparison:
|
|
612
|
+
band: Band
|
|
613
|
+
value: float
|
|
614
|
+
verdict: Verdict
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _two_sided_verdict(value: float, *, lo: float, hi: float, better: Better) -> Verdict:
|
|
618
|
+
"""WITHIN / IMPROVED / REGRESSED for a value against a closed ``[lo, hi]`` envelope.
|
|
619
|
+
|
|
620
|
+
Shared by the committed-band gate (:func:`band_verdict`) and A/B experiments
|
|
621
|
+
(:func:`classify_change`) so both speak one verdict language.
|
|
622
|
+
"""
|
|
623
|
+
if value < lo:
|
|
624
|
+
return Verdict.REGRESSED if better is Better.HIGHER else Verdict.IMPROVED
|
|
625
|
+
if value > hi:
|
|
626
|
+
return Verdict.IMPROVED if better is Better.HIGHER else Verdict.REGRESSED
|
|
627
|
+
return Verdict.WITHIN
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def band_verdict(band: Band, value: float) -> Verdict:
|
|
631
|
+
"""Two-sided verdict; the interval is closed (values on the edge are WITHIN)."""
|
|
632
|
+
return _two_sided_verdict(value, lo=band.lo, hi=band.hi, better=band.better)
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def compare_to_band(band: Band, value: float, *, fingerprint: str) -> BandComparison:
|
|
636
|
+
"""Verdict for one measured value, refusing cross-runner-class comparisons.
|
|
637
|
+
|
|
638
|
+
A runner change must force a visible recalibration, never a silent shift —
|
|
639
|
+
so a fingerprint mismatch is a refusal, not a verdict.
|
|
640
|
+
"""
|
|
641
|
+
if fingerprint != band.provenance.fingerprint:
|
|
642
|
+
raise FingerprintMismatch(
|
|
643
|
+
f"{band.metric} band for ({band.row}, {band.profile}) was calibrated on runner class "
|
|
644
|
+
f"{band.provenance.fingerprint!r}, but this run comes from {fingerprint!r}. "
|
|
645
|
+
f"Bands never transfer across runner classes — {RECALIBRATE_HINT}"
|
|
646
|
+
)
|
|
647
|
+
return BandComparison(band=band, value=float(value), verdict=band_verdict(band, float(value)))
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def save_bands(path: Path, bands: Iterable[Band]) -> None:
|
|
651
|
+
"""Write the band store: one JSON line per (row, profile, metric), stably ordered.
|
|
652
|
+
|
|
653
|
+
The band diff is part of the reviewed change (RFC 0007 §2), so ordering and
|
|
654
|
+
key order are deterministic — a ratchet shows up as a minimal, readable diff.
|
|
655
|
+
"""
|
|
656
|
+
ordered = sorted(bands, key=lambda band: band.key)
|
|
657
|
+
duplicates = [key for key, count in _count_keys(ordered).items() if count > 1]
|
|
658
|
+
if duplicates:
|
|
659
|
+
raise BandError(f"duplicate band keys {duplicates!r}; one band per (row, profile, metric)")
|
|
660
|
+
lines = [
|
|
661
|
+
json.dumps(
|
|
662
|
+
{
|
|
663
|
+
"schema": BAND_SCHEMA,
|
|
664
|
+
"row": band.row,
|
|
665
|
+
"profile": band.profile,
|
|
666
|
+
"metric": band.metric,
|
|
667
|
+
"lo": band.lo,
|
|
668
|
+
"hi": band.hi,
|
|
669
|
+
"better": band.better.value,
|
|
670
|
+
"provenance": asdict(band.provenance),
|
|
671
|
+
},
|
|
672
|
+
sort_keys=True,
|
|
673
|
+
)
|
|
674
|
+
for band in ordered
|
|
675
|
+
]
|
|
676
|
+
path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def load_bands(path: Path) -> list[Band]:
|
|
680
|
+
bands: list[Band] = []
|
|
681
|
+
for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
682
|
+
if not line.strip():
|
|
683
|
+
continue
|
|
684
|
+
raw = json.loads(line)
|
|
685
|
+
schema = raw.get("schema")
|
|
686
|
+
if schema != BAND_SCHEMA:
|
|
687
|
+
raise BandError(
|
|
688
|
+
f"{path}:{line_no}: band schema {schema!r} is not supported (expected {BAND_SCHEMA}). "
|
|
689
|
+
f"v1 budget entries were removed without a migration shim — {RECALIBRATE_HINT}"
|
|
690
|
+
)
|
|
691
|
+
bands.append(
|
|
692
|
+
Band(
|
|
693
|
+
row=str(raw["row"]),
|
|
694
|
+
profile=str(raw["profile"]),
|
|
695
|
+
metric=str(raw["metric"]),
|
|
696
|
+
lo=float(raw["lo"]),
|
|
697
|
+
hi=float(raw["hi"]),
|
|
698
|
+
better=Better(raw["better"]),
|
|
699
|
+
provenance=BandProvenance(**raw["provenance"]),
|
|
700
|
+
)
|
|
701
|
+
)
|
|
702
|
+
duplicates = [key for key, count in _count_keys(bands).items() if count > 1]
|
|
703
|
+
if duplicates:
|
|
704
|
+
raise BandError(f"{path}: duplicate band keys {duplicates!r}; one band per (row, profile, metric)")
|
|
705
|
+
return bands
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def _count_keys(bands: Iterable[Band]) -> dict[tuple[str, str, str], int]:
|
|
709
|
+
counts: dict[tuple[str, str, str], int] = {}
|
|
710
|
+
for band in bands:
|
|
711
|
+
counts[band.key] = counts.get(band.key, 0) + 1
|
|
712
|
+
return counts
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def find_band(bands: Iterable[Band], *, row: str, profile: str, metric: str) -> Band | None:
|
|
716
|
+
for band in bands:
|
|
717
|
+
if band.key == (row, profile, metric):
|
|
718
|
+
return band
|
|
719
|
+
return None
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
def bands_for(bands: Iterable[Band], *, row: str, profile: str) -> list[Band]:
|
|
723
|
+
return [band for band in bands if band.row == row and band.profile == profile]
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
def ratchet_band(
|
|
727
|
+
existing: Band | None,
|
|
728
|
+
values: Sequence[float],
|
|
729
|
+
*,
|
|
730
|
+
row: str,
|
|
731
|
+
profile: str,
|
|
732
|
+
metric: str,
|
|
733
|
+
better: Better,
|
|
734
|
+
fingerprint: str,
|
|
735
|
+
window_s: float,
|
|
736
|
+
source_sha: str,
|
|
737
|
+
ratcheted_at: str,
|
|
738
|
+
note: str = "",
|
|
739
|
+
recalibrate: bool = False,
|
|
740
|
+
k: float = DEFAULT_WIDTH_K,
|
|
741
|
+
floor: float = 0.0,
|
|
742
|
+
floor_frac: float = DEFAULT_FLOOR_FRAC,
|
|
743
|
+
) -> Band:
|
|
744
|
+
"""The only way a band changes (RFC 0007 §2). Center = median of ``values``.
|
|
745
|
+
|
|
746
|
+
Plain ratchet re-centers inside the calibrated width and only turns one way:
|
|
747
|
+
the worse edge may move toward better, never toward worse, and the width and
|
|
748
|
+
calibration provenance (σ, floor, k, fingerprint, window, repeats) are
|
|
749
|
+
preserved. ``recalibrate`` recomputes the width from the given runs — K fresh
|
|
750
|
+
repeats — with half-width ``max(k·σ, floor)``; that is the only path that may
|
|
751
|
+
widen a band, move it toward worse, or change its runner class.
|
|
752
|
+
"""
|
|
753
|
+
if not values:
|
|
754
|
+
raise BandError(f"no values for {metric!r} — a ratchet needs at least one run")
|
|
755
|
+
center = _median([float(value) for value in values])
|
|
756
|
+
|
|
757
|
+
if recalibrate:
|
|
758
|
+
sigma = _sample_stdev([float(value) for value in values])
|
|
759
|
+
floor_used = max(float(floor), float(floor_frac) * abs(center))
|
|
760
|
+
half_width = max(float(k) * sigma, floor_used)
|
|
761
|
+
if half_width <= 0.0:
|
|
762
|
+
raise BandError(
|
|
763
|
+
f"recalibrated half-width for {metric!r} is 0 (σ=0 from {len(values)} run(s), floor=0) — "
|
|
764
|
+
"pass --floor or --floor-frac so the band has a width"
|
|
765
|
+
)
|
|
766
|
+
provenance = BandProvenance(
|
|
767
|
+
fingerprint=fingerprint,
|
|
768
|
+
window_s=float(window_s),
|
|
769
|
+
repeats=len(values),
|
|
770
|
+
sigma=sigma,
|
|
771
|
+
floor=floor_used,
|
|
772
|
+
k=float(k),
|
|
773
|
+
source_sha=source_sha,
|
|
774
|
+
ratcheted_at=ratcheted_at,
|
|
775
|
+
note=note,
|
|
776
|
+
)
|
|
777
|
+
return Band(
|
|
778
|
+
row=row,
|
|
779
|
+
profile=profile,
|
|
780
|
+
metric=metric,
|
|
781
|
+
lo=center - half_width,
|
|
782
|
+
hi=center + half_width,
|
|
783
|
+
better=better,
|
|
784
|
+
provenance=provenance,
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
if existing is None:
|
|
788
|
+
raise BandError(
|
|
789
|
+
f"no committed band for ({row}, {profile}, {metric}) — a first band is a calibration; "
|
|
790
|
+
"rerun with --recalibrate"
|
|
791
|
+
)
|
|
792
|
+
if better is not existing.better:
|
|
793
|
+
raise BandError(
|
|
794
|
+
f"better-direction for {metric!r} changed ({existing.better.value} → {better.value}); "
|
|
795
|
+
"that is a redesign of the metric — rerun with --recalibrate"
|
|
796
|
+
)
|
|
797
|
+
if fingerprint != existing.provenance.fingerprint:
|
|
798
|
+
raise FingerprintMismatch(
|
|
799
|
+
f"cannot ratchet the ({row}, {profile}, {metric}) band from runner class {fingerprint!r}: "
|
|
800
|
+
f"its width was calibrated on {existing.provenance.fingerprint!r}. {RECALIBRATE_HINT}"
|
|
801
|
+
)
|
|
802
|
+
|
|
803
|
+
half_width = existing.half_width
|
|
804
|
+
lo, hi = center - half_width, center + half_width
|
|
805
|
+
if existing.better is Better.HIGHER and lo < existing.lo:
|
|
806
|
+
raise WideningRefused(
|
|
807
|
+
f"ratchet would move the {metric!r} floor toward worse ({existing.lo:g} → {lo:g}). "
|
|
808
|
+
"A ratchet only tightens; a deliberate move toward worse needs --recalibrate"
|
|
809
|
+
)
|
|
810
|
+
if existing.better is Better.LOWER and hi > existing.hi:
|
|
811
|
+
raise WideningRefused(
|
|
812
|
+
f"ratchet would move the {metric!r} ceiling toward worse ({existing.hi:g} → {hi:g}). "
|
|
813
|
+
"A ratchet only tightens; a deliberate move toward worse needs --recalibrate"
|
|
814
|
+
)
|
|
815
|
+
provenance = replace(existing.provenance, source_sha=source_sha, ratcheted_at=ratcheted_at, note=note)
|
|
816
|
+
return Band(row=row, profile=profile, metric=metric, lo=lo, hi=hi, better=existing.better, provenance=provenance)
|
|
817
|
+
|
|
818
|
+
|
|
819
|
+
def _sample_stdev(values: Sequence[float]) -> float:
|
|
820
|
+
if len(values) < 2:
|
|
821
|
+
return 0.0
|
|
822
|
+
mean = sum(values) / len(values)
|
|
823
|
+
return math.sqrt(sum((value - mean) ** 2 for value in values) / (len(values) - 1))
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def runner_fingerprint(env: Mapping[str, str] | None = None) -> str:
|
|
827
|
+
"""Runner-class fingerprint recorded in bands and in every ``result.json``.
|
|
828
|
+
|
|
829
|
+
``ROSOTACOM_RUNNER_CLASS`` names the class explicitly (the private bench pair
|
|
830
|
+
sets it); GitHub-hosted runners collapse to one class per OS/arch (their CPU
|
|
831
|
+
models vary run-to-run, the class does not); anything else is pinned to the
|
|
832
|
+
host, so an ad-hoc machine never silently compares against CI bands.
|
|
833
|
+
"""
|
|
834
|
+
env = os.environ if env is None else env
|
|
835
|
+
explicit = env.get("ROSOTACOM_RUNNER_CLASS", "").strip()
|
|
836
|
+
if explicit:
|
|
837
|
+
return explicit
|
|
838
|
+
system = platform.system().lower()
|
|
839
|
+
machine = platform.machine().lower()
|
|
840
|
+
if env.get("RUNNER_ENVIRONMENT", "") == "github-hosted":
|
|
841
|
+
return f"github-hosted-{system}-{machine}"
|
|
842
|
+
return f"host-{socket.gethostname()}-{system}-{machine}"
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
# Better-directions for the metrics the extraction below produces. New banded
|
|
846
|
+
# metrics extend this table (or pass an explicit direction at ratchet time).
|
|
847
|
+
_DEFAULT_BETTER: dict[str, Better] = {
|
|
848
|
+
"t_recover_s": Better.LOWER,
|
|
849
|
+
"t_steady_s": Better.LOWER,
|
|
850
|
+
"recovery_burst": Better.LOWER,
|
|
851
|
+
"lost_during_outage_total": Better.LOWER,
|
|
852
|
+
"completeness_pct": Better.HIGHER,
|
|
853
|
+
"loss_pct": Better.LOWER,
|
|
854
|
+
"latency_p50_ms": Better.LOWER,
|
|
855
|
+
"latency_p95_ms": Better.LOWER,
|
|
856
|
+
"payload_bandwidth_bps": Better.HIGHER,
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def default_better(metric: str) -> Better:
|
|
861
|
+
if metric.startswith("capacity_"):
|
|
862
|
+
return Better.HIGHER
|
|
863
|
+
better = _DEFAULT_BETTER.get(metric)
|
|
864
|
+
if better is None:
|
|
865
|
+
raise BandError(f"no default better-direction for metric {metric!r}; pass --better higher|lower")
|
|
866
|
+
return better
|
|
867
|
+
|
|
868
|
+
|
|
869
|
+
def _probe_metrics_from_result(doc: Mapping[str, Any]) -> dict[str, float]:
|
|
870
|
+
"""Aggregate a probe run's per-attempt topic summaries into band metrics.
|
|
871
|
+
|
|
872
|
+
``loss_pct`` is delivered-weighted across every attempt and topic (the
|
|
873
|
+
bottleneck-dominated gate metric); the latency percentiles are medians of
|
|
874
|
+
the per-attempt summaries (host-timing-dominated — monitor material on
|
|
875
|
+
shared runners, RFC 0007 §3).
|
|
876
|
+
"""
|
|
877
|
+
attempts = (doc.get("measurements") or {}).get("attempts") or []
|
|
878
|
+
expected = 0
|
|
879
|
+
lost = 0
|
|
880
|
+
p50s: list[float] = []
|
|
881
|
+
p95s: list[float] = []
|
|
882
|
+
bandwidth_by_bin: dict[tuple[int, float, float], float] = {}
|
|
883
|
+
for attempt in attempts:
|
|
884
|
+
for topic_row in attempt.get("topics") or []:
|
|
885
|
+
if topic_row.get("expected") is None:
|
|
886
|
+
continue
|
|
887
|
+
expected += int(topic_row["expected"])
|
|
888
|
+
lost += int(topic_row.get("lost") or 0)
|
|
889
|
+
latency = topic_row.get("latency_ms") or {}
|
|
890
|
+
if latency.get("p50") is not None:
|
|
891
|
+
p50s.append(float(latency["p50"]))
|
|
892
|
+
if latency.get("p95") is not None:
|
|
893
|
+
p95s.append(float(latency["p95"]))
|
|
894
|
+
for bin_row in (doc.get("measurements") or {}).get("time_bins") or []:
|
|
895
|
+
bandwidth = bin_row.get("payload_bandwidth_bps")
|
|
896
|
+
if bandwidth is None:
|
|
897
|
+
continue
|
|
898
|
+
key = (
|
|
899
|
+
int(bin_row.get("attempt") or 0),
|
|
900
|
+
float(bin_row.get("bin_start_s") or 0.0),
|
|
901
|
+
float(bin_row.get("bin_end_s") or 0.0),
|
|
902
|
+
)
|
|
903
|
+
bandwidth_by_bin[key] = bandwidth_by_bin.get(key, 0.0) + float(bandwidth)
|
|
904
|
+
if expected <= 0:
|
|
905
|
+
raise BandError("probe run carries no per-topic expected/lost counts — there is nothing to band")
|
|
906
|
+
delivered = expected - lost
|
|
907
|
+
metrics = {
|
|
908
|
+
"completeness_pct": 100.0 * delivered / expected,
|
|
909
|
+
"loss_pct": 100.0 * lost / expected,
|
|
910
|
+
}
|
|
911
|
+
if p50s:
|
|
912
|
+
metrics["latency_p50_ms"] = _median(p50s)
|
|
913
|
+
if p95s:
|
|
914
|
+
metrics["latency_p95_ms"] = _median(p95s)
|
|
915
|
+
if bandwidth_by_bin:
|
|
916
|
+
metrics["payload_bandwidth_bps"] = _median(list(bandwidth_by_bin.values()))
|
|
917
|
+
return metrics
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
def metrics_from_result(doc: Mapping[str, Any]) -> dict[str, float]:
|
|
921
|
+
"""The band-comparable metrics of one self-contained ``result.json``.
|
|
922
|
+
|
|
923
|
+
Covers the deterministic genres the gate bands today: ``capacity`` (the
|
|
924
|
+
breakpoint), ``probe`` (loss/completeness plus latency percentiles) and
|
|
925
|
+
``recovery`` (the RFC 0005 recovery metric set). The benched-set registry
|
|
926
|
+
(RFC 0007 §4) picks which of these actually gate per row.
|
|
927
|
+
"""
|
|
928
|
+
genre = doc.get("genre")
|
|
929
|
+
if genre == "probe":
|
|
930
|
+
return _probe_metrics_from_result(doc)
|
|
931
|
+
if genre == "capacity":
|
|
932
|
+
knob = str((doc.get("configuration") or {}).get("knob") or "size")
|
|
933
|
+
capacity = (doc.get("result") or {}).get("capacity")
|
|
934
|
+
if capacity is None:
|
|
935
|
+
raise BandError("capacity run found no passing probe — there is no value to band")
|
|
936
|
+
return {f"capacity_{knob}": float(capacity)}
|
|
937
|
+
if genre == "recovery":
|
|
938
|
+
result = doc.get("result") or {}
|
|
939
|
+
metrics: dict[str, float] = {}
|
|
940
|
+
for metric, field_name in (("t_recover_s", "t_recover"), ("t_steady_s", "t_steady")):
|
|
941
|
+
if result.get(field_name) is not None:
|
|
942
|
+
metrics[metric] = float(result[field_name])
|
|
943
|
+
if result.get("recovery_burst") is not None:
|
|
944
|
+
metrics["recovery_burst"] = float(result["recovery_burst"])
|
|
945
|
+
lost = result.get("lost_during_outage") or {}
|
|
946
|
+
metrics["lost_during_outage_total"] = float(sum(int(count) for count in lost.values()))
|
|
947
|
+
return metrics
|
|
948
|
+
raise BandError(f"genre {genre!r} has no band metrics yet — banded rows cover probe, capacity and recovery today")
|
|
949
|
+
|
|
950
|
+
|
|
951
|
+
def result_row_id(doc: Mapping[str, Any]) -> str:
|
|
952
|
+
"""Canonical row id of a run until the benched-set registry formalizes rows."""
|
|
953
|
+
genre = str(doc.get("genre") or "")
|
|
954
|
+
if genre == "capacity":
|
|
955
|
+
knob = str((doc.get("configuration") or {}).get("knob") or "size")
|
|
956
|
+
return f"capacity-{knob}"
|
|
957
|
+
return genre or "unknown"
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def result_profile(doc: Mapping[str, Any]) -> str:
|
|
961
|
+
profile = (doc.get("configuration") or {}).get("profile")
|
|
962
|
+
if not profile:
|
|
963
|
+
raise BandError("result.json carries no configuration.profile — cannot key a band without one")
|
|
964
|
+
return str(profile)
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
def result_fingerprint(doc: Mapping[str, Any]) -> str:
|
|
968
|
+
fingerprint = (doc.get("runner") or {}).get("fingerprint")
|
|
969
|
+
if not fingerprint:
|
|
970
|
+
raise BandError("result.json carries no runner fingerprint — it predates band schema v2; re-run the benchmark")
|
|
971
|
+
return str(fingerprint)
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def result_window_s(doc: Mapping[str, Any]) -> float:
|
|
975
|
+
return float((doc.get("configuration") or {}).get("duration_s") or 0.0)
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def result_sha(doc: Mapping[str, Any]) -> str:
|
|
979
|
+
return str(doc.get("sha") or "unknown")
|
|
980
|
+
|
|
981
|
+
|
|
982
|
+
# --------------------------------------------------------------------------- #
|
|
983
|
+
# Fixed probe — time-binned connection characterization
|
|
984
|
+
# --------------------------------------------------------------------------- #
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
@dataclass(frozen=True)
|
|
988
|
+
class ProbeBin:
|
|
989
|
+
topic: str
|
|
990
|
+
bin_start_s: float
|
|
991
|
+
bin_end_s: float
|
|
992
|
+
expected: int
|
|
993
|
+
delivered: int
|
|
994
|
+
lost: int
|
|
995
|
+
loss_pct: float
|
|
996
|
+
delivered_hz: float
|
|
997
|
+
expected_hz: float
|
|
998
|
+
payload_bandwidth_bps: float
|
|
999
|
+
mean_size_bytes: float | None
|
|
1000
|
+
latency_p50_ms: float | None
|
|
1001
|
+
latency_p95_ms: float | None
|
|
1002
|
+
jitter_p50_ms: float | None
|
|
1003
|
+
jitter_p95_ms: float | None
|
|
1004
|
+
inter_arrival_p50_ms: float | None
|
|
1005
|
+
inter_arrival_p95_ms: float | None
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _percentile(values: list[float], percentile: float) -> float | None:
|
|
1009
|
+
if not values:
|
|
1010
|
+
return None
|
|
1011
|
+
ordered = sorted(values)
|
|
1012
|
+
rank = max(0, min(len(ordered) - 1, math.ceil(percentile * len(ordered)) - 1))
|
|
1013
|
+
return round(ordered[rank], 3)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def _transit_topic_label(record: dict[str, Any]) -> str:
|
|
1017
|
+
topic = str(record.get("topic") or "")
|
|
1018
|
+
source = str(record.get("source") or "")
|
|
1019
|
+
target = str(record.get("target") or "")
|
|
1020
|
+
return f"{source}->{target}:{topic}" if source or target else topic
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _section_ms(record: dict[str, Any], *names: str) -> float | None:
|
|
1024
|
+
sections = record.get("sections")
|
|
1025
|
+
if not isinstance(sections, dict):
|
|
1026
|
+
return None
|
|
1027
|
+
for name in names:
|
|
1028
|
+
value = sections.get(name)
|
|
1029
|
+
if value is not None:
|
|
1030
|
+
return float(value)
|
|
1031
|
+
return None
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def _infer_nominal_period_s(records: Sequence[dict[str, Any]], *, time_field: str) -> float | None:
|
|
1035
|
+
candidates: list[float] = []
|
|
1036
|
+
grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
|
|
1037
|
+
for record in records:
|
|
1038
|
+
key = (
|
|
1039
|
+
str(record.get("source") or ""),
|
|
1040
|
+
str(record.get("target") or ""),
|
|
1041
|
+
str(record.get("topic") or ""),
|
|
1042
|
+
)
|
|
1043
|
+
grouped.setdefault(key, []).append(record)
|
|
1044
|
+
|
|
1045
|
+
for stream in grouped.values():
|
|
1046
|
+
stamped = sorted(
|
|
1047
|
+
(record for record in stream if record.get(time_field) is not None),
|
|
1048
|
+
key=lambda record: int(record["seq"]),
|
|
1049
|
+
)
|
|
1050
|
+
for previous, current in zip(stamped, stamped[1:], strict=False):
|
|
1051
|
+
seq_delta = int(current["seq"]) - int(previous["seq"])
|
|
1052
|
+
time_delta = float(current[time_field]) - float(previous[time_field])
|
|
1053
|
+
if seq_delta > 0 and time_delta >= 0.0:
|
|
1054
|
+
candidates.append(time_delta / seq_delta)
|
|
1055
|
+
if not candidates:
|
|
1056
|
+
return None
|
|
1057
|
+
ordered = sorted(candidates)
|
|
1058
|
+
return ordered[len(ordered) // 2]
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def _median(values: Sequence[float]) -> float:
|
|
1062
|
+
ordered = sorted(values)
|
|
1063
|
+
n = len(ordered)
|
|
1064
|
+
mid = n // 2
|
|
1065
|
+
return ordered[mid] if n % 2 else 0.5 * (ordered[mid - 1] + ordered[mid])
|
|
1066
|
+
|
|
1067
|
+
|
|
1068
|
+
def _mad(values: Sequence[float]) -> float:
|
|
1069
|
+
"""Median absolute deviation — a spread estimate that ignores outliers."""
|
|
1070
|
+
center = _median(values)
|
|
1071
|
+
return _median([abs(v - center) for v in values])
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
# A fixed-probe sample is either INCLUDED (the impaired regime under test) or
|
|
1075
|
+
# EXCLUDED (the start-up: warm-up floor, the partial packet, and any startup drop).
|
|
1076
|
+
INCLUDED = "included"
|
|
1077
|
+
EXCLUDED = "excluded"
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def find_probe_onset(
|
|
1081
|
+
samples: Sequence[float | None],
|
|
1082
|
+
*,
|
|
1083
|
+
min_points: int = 12,
|
|
1084
|
+
step_window: int = 6,
|
|
1085
|
+
min_step_ratio: float = 3.0,
|
|
1086
|
+
min_step_ms: float = 15.0,
|
|
1087
|
+
settle_fraction: float = 0.6,
|
|
1088
|
+
settle_window: int = 6,
|
|
1089
|
+
) -> int | None:
|
|
1090
|
+
"""Index in ``samples`` where the clean impaired regime begins, or ``None``.
|
|
1091
|
+
|
|
1092
|
+
``samples`` is one stream ordered by send time; each entry is a delivered
|
|
1093
|
+
packet's latency in ms, or ``None`` for a lost packet.
|
|
1094
|
+
|
|
1095
|
+
A fixed probe records packets before the tc/netem shaping is applied (the low
|
|
1096
|
+
transport floor), then a brief transition where the in-flight packet sees a
|
|
1097
|
+
partial delay and/or is dropped as the qdisc changes, then the impaired regime
|
|
1098
|
+
under test (a plateau *or* a rising bufferbloat ramp). This returns the first
|
|
1099
|
+
index from which the stream is *cleanly* in the impaired regime — a sustained
|
|
1100
|
+
window whose latencies are at/above a threshold set between the floor and the
|
|
1101
|
+
impaired level, with no loss dragging the window down. Everything before it
|
|
1102
|
+
(warm-up, the immature partial packet, and any startup drop) is start-up and
|
|
1103
|
+
should be excluded, so plot and summary agree on a clean measurement window.
|
|
1104
|
+
|
|
1105
|
+
Detection is conservative: without a clear low-floor→high step (``impaired`` at
|
|
1106
|
+
least ``min_step_ratio``× **or** ``min_step_ms`` above the floor) it returns
|
|
1107
|
+
``None`` and nothing is excluded — e.g. impairment live from the first packet,
|
|
1108
|
+
a no-op profile, or a smooth ramp with no warm-up.
|
|
1109
|
+
"""
|
|
1110
|
+
n = len(samples)
|
|
1111
|
+
delivered = [s for s in samples if s is not None]
|
|
1112
|
+
if len(delivered) < min_points:
|
|
1113
|
+
return None
|
|
1114
|
+
|
|
1115
|
+
window = max(1, min(step_window, len(delivered) // 2))
|
|
1116
|
+
# Sharpest upward step on the delivered latencies: the largest ratio between
|
|
1117
|
+
# the windowed median just after and just before an index. The un-impaired
|
|
1118
|
+
# floor is the lowest level, so the warm-up→impaired jump dominates, and
|
|
1119
|
+
# windowed medians stop a single ramp spike from masquerading as the step.
|
|
1120
|
+
step_index: int | None = None
|
|
1121
|
+
best_ratio = 1.0
|
|
1122
|
+
for i in range(window, len(delivered) - window):
|
|
1123
|
+
pre = _median(delivered[i - window : i])
|
|
1124
|
+
post = _median(delivered[i : i + window])
|
|
1125
|
+
if pre <= 0.0:
|
|
1126
|
+
continue
|
|
1127
|
+
ratio = post / pre
|
|
1128
|
+
if ratio > best_ratio:
|
|
1129
|
+
best_ratio = ratio
|
|
1130
|
+
step_index = i
|
|
1131
|
+
if step_index is None:
|
|
1132
|
+
return None
|
|
1133
|
+
|
|
1134
|
+
floor = _median(delivered[max(0, step_index - window) : step_index])
|
|
1135
|
+
impaired = _median(delivered[step_index : step_index + window])
|
|
1136
|
+
step_is_real = impaired >= floor * min_step_ratio or impaired - floor >= min_step_ms
|
|
1137
|
+
if best_ratio < min_step_ratio or not step_is_real:
|
|
1138
|
+
return None
|
|
1139
|
+
|
|
1140
|
+
# Threshold between floor and impaired level; ``settle_fraction`` sits it above
|
|
1141
|
+
# the partial packet so the immature transition sample is excluded.
|
|
1142
|
+
impaired_lo = floor + settle_fraction * (impaired - floor)
|
|
1143
|
+
hold = max(1, min(settle_window, len(delivered) // 2))
|
|
1144
|
+
|
|
1145
|
+
# Onset = first delivered sample at/above the regime that *starts* a sustained
|
|
1146
|
+
# window mostly at/above it. A startup drop (``None``) or the partial packet
|
|
1147
|
+
# keeps its window from qualifying, so onset lands on the first mature packet
|
|
1148
|
+
# after them — the point from which the measurement is clean.
|
|
1149
|
+
for i in range(n):
|
|
1150
|
+
value = samples[i]
|
|
1151
|
+
if value is None or value < impaired_lo:
|
|
1152
|
+
continue
|
|
1153
|
+
ahead = samples[i : i + hold]
|
|
1154
|
+
if len(ahead) < hold:
|
|
1155
|
+
break
|
|
1156
|
+
good = sum(1 for s in ahead if s is not None and s >= impaired_lo)
|
|
1157
|
+
if good >= 0.7 * hold:
|
|
1158
|
+
return i
|
|
1159
|
+
return None
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
def _stream_samples(
|
|
1163
|
+
stream: Sequence[dict[str, Any]],
|
|
1164
|
+
*,
|
|
1165
|
+
seq0: int,
|
|
1166
|
+
t0: float,
|
|
1167
|
+
period_s: float,
|
|
1168
|
+
time_field: str,
|
|
1169
|
+
) -> list[tuple[float, float | None]]:
|
|
1170
|
+
"""One seq-ordered stream as ``(send_time, latency_or_None)`` — ``None`` = lost."""
|
|
1171
|
+
samples: list[tuple[float, float | None]] = []
|
|
1172
|
+
for record in stream:
|
|
1173
|
+
send_t = _send_time(record, seq0=seq0, t0=t0, period_s=period_s, time_field=time_field)
|
|
1174
|
+
latency = (
|
|
1175
|
+
None if record.get("status") == "lost" else _section_ms(record, "ota_hop_ms", "ota_hop_uncorrected_ms")
|
|
1176
|
+
)
|
|
1177
|
+
samples.append((send_t, latency))
|
|
1178
|
+
return samples
|
|
1179
|
+
|
|
1180
|
+
|
|
1181
|
+
def exclude_probe_warmup(
|
|
1182
|
+
joined_records: Sequence[dict[str, Any]],
|
|
1183
|
+
*,
|
|
1184
|
+
nominal_period_s: float | None = None,
|
|
1185
|
+
time_field: str = "t_wrap",
|
|
1186
|
+
) -> tuple[list[dict[str, Any]], float | None]:
|
|
1187
|
+
"""Drop the start-up prefix from already-joined transit records.
|
|
1188
|
+
|
|
1189
|
+
Returns ``(included_records, onset_send_time)`` where ``onset_send_time`` is
|
|
1190
|
+
the earliest per-stream impairment onset on the publish timeline — the origin
|
|
1191
|
+
that re-anchors "time since first publish" so ``t=0`` is the clean measurement
|
|
1192
|
+
start. Every record before its stream's onset (warm-up, the partial packet,
|
|
1193
|
+
and any startup drop) is dropped, so the summary and the raw plot agree on the
|
|
1194
|
+
same window. Streams with no clear warm-up step are kept whole; when no stream
|
|
1195
|
+
shows a step, nothing is dropped and ``onset_send_time`` is ``None``.
|
|
1196
|
+
"""
|
|
1197
|
+
joined = list(joined_records)
|
|
1198
|
+
if not joined:
|
|
1199
|
+
return [], None
|
|
1200
|
+
period_s = (
|
|
1201
|
+
nominal_period_s if nominal_period_s is not None else _infer_nominal_period_s(joined, time_field=time_field)
|
|
1202
|
+
) or 0.0
|
|
1203
|
+
|
|
1204
|
+
grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
|
|
1205
|
+
for record in joined:
|
|
1206
|
+
key = (
|
|
1207
|
+
str(record.get("source") or ""),
|
|
1208
|
+
str(record.get("target") or ""),
|
|
1209
|
+
str(record.get("topic") or ""),
|
|
1210
|
+
)
|
|
1211
|
+
grouped.setdefault(key, []).append(record)
|
|
1212
|
+
|
|
1213
|
+
kept: list[dict[str, Any]] = []
|
|
1214
|
+
onsets: list[float] = []
|
|
1215
|
+
for _key, stream in grouped.items():
|
|
1216
|
+
stream = sorted(stream, key=lambda record: int(record["seq"]))
|
|
1217
|
+
stamped = [record for record in stream if record.get(time_field) is not None]
|
|
1218
|
+
anchor = min(stamped or stream, key=lambda record: int(record["seq"]))
|
|
1219
|
+
seq0 = int(anchor["seq"])
|
|
1220
|
+
t0 = float(anchor.get(time_field) or 0.0)
|
|
1221
|
+
samples = _stream_samples(stream, seq0=seq0, t0=t0, period_s=period_s, time_field=time_field)
|
|
1222
|
+
onset_index = find_probe_onset([latency for _, latency in samples])
|
|
1223
|
+
if onset_index is None:
|
|
1224
|
+
kept.extend(stream)
|
|
1225
|
+
continue
|
|
1226
|
+
onset_send = samples[onset_index][0]
|
|
1227
|
+
onsets.append(onset_send)
|
|
1228
|
+
kept.extend(record for record, (send_t, _lat) in zip(stream, samples, strict=True) if send_t >= onset_send)
|
|
1229
|
+
return kept, (min(onsets) if onsets else None)
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
def characterize_probe_records(
|
|
1233
|
+
records: Iterable[dict[str, Any]],
|
|
1234
|
+
*,
|
|
1235
|
+
bin_s: float = 1.0,
|
|
1236
|
+
nominal_period_s: float | None = None,
|
|
1237
|
+
topic: str = "",
|
|
1238
|
+
time_field: str = "t_wrap",
|
|
1239
|
+
exclude_warmup: bool = True,
|
|
1240
|
+
) -> list[dict[str, Any]]:
|
|
1241
|
+
"""Summarize one fixed probe as per-topic time bins.
|
|
1242
|
+
|
|
1243
|
+
Bins use publish time, not arrival time, so latency/loss spikes line up with
|
|
1244
|
+
the traffic that caused them. Lost records often lack timestamps; when a
|
|
1245
|
+
nominal period is supplied (usually ``1 / rate_hz``), their send time is
|
|
1246
|
+
reconstructed from sequence number just like the recovery metrics do.
|
|
1247
|
+
|
|
1248
|
+
With ``exclude_warmup`` (the default) the un-impaired warm-up plateau and the
|
|
1249
|
+
partial-impairment transition packet are dropped via
|
|
1250
|
+
:func:`exclude_probe_warmup`, and the bins are aligned to the impairment
|
|
1251
|
+
onset so ``bin_start_s == 0`` is where shaping is fully applied. Set it False
|
|
1252
|
+
to bin the raw timeline from the first publish.
|
|
1253
|
+
"""
|
|
1254
|
+
if bin_s <= 0.0:
|
|
1255
|
+
raise ValueError("bin_s must be > 0.")
|
|
1256
|
+
|
|
1257
|
+
from .transit import join_transit_records
|
|
1258
|
+
|
|
1259
|
+
joined_records = join_transit_records(records)
|
|
1260
|
+
if not joined_records:
|
|
1261
|
+
return []
|
|
1262
|
+
period_s = (
|
|
1263
|
+
nominal_period_s
|
|
1264
|
+
if nominal_period_s is not None
|
|
1265
|
+
else _infer_nominal_period_s(joined_records, time_field=time_field)
|
|
1266
|
+
)
|
|
1267
|
+
period_s = float(period_s or 0.0)
|
|
1268
|
+
|
|
1269
|
+
if exclude_warmup:
|
|
1270
|
+
joined_records, _onset = exclude_probe_warmup(joined_records, nominal_period_s=period_s, time_field=time_field)
|
|
1271
|
+
if not joined_records:
|
|
1272
|
+
return []
|
|
1273
|
+
|
|
1274
|
+
grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
|
|
1275
|
+
for record in joined_records:
|
|
1276
|
+
key = (
|
|
1277
|
+
str(record.get("source") or ""),
|
|
1278
|
+
str(record.get("target") or ""),
|
|
1279
|
+
str(record.get("topic") or ""),
|
|
1280
|
+
)
|
|
1281
|
+
grouped.setdefault(key, []).append(record)
|
|
1282
|
+
|
|
1283
|
+
expanded: list[tuple[str, dict[str, Any], float]] = []
|
|
1284
|
+
for _key, stream in sorted(grouped.items()):
|
|
1285
|
+
stream = sorted(stream, key=lambda record: int(record["seq"]))
|
|
1286
|
+
stamped = [record for record in stream if record.get(time_field) is not None]
|
|
1287
|
+
anchor = min(stamped or stream, key=lambda record: int(record["seq"]))
|
|
1288
|
+
seq0 = int(anchor["seq"])
|
|
1289
|
+
t0 = float(anchor.get(time_field) or 0.0)
|
|
1290
|
+
for record in stream:
|
|
1291
|
+
label = _transit_topic_label(record)
|
|
1292
|
+
if topic and topic not in {label, str(record.get("topic") or "")}:
|
|
1293
|
+
continue
|
|
1294
|
+
expanded.append(
|
|
1295
|
+
(
|
|
1296
|
+
label,
|
|
1297
|
+
record,
|
|
1298
|
+
_send_time(record, seq0=seq0, t0=t0, period_s=period_s, time_field=time_field),
|
|
1299
|
+
)
|
|
1300
|
+
)
|
|
1301
|
+
if not expanded:
|
|
1302
|
+
return []
|
|
1303
|
+
|
|
1304
|
+
first_send_s = min(send_t for _label, _record, send_t in expanded)
|
|
1305
|
+
buckets: dict[tuple[str, float], dict[str, Any]] = {}
|
|
1306
|
+
for label, record, send_t in expanded:
|
|
1307
|
+
relative_s = max(0.0, send_t - first_send_s)
|
|
1308
|
+
bucket_start = math.floor(relative_s / bin_s) * bin_s
|
|
1309
|
+
bucket = buckets.setdefault(
|
|
1310
|
+
(label, bucket_start),
|
|
1311
|
+
{
|
|
1312
|
+
"topic": label,
|
|
1313
|
+
"bin_start_s": bucket_start,
|
|
1314
|
+
"bin_end_s": bucket_start + bin_s,
|
|
1315
|
+
"expected": 0,
|
|
1316
|
+
"delivered": 0,
|
|
1317
|
+
"lost": 0,
|
|
1318
|
+
"payload_bytes": 0.0,
|
|
1319
|
+
"latencies": [],
|
|
1320
|
+
"jitters": [],
|
|
1321
|
+
"inter_arrivals": [],
|
|
1322
|
+
},
|
|
1323
|
+
)
|
|
1324
|
+
bucket["expected"] += 1
|
|
1325
|
+
if record.get("status") == "lost":
|
|
1326
|
+
bucket["lost"] += 1
|
|
1327
|
+
continue
|
|
1328
|
+
bucket["delivered"] += 1
|
|
1329
|
+
if record.get("size_bytes") is not None:
|
|
1330
|
+
bucket["payload_bytes"] += float(record["size_bytes"])
|
|
1331
|
+
latency_ms = _section_ms(record, "ota_hop_ms", "ota_hop_uncorrected_ms")
|
|
1332
|
+
if latency_ms is not None:
|
|
1333
|
+
bucket["latencies"].append(latency_ms)
|
|
1334
|
+
if record.get("jitter_ms") is not None:
|
|
1335
|
+
bucket["jitters"].append(float(record["jitter_ms"]))
|
|
1336
|
+
if record.get("inter_arrival_ms") is not None:
|
|
1337
|
+
bucket["inter_arrivals"].append(float(record["inter_arrival_ms"]))
|
|
1338
|
+
|
|
1339
|
+
rows: list[dict[str, Any]] = []
|
|
1340
|
+
for (_label, _bucket_start), bucket in sorted(buckets.items()):
|
|
1341
|
+
expected = int(bucket["expected"])
|
|
1342
|
+
delivered = int(bucket["delivered"])
|
|
1343
|
+
lost = int(bucket["lost"])
|
|
1344
|
+
payload_bytes = float(bucket["payload_bytes"])
|
|
1345
|
+
row = ProbeBin(
|
|
1346
|
+
topic=str(bucket["topic"]),
|
|
1347
|
+
bin_start_s=round(float(bucket["bin_start_s"]), 6),
|
|
1348
|
+
bin_end_s=round(float(bucket["bin_end_s"]), 6),
|
|
1349
|
+
expected=expected,
|
|
1350
|
+
delivered=delivered,
|
|
1351
|
+
lost=lost,
|
|
1352
|
+
loss_pct=round(100.0 * lost / expected, 3) if expected else 0.0,
|
|
1353
|
+
delivered_hz=round(delivered / bin_s, 3),
|
|
1354
|
+
expected_hz=round(expected / bin_s, 3),
|
|
1355
|
+
payload_bandwidth_bps=round(payload_bytes * 8.0 / bin_s, 3),
|
|
1356
|
+
mean_size_bytes=round(payload_bytes / delivered, 3) if delivered else None,
|
|
1357
|
+
latency_p50_ms=_percentile(bucket["latencies"], 0.50),
|
|
1358
|
+
latency_p95_ms=_percentile(bucket["latencies"], 0.95),
|
|
1359
|
+
jitter_p50_ms=_percentile(bucket["jitters"], 0.50),
|
|
1360
|
+
jitter_p95_ms=_percentile(bucket["jitters"], 0.95),
|
|
1361
|
+
inter_arrival_p50_ms=_percentile(bucket["inter_arrivals"], 0.50),
|
|
1362
|
+
inter_arrival_p95_ms=_percentile(bucket["inter_arrivals"], 0.95),
|
|
1363
|
+
)
|
|
1364
|
+
rows.append(asdict(row))
|
|
1365
|
+
return rows
|
|
1366
|
+
|
|
1367
|
+
|
|
1368
|
+
# --------------------------------------------------------------------------- #
|
|
1369
|
+
# Genre 2 — perturbation / recovery metrics
|
|
1370
|
+
# --------------------------------------------------------------------------- #
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
@dataclass(frozen=True)
|
|
1374
|
+
class OutageWindow:
|
|
1375
|
+
start: float # seconds on the publish timeline
|
|
1376
|
+
end: float # restore time
|
|
1377
|
+
|
|
1378
|
+
|
|
1379
|
+
@dataclass(frozen=True)
|
|
1380
|
+
class RecoveryMetrics:
|
|
1381
|
+
t_recover: float | None # restore -> first message arriving after restore
|
|
1382
|
+
t_steady: float | None # restore -> inter-arrival back within tolerance
|
|
1383
|
+
recovery_burst: int # messages arriving in the burst window after restore
|
|
1384
|
+
lost_during_outage: dict[str, int] # per topic
|
|
1385
|
+
latched_rearrival: dict[str, bool] # per latched topic
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
def _send_time(record: dict[str, Any], *, seq0: int, t0: float, period_s: float, time_field: str) -> float:
|
|
1389
|
+
"""Position on the publish timeline; reconstruct lost messages from the schedule.
|
|
1390
|
+
|
|
1391
|
+
Delivered records carry ``t_wrap`` (publish time). Lost records carry no
|
|
1392
|
+
timestamp, so their position is the nominal send time ``t0 + (seq - seq0)·T``.
|
|
1393
|
+
"""
|
|
1394
|
+
stamp = record.get(time_field)
|
|
1395
|
+
if stamp is not None:
|
|
1396
|
+
return float(stamp)
|
|
1397
|
+
return t0 + (int(record["seq"]) - seq0) * period_s
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def _arrival_time(send_t: float, record: dict[str, Any]) -> float:
|
|
1401
|
+
sections = record.get("sections") or {}
|
|
1402
|
+
ota = sections.get("ota_hop_ms")
|
|
1403
|
+
if ota is None:
|
|
1404
|
+
ota = sections.get("ota_hop_uncorrected_ms")
|
|
1405
|
+
return send_t + (float(ota) / 1000.0 if ota is not None else 0.0)
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def recovery_metrics(
|
|
1409
|
+
records: Iterable[dict[str, Any]],
|
|
1410
|
+
outage: OutageWindow,
|
|
1411
|
+
*,
|
|
1412
|
+
nominal_period_s: float,
|
|
1413
|
+
time_field: str = "t_wrap",
|
|
1414
|
+
steady_factor: float = 1.5,
|
|
1415
|
+
burst_window_s: float = 1.0,
|
|
1416
|
+
latched_topics: Iterable[str] = (),
|
|
1417
|
+
) -> RecoveryMetrics:
|
|
1418
|
+
"""Extract recovery dynamics from a timeline of RFC 0003 transit records.
|
|
1419
|
+
|
|
1420
|
+
``t_recover`` is the delay from restore to the first message arriving at/after
|
|
1421
|
+
restore; ``recovery_burst`` counts arrivals within ``burst_window_s`` of restore
|
|
1422
|
+
(the reconnect burst the baseline calls "the part that actually hurts");
|
|
1423
|
+
``t_steady`` is the delay until inter-arrival returns within
|
|
1424
|
+
``steady_factor × nominal_period``; ``lost_during_outage`` counts per-topic lost
|
|
1425
|
+
messages whose send time falls in the outage; ``latched_rearrival`` reports
|
|
1426
|
+
whether each latched topic delivered a value again after restore.
|
|
1427
|
+
"""
|
|
1428
|
+
records = list(records)
|
|
1429
|
+
if not records:
|
|
1430
|
+
return RecoveryMetrics(None, None, 0, {}, {topic: False for topic in latched_topics})
|
|
1431
|
+
|
|
1432
|
+
stamped = [record for record in records if record.get(time_field) is not None]
|
|
1433
|
+
anchor = min(stamped or records, key=lambda record: int(record["seq"]))
|
|
1434
|
+
seq0 = int(anchor["seq"])
|
|
1435
|
+
t0 = float(anchor.get(time_field) or 0.0)
|
|
1436
|
+
|
|
1437
|
+
lost_during_outage: dict[str, int] = {}
|
|
1438
|
+
delivered: list[tuple[float, dict[str, Any]]] = []
|
|
1439
|
+
for record in records:
|
|
1440
|
+
send_t = _send_time(record, seq0=seq0, t0=t0, period_s=nominal_period_s, time_field=time_field)
|
|
1441
|
+
topic = str(record.get("topic") or "")
|
|
1442
|
+
if record.get("status") == "lost":
|
|
1443
|
+
if outage.start <= send_t <= outage.end:
|
|
1444
|
+
lost_during_outage[topic] = lost_during_outage.get(topic, 0) + 1
|
|
1445
|
+
else:
|
|
1446
|
+
delivered.append((_arrival_time(send_t, record), record))
|
|
1447
|
+
|
|
1448
|
+
delivered.sort(key=lambda item: item[0])
|
|
1449
|
+
after = [arrival for arrival, _ in delivered if arrival >= outage.end]
|
|
1450
|
+
|
|
1451
|
+
t_recover = after[0] - outage.end if after else None
|
|
1452
|
+
recovery_burst = sum(1 for arrival in after if arrival <= outage.end + burst_window_s)
|
|
1453
|
+
|
|
1454
|
+
# Steady state is inter-arrival back *near* the nominal period — a symmetric band.
|
|
1455
|
+
# A reconnect burst arrives in a clump (gaps far below nominal), so it is not steady.
|
|
1456
|
+
t_steady: float | None = None
|
|
1457
|
+
low_gap = nominal_period_s / steady_factor
|
|
1458
|
+
high_gap = nominal_period_s * steady_factor
|
|
1459
|
+
for index in range(1, len(after)):
|
|
1460
|
+
if low_gap <= after[index] - after[index - 1] <= high_gap:
|
|
1461
|
+
t_steady = after[index] - outage.end
|
|
1462
|
+
break
|
|
1463
|
+
|
|
1464
|
+
latched_rearrival = {
|
|
1465
|
+
topic: any(record.get("topic") == topic and arrival >= outage.end for arrival, record in delivered)
|
|
1466
|
+
for topic in latched_topics
|
|
1467
|
+
}
|
|
1468
|
+
return RecoveryMetrics(t_recover, t_steady, recovery_burst, lost_during_outage, latched_rearrival)
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
# --------------------------------------------------------------------------- #
|
|
1472
|
+
# Genre 1 — coarse linear ramp (monitor-only trend)
|
|
1473
|
+
# --------------------------------------------------------------------------- #
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
@dataclass(frozen=True)
|
|
1477
|
+
class RampPoint:
|
|
1478
|
+
value: float
|
|
1479
|
+
metric: float
|
|
1480
|
+
|
|
1481
|
+
|
|
1482
|
+
def linear_ramp(values: Iterable[float], measure: Callable[[float], float]) -> list[RampPoint]:
|
|
1483
|
+
"""The whole response curve (latency-vs-load). Monitor-only: trended, never gated."""
|
|
1484
|
+
return [RampPoint(float(value), float(measure(value))) for value in values]
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
# --------------------------------------------------------------------------- #
|
|
1488
|
+
# A/B tuning experiments (#22): candidate config vs baseline, same load+profile
|
|
1489
|
+
# --------------------------------------------------------------------------- #
|
|
1490
|
+
#
|
|
1491
|
+
# The regression gate asks "is this run within a committed envelope?"; an A/B
|
|
1492
|
+
# experiment asks the sibling question "is candidate config B better, worse or
|
|
1493
|
+
# unchanged vs baseline config A on the *same* load and profile?". Both are
|
|
1494
|
+
# two-sided and both speak the same Better/Verdict language: an A/B cell is just
|
|
1495
|
+
# :func:`_two_sided_verdict` on the ephemeral ``[baseline +/- tolerance]``
|
|
1496
|
+
# envelope. This layer is pure and host-testable; the live runs (Docker graphs,
|
|
1497
|
+
# emulated profiles) live in the CLI, injected as a ``run_point`` (RFC 0005).
|
|
1498
|
+
|
|
1499
|
+
# The per-topic metrics an A/B experiment watches, with their better-direction.
|
|
1500
|
+
# Names mirror the gate's band metrics (``_DEFAULT_BETTER``) so a verdict reads
|
|
1501
|
+
# the same in both places. Read off ``transit.summarize_transit_records``.
|
|
1502
|
+
AB_METRICS: dict[str, Better] = {
|
|
1503
|
+
"completeness_pct": Better.HIGHER,
|
|
1504
|
+
"loss_pct": Better.LOWER,
|
|
1505
|
+
"latency_p50_ms": Better.LOWER,
|
|
1506
|
+
"latency_p95_ms": Better.LOWER,
|
|
1507
|
+
"jitter_p50_ms": Better.LOWER,
|
|
1508
|
+
"jitter_p95_ms": Better.LOWER,
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
# The default watched set: the completeness/loss pair (bottleneck-dominated —
|
|
1512
|
+
# counted off exact sequence numbers, so trustworthy even on a shared runner)
|
|
1513
|
+
# plus the p95 tails (host-timing-dominated — directional hints, which is why the
|
|
1514
|
+
# spread is always reported, not just the median).
|
|
1515
|
+
DEFAULT_AB_METRICS: tuple[str, ...] = ("completeness_pct", "loss_pct", "latency_p95_ms", "jitter_p95_ms")
|
|
1516
|
+
|
|
1517
|
+
# A candidate median within this relative OR the metric's absolute half-width of
|
|
1518
|
+
# the baseline median is called unchanged (WITHIN). The absolute floors keep
|
|
1519
|
+
# sub-millisecond tail noise on a tiny baseline from reading as a change; loss
|
|
1520
|
+
# and completeness are exact percentages and get no floor (a real 1% delta is
|
|
1521
|
+
# signal).
|
|
1522
|
+
DEFAULT_AB_REL_TOLERANCE = 0.10
|
|
1523
|
+
DEFAULT_AB_ABS_TOLERANCE: dict[str, float] = {
|
|
1524
|
+
"latency_p50_ms": 1.0,
|
|
1525
|
+
"latency_p95_ms": 2.0,
|
|
1526
|
+
"jitter_p50_ms": 1.0,
|
|
1527
|
+
"jitter_p95_ms": 2.0,
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
def ab_better(metric: str) -> Better:
|
|
1532
|
+
"""The better-direction of an A/B-watched metric (raises on an unknown one)."""
|
|
1533
|
+
better = AB_METRICS.get(metric)
|
|
1534
|
+
if better is None:
|
|
1535
|
+
raise BandError(f"metric {metric!r} is not an A/B-watched metric; known: {sorted(AB_METRICS)}")
|
|
1536
|
+
return better
|
|
1537
|
+
|
|
1538
|
+
|
|
1539
|
+
def _bare_topic(label: str) -> str:
|
|
1540
|
+
"""``a->b:/topic`` -> ``/topic``; a plain topic label is returned unchanged."""
|
|
1541
|
+
return label.split(":", 1)[1] if ":" in label else label
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
def ab_metrics_from_topic(topic_summary: Mapping[str, Any]) -> dict[str, float | None]:
|
|
1545
|
+
"""Pull the A/B-watched metrics out of one topic's transit-summary block.
|
|
1546
|
+
|
|
1547
|
+
A metric is ``None`` when the run produced no sample for it (e.g. latency
|
|
1548
|
+
with 100% loss); the aggregator drops ``None`` holes from a spread.
|
|
1549
|
+
"""
|
|
1550
|
+
expected = topic_summary.get("expected")
|
|
1551
|
+
delivered = topic_summary.get("delivered")
|
|
1552
|
+
completeness: float | None = None
|
|
1553
|
+
if expected and delivered is not None:
|
|
1554
|
+
completeness = round(100.0 * float(delivered) / float(expected), 3)
|
|
1555
|
+
ota = topic_summary.get("ota_hop_ms") or {}
|
|
1556
|
+
jitter = topic_summary.get("jitter_ms") or {}
|
|
1557
|
+
|
|
1558
|
+
def opt(value: Any) -> float | None:
|
|
1559
|
+
return None if value is None else float(value)
|
|
1560
|
+
|
|
1561
|
+
return {
|
|
1562
|
+
"completeness_pct": completeness,
|
|
1563
|
+
"loss_pct": opt(topic_summary.get("loss_pct")),
|
|
1564
|
+
"latency_p50_ms": opt(ota.get("p50")),
|
|
1565
|
+
"latency_p95_ms": opt(ota.get("p95")),
|
|
1566
|
+
"jitter_p50_ms": opt(jitter.get("p50")),
|
|
1567
|
+
"jitter_p95_ms": opt(jitter.get("p95")),
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
|
|
1571
|
+
@dataclass(frozen=True)
|
|
1572
|
+
class AbTolerance:
|
|
1573
|
+
"""Half-width of the unchanged band around a baseline value: ``max(abs, rel*|baseline|)``."""
|
|
1574
|
+
|
|
1575
|
+
rel: float = DEFAULT_AB_REL_TOLERANCE
|
|
1576
|
+
abs: float = 0.0
|
|
1577
|
+
|
|
1578
|
+
def half_width(self, baseline: float) -> float:
|
|
1579
|
+
return max(self.abs, abs(baseline) * self.rel)
|
|
1580
|
+
|
|
1581
|
+
|
|
1582
|
+
def default_ab_tolerance(metric: str, *, rel: float = DEFAULT_AB_REL_TOLERANCE) -> AbTolerance:
|
|
1583
|
+
"""The default tolerance for a metric: ``rel`` plus the metric's absolute floor."""
|
|
1584
|
+
return AbTolerance(rel=rel, abs=DEFAULT_AB_ABS_TOLERANCE.get(metric, 0.0))
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
def classify_change(baseline: float, candidate: float, *, better: Better, tol: AbTolerance) -> Verdict:
|
|
1588
|
+
"""Candidate vs baseline as WITHIN / IMPROVED / REGRESSED.
|
|
1589
|
+
|
|
1590
|
+
``_two_sided_verdict`` on the ephemeral ``[baseline +/- tol]`` envelope, so an
|
|
1591
|
+
A/B cell classifies identically to a committed-band gate cell.
|
|
1592
|
+
"""
|
|
1593
|
+
half = tol.half_width(baseline)
|
|
1594
|
+
return _two_sided_verdict(candidate, lo=baseline - half, hi=baseline + half, better=better)
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
def ab_schedule(configs: Sequence[str], repeats: int) -> list[tuple[str, int]]:
|
|
1598
|
+
"""Deterministic interleaving of ``configs`` over ``repeats`` as ``(config, repeat)``.
|
|
1599
|
+
|
|
1600
|
+
Every repeat runs each config exactly once; the starting config rotates each
|
|
1601
|
+
repeat so no config is always measured first. Interleaving (not "all of A,
|
|
1602
|
+
then all of B") spreads any slow host-drift across every config, and the
|
|
1603
|
+
rotation removes the residual first-slot warm-up bias — both guard the verdict
|
|
1604
|
+
against drift being mistaken for a config effect. Deterministic by
|
|
1605
|
+
construction, so a verdict is reproducible from the same runs.
|
|
1606
|
+
"""
|
|
1607
|
+
if repeats < 1:
|
|
1608
|
+
raise ValueError("repeats must be >= 1.")
|
|
1609
|
+
if not configs:
|
|
1610
|
+
raise ValueError("need at least one config.")
|
|
1611
|
+
order: list[tuple[str, int]] = []
|
|
1612
|
+
count = len(configs)
|
|
1613
|
+
for repeat in range(1, repeats + 1):
|
|
1614
|
+
rotation = (repeat - 1) % count
|
|
1615
|
+
rotated = [*configs[rotation:], *configs[:rotation]]
|
|
1616
|
+
order.extend((config, repeat) for config in rotated)
|
|
1617
|
+
return order
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
@dataclass(frozen=True)
|
|
1621
|
+
class MetricSpread:
|
|
1622
|
+
"""Repeat-to-repeat spread of one metric for one config+topic."""
|
|
1623
|
+
|
|
1624
|
+
n: int # repeats that produced a value
|
|
1625
|
+
min: float
|
|
1626
|
+
median: float
|
|
1627
|
+
max: float
|
|
1628
|
+
values: list[float] # per-repeat, in execution order
|
|
1629
|
+
|
|
1630
|
+
|
|
1631
|
+
def _metric_spread(values: Sequence[float]) -> MetricSpread | None:
|
|
1632
|
+
vals = [float(v) for v in values]
|
|
1633
|
+
if not vals:
|
|
1634
|
+
return None
|
|
1635
|
+
return MetricSpread(
|
|
1636
|
+
n=len(vals),
|
|
1637
|
+
min=round(min(vals), 3),
|
|
1638
|
+
median=round(_median(vals), 3),
|
|
1639
|
+
max=round(max(vals), 3),
|
|
1640
|
+
values=[round(v, 3) for v in vals],
|
|
1641
|
+
)
|
|
1642
|
+
|
|
1643
|
+
|
|
1644
|
+
@dataclass(frozen=True)
|
|
1645
|
+
class AbRun:
|
|
1646
|
+
"""One executed run in an experiment: which config, which 1-based repeat, its summary."""
|
|
1647
|
+
|
|
1648
|
+
config: str
|
|
1649
|
+
repeat: int
|
|
1650
|
+
summary: Mapping[str, Any] # transit.summarize_transit_records output
|
|
1651
|
+
|
|
1652
|
+
|
|
1653
|
+
@dataclass(frozen=True)
|
|
1654
|
+
class AbCell:
|
|
1655
|
+
"""One (topic, metric) candidate-vs-baseline verdict, both spreads carried."""
|
|
1656
|
+
|
|
1657
|
+
topic: str
|
|
1658
|
+
metric: str
|
|
1659
|
+
better: str # Better value
|
|
1660
|
+
baseline: MetricSpread | None
|
|
1661
|
+
candidate: MetricSpread | None
|
|
1662
|
+
verdict: str | None # Verdict value; None when a side has no sample
|
|
1663
|
+
delta: float | None # candidate.median - baseline.median
|
|
1664
|
+
separated: bool | None # spreads disjoint => effect separable at this N
|
|
1665
|
+
|
|
1666
|
+
|
|
1667
|
+
@dataclass(frozen=True)
|
|
1668
|
+
class CandidateReport:
|
|
1669
|
+
"""A candidate config's full comparison against the baseline."""
|
|
1670
|
+
|
|
1671
|
+
config: str
|
|
1672
|
+
cells: list[AbCell]
|
|
1673
|
+
regressed: list[list[str]] # [topic, metric] pairs that regressed beyond tolerance
|
|
1674
|
+
improved: list[list[str]] # [topic, metric] pairs that improved beyond tolerance
|
|
1675
|
+
dropped_topics: list[str] # topics the baseline delivered that this config did not
|
|
1676
|
+
passed: bool # no regression and no dropped topic
|
|
1677
|
+
|
|
1678
|
+
|
|
1679
|
+
@dataclass(frozen=True)
|
|
1680
|
+
class AbReport:
|
|
1681
|
+
baseline: str
|
|
1682
|
+
metrics: list[str]
|
|
1683
|
+
repeats: int
|
|
1684
|
+
candidates: list[CandidateReport]
|
|
1685
|
+
passed: bool # every candidate passed
|
|
1686
|
+
|
|
1687
|
+
|
|
1688
|
+
def _collect_ab_metrics(
|
|
1689
|
+
runs: Iterable[AbRun], *, metrics: Sequence[str], topic_filter: str
|
|
1690
|
+
) -> dict[str, dict[str, dict[str, list[float]]]]:
|
|
1691
|
+
"""``{config: {topic: {metric: [per-repeat values]}}}`` in execution order."""
|
|
1692
|
+
collected: dict[str, dict[str, dict[str, list[float]]]] = {}
|
|
1693
|
+
for run in runs:
|
|
1694
|
+
topics = run.summary.get("topics") or {}
|
|
1695
|
+
for topic_label, topic_summary in topics.items():
|
|
1696
|
+
if topic_filter and topic_filter not in {topic_label, _bare_topic(topic_label)}:
|
|
1697
|
+
continue
|
|
1698
|
+
values = ab_metrics_from_topic(topic_summary)
|
|
1699
|
+
per_topic = collected.setdefault(run.config, {}).setdefault(topic_label, {})
|
|
1700
|
+
for metric in metrics:
|
|
1701
|
+
value = values.get(metric)
|
|
1702
|
+
if value is not None:
|
|
1703
|
+
per_topic.setdefault(metric, []).append(value)
|
|
1704
|
+
return collected
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
def ab_verdict(
|
|
1708
|
+
runs: Sequence[AbRun],
|
|
1709
|
+
*,
|
|
1710
|
+
baseline: str,
|
|
1711
|
+
metrics: Sequence[str] = DEFAULT_AB_METRICS,
|
|
1712
|
+
tolerances: Mapping[str, AbTolerance] | None = None,
|
|
1713
|
+
topic: str = "",
|
|
1714
|
+
) -> AbReport:
|
|
1715
|
+
"""Classify every candidate config against the baseline, per topic and metric.
|
|
1716
|
+
|
|
1717
|
+
Comparison is on the per-config **median** of each metric across repeats (one
|
|
1718
|
+
unlucky repeat cannot flip a verdict), and both spreads travel with the cell
|
|
1719
|
+
so a reader sees whether the medians are actually separable at this N
|
|
1720
|
+
(``separated``). A candidate ``passed`` iff nothing regressed beyond tolerance
|
|
1721
|
+
and it did not drop a topic the baseline delivered.
|
|
1722
|
+
"""
|
|
1723
|
+
metric_list = list(metrics)
|
|
1724
|
+
tol_map = {metric: (tolerances or {}).get(metric) or default_ab_tolerance(metric) for metric in metric_list}
|
|
1725
|
+
ordered = sorted(runs, key=lambda run: (run.config, run.repeat))
|
|
1726
|
+
collected = _collect_ab_metrics(ordered, metrics=metric_list, topic_filter=topic)
|
|
1727
|
+
|
|
1728
|
+
if baseline not in collected:
|
|
1729
|
+
raise BandError(f"baseline config {baseline!r} produced no runs to compare against.")
|
|
1730
|
+
|
|
1731
|
+
configs_in_order: list[str] = []
|
|
1732
|
+
for run in ordered:
|
|
1733
|
+
if run.config not in configs_in_order:
|
|
1734
|
+
configs_in_order.append(run.config)
|
|
1735
|
+
repeats = max((run.repeat for run in ordered), default=0)
|
|
1736
|
+
baseline_topics = collected[baseline]
|
|
1737
|
+
|
|
1738
|
+
candidates: list[CandidateReport] = []
|
|
1739
|
+
for config in configs_in_order:
|
|
1740
|
+
if config == baseline:
|
|
1741
|
+
continue
|
|
1742
|
+
candidate_topics = collected.get(config, {})
|
|
1743
|
+
cells: list[AbCell] = []
|
|
1744
|
+
regressed: list[list[str]] = []
|
|
1745
|
+
improved: list[list[str]] = []
|
|
1746
|
+
dropped: list[str] = []
|
|
1747
|
+
for topic_label in sorted(baseline_topics):
|
|
1748
|
+
candidate_metrics = candidate_topics.get(topic_label)
|
|
1749
|
+
if candidate_metrics is None:
|
|
1750
|
+
dropped.append(topic_label)
|
|
1751
|
+
continue
|
|
1752
|
+
baseline_metrics = baseline_topics[topic_label]
|
|
1753
|
+
for metric in metric_list:
|
|
1754
|
+
better = ab_better(metric)
|
|
1755
|
+
baseline_spread = _metric_spread(baseline_metrics.get(metric, []))
|
|
1756
|
+
candidate_spread = _metric_spread(candidate_metrics.get(metric, []))
|
|
1757
|
+
verdict: Verdict | None = None
|
|
1758
|
+
delta: float | None = None
|
|
1759
|
+
separated: bool | None = None
|
|
1760
|
+
if baseline_spread is not None and candidate_spread is not None:
|
|
1761
|
+
verdict = classify_change(
|
|
1762
|
+
baseline_spread.median, candidate_spread.median, better=better, tol=tol_map[metric]
|
|
1763
|
+
)
|
|
1764
|
+
delta = round(candidate_spread.median - baseline_spread.median, 3)
|
|
1765
|
+
separated = candidate_spread.max < baseline_spread.min or baseline_spread.max < candidate_spread.min
|
|
1766
|
+
if verdict is Verdict.REGRESSED:
|
|
1767
|
+
regressed.append([topic_label, metric])
|
|
1768
|
+
elif verdict is Verdict.IMPROVED:
|
|
1769
|
+
improved.append([topic_label, metric])
|
|
1770
|
+
cells.append(
|
|
1771
|
+
AbCell(
|
|
1772
|
+
topic=topic_label,
|
|
1773
|
+
metric=metric,
|
|
1774
|
+
better=better.value,
|
|
1775
|
+
baseline=baseline_spread,
|
|
1776
|
+
candidate=candidate_spread,
|
|
1777
|
+
verdict=None if verdict is None else verdict.value,
|
|
1778
|
+
delta=delta,
|
|
1779
|
+
separated=separated,
|
|
1780
|
+
)
|
|
1781
|
+
)
|
|
1782
|
+
candidates.append(
|
|
1783
|
+
CandidateReport(
|
|
1784
|
+
config=config,
|
|
1785
|
+
cells=cells,
|
|
1786
|
+
regressed=regressed,
|
|
1787
|
+
improved=improved,
|
|
1788
|
+
dropped_topics=dropped,
|
|
1789
|
+
passed=not regressed and not dropped,
|
|
1790
|
+
)
|
|
1791
|
+
)
|
|
1792
|
+
|
|
1793
|
+
return AbReport(
|
|
1794
|
+
baseline=baseline,
|
|
1795
|
+
metrics=metric_list,
|
|
1796
|
+
repeats=repeats,
|
|
1797
|
+
candidates=candidates,
|
|
1798
|
+
passed=all(candidate.passed for candidate in candidates),
|
|
1799
|
+
)
|
|
1800
|
+
|
|
1801
|
+
|
|
1802
|
+
def _fmt_spread(spread: MetricSpread | None) -> str:
|
|
1803
|
+
if spread is None:
|
|
1804
|
+
return "n/a"
|
|
1805
|
+
if spread.min == spread.max:
|
|
1806
|
+
return f"{spread.median:g}"
|
|
1807
|
+
return f"{spread.median:g} ({spread.min:g}–{spread.max:g})"
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
def render_ab_markdown(report: AbReport) -> str:
|
|
1811
|
+
"""A self-describing markdown table per candidate: the paper-grade byproduct."""
|
|
1812
|
+
lines: list[str] = []
|
|
1813
|
+
verdict_word = "PASS" if report.passed else "FAIL"
|
|
1814
|
+
lines.append(f"# A/B verdict: **{verdict_word}** (baseline `{report.baseline}`, {report.repeats} repeats)")
|
|
1815
|
+
lines.append("")
|
|
1816
|
+
lines.append(
|
|
1817
|
+
"Cells compare the candidate median against the baseline median; "
|
|
1818
|
+
"`baseline`/`candidate` show `median (min–max)` across repeats. "
|
|
1819
|
+
"`sep?` is yes when the two spreads do not overlap (the effect is "
|
|
1820
|
+
"separable at this repeat count)."
|
|
1821
|
+
)
|
|
1822
|
+
for candidate in report.candidates:
|
|
1823
|
+
lines.append("")
|
|
1824
|
+
status = "PASS" if candidate.passed else "FAIL"
|
|
1825
|
+
lines.append(f"## `{candidate.config}` vs `{report.baseline}` — **{status}**")
|
|
1826
|
+
if candidate.dropped_topics:
|
|
1827
|
+
lines.append("")
|
|
1828
|
+
lines.append(
|
|
1829
|
+
f"Dropped topics (baseline delivered, candidate did not): {', '.join(candidate.dropped_topics)}"
|
|
1830
|
+
)
|
|
1831
|
+
lines.append("")
|
|
1832
|
+
lines.append("| topic | metric | better | baseline | candidate | Δ | sep? | verdict |")
|
|
1833
|
+
lines.append("|---|---|---|---|---|---|---|---|")
|
|
1834
|
+
for cell in candidate.cells:
|
|
1835
|
+
delta = "" if cell.delta is None else f"{cell.delta:+g}"
|
|
1836
|
+
sep = "" if cell.separated is None else ("yes" if cell.separated else "no")
|
|
1837
|
+
verdict = cell.verdict or "n/a"
|
|
1838
|
+
lines.append(
|
|
1839
|
+
f"| {cell.topic} | {cell.metric} | {cell.better} | "
|
|
1840
|
+
f"{_fmt_spread(cell.baseline)} | {_fmt_spread(cell.candidate)} | {delta} | {sep} | {verdict} |"
|
|
1841
|
+
)
|
|
1842
|
+
return "\n".join(lines)
|