rosotacom-dev 2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. rosotacom/__init__.py +32 -0
  2. rosotacom/__main__.py +8 -0
  3. rosotacom/anonymize.py +360 -0
  4. rosotacom/bag_ground_truth.py +395 -0
  5. rosotacom/benched_set.py +492 -0
  6. rosotacom/benchmark.py +1842 -0
  7. rosotacom/bundle_check.py +517 -0
  8. rosotacom/cli.py +8637 -0
  9. rosotacom/cli_benchmark.py +8211 -0
  10. rosotacom/deployment.py +196 -0
  11. rosotacom/ffmpeg_packet.py +123 -0
  12. rosotacom/forensics.py +1172 -0
  13. rosotacom/geomap.py +848 -0
  14. rosotacom/network_profiles.py +563 -0
  15. rosotacom/network_shaper.py +127 -0
  16. rosotacom/plots.py +609 -0
  17. rosotacom/py.typed +0 -0
  18. rosotacom/resources/benched-set.yaml +317 -0
  19. rosotacom/resources/examples/.gitignore +1 -0
  20. rosotacom/resources/examples/README.md +163 -0
  21. rosotacom/resources/examples/__init__.py +1 -0
  22. rosotacom/resources/examples/bags/8_drop_reference/metadata.yaml +17 -0
  23. rosotacom/resources/examples/deployment.example.yaml +14 -0
  24. rosotacom/resources/examples/profiles.yaml +85 -0
  25. rosotacom/resources/examples/ros2docker.json +33 -0
  26. rosotacom/resources/examples/rosotacom.yaml +12 -0
  27. rosotacom/resources/examples/scenarios/2_native_chatter/scenario-definition.yaml +10 -0
  28. rosotacom/resources/examples/scripts/1_heartbeat/run_machine_a.sh +9 -0
  29. rosotacom/resources/examples/scripts/1_heartbeat/run_machine_b.sh +9 -0
  30. rosotacom/resources/examples/scripts/2_native_chatter/machine_a/external.ros2docker.json +21 -0
  31. rosotacom/resources/examples/scripts/2_native_chatter/machine_a/run_communication.sh +9 -0
  32. rosotacom/resources/examples/scripts/2_native_chatter/machine_a/run_external.py +15 -0
  33. rosotacom/resources/examples/scripts/2_native_chatter/machine_a/wait_for_echo_topic.sh +10 -0
  34. rosotacom/resources/examples/scripts/2_native_chatter/machine_b/external.ros2docker.json +19 -0
  35. rosotacom/resources/examples/scripts/2_native_chatter/machine_b/run_communication.sh +9 -0
  36. rosotacom/resources/examples/scripts/2_native_chatter/machine_b/run_external.py +15 -0
  37. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/external.ros2docker.json +12 -0
  38. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/external_session.yaml +9 -0
  39. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/foxglove.json +89 -0
  40. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/run_communication.sh +9 -0
  41. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_a/run_external.py +15 -0
  42. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/costmap_1second/costmap_1second_0.mcap +0 -0
  43. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/costmap_1second/metadata.yaml +44 -0
  44. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/external.ros2docker.json +13 -0
  45. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/run_communication.sh +9 -0
  46. rosotacom/resources/examples/scripts/3_comp_occ_grid/machine_b/run_external.py +15 -0
  47. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/external.ros2docker.json +12 -0
  48. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/external_session.yaml +9 -0
  49. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/foxglove.json +89 -0
  50. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/run_communication.sh +9 -0
  51. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_a/run_external.py +15 -0
  52. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/costmap_1second/costmap_1second_0.mcap +0 -0
  53. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/costmap_1second/metadata.yaml +44 -0
  54. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/external.ros2docker.json +13 -0
  55. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/run_communication.sh +9 -0
  56. rosotacom/resources/examples/scripts/4_comp_occ_grid_zen/machine_b/run_external.py +15 -0
  57. rosotacom/resources/examples/scripts/5_sized_payload/machine_a/external.ros2docker.json +20 -0
  58. rosotacom/resources/examples/scripts/5_sized_payload/machine_a/external_session.yaml +22 -0
  59. rosotacom/resources/examples/scripts/5_sized_payload/machine_a/run_communication.sh +9 -0
  60. rosotacom/resources/examples/scripts/5_sized_payload/machine_a/run_external.py +15 -0
  61. rosotacom/resources/examples/scripts/5_sized_payload/machine_b/external.ros2docker.json +20 -0
  62. rosotacom/resources/examples/scripts/5_sized_payload/machine_b/external_session.yaml +22 -0
  63. rosotacom/resources/examples/scripts/5_sized_payload/machine_b/run_communication.sh +9 -0
  64. rosotacom/resources/examples/scripts/5_sized_payload/machine_b/run_external.py +15 -0
  65. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/external.ros2docker.json +18 -0
  66. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/external_session.yaml +22 -0
  67. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/run_communication.sh +9 -0
  68. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_a/run_external.py +15 -0
  69. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/external.ros2docker.json +18 -0
  70. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/external_session.yaml +22 -0
  71. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/run_communication.sh +9 -0
  72. rosotacom/resources/examples/scripts/6_sized_payload_zen/machine_b/run_external.py +15 -0
  73. rosotacom/resources/examples/sessions/10_restamp/session-definition.yaml +39 -0
  74. rosotacom/resources/examples/sessions/11_trickle/session-definition.yaml +42 -0
  75. rosotacom/resources/examples/sessions/12_content_integrity/session-definition.yaml +33 -0
  76. rosotacom/resources/examples/sessions/13_link_latency/session-definition.yaml +42 -0
  77. rosotacom/resources/examples/sessions/14_remote_assist_anonymized/session-definition.yaml +211 -0
  78. rosotacom/resources/examples/sessions/15_remote_assist_anonymized_costmap/session-definition.yaml +59 -0
  79. rosotacom/resources/examples/sessions/16_remote_assist_anonymized_camera/session-definition.yaml +61 -0
  80. rosotacom/resources/examples/sessions/17_synthetic_camera_quality/session-definition.yaml +57 -0
  81. rosotacom/resources/examples/sessions/1_heartbeat/session-definition.yaml +26 -0
  82. rosotacom/resources/examples/sessions/1_heartbeat_status/session-definition.yaml +37 -0
  83. rosotacom/resources/examples/sessions/2_native_chatter/session-definition.yaml +34 -0
  84. rosotacom/resources/examples/sessions/3_comp_occ_grid/session-definition.yaml +43 -0
  85. rosotacom/resources/examples/sessions/4_comp_occ_grid_zen/session-definition.yaml +43 -0
  86. rosotacom/resources/examples/sessions/5_sized_payload/session-definition.yaml +39 -0
  87. rosotacom/resources/examples/sessions/6_sized_payload_zen/session-definition.yaml +50 -0
  88. rosotacom/resources/examples/sessions/7_latched_static/session-definition.yaml +49 -0
  89. rosotacom/resources/examples/sessions/8_drop/session-definition.yaml +37 -0
  90. rosotacom/resources/examples/sessions/9_throttle/session-definition.yaml +37 -0
  91. rosotacom/resources/examples/sessions/bench_1_1_capacity/session-definition.yaml +42 -0
  92. rosotacom/resources/examples/sessions/bench_1_2_load_sweep/session-definition.yaml +51 -0
  93. rosotacom/resources/examples/sessions/bench_1_3_ramp/session-definition.yaml +41 -0
  94. rosotacom/resources/examples/sessions/bench_1_4_recovery/session-definition.yaml +47 -0
  95. rosotacom/resources/examples/sessions/bench_1_5_camera/session-definition.yaml +45 -0
  96. rosotacom/resources/ros2docker.json.example +33 -0
  97. rosotacom/resources/ws/.vscode/c_cpp_properties.json +20 -0
  98. rosotacom/resources/ws/.vscode/settings.json +8 -0
  99. rosotacom/resources/ws/ota_configs/archive/fastdds_easy_mode.xml.template +32 -0
  100. rosotacom/resources/ws/ota_configs/archive/fastdds_initial_peer_list.xml.template +21 -0
  101. rosotacom/resources/ws/ota_configs/archive/fastdds_v2.xml.template +17 -0
  102. rosotacom/resources/ws/ota_configs/archive/fastdds_v3.xml.template +48 -0
  103. rosotacom/resources/ws/ota_configs/archive/fastdds_v4.xml.template +95 -0
  104. rosotacom/resources/ws/ota_configs/archive/fastdds_v5.xml.template +64 -0
  105. rosotacom/resources/ws/ota_configs/create_zenoh_json5.py +56 -0
  106. rosotacom/resources/ws/ota_configs/cyclonedds_minimal.xml.template +9 -0
  107. rosotacom/resources/ws/ota_configs/cyclonedds_tuned.xml.template +33 -0
  108. rosotacom/resources/ws/ota_configs/fastdds_unicast.xml.template +71 -0
  109. rosotacom/resources/ws/ota_configs/get_ota_xml.py +214 -0
  110. rosotacom/resources/ws/ota_configs/utils.py +74 -0
  111. rosotacom/resources/ws/ota_configs/zenoh.json5.template +32 -0
  112. rosotacom/resources/ws/ros2src/.gitkeep +0 -0
  113. rosotacom/resources/ws/ros2src/com_cpp/CMakeLists.txt +67 -0
  114. rosotacom/resources/ws/ros2src/com_cpp/COLCON_IGNORE +0 -0
  115. rosotacom/resources/ws/ros2src/com_cpp/LICENSE.txt +25 -0
  116. rosotacom/resources/ws/ros2src/com_cpp/package.xml +37 -0
  117. rosotacom/resources/ws/ros2src/com_cpp/src/hz_monitor.cpp +192 -0
  118. rosotacom/resources/ws/ros2src/com_cpp/src/universal_compressor.cpp +283 -0
  119. rosotacom/resources/ws/ros2src/com_cpp/src/universal_decompressor.cpp +306 -0
  120. rosotacom/resources/ws/ros2src/com_msgs/CMakeLists.txt +22 -0
  121. rosotacom/resources/ws/ros2src/com_msgs/LICENSE.txt +25 -0
  122. rosotacom/resources/ws/ros2src/com_msgs/msg/CompressedData.msg +3 -0
  123. rosotacom/resources/ws/ros2src/com_msgs/msg/EchoHeartbeat.msg +10 -0
  124. rosotacom/resources/ws/ros2src/com_msgs/msg/OtaStamped.msg +4 -0
  125. rosotacom/resources/ws/ros2src/com_msgs/msg/SizedPayload.msg +4 -0
  126. rosotacom/resources/ws/ros2src/com_msgs/package.xml +25 -0
  127. rosotacom/resources/ws/ros2src/com_py/LICENSE.txt +25 -0
  128. rosotacom/resources/ws/ros2src/com_py/com_py/__init__.py +0 -0
  129. rosotacom/resources/ws/ros2src/com_py/com_py/base_bridge_relay.py +106 -0
  130. rosotacom/resources/ws/ros2src/com_py/com_py/bridge_in.py +76 -0
  131. rosotacom/resources/ws/ros2src/com_py/com_py/bridge_out.py +70 -0
  132. rosotacom/resources/ws/ros2src/com_py/com_py/heartbeat_echo.py +221 -0
  133. rosotacom/resources/ws/ros2src/com_py/com_py/hz_monitor.py +178 -0
  134. rosotacom/resources/ws/ros2src/com_py/com_py/image_bandwidth_reducer.py +218 -0
  135. rosotacom/resources/ws/ros2src/com_py/com_py/image_pixel_capper.py +370 -0
  136. rosotacom/resources/ws/ros2src/com_py/com_py/lanelet_saver.py +80 -0
  137. rosotacom/resources/ws/ros2src/com_py/com_py/latch_relay.py +205 -0
  138. rosotacom/resources/ws/ros2src/com_py/com_py/legacy/costmap_remapper.py +73 -0
  139. rosotacom/resources/ws/ros2src/com_py/com_py/legacy/marker_remapper.py +93 -0
  140. rosotacom/resources/ws/ros2src/com_py/com_py/legacy/speed_test.py +107 -0
  141. rosotacom/resources/ws/ros2src/com_py/com_py/legacy/tf_remapper.py +156 -0
  142. rosotacom/resources/ws/ros2src/com_py/com_py/legacy/traffic_monitor.py +91 -0
  143. rosotacom/resources/ws/ros2src/com_py/com_py/legacy/wifi_network_analysis_1.py +213 -0
  144. rosotacom/resources/ws/ros2src/com_py/com_py/link_bytes.py +177 -0
  145. rosotacom/resources/ws/ros2src/com_py/com_py/link_trace.py +186 -0
  146. rosotacom/resources/ws/ros2src/com_py/com_py/local_global_frame_bridge.py +465 -0
  147. rosotacom/resources/ws/ros2src/com_py/com_py/localization_map_split_reduce_saver.py +258 -0
  148. rosotacom/resources/ws/ros2src/com_py/com_py/msg_transform.py +277 -0
  149. rosotacom/resources/ws/ros2src/com_py/com_py/pair_management.py +87 -0
  150. rosotacom/resources/ws/ros2src/com_py/com_py/pub_sub_pair.py +145 -0
  151. rosotacom/resources/ws/ros2src/com_py/com_py/qos.py +254 -0
  152. rosotacom/resources/ws/ros2src/com_py/com_py/relay_in.py +78 -0
  153. rosotacom/resources/ws/ros2src/com_py/com_py/relay_out.py +75 -0
  154. rosotacom/resources/ws/ros2src/com_py/com_py/restamp.py +191 -0
  155. rosotacom/resources/ws/ros2src/com_py/com_py/sized_publisher.py +244 -0
  156. rosotacom/resources/ws/ros2src/com_py/com_py/stage_latency.py +64 -0
  157. rosotacom/resources/ws/ros2src/com_py/com_py/stage_latency_core.py +51 -0
  158. rosotacom/resources/ws/ros2src/com_py/com_py/status_overview.py +476 -0
  159. rosotacom/resources/ws/ros2src/com_py/com_py/status_overview_core.py +989 -0
  160. rosotacom/resources/ws/ros2src/com_py/com_py/tf_map_to_cart_saver.py +99 -0
  161. rosotacom/resources/ws/ros2src/com_py/com_py/topic_monitor.py +327 -0
  162. rosotacom/resources/ws/ros2src/com_py/com_py/topic_resolution.py +171 -0
  163. rosotacom/resources/ws/ros2src/com_py/com_py/trickle.py +183 -0
  164. rosotacom/resources/ws/ros2src/com_py/com_py/universal_compressor.py +653 -0
  165. rosotacom/resources/ws/ros2src/com_py/com_py/universal_decompressor.py +372 -0
  166. rosotacom/resources/ws/ros2src/com_py/com_py/universal_ota_unwrapper.py +191 -0
  167. rosotacom/resources/ws/ros2src/com_py/com_py/universal_ota_wrapper.py +154 -0
  168. rosotacom/resources/ws/ros2src/com_py/package.xml +34 -0
  169. rosotacom/resources/ws/ros2src/com_py/resource/com_py +0 -0
  170. rosotacom/resources/ws/ros2src/com_py/setup.cfg +6 -0
  171. rosotacom/resources/ws/ros2src/com_py/setup.py +55 -0
  172. rosotacom/resources/ws/ros2src/com_py/test/test_copyright.py +25 -0
  173. rosotacom/resources/ws/ros2src/com_py/test/test_flake8.py +25 -0
  174. rosotacom/resources/ws/ros2src/com_py/test/test_pep257.py +23 -0
  175. rosotacom/resources/ws/session/content/base/session_plugin_base.yaml +633 -0
  176. rosotacom/resources/ws/session/content/network/up_connectivity.py +364 -0
  177. rosotacom/resources/ws/session/content/network/wifi_network_analysis_2.py +215 -0
  178. rosotacom/resources/ws/session/creation/anonymize_bag.py +156 -0
  179. rosotacom/resources/ws/session/creation/catmux_log_setup.sh +65 -0
  180. rosotacom/resources/ws/session/creation/create_session_yaml.py +145 -0
  181. rosotacom/resources/ws/session/creation/generate_session_files.py +3217 -0
  182. rosotacom/resources/ws/session/creation/run_session.py +447 -0
  183. rosotacom/resources/ws/session/creation/run_session_in_external_terminal.py +54 -0
  184. rosotacom/resources/ws/session/creation/strip_ansi.py +42 -0
  185. rosotacom/status_eval.py +475 -0
  186. rosotacom/stream_stats.py +650 -0
  187. rosotacom/trace_profiles.py +561 -0
  188. rosotacom/transit.py +249 -0
  189. rosotacom/video_quality.py +906 -0
  190. rosotacom_dev-2.2.dist-info/METADATA +874 -0
  191. rosotacom_dev-2.2.dist-info/RECORD +195 -0
  192. rosotacom_dev-2.2.dist-info/WHEEL +5 -0
  193. rosotacom_dev-2.2.dist-info/entry_points.txt +4 -0
  194. rosotacom_dev-2.2.dist-info/licenses/LICENSE.txt +25 -0
  195. rosotacom_dev-2.2.dist-info/top_level.txt +1 -0
rosotacom/benchmark.py ADDED
@@ -0,0 +1,1842 @@
1
+ """RFC 0005 — benchmark genres (sweep/capacity & perturbation/recovery).
2
+
3
+ Pure, host-testable driver and verdict logic. The *runs* themselves (live ROS
4
+ graphs, emulated profiles, the nightly runner topology) are non-deterministic and
5
+ FZI-private — they live in the harness, not here (RFC 0005, "CI distribution").
6
+ What lives here is everything that can be exercised by a deterministic host test:
7
+
8
+ * size-pattern expansion (the a/b irregular-size load, mirrors ``sized_publisher``);
9
+ * the capacity binary-search driver and its oracle (``loss < p`` and ``latency < L``);
10
+ * sweep bounds + the shared-link guard (an unshaped run never saturates the LAN);
11
+ * the committed band store, the two-sided compare, and the ratchet (RFC 0007);
12
+ * recovery-metric extraction from a timeline of RFC 0003 transit records;
13
+ * fixed-probe time-bin characterization for latency/loss/Hz/bandwidth plots;
14
+ * the coarse linear-ramp curve (monitor-only trend).
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import math
21
+ import os
22
+ import platform
23
+ import re
24
+ import socket
25
+ from collections.abc import Callable, Iterable, Mapping, Sequence
26
+ from dataclasses import asdict, dataclass, field, replace
27
+ from enum import Enum
28
+ from pathlib import Path
29
+ from typing import Any
30
+
31
+ # --------------------------------------------------------------------------- #
32
+ # Genre 1 — load driver: a/b size pattern
33
+ # --------------------------------------------------------------------------- #
34
+
35
+
36
+ def parse_size_pattern(pattern: str) -> list[str]:
37
+ """Parse ``"a*4,b*1"`` / ``"ax4,bx1"`` into a cyclic list of ``a``/``b`` tokens.
38
+
39
+ Mirrors ``sized_publisher._parse_pattern`` so the host test validates the same
40
+ generation logic the ROS node uses. An empty pattern means a single ``a`` size.
41
+ """
42
+ if not pattern.strip():
43
+ return ["a"]
44
+
45
+ tokens: list[str] = []
46
+ for raw_token in pattern.split(","):
47
+ token = raw_token.strip().lower()
48
+ match = re.fullmatch(r"([a-z])(?:[*x](\d+))?", token)
49
+ if not match:
50
+ raise ValueError(f"Invalid pattern token '{raw_token}'. Use tokens like 'a', 'b', 'ax4', or 'b*2'.")
51
+ count = int(match.group(2) or "1")
52
+ if count < 1:
53
+ raise ValueError(f"Pattern token '{raw_token}' must repeat at least once.")
54
+ tokens.extend([match.group(1)] * count)
55
+ return tokens
56
+
57
+
58
+ def parse_payload_size_bytes(raw: str) -> int:
59
+ """Parse a payload byte size with optional decimal/binary units."""
60
+ text = raw.strip()
61
+ match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([A-Za-z]*)", text)
62
+ if not match:
63
+ raise ValueError(f"Invalid payload size {raw!r}. Use values like 20000, 20KB, or 20KiB.")
64
+
65
+ value = float(match.group(1))
66
+ unit = match.group(2).lower()
67
+ units = {
68
+ "": 1,
69
+ "b": 1,
70
+ "k": 1_000,
71
+ "kb": 1_000,
72
+ "m": 1_000_000,
73
+ "mb": 1_000_000,
74
+ "g": 1_000_000_000,
75
+ "gb": 1_000_000_000,
76
+ "ki": 1024,
77
+ "kib": 1024,
78
+ "mi": 1024 * 1024,
79
+ "mib": 1024 * 1024,
80
+ "gi": 1024 * 1024 * 1024,
81
+ "gib": 1024 * 1024 * 1024,
82
+ }
83
+ if unit not in units:
84
+ raise ValueError(f"Invalid payload size unit {match.group(2)!r}. Use B, KB, MB, KiB, or MiB.")
85
+ bytes_value = value * units[unit]
86
+ if bytes_value < 0 or not bytes_value.is_integer():
87
+ raise ValueError(f"Payload size {raw!r} does not resolve to a whole non-negative byte count.")
88
+ return int(bytes_value)
89
+
90
+
91
+ def _compress_size_pattern(tokens: Sequence[str]) -> str:
92
+ compressed: list[str] = []
93
+ current = tokens[0]
94
+ count = 1
95
+ for token in tokens[1:]:
96
+ if token == current:
97
+ count += 1
98
+ continue
99
+ compressed.append(f"{current}*{count}")
100
+ current = token
101
+ count = 1
102
+ compressed.append(f"{current}*{count}")
103
+ return ",".join(compressed)
104
+
105
+
106
+ def parse_size_pattern_load(pattern: str) -> dict[str, Any]:
107
+ """Parse ``1x20KB+1x0KB`` into ``sized_publisher`` load parameters.
108
+
109
+ Pattern terms are ``COUNTxSIZE`` or just ``SIZE``, separated by ``+`` or
110
+ ``,``. Decimal units (KB/MB/GB) use powers of 1000; binary units
111
+ (KiB/MiB/GiB) use powers of 1024.
112
+ """
113
+ if not pattern.strip():
114
+ raise ValueError("size pattern must not be empty.")
115
+
116
+ label_by_size: dict[int, str] = {}
117
+ size_by_label: dict[str, int] = {}
118
+ tokens: list[str] = []
119
+
120
+ for raw_term in re.split(r"[+,]", pattern):
121
+ term = raw_term.strip()
122
+ if not term:
123
+ raise ValueError(f"Invalid size pattern {pattern!r}: empty term.")
124
+ match = re.fullmatch(r"(?:(\d+)\s*[x*]\s*)?(.+)", term)
125
+ if not match:
126
+ raise ValueError(f"Invalid size pattern term {raw_term!r}. Use terms like 1x20KB or 0KB.")
127
+ count = int(match.group(1) or "1")
128
+ if count < 1:
129
+ raise ValueError(f"Size pattern term {raw_term!r} must repeat at least once.")
130
+ size = parse_payload_size_bytes(match.group(2))
131
+ label = label_by_size.get(size)
132
+ if label is None:
133
+ label = chr(ord("a") + len(label_by_size))
134
+ label_by_size[size] = label
135
+ size_by_label[label] = size
136
+ tokens.extend([label] * count)
137
+
138
+ sizes_list = [size_by_label[t] for t in tokens]
139
+
140
+ load: dict[str, Any] = {
141
+ "sizes": sizes_list,
142
+ "pattern": _compress_size_pattern(tokens),
143
+ "size_pattern": pattern,
144
+ }
145
+ for label, size in size_by_label.items():
146
+ load[f"size_{label}"] = size
147
+
148
+ return load
149
+
150
+
151
+ def expand_size_pattern(pattern: str, size_a: int, size_b: int | None = None, **kwargs: Any) -> list[int]:
152
+ """Expand a pattern into the concrete cyclic byte-size sequence it publishes."""
153
+ if size_a < 0:
154
+ raise ValueError("size_a must be >= 0.")
155
+ tokens = parse_size_pattern(pattern)
156
+ sizes = {"a": size_a, "b": size_b}
157
+ sizes.update(kwargs)
158
+ for token in tokens:
159
+ if token not in sizes or sizes[token] is None:
160
+ raise ValueError(f"Pattern references size {token!r} but it was not provided.")
161
+ return [int(sizes[token]) for token in tokens] # type: ignore[arg-type]
162
+
163
+
164
+ def pattern_mean_bytes(pattern: str, size_a: int, size_b: int | None = None, **kwargs: Any) -> float:
165
+ """Mean payload of one pattern cycle — the basis for the offered-bandwidth bound."""
166
+ sizes = expand_size_pattern(pattern, size_a, size_b, **kwargs)
167
+ return sum(sizes) / len(sizes)
168
+
169
+
170
+ # --------------------------------------------------------------------------- #
171
+ # Genre 1 — oracle: loss < p AND latency < L over a window
172
+ # --------------------------------------------------------------------------- #
173
+
174
+
175
+ @dataclass(frozen=True)
176
+ class OracleThresholds:
177
+ """Pass/fail bound for a capacity probe, read off the RFC 0003 backbone."""
178
+
179
+ max_loss_pct: float
180
+ max_latency_ms: float
181
+ latency_quantile: str = "p95"
182
+
183
+
184
+ def oracle_passes(loss_pct: float, latency_ms: float | None, thresholds: OracleThresholds) -> bool:
185
+ """A point passes iff loss and the chosen latency quantile are both under bound.
186
+
187
+ ``latency_ms is None`` means nothing was delivered in the window — a failure.
188
+ """
189
+ if latency_ms is None:
190
+ return False
191
+ return loss_pct <= thresholds.max_loss_pct and latency_ms <= thresholds.max_latency_ms
192
+
193
+
194
+ def oracle_passes_topic(topic_summary: dict[str, Any], thresholds: OracleThresholds) -> bool:
195
+ """Apply the oracle to one topic of ``transit.summarize_transit_records`` output."""
196
+ loss_pct = float(topic_summary.get("loss_pct", 100.0))
197
+ latency = (topic_summary.get("ota_hop_ms") or {}).get(thresholds.latency_quantile)
198
+ return oracle_passes(loss_pct, None if latency is None else float(latency), thresholds)
199
+
200
+
201
+ # --------------------------------------------------------------------------- #
202
+ # Bag-as-load oracle: a whole replay contract, judged per topic
203
+ # --------------------------------------------------------------------------- #
204
+ #
205
+ # When the benchmark *load* is a bag replay (a real session/scenario) rather than
206
+ # a single synthetic stream, one probe point delivers many topics at once. The
207
+ # verdict is then per topic — each contract topic must clear the network-loss and
208
+ # latency bound *and* deliver a high-enough share of the bag's known messages
209
+ # (completeness ground truth from ``bag_ground_truth``) — aggregated to a run
210
+ # verdict: the run passes iff every required topic passes, and the failing topics
211
+ # are named. This mirrors ``oracle_passes_topic`` for the loss/latency part and
212
+ # adds the completeness gate the known-source replay makes possible (RFC 0002,
213
+ # "Live vs replay"). It is pure so the search over replay verdicts is host-tested.
214
+
215
+
216
+ @dataclass(frozen=True)
217
+ class TopicVerdict:
218
+ """Per-topic verdict of one replay run against the contract + bag ground truth."""
219
+
220
+ topic: str
221
+ passes: bool
222
+ expected: int | None # bag ground-truth count when known, else the run's own expected
223
+ delivered: int | None
224
+ lost: int | None
225
+ reordered: int | None
226
+ loss_pct: float
227
+ completeness: float | None # delivered / bag-count; None when no ground truth
228
+ latency_p95_ms: float | None
229
+ jitter_p95_ms: float | None
230
+ loss_free: bool # zero network loss AND (when known) complete against the bag
231
+ latency_ok: bool
232
+ reason: str # "ok" or the first failing check, for the failing-topics list
233
+
234
+
235
+ @dataclass(frozen=True)
236
+ class BagRunVerdict:
237
+ """Run verdict aggregated over every required contract topic of one replay."""
238
+
239
+ passes: bool
240
+ loss_free: bool
241
+ latency_ok: bool
242
+ topics: tuple[TopicVerdict, ...]
243
+ failing_topics: tuple[str, ...]
244
+ representative_topic: str
245
+ # Row aggregates (summed counts; worst-case rate metrics across topics):
246
+ expected: int
247
+ delivered: int
248
+ lost: int
249
+ reordered: int
250
+ loss_pct: float
251
+ latency_p95_ms: float | None
252
+ jitter_p95_ms: float | None
253
+
254
+
255
+ def summary_topic_name(label: str) -> str:
256
+ """The bare ROS topic of a ``summarize_transit_records`` label.
257
+
258
+ Labels are ``"<source>-><target>:<topic>"`` when directions are known (peer
259
+ identities carry no ``:``) and just ``"<topic>"`` otherwise; ROS topics never
260
+ contain ``:``, so the topic is everything after the first ``:``.
261
+ """
262
+ if "->" in label and ":" in label:
263
+ return label.split(":", 1)[1]
264
+ return label
265
+
266
+
267
+ def _topic_data_by_name(summary: dict[str, Any]) -> dict[str, tuple[str, dict[str, Any]]]:
268
+ """Index a run summary by bare topic name, keeping the richest per-topic row.
269
+
270
+ A contract topic flows one direction, but if a name appears under several
271
+ labels the entry with the most delivered messages is the real stream.
272
+ """
273
+ topics = summary.get("topics")
274
+ if not isinstance(topics, dict):
275
+ return {}
276
+ indexed: dict[str, tuple[str, dict[str, Any]]] = {}
277
+ for label, data in topics.items():
278
+ if not isinstance(data, dict):
279
+ continue
280
+ name = summary_topic_name(str(label))
281
+ current = indexed.get(name)
282
+ if current is None or int(data.get("delivered") or 0) > int(current[1].get("delivered") or 0):
283
+ indexed[name] = (str(label), data)
284
+ return indexed
285
+
286
+
287
+ def evaluate_bag_run(
288
+ summary: dict[str, Any],
289
+ *,
290
+ thresholds: OracleThresholds,
291
+ ground_truth: dict[str, dict[str, Any]] | None = None,
292
+ min_completeness: float = 0.95,
293
+ required_topics: Sequence[str] | None = None,
294
+ ) -> BagRunVerdict:
295
+ """Judge one replay run's transit summary against the whole contract.
296
+
297
+ ``required_topics`` is the set the run must deliver (the session's carried
298
+ topics); when omitted it is taken from ``ground_truth`` if given, else from
299
+ every topic in the summary. ``ground_truth`` maps ROS topic -> bag facts
300
+ (``bag_ground_truth``); when a topic has a bag ``count`` the completeness gate
301
+ (``delivered / count >= min_completeness``) applies on top of the loss/latency
302
+ oracle. A required topic missing from the summary counts as fully lost.
303
+ """
304
+ if not 0.0 < min_completeness <= 1.0:
305
+ raise ValueError("min_completeness must be within (0, 1].")
306
+ indexed = _topic_data_by_name(summary)
307
+ if required_topics is not None:
308
+ names = [str(name) for name in required_topics]
309
+ elif ground_truth is not None:
310
+ names = sorted(name for name in ground_truth if name in indexed) or sorted(indexed)
311
+ else:
312
+ names = sorted(indexed)
313
+
314
+ quantile = thresholds.latency_quantile
315
+ verdicts: list[TopicVerdict] = []
316
+ for name in names:
317
+ label, data = indexed.get(name, (name, {}))
318
+ gt = (ground_truth or {}).get(name) or {}
319
+ gt_count = int(gt.get("count") or 0) if gt else 0
320
+
321
+ delivered = data.get("delivered")
322
+ delivered_n = int(delivered) if delivered is not None else 0
323
+ lost = data.get("lost")
324
+ reordered = data.get("reordered")
325
+ run_expected = data.get("expected")
326
+ loss_pct = float(data.get("loss_pct", 100.0)) if data else 100.0
327
+ latency_p95 = (data.get("ota_hop_ms") or {}).get(quantile) if data else None
328
+ jitter_p95 = (data.get("jitter_ms") or {}).get("p95") if data else None
329
+ latency_val = None if latency_p95 is None else float(latency_p95)
330
+
331
+ expected_out = gt_count if gt_count > 0 else (int(run_expected) if run_expected is not None else None)
332
+ completeness = (delivered_n / gt_count) if gt_count > 0 else None
333
+
334
+ latency_ok = latency_val is not None and latency_val <= thresholds.max_latency_ms
335
+ loss_ok = loss_pct <= thresholds.max_loss_pct
336
+ no_network_loss = loss_pct <= 0.0 and int(lost or 0) == 0
337
+ complete = completeness is None or completeness >= min_completeness
338
+ passes = bool(data) and loss_ok and latency_ok and complete
339
+ loss_free = bool(data) and no_network_loss and complete
340
+
341
+ if not data:
342
+ reason = "absent"
343
+ elif not complete:
344
+ reason = f"incomplete({completeness:.3f}<{min_completeness:g})"
345
+ elif not loss_ok:
346
+ reason = f"loss({loss_pct:.3g}%>{thresholds.max_loss_pct:g}%)"
347
+ elif not latency_ok:
348
+ if latency_val is None:
349
+ reason = "latency(none)"
350
+ else:
351
+ reason = f"latency({latency_val:.3g}>{thresholds.max_latency_ms:g}ms)"
352
+ else:
353
+ reason = "ok"
354
+
355
+ verdicts.append(
356
+ TopicVerdict(
357
+ topic=name,
358
+ passes=passes,
359
+ expected=expected_out,
360
+ delivered=delivered_n if data else 0,
361
+ lost=int(lost) if lost is not None else (expected_out if not data and expected_out else None),
362
+ reordered=int(reordered) if reordered is not None else None,
363
+ loss_pct=loss_pct,
364
+ completeness=round(completeness, 6) if completeness is not None else None,
365
+ latency_p95_ms=latency_val,
366
+ jitter_p95_ms=None if jitter_p95 is None else float(jitter_p95),
367
+ loss_free=loss_free,
368
+ latency_ok=latency_ok,
369
+ reason=reason,
370
+ )
371
+ )
372
+
373
+ failing = tuple(v.topic for v in verdicts if not v.passes)
374
+ passes = bool(verdicts) and not failing
375
+ loss_free = bool(verdicts) and all(v.loss_free for v in verdicts)
376
+ latency_ok = bool(verdicts) and all(v.latency_ok for v in verdicts)
377
+
378
+ expected_total = sum(int(v.expected) for v in verdicts if v.expected is not None)
379
+ delivered_total = sum(int(v.delivered) for v in verdicts if v.delivered is not None)
380
+ lost_total = sum(int(v.lost) for v in verdicts if v.lost is not None)
381
+ reordered_total = sum(int(v.reordered) for v in verdicts if v.reordered is not None)
382
+ loss_values = [v.loss_pct for v in verdicts]
383
+ latency_values = [v.latency_p95_ms for v in verdicts if v.latency_p95_ms is not None]
384
+ jitter_values = [v.jitter_p95_ms for v in verdicts if v.jitter_p95_ms is not None]
385
+ representative = failing[0] if failing else (verdicts[0].topic if verdicts else "")
386
+
387
+ return BagRunVerdict(
388
+ passes=passes,
389
+ loss_free=loss_free,
390
+ latency_ok=latency_ok,
391
+ topics=tuple(verdicts),
392
+ failing_topics=failing,
393
+ representative_topic=representative,
394
+ expected=expected_total,
395
+ delivered=delivered_total,
396
+ lost=lost_total,
397
+ reordered=reordered_total,
398
+ loss_pct=max(loss_values) if loss_values else 100.0,
399
+ latency_p95_ms=max(latency_values) if latency_values else None,
400
+ jitter_p95_ms=max(jitter_values) if jitter_values else None,
401
+ )
402
+
403
+
404
+ # --------------------------------------------------------------------------- #
405
+ # Genre 1 — sweep bounds + shared-link guard
406
+ # --------------------------------------------------------------------------- #
407
+
408
+
409
+ @dataclass(frozen=True)
410
+ class SweepBounds:
411
+ """Operationally-relevant ceilings; a sweep stops here, not "until it breaks".
412
+
413
+ ``max_bandwidth_bps`` is the offered-load ceiling that keeps an unshaped / LAN
414
+ run from saturating the shared network (RFC 0005, "Bounded, not unbounded").
415
+ """
416
+
417
+ max_size: int | None = None
418
+ max_rate: float | None = None
419
+ max_bandwidth_bps: float | None = None
420
+
421
+
422
+ def offered_bandwidth_bps(size_bytes: float, rate_hz: float) -> float:
423
+ return size_bytes * 8.0 * rate_hz
424
+
425
+
426
+ def within_shared_link_budget(size_bytes: float, rate_hz: float, bounds: SweepBounds) -> bool:
427
+ if bounds.max_bandwidth_bps is None:
428
+ return True
429
+ return offered_bandwidth_bps(size_bytes, rate_hz) <= bounds.max_bandwidth_bps
430
+
431
+
432
+ def guard_shared_link(size_bytes: float, rate_hz: float, bounds: SweepBounds, *, shared_link: bool) -> None:
433
+ """Raise if a *shared* (unshaped/LAN) run would exceed the offered-load budget."""
434
+ if shared_link and not within_shared_link_budget(size_bytes, rate_hz, bounds):
435
+ raise ValueError(
436
+ f"offered load {offered_bandwidth_bps(size_bytes, rate_hz):.0f} bps at "
437
+ f"size={size_bytes} B, rate={rate_hz} Hz would saturate the shared link "
438
+ f"(budget {bounds.max_bandwidth_bps:.0f} bps)"
439
+ )
440
+
441
+
442
+ def size_ceiling(bounds: SweepBounds, *, rate_hz: float) -> int | None:
443
+ """Largest payload the sweep may probe at ``rate_hz`` under the configured bounds."""
444
+ ceilings: list[int] = []
445
+ if bounds.max_size is not None:
446
+ ceilings.append(int(bounds.max_size))
447
+ if bounds.max_bandwidth_bps is not None and rate_hz > 0:
448
+ ceilings.append(int(bounds.max_bandwidth_bps // (8.0 * rate_hz)))
449
+ return min(ceilings) if ceilings else None
450
+
451
+
452
+ # --------------------------------------------------------------------------- #
453
+ # Genre 1 — capacity binary search
454
+ # --------------------------------------------------------------------------- #
455
+
456
+
457
+ @dataclass(frozen=True)
458
+ class CapacitySlice:
459
+ """A capacity is a slice of (size × rate × profile); always state the slice."""
460
+
461
+ profile: str
462
+ knob: str # the swept parameter, e.g. "size" or "rate"
463
+ fixed: dict[str, float] = field(default_factory=dict)
464
+
465
+
466
+ @dataclass(frozen=True)
467
+ class CapacityResult:
468
+ slice: CapacitySlice
469
+ capacity: int | None # highest passing value, or None if even ``low`` fails
470
+
471
+
472
+ def capacity_binary_search(low: int, high: int, probe: Callable[[int], bool]) -> int | None:
473
+ """Highest integer value in ``[low, high]`` for which ``probe`` passes.
474
+
475
+ Assumes monotonicity (passing at smaller values) — see the RFC's "honest
476
+ limits": real cellular's non-monotonic loss can fool a single search, so the
477
+ live driver confirms with repeats/medians. Returns ``None`` if ``low`` fails.
478
+ """
479
+ if low > high:
480
+ raise ValueError("low must be <= high")
481
+ if not probe(low):
482
+ return None
483
+ best = low
484
+ lo, hi = low + 1, high
485
+ while lo <= hi:
486
+ mid = (lo + hi) // 2
487
+ if probe(mid):
488
+ best = mid
489
+ lo = mid + 1
490
+ else:
491
+ hi = mid - 1
492
+ return best
493
+
494
+
495
+ def find_capacity(
496
+ slice_: CapacitySlice,
497
+ low: int,
498
+ high: int,
499
+ probe: Callable[[int], bool],
500
+ *,
501
+ bounds: SweepBounds | None = None,
502
+ rate_hz: float | None = None,
503
+ ) -> CapacityResult:
504
+ """Bounded capacity search: clamp ``high`` to the sweep ceiling, then search.
505
+
506
+ When ``bounds`` and ``rate_hz`` are given the swept ceiling never exceeds the
507
+ offered-load budget, so a sweep on a shared link cannot be asked to saturate it.
508
+ """
509
+ if bounds is not None and rate_hz is not None:
510
+ ceiling = size_ceiling(bounds, rate_hz=rate_hz)
511
+ if ceiling is not None:
512
+ high = min(high, ceiling)
513
+ return CapacityResult(slice=slice_, capacity=capacity_binary_search(low, high, probe))
514
+
515
+
516
+ # --------------------------------------------------------------------------- #
517
+ # Genre 1/2 — committed bands + the ratchet (RFC 0007, the analogue of ``expect``)
518
+ # --------------------------------------------------------------------------- #
519
+
520
+ BAND_SCHEMA = 2
521
+
522
+ # Half-width = max(k·σ, floor). The defaults are starting points; the runner-class
523
+ # calibration (RFC 0007 checklist) settles them empirically per metric.
524
+ DEFAULT_WIDTH_K = 3.0
525
+ DEFAULT_FLOOR_FRAC = 0.02
526
+
527
+ # Fingerprint for bands rewritten mechanically without measured runs (e.g. the v1
528
+ # → v2 schema rewrite). It never matches a real runner class, so ``compare``
529
+ # refuses these bands until a real calibration replaces them.
530
+ UNCALIBRATED_FINGERPRINT = "uncalibrated"
531
+
532
+ RECALIBRATE_HINT = "recalibrate on the target runner class: rosotacom benchmark ratchet <result.json ...> --recalibrate"
533
+
534
+
535
+ class Better(str, Enum):
536
+ """A metric's better-direction: which way out of the band is an improvement."""
537
+
538
+ HIGHER = "higher" # capacity numbers, completeness
539
+ LOWER = "lower" # latency, loss, recovery times/bursts
540
+
541
+
542
+ class Verdict(str, Enum):
543
+ WITHIN = "WITHIN"
544
+ REGRESSED = "REGRESSED" # out of band on the worse side
545
+ IMPROVED = "IMPROVED" # out of band on the better side — gate-red too: ratchet it
546
+
547
+
548
+ class BandError(ValueError):
549
+ """A band refusal. Every path that declines to compare or ratchet raises this."""
550
+
551
+
552
+ class FingerprintMismatch(BandError):
553
+ """Bands never transfer across runner classes (RFC 0007 §3)."""
554
+
555
+
556
+ class WideningRefused(BandError):
557
+ """A plain ratchet only turns one way; moving toward worse needs ``--recalibrate``."""
558
+
559
+
560
+ @dataclass(frozen=True)
561
+ class BandProvenance:
562
+ """Where a band's width comes from — a band without provenance is a guess."""
563
+
564
+ fingerprint: str # runner class whose variance calibrated the width
565
+ window_s: float # measurement window of one calibration run
566
+ repeats: int # K calibration runs behind sigma
567
+ sigma: float # run-to-run standard deviation across the K runs
568
+ floor: float # minimum half-width; keeps one lucky calibration from minting an impossibly tight band
569
+ k: float # half-width = max(k·sigma, floor)
570
+ source_sha: str # commit whose run(s) last moved the band
571
+ ratcheted_at: str # ISO timestamp of the last ratchet
572
+ note: str = "" # one-line cause note for the last move ("gop default changed → smaller keyframes")
573
+
574
+
575
+ @dataclass(frozen=True)
576
+ class Band:
577
+ """Committed two-sided envelope for one ``(row, profile, metric)``.
578
+
579
+ Leaving ``[lo, hi]`` toward worse is ``REGRESSED``; toward better is
580
+ ``IMPROVED`` — and in a gate lane both are red, the latter with the exact
581
+ ratchet command. Bands are never hand-edited: they change only through
582
+ :func:`ratchet_band`.
583
+ """
584
+
585
+ row: str
586
+ profile: str
587
+ metric: str
588
+ lo: float
589
+ hi: float
590
+ better: Better
591
+ provenance: BandProvenance
592
+
593
+ def __post_init__(self) -> None:
594
+ if self.lo > self.hi:
595
+ raise BandError(f"band [{self.lo}, {self.hi}] for {self.metric!r} is inverted (lo > hi)")
596
+
597
+ @property
598
+ def center(self) -> float:
599
+ return 0.5 * (self.lo + self.hi)
600
+
601
+ @property
602
+ def half_width(self) -> float:
603
+ return 0.5 * (self.hi - self.lo)
604
+
605
+ @property
606
+ def key(self) -> tuple[str, str, str]:
607
+ return (self.row, self.profile, self.metric)
608
+
609
+
610
+ @dataclass(frozen=True)
611
+ class BandComparison:
612
+ band: Band
613
+ value: float
614
+ verdict: Verdict
615
+
616
+
617
+ def _two_sided_verdict(value: float, *, lo: float, hi: float, better: Better) -> Verdict:
618
+ """WITHIN / IMPROVED / REGRESSED for a value against a closed ``[lo, hi]`` envelope.
619
+
620
+ Shared by the committed-band gate (:func:`band_verdict`) and A/B experiments
621
+ (:func:`classify_change`) so both speak one verdict language.
622
+ """
623
+ if value < lo:
624
+ return Verdict.REGRESSED if better is Better.HIGHER else Verdict.IMPROVED
625
+ if value > hi:
626
+ return Verdict.IMPROVED if better is Better.HIGHER else Verdict.REGRESSED
627
+ return Verdict.WITHIN
628
+
629
+
630
+ def band_verdict(band: Band, value: float) -> Verdict:
631
+ """Two-sided verdict; the interval is closed (values on the edge are WITHIN)."""
632
+ return _two_sided_verdict(value, lo=band.lo, hi=band.hi, better=band.better)
633
+
634
+
635
+ def compare_to_band(band: Band, value: float, *, fingerprint: str) -> BandComparison:
636
+ """Verdict for one measured value, refusing cross-runner-class comparisons.
637
+
638
+ A runner change must force a visible recalibration, never a silent shift —
639
+ so a fingerprint mismatch is a refusal, not a verdict.
640
+ """
641
+ if fingerprint != band.provenance.fingerprint:
642
+ raise FingerprintMismatch(
643
+ f"{band.metric} band for ({band.row}, {band.profile}) was calibrated on runner class "
644
+ f"{band.provenance.fingerprint!r}, but this run comes from {fingerprint!r}. "
645
+ f"Bands never transfer across runner classes — {RECALIBRATE_HINT}"
646
+ )
647
+ return BandComparison(band=band, value=float(value), verdict=band_verdict(band, float(value)))
648
+
649
+
650
+ def save_bands(path: Path, bands: Iterable[Band]) -> None:
651
+ """Write the band store: one JSON line per (row, profile, metric), stably ordered.
652
+
653
+ The band diff is part of the reviewed change (RFC 0007 §2), so ordering and
654
+ key order are deterministic — a ratchet shows up as a minimal, readable diff.
655
+ """
656
+ ordered = sorted(bands, key=lambda band: band.key)
657
+ duplicates = [key for key, count in _count_keys(ordered).items() if count > 1]
658
+ if duplicates:
659
+ raise BandError(f"duplicate band keys {duplicates!r}; one band per (row, profile, metric)")
660
+ lines = [
661
+ json.dumps(
662
+ {
663
+ "schema": BAND_SCHEMA,
664
+ "row": band.row,
665
+ "profile": band.profile,
666
+ "metric": band.metric,
667
+ "lo": band.lo,
668
+ "hi": band.hi,
669
+ "better": band.better.value,
670
+ "provenance": asdict(band.provenance),
671
+ },
672
+ sort_keys=True,
673
+ )
674
+ for band in ordered
675
+ ]
676
+ path.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
677
+
678
+
679
+ def load_bands(path: Path) -> list[Band]:
680
+ bands: list[Band] = []
681
+ for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
682
+ if not line.strip():
683
+ continue
684
+ raw = json.loads(line)
685
+ schema = raw.get("schema")
686
+ if schema != BAND_SCHEMA:
687
+ raise BandError(
688
+ f"{path}:{line_no}: band schema {schema!r} is not supported (expected {BAND_SCHEMA}). "
689
+ f"v1 budget entries were removed without a migration shim — {RECALIBRATE_HINT}"
690
+ )
691
+ bands.append(
692
+ Band(
693
+ row=str(raw["row"]),
694
+ profile=str(raw["profile"]),
695
+ metric=str(raw["metric"]),
696
+ lo=float(raw["lo"]),
697
+ hi=float(raw["hi"]),
698
+ better=Better(raw["better"]),
699
+ provenance=BandProvenance(**raw["provenance"]),
700
+ )
701
+ )
702
+ duplicates = [key for key, count in _count_keys(bands).items() if count > 1]
703
+ if duplicates:
704
+ raise BandError(f"{path}: duplicate band keys {duplicates!r}; one band per (row, profile, metric)")
705
+ return bands
706
+
707
+
708
+ def _count_keys(bands: Iterable[Band]) -> dict[tuple[str, str, str], int]:
709
+ counts: dict[tuple[str, str, str], int] = {}
710
+ for band in bands:
711
+ counts[band.key] = counts.get(band.key, 0) + 1
712
+ return counts
713
+
714
+
715
+ def find_band(bands: Iterable[Band], *, row: str, profile: str, metric: str) -> Band | None:
716
+ for band in bands:
717
+ if band.key == (row, profile, metric):
718
+ return band
719
+ return None
720
+
721
+
722
+ def bands_for(bands: Iterable[Band], *, row: str, profile: str) -> list[Band]:
723
+ return [band for band in bands if band.row == row and band.profile == profile]
724
+
725
+
726
+ def ratchet_band(
727
+ existing: Band | None,
728
+ values: Sequence[float],
729
+ *,
730
+ row: str,
731
+ profile: str,
732
+ metric: str,
733
+ better: Better,
734
+ fingerprint: str,
735
+ window_s: float,
736
+ source_sha: str,
737
+ ratcheted_at: str,
738
+ note: str = "",
739
+ recalibrate: bool = False,
740
+ k: float = DEFAULT_WIDTH_K,
741
+ floor: float = 0.0,
742
+ floor_frac: float = DEFAULT_FLOOR_FRAC,
743
+ ) -> Band:
744
+ """The only way a band changes (RFC 0007 §2). Center = median of ``values``.
745
+
746
+ Plain ratchet re-centers inside the calibrated width and only turns one way:
747
+ the worse edge may move toward better, never toward worse, and the width and
748
+ calibration provenance (σ, floor, k, fingerprint, window, repeats) are
749
+ preserved. ``recalibrate`` recomputes the width from the given runs — K fresh
750
+ repeats — with half-width ``max(k·σ, floor)``; that is the only path that may
751
+ widen a band, move it toward worse, or change its runner class.
752
+ """
753
+ if not values:
754
+ raise BandError(f"no values for {metric!r} — a ratchet needs at least one run")
755
+ center = _median([float(value) for value in values])
756
+
757
+ if recalibrate:
758
+ sigma = _sample_stdev([float(value) for value in values])
759
+ floor_used = max(float(floor), float(floor_frac) * abs(center))
760
+ half_width = max(float(k) * sigma, floor_used)
761
+ if half_width <= 0.0:
762
+ raise BandError(
763
+ f"recalibrated half-width for {metric!r} is 0 (σ=0 from {len(values)} run(s), floor=0) — "
764
+ "pass --floor or --floor-frac so the band has a width"
765
+ )
766
+ provenance = BandProvenance(
767
+ fingerprint=fingerprint,
768
+ window_s=float(window_s),
769
+ repeats=len(values),
770
+ sigma=sigma,
771
+ floor=floor_used,
772
+ k=float(k),
773
+ source_sha=source_sha,
774
+ ratcheted_at=ratcheted_at,
775
+ note=note,
776
+ )
777
+ return Band(
778
+ row=row,
779
+ profile=profile,
780
+ metric=metric,
781
+ lo=center - half_width,
782
+ hi=center + half_width,
783
+ better=better,
784
+ provenance=provenance,
785
+ )
786
+
787
+ if existing is None:
788
+ raise BandError(
789
+ f"no committed band for ({row}, {profile}, {metric}) — a first band is a calibration; "
790
+ "rerun with --recalibrate"
791
+ )
792
+ if better is not existing.better:
793
+ raise BandError(
794
+ f"better-direction for {metric!r} changed ({existing.better.value} → {better.value}); "
795
+ "that is a redesign of the metric — rerun with --recalibrate"
796
+ )
797
+ if fingerprint != existing.provenance.fingerprint:
798
+ raise FingerprintMismatch(
799
+ f"cannot ratchet the ({row}, {profile}, {metric}) band from runner class {fingerprint!r}: "
800
+ f"its width was calibrated on {existing.provenance.fingerprint!r}. {RECALIBRATE_HINT}"
801
+ )
802
+
803
+ half_width = existing.half_width
804
+ lo, hi = center - half_width, center + half_width
805
+ if existing.better is Better.HIGHER and lo < existing.lo:
806
+ raise WideningRefused(
807
+ f"ratchet would move the {metric!r} floor toward worse ({existing.lo:g} → {lo:g}). "
808
+ "A ratchet only tightens; a deliberate move toward worse needs --recalibrate"
809
+ )
810
+ if existing.better is Better.LOWER and hi > existing.hi:
811
+ raise WideningRefused(
812
+ f"ratchet would move the {metric!r} ceiling toward worse ({existing.hi:g} → {hi:g}). "
813
+ "A ratchet only tightens; a deliberate move toward worse needs --recalibrate"
814
+ )
815
+ provenance = replace(existing.provenance, source_sha=source_sha, ratcheted_at=ratcheted_at, note=note)
816
+ return Band(row=row, profile=profile, metric=metric, lo=lo, hi=hi, better=existing.better, provenance=provenance)
817
+
818
+
819
+ def _sample_stdev(values: Sequence[float]) -> float:
820
+ if len(values) < 2:
821
+ return 0.0
822
+ mean = sum(values) / len(values)
823
+ return math.sqrt(sum((value - mean) ** 2 for value in values) / (len(values) - 1))
824
+
825
+
826
+ def runner_fingerprint(env: Mapping[str, str] | None = None) -> str:
827
+ """Runner-class fingerprint recorded in bands and in every ``result.json``.
828
+
829
+ ``ROSOTACOM_RUNNER_CLASS`` names the class explicitly (the private bench pair
830
+ sets it); GitHub-hosted runners collapse to one class per OS/arch (their CPU
831
+ models vary run-to-run, the class does not); anything else is pinned to the
832
+ host, so an ad-hoc machine never silently compares against CI bands.
833
+ """
834
+ env = os.environ if env is None else env
835
+ explicit = env.get("ROSOTACOM_RUNNER_CLASS", "").strip()
836
+ if explicit:
837
+ return explicit
838
+ system = platform.system().lower()
839
+ machine = platform.machine().lower()
840
+ if env.get("RUNNER_ENVIRONMENT", "") == "github-hosted":
841
+ return f"github-hosted-{system}-{machine}"
842
+ return f"host-{socket.gethostname()}-{system}-{machine}"
843
+
844
+
845
+ # Better-directions for the metrics the extraction below produces. New banded
846
+ # metrics extend this table (or pass an explicit direction at ratchet time).
847
+ _DEFAULT_BETTER: dict[str, Better] = {
848
+ "t_recover_s": Better.LOWER,
849
+ "t_steady_s": Better.LOWER,
850
+ "recovery_burst": Better.LOWER,
851
+ "lost_during_outage_total": Better.LOWER,
852
+ "completeness_pct": Better.HIGHER,
853
+ "loss_pct": Better.LOWER,
854
+ "latency_p50_ms": Better.LOWER,
855
+ "latency_p95_ms": Better.LOWER,
856
+ "payload_bandwidth_bps": Better.HIGHER,
857
+ }
858
+
859
+
860
+ def default_better(metric: str) -> Better:
861
+ if metric.startswith("capacity_"):
862
+ return Better.HIGHER
863
+ better = _DEFAULT_BETTER.get(metric)
864
+ if better is None:
865
+ raise BandError(f"no default better-direction for metric {metric!r}; pass --better higher|lower")
866
+ return better
867
+
868
+
869
+ def _probe_metrics_from_result(doc: Mapping[str, Any]) -> dict[str, float]:
870
+ """Aggregate a probe run's per-attempt topic summaries into band metrics.
871
+
872
+ ``loss_pct`` is delivered-weighted across every attempt and topic (the
873
+ bottleneck-dominated gate metric); the latency percentiles are medians of
874
+ the per-attempt summaries (host-timing-dominated — monitor material on
875
+ shared runners, RFC 0007 §3).
876
+ """
877
+ attempts = (doc.get("measurements") or {}).get("attempts") or []
878
+ expected = 0
879
+ lost = 0
880
+ p50s: list[float] = []
881
+ p95s: list[float] = []
882
+ bandwidth_by_bin: dict[tuple[int, float, float], float] = {}
883
+ for attempt in attempts:
884
+ for topic_row in attempt.get("topics") or []:
885
+ if topic_row.get("expected") is None:
886
+ continue
887
+ expected += int(topic_row["expected"])
888
+ lost += int(topic_row.get("lost") or 0)
889
+ latency = topic_row.get("latency_ms") or {}
890
+ if latency.get("p50") is not None:
891
+ p50s.append(float(latency["p50"]))
892
+ if latency.get("p95") is not None:
893
+ p95s.append(float(latency["p95"]))
894
+ for bin_row in (doc.get("measurements") or {}).get("time_bins") or []:
895
+ bandwidth = bin_row.get("payload_bandwidth_bps")
896
+ if bandwidth is None:
897
+ continue
898
+ key = (
899
+ int(bin_row.get("attempt") or 0),
900
+ float(bin_row.get("bin_start_s") or 0.0),
901
+ float(bin_row.get("bin_end_s") or 0.0),
902
+ )
903
+ bandwidth_by_bin[key] = bandwidth_by_bin.get(key, 0.0) + float(bandwidth)
904
+ if expected <= 0:
905
+ raise BandError("probe run carries no per-topic expected/lost counts — there is nothing to band")
906
+ delivered = expected - lost
907
+ metrics = {
908
+ "completeness_pct": 100.0 * delivered / expected,
909
+ "loss_pct": 100.0 * lost / expected,
910
+ }
911
+ if p50s:
912
+ metrics["latency_p50_ms"] = _median(p50s)
913
+ if p95s:
914
+ metrics["latency_p95_ms"] = _median(p95s)
915
+ if bandwidth_by_bin:
916
+ metrics["payload_bandwidth_bps"] = _median(list(bandwidth_by_bin.values()))
917
+ return metrics
918
+
919
+
920
+ def metrics_from_result(doc: Mapping[str, Any]) -> dict[str, float]:
921
+ """The band-comparable metrics of one self-contained ``result.json``.
922
+
923
+ Covers the deterministic genres the gate bands today: ``capacity`` (the
924
+ breakpoint), ``probe`` (loss/completeness plus latency percentiles) and
925
+ ``recovery`` (the RFC 0005 recovery metric set). The benched-set registry
926
+ (RFC 0007 §4) picks which of these actually gate per row.
927
+ """
928
+ genre = doc.get("genre")
929
+ if genre == "probe":
930
+ return _probe_metrics_from_result(doc)
931
+ if genre == "capacity":
932
+ knob = str((doc.get("configuration") or {}).get("knob") or "size")
933
+ capacity = (doc.get("result") or {}).get("capacity")
934
+ if capacity is None:
935
+ raise BandError("capacity run found no passing probe — there is no value to band")
936
+ return {f"capacity_{knob}": float(capacity)}
937
+ if genre == "recovery":
938
+ result = doc.get("result") or {}
939
+ metrics: dict[str, float] = {}
940
+ for metric, field_name in (("t_recover_s", "t_recover"), ("t_steady_s", "t_steady")):
941
+ if result.get(field_name) is not None:
942
+ metrics[metric] = float(result[field_name])
943
+ if result.get("recovery_burst") is not None:
944
+ metrics["recovery_burst"] = float(result["recovery_burst"])
945
+ lost = result.get("lost_during_outage") or {}
946
+ metrics["lost_during_outage_total"] = float(sum(int(count) for count in lost.values()))
947
+ return metrics
948
+ raise BandError(f"genre {genre!r} has no band metrics yet — banded rows cover probe, capacity and recovery today")
949
+
950
+
951
+ def result_row_id(doc: Mapping[str, Any]) -> str:
952
+ """Canonical row id of a run until the benched-set registry formalizes rows."""
953
+ genre = str(doc.get("genre") or "")
954
+ if genre == "capacity":
955
+ knob = str((doc.get("configuration") or {}).get("knob") or "size")
956
+ return f"capacity-{knob}"
957
+ return genre or "unknown"
958
+
959
+
960
+ def result_profile(doc: Mapping[str, Any]) -> str:
961
+ profile = (doc.get("configuration") or {}).get("profile")
962
+ if not profile:
963
+ raise BandError("result.json carries no configuration.profile — cannot key a band without one")
964
+ return str(profile)
965
+
966
+
967
+ def result_fingerprint(doc: Mapping[str, Any]) -> str:
968
+ fingerprint = (doc.get("runner") or {}).get("fingerprint")
969
+ if not fingerprint:
970
+ raise BandError("result.json carries no runner fingerprint — it predates band schema v2; re-run the benchmark")
971
+ return str(fingerprint)
972
+
973
+
974
+ def result_window_s(doc: Mapping[str, Any]) -> float:
975
+ return float((doc.get("configuration") or {}).get("duration_s") or 0.0)
976
+
977
+
978
+ def result_sha(doc: Mapping[str, Any]) -> str:
979
+ return str(doc.get("sha") or "unknown")
980
+
981
+
982
+ # --------------------------------------------------------------------------- #
983
+ # Fixed probe — time-binned connection characterization
984
+ # --------------------------------------------------------------------------- #
985
+
986
+
987
+ @dataclass(frozen=True)
988
+ class ProbeBin:
989
+ topic: str
990
+ bin_start_s: float
991
+ bin_end_s: float
992
+ expected: int
993
+ delivered: int
994
+ lost: int
995
+ loss_pct: float
996
+ delivered_hz: float
997
+ expected_hz: float
998
+ payload_bandwidth_bps: float
999
+ mean_size_bytes: float | None
1000
+ latency_p50_ms: float | None
1001
+ latency_p95_ms: float | None
1002
+ jitter_p50_ms: float | None
1003
+ jitter_p95_ms: float | None
1004
+ inter_arrival_p50_ms: float | None
1005
+ inter_arrival_p95_ms: float | None
1006
+
1007
+
1008
+ def _percentile(values: list[float], percentile: float) -> float | None:
1009
+ if not values:
1010
+ return None
1011
+ ordered = sorted(values)
1012
+ rank = max(0, min(len(ordered) - 1, math.ceil(percentile * len(ordered)) - 1))
1013
+ return round(ordered[rank], 3)
1014
+
1015
+
1016
+ def _transit_topic_label(record: dict[str, Any]) -> str:
1017
+ topic = str(record.get("topic") or "")
1018
+ source = str(record.get("source") or "")
1019
+ target = str(record.get("target") or "")
1020
+ return f"{source}->{target}:{topic}" if source or target else topic
1021
+
1022
+
1023
+ def _section_ms(record: dict[str, Any], *names: str) -> float | None:
1024
+ sections = record.get("sections")
1025
+ if not isinstance(sections, dict):
1026
+ return None
1027
+ for name in names:
1028
+ value = sections.get(name)
1029
+ if value is not None:
1030
+ return float(value)
1031
+ return None
1032
+
1033
+
1034
+ def _infer_nominal_period_s(records: Sequence[dict[str, Any]], *, time_field: str) -> float | None:
1035
+ candidates: list[float] = []
1036
+ grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
1037
+ for record in records:
1038
+ key = (
1039
+ str(record.get("source") or ""),
1040
+ str(record.get("target") or ""),
1041
+ str(record.get("topic") or ""),
1042
+ )
1043
+ grouped.setdefault(key, []).append(record)
1044
+
1045
+ for stream in grouped.values():
1046
+ stamped = sorted(
1047
+ (record for record in stream if record.get(time_field) is not None),
1048
+ key=lambda record: int(record["seq"]),
1049
+ )
1050
+ for previous, current in zip(stamped, stamped[1:], strict=False):
1051
+ seq_delta = int(current["seq"]) - int(previous["seq"])
1052
+ time_delta = float(current[time_field]) - float(previous[time_field])
1053
+ if seq_delta > 0 and time_delta >= 0.0:
1054
+ candidates.append(time_delta / seq_delta)
1055
+ if not candidates:
1056
+ return None
1057
+ ordered = sorted(candidates)
1058
+ return ordered[len(ordered) // 2]
1059
+
1060
+
1061
+ def _median(values: Sequence[float]) -> float:
1062
+ ordered = sorted(values)
1063
+ n = len(ordered)
1064
+ mid = n // 2
1065
+ return ordered[mid] if n % 2 else 0.5 * (ordered[mid - 1] + ordered[mid])
1066
+
1067
+
1068
+ def _mad(values: Sequence[float]) -> float:
1069
+ """Median absolute deviation — a spread estimate that ignores outliers."""
1070
+ center = _median(values)
1071
+ return _median([abs(v - center) for v in values])
1072
+
1073
+
1074
+ # A fixed-probe sample is either INCLUDED (the impaired regime under test) or
1075
+ # EXCLUDED (the start-up: warm-up floor, the partial packet, and any startup drop).
1076
+ INCLUDED = "included"
1077
+ EXCLUDED = "excluded"
1078
+
1079
+
1080
+ def find_probe_onset(
1081
+ samples: Sequence[float | None],
1082
+ *,
1083
+ min_points: int = 12,
1084
+ step_window: int = 6,
1085
+ min_step_ratio: float = 3.0,
1086
+ min_step_ms: float = 15.0,
1087
+ settle_fraction: float = 0.6,
1088
+ settle_window: int = 6,
1089
+ ) -> int | None:
1090
+ """Index in ``samples`` where the clean impaired regime begins, or ``None``.
1091
+
1092
+ ``samples`` is one stream ordered by send time; each entry is a delivered
1093
+ packet's latency in ms, or ``None`` for a lost packet.
1094
+
1095
+ A fixed probe records packets before the tc/netem shaping is applied (the low
1096
+ transport floor), then a brief transition where the in-flight packet sees a
1097
+ partial delay and/or is dropped as the qdisc changes, then the impaired regime
1098
+ under test (a plateau *or* a rising bufferbloat ramp). This returns the first
1099
+ index from which the stream is *cleanly* in the impaired regime — a sustained
1100
+ window whose latencies are at/above a threshold set between the floor and the
1101
+ impaired level, with no loss dragging the window down. Everything before it
1102
+ (warm-up, the immature partial packet, and any startup drop) is start-up and
1103
+ should be excluded, so plot and summary agree on a clean measurement window.
1104
+
1105
+ Detection is conservative: without a clear low-floor→high step (``impaired`` at
1106
+ least ``min_step_ratio``× **or** ``min_step_ms`` above the floor) it returns
1107
+ ``None`` and nothing is excluded — e.g. impairment live from the first packet,
1108
+ a no-op profile, or a smooth ramp with no warm-up.
1109
+ """
1110
+ n = len(samples)
1111
+ delivered = [s for s in samples if s is not None]
1112
+ if len(delivered) < min_points:
1113
+ return None
1114
+
1115
+ window = max(1, min(step_window, len(delivered) // 2))
1116
+ # Sharpest upward step on the delivered latencies: the largest ratio between
1117
+ # the windowed median just after and just before an index. The un-impaired
1118
+ # floor is the lowest level, so the warm-up→impaired jump dominates, and
1119
+ # windowed medians stop a single ramp spike from masquerading as the step.
1120
+ step_index: int | None = None
1121
+ best_ratio = 1.0
1122
+ for i in range(window, len(delivered) - window):
1123
+ pre = _median(delivered[i - window : i])
1124
+ post = _median(delivered[i : i + window])
1125
+ if pre <= 0.0:
1126
+ continue
1127
+ ratio = post / pre
1128
+ if ratio > best_ratio:
1129
+ best_ratio = ratio
1130
+ step_index = i
1131
+ if step_index is None:
1132
+ return None
1133
+
1134
+ floor = _median(delivered[max(0, step_index - window) : step_index])
1135
+ impaired = _median(delivered[step_index : step_index + window])
1136
+ step_is_real = impaired >= floor * min_step_ratio or impaired - floor >= min_step_ms
1137
+ if best_ratio < min_step_ratio or not step_is_real:
1138
+ return None
1139
+
1140
+ # Threshold between floor and impaired level; ``settle_fraction`` sits it above
1141
+ # the partial packet so the immature transition sample is excluded.
1142
+ impaired_lo = floor + settle_fraction * (impaired - floor)
1143
+ hold = max(1, min(settle_window, len(delivered) // 2))
1144
+
1145
+ # Onset = first delivered sample at/above the regime that *starts* a sustained
1146
+ # window mostly at/above it. A startup drop (``None``) or the partial packet
1147
+ # keeps its window from qualifying, so onset lands on the first mature packet
1148
+ # after them — the point from which the measurement is clean.
1149
+ for i in range(n):
1150
+ value = samples[i]
1151
+ if value is None or value < impaired_lo:
1152
+ continue
1153
+ ahead = samples[i : i + hold]
1154
+ if len(ahead) < hold:
1155
+ break
1156
+ good = sum(1 for s in ahead if s is not None and s >= impaired_lo)
1157
+ if good >= 0.7 * hold:
1158
+ return i
1159
+ return None
1160
+
1161
+
1162
+ def _stream_samples(
1163
+ stream: Sequence[dict[str, Any]],
1164
+ *,
1165
+ seq0: int,
1166
+ t0: float,
1167
+ period_s: float,
1168
+ time_field: str,
1169
+ ) -> list[tuple[float, float | None]]:
1170
+ """One seq-ordered stream as ``(send_time, latency_or_None)`` — ``None`` = lost."""
1171
+ samples: list[tuple[float, float | None]] = []
1172
+ for record in stream:
1173
+ send_t = _send_time(record, seq0=seq0, t0=t0, period_s=period_s, time_field=time_field)
1174
+ latency = (
1175
+ None if record.get("status") == "lost" else _section_ms(record, "ota_hop_ms", "ota_hop_uncorrected_ms")
1176
+ )
1177
+ samples.append((send_t, latency))
1178
+ return samples
1179
+
1180
+
1181
+ def exclude_probe_warmup(
1182
+ joined_records: Sequence[dict[str, Any]],
1183
+ *,
1184
+ nominal_period_s: float | None = None,
1185
+ time_field: str = "t_wrap",
1186
+ ) -> tuple[list[dict[str, Any]], float | None]:
1187
+ """Drop the start-up prefix from already-joined transit records.
1188
+
1189
+ Returns ``(included_records, onset_send_time)`` where ``onset_send_time`` is
1190
+ the earliest per-stream impairment onset on the publish timeline — the origin
1191
+ that re-anchors "time since first publish" so ``t=0`` is the clean measurement
1192
+ start. Every record before its stream's onset (warm-up, the partial packet,
1193
+ and any startup drop) is dropped, so the summary and the raw plot agree on the
1194
+ same window. Streams with no clear warm-up step are kept whole; when no stream
1195
+ shows a step, nothing is dropped and ``onset_send_time`` is ``None``.
1196
+ """
1197
+ joined = list(joined_records)
1198
+ if not joined:
1199
+ return [], None
1200
+ period_s = (
1201
+ nominal_period_s if nominal_period_s is not None else _infer_nominal_period_s(joined, time_field=time_field)
1202
+ ) or 0.0
1203
+
1204
+ grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
1205
+ for record in joined:
1206
+ key = (
1207
+ str(record.get("source") or ""),
1208
+ str(record.get("target") or ""),
1209
+ str(record.get("topic") or ""),
1210
+ )
1211
+ grouped.setdefault(key, []).append(record)
1212
+
1213
+ kept: list[dict[str, Any]] = []
1214
+ onsets: list[float] = []
1215
+ for _key, stream in grouped.items():
1216
+ stream = sorted(stream, key=lambda record: int(record["seq"]))
1217
+ stamped = [record for record in stream if record.get(time_field) is not None]
1218
+ anchor = min(stamped or stream, key=lambda record: int(record["seq"]))
1219
+ seq0 = int(anchor["seq"])
1220
+ t0 = float(anchor.get(time_field) or 0.0)
1221
+ samples = _stream_samples(stream, seq0=seq0, t0=t0, period_s=period_s, time_field=time_field)
1222
+ onset_index = find_probe_onset([latency for _, latency in samples])
1223
+ if onset_index is None:
1224
+ kept.extend(stream)
1225
+ continue
1226
+ onset_send = samples[onset_index][0]
1227
+ onsets.append(onset_send)
1228
+ kept.extend(record for record, (send_t, _lat) in zip(stream, samples, strict=True) if send_t >= onset_send)
1229
+ return kept, (min(onsets) if onsets else None)
1230
+
1231
+
1232
+ def characterize_probe_records(
1233
+ records: Iterable[dict[str, Any]],
1234
+ *,
1235
+ bin_s: float = 1.0,
1236
+ nominal_period_s: float | None = None,
1237
+ topic: str = "",
1238
+ time_field: str = "t_wrap",
1239
+ exclude_warmup: bool = True,
1240
+ ) -> list[dict[str, Any]]:
1241
+ """Summarize one fixed probe as per-topic time bins.
1242
+
1243
+ Bins use publish time, not arrival time, so latency/loss spikes line up with
1244
+ the traffic that caused them. Lost records often lack timestamps; when a
1245
+ nominal period is supplied (usually ``1 / rate_hz``), their send time is
1246
+ reconstructed from sequence number just like the recovery metrics do.
1247
+
1248
+ With ``exclude_warmup`` (the default) the un-impaired warm-up plateau and the
1249
+ partial-impairment transition packet are dropped via
1250
+ :func:`exclude_probe_warmup`, and the bins are aligned to the impairment
1251
+ onset so ``bin_start_s == 0`` is where shaping is fully applied. Set it False
1252
+ to bin the raw timeline from the first publish.
1253
+ """
1254
+ if bin_s <= 0.0:
1255
+ raise ValueError("bin_s must be > 0.")
1256
+
1257
+ from .transit import join_transit_records
1258
+
1259
+ joined_records = join_transit_records(records)
1260
+ if not joined_records:
1261
+ return []
1262
+ period_s = (
1263
+ nominal_period_s
1264
+ if nominal_period_s is not None
1265
+ else _infer_nominal_period_s(joined_records, time_field=time_field)
1266
+ )
1267
+ period_s = float(period_s or 0.0)
1268
+
1269
+ if exclude_warmup:
1270
+ joined_records, _onset = exclude_probe_warmup(joined_records, nominal_period_s=period_s, time_field=time_field)
1271
+ if not joined_records:
1272
+ return []
1273
+
1274
+ grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
1275
+ for record in joined_records:
1276
+ key = (
1277
+ str(record.get("source") or ""),
1278
+ str(record.get("target") or ""),
1279
+ str(record.get("topic") or ""),
1280
+ )
1281
+ grouped.setdefault(key, []).append(record)
1282
+
1283
+ expanded: list[tuple[str, dict[str, Any], float]] = []
1284
+ for _key, stream in sorted(grouped.items()):
1285
+ stream = sorted(stream, key=lambda record: int(record["seq"]))
1286
+ stamped = [record for record in stream if record.get(time_field) is not None]
1287
+ anchor = min(stamped or stream, key=lambda record: int(record["seq"]))
1288
+ seq0 = int(anchor["seq"])
1289
+ t0 = float(anchor.get(time_field) or 0.0)
1290
+ for record in stream:
1291
+ label = _transit_topic_label(record)
1292
+ if topic and topic not in {label, str(record.get("topic") or "")}:
1293
+ continue
1294
+ expanded.append(
1295
+ (
1296
+ label,
1297
+ record,
1298
+ _send_time(record, seq0=seq0, t0=t0, period_s=period_s, time_field=time_field),
1299
+ )
1300
+ )
1301
+ if not expanded:
1302
+ return []
1303
+
1304
+ first_send_s = min(send_t for _label, _record, send_t in expanded)
1305
+ buckets: dict[tuple[str, float], dict[str, Any]] = {}
1306
+ for label, record, send_t in expanded:
1307
+ relative_s = max(0.0, send_t - first_send_s)
1308
+ bucket_start = math.floor(relative_s / bin_s) * bin_s
1309
+ bucket = buckets.setdefault(
1310
+ (label, bucket_start),
1311
+ {
1312
+ "topic": label,
1313
+ "bin_start_s": bucket_start,
1314
+ "bin_end_s": bucket_start + bin_s,
1315
+ "expected": 0,
1316
+ "delivered": 0,
1317
+ "lost": 0,
1318
+ "payload_bytes": 0.0,
1319
+ "latencies": [],
1320
+ "jitters": [],
1321
+ "inter_arrivals": [],
1322
+ },
1323
+ )
1324
+ bucket["expected"] += 1
1325
+ if record.get("status") == "lost":
1326
+ bucket["lost"] += 1
1327
+ continue
1328
+ bucket["delivered"] += 1
1329
+ if record.get("size_bytes") is not None:
1330
+ bucket["payload_bytes"] += float(record["size_bytes"])
1331
+ latency_ms = _section_ms(record, "ota_hop_ms", "ota_hop_uncorrected_ms")
1332
+ if latency_ms is not None:
1333
+ bucket["latencies"].append(latency_ms)
1334
+ if record.get("jitter_ms") is not None:
1335
+ bucket["jitters"].append(float(record["jitter_ms"]))
1336
+ if record.get("inter_arrival_ms") is not None:
1337
+ bucket["inter_arrivals"].append(float(record["inter_arrival_ms"]))
1338
+
1339
+ rows: list[dict[str, Any]] = []
1340
+ for (_label, _bucket_start), bucket in sorted(buckets.items()):
1341
+ expected = int(bucket["expected"])
1342
+ delivered = int(bucket["delivered"])
1343
+ lost = int(bucket["lost"])
1344
+ payload_bytes = float(bucket["payload_bytes"])
1345
+ row = ProbeBin(
1346
+ topic=str(bucket["topic"]),
1347
+ bin_start_s=round(float(bucket["bin_start_s"]), 6),
1348
+ bin_end_s=round(float(bucket["bin_end_s"]), 6),
1349
+ expected=expected,
1350
+ delivered=delivered,
1351
+ lost=lost,
1352
+ loss_pct=round(100.0 * lost / expected, 3) if expected else 0.0,
1353
+ delivered_hz=round(delivered / bin_s, 3),
1354
+ expected_hz=round(expected / bin_s, 3),
1355
+ payload_bandwidth_bps=round(payload_bytes * 8.0 / bin_s, 3),
1356
+ mean_size_bytes=round(payload_bytes / delivered, 3) if delivered else None,
1357
+ latency_p50_ms=_percentile(bucket["latencies"], 0.50),
1358
+ latency_p95_ms=_percentile(bucket["latencies"], 0.95),
1359
+ jitter_p50_ms=_percentile(bucket["jitters"], 0.50),
1360
+ jitter_p95_ms=_percentile(bucket["jitters"], 0.95),
1361
+ inter_arrival_p50_ms=_percentile(bucket["inter_arrivals"], 0.50),
1362
+ inter_arrival_p95_ms=_percentile(bucket["inter_arrivals"], 0.95),
1363
+ )
1364
+ rows.append(asdict(row))
1365
+ return rows
1366
+
1367
+
1368
+ # --------------------------------------------------------------------------- #
1369
+ # Genre 2 — perturbation / recovery metrics
1370
+ # --------------------------------------------------------------------------- #
1371
+
1372
+
1373
+ @dataclass(frozen=True)
1374
+ class OutageWindow:
1375
+ start: float # seconds on the publish timeline
1376
+ end: float # restore time
1377
+
1378
+
1379
+ @dataclass(frozen=True)
1380
+ class RecoveryMetrics:
1381
+ t_recover: float | None # restore -> first message arriving after restore
1382
+ t_steady: float | None # restore -> inter-arrival back within tolerance
1383
+ recovery_burst: int # messages arriving in the burst window after restore
1384
+ lost_during_outage: dict[str, int] # per topic
1385
+ latched_rearrival: dict[str, bool] # per latched topic
1386
+
1387
+
1388
+ def _send_time(record: dict[str, Any], *, seq0: int, t0: float, period_s: float, time_field: str) -> float:
1389
+ """Position on the publish timeline; reconstruct lost messages from the schedule.
1390
+
1391
+ Delivered records carry ``t_wrap`` (publish time). Lost records carry no
1392
+ timestamp, so their position is the nominal send time ``t0 + (seq - seq0)·T``.
1393
+ """
1394
+ stamp = record.get(time_field)
1395
+ if stamp is not None:
1396
+ return float(stamp)
1397
+ return t0 + (int(record["seq"]) - seq0) * period_s
1398
+
1399
+
1400
+ def _arrival_time(send_t: float, record: dict[str, Any]) -> float:
1401
+ sections = record.get("sections") or {}
1402
+ ota = sections.get("ota_hop_ms")
1403
+ if ota is None:
1404
+ ota = sections.get("ota_hop_uncorrected_ms")
1405
+ return send_t + (float(ota) / 1000.0 if ota is not None else 0.0)
1406
+
1407
+
1408
+ def recovery_metrics(
1409
+ records: Iterable[dict[str, Any]],
1410
+ outage: OutageWindow,
1411
+ *,
1412
+ nominal_period_s: float,
1413
+ time_field: str = "t_wrap",
1414
+ steady_factor: float = 1.5,
1415
+ burst_window_s: float = 1.0,
1416
+ latched_topics: Iterable[str] = (),
1417
+ ) -> RecoveryMetrics:
1418
+ """Extract recovery dynamics from a timeline of RFC 0003 transit records.
1419
+
1420
+ ``t_recover`` is the delay from restore to the first message arriving at/after
1421
+ restore; ``recovery_burst`` counts arrivals within ``burst_window_s`` of restore
1422
+ (the reconnect burst the baseline calls "the part that actually hurts");
1423
+ ``t_steady`` is the delay until inter-arrival returns within
1424
+ ``steady_factor × nominal_period``; ``lost_during_outage`` counts per-topic lost
1425
+ messages whose send time falls in the outage; ``latched_rearrival`` reports
1426
+ whether each latched topic delivered a value again after restore.
1427
+ """
1428
+ records = list(records)
1429
+ if not records:
1430
+ return RecoveryMetrics(None, None, 0, {}, {topic: False for topic in latched_topics})
1431
+
1432
+ stamped = [record for record in records if record.get(time_field) is not None]
1433
+ anchor = min(stamped or records, key=lambda record: int(record["seq"]))
1434
+ seq0 = int(anchor["seq"])
1435
+ t0 = float(anchor.get(time_field) or 0.0)
1436
+
1437
+ lost_during_outage: dict[str, int] = {}
1438
+ delivered: list[tuple[float, dict[str, Any]]] = []
1439
+ for record in records:
1440
+ send_t = _send_time(record, seq0=seq0, t0=t0, period_s=nominal_period_s, time_field=time_field)
1441
+ topic = str(record.get("topic") or "")
1442
+ if record.get("status") == "lost":
1443
+ if outage.start <= send_t <= outage.end:
1444
+ lost_during_outage[topic] = lost_during_outage.get(topic, 0) + 1
1445
+ else:
1446
+ delivered.append((_arrival_time(send_t, record), record))
1447
+
1448
+ delivered.sort(key=lambda item: item[0])
1449
+ after = [arrival for arrival, _ in delivered if arrival >= outage.end]
1450
+
1451
+ t_recover = after[0] - outage.end if after else None
1452
+ recovery_burst = sum(1 for arrival in after if arrival <= outage.end + burst_window_s)
1453
+
1454
+ # Steady state is inter-arrival back *near* the nominal period — a symmetric band.
1455
+ # A reconnect burst arrives in a clump (gaps far below nominal), so it is not steady.
1456
+ t_steady: float | None = None
1457
+ low_gap = nominal_period_s / steady_factor
1458
+ high_gap = nominal_period_s * steady_factor
1459
+ for index in range(1, len(after)):
1460
+ if low_gap <= after[index] - after[index - 1] <= high_gap:
1461
+ t_steady = after[index] - outage.end
1462
+ break
1463
+
1464
+ latched_rearrival = {
1465
+ topic: any(record.get("topic") == topic and arrival >= outage.end for arrival, record in delivered)
1466
+ for topic in latched_topics
1467
+ }
1468
+ return RecoveryMetrics(t_recover, t_steady, recovery_burst, lost_during_outage, latched_rearrival)
1469
+
1470
+
1471
+ # --------------------------------------------------------------------------- #
1472
+ # Genre 1 — coarse linear ramp (monitor-only trend)
1473
+ # --------------------------------------------------------------------------- #
1474
+
1475
+
1476
+ @dataclass(frozen=True)
1477
+ class RampPoint:
1478
+ value: float
1479
+ metric: float
1480
+
1481
+
1482
+ def linear_ramp(values: Iterable[float], measure: Callable[[float], float]) -> list[RampPoint]:
1483
+ """The whole response curve (latency-vs-load). Monitor-only: trended, never gated."""
1484
+ return [RampPoint(float(value), float(measure(value))) for value in values]
1485
+
1486
+
1487
+ # --------------------------------------------------------------------------- #
1488
+ # A/B tuning experiments (#22): candidate config vs baseline, same load+profile
1489
+ # --------------------------------------------------------------------------- #
1490
+ #
1491
+ # The regression gate asks "is this run within a committed envelope?"; an A/B
1492
+ # experiment asks the sibling question "is candidate config B better, worse or
1493
+ # unchanged vs baseline config A on the *same* load and profile?". Both are
1494
+ # two-sided and both speak the same Better/Verdict language: an A/B cell is just
1495
+ # :func:`_two_sided_verdict` on the ephemeral ``[baseline +/- tolerance]``
1496
+ # envelope. This layer is pure and host-testable; the live runs (Docker graphs,
1497
+ # emulated profiles) live in the CLI, injected as a ``run_point`` (RFC 0005).
1498
+
1499
+ # The per-topic metrics an A/B experiment watches, with their better-direction.
1500
+ # Names mirror the gate's band metrics (``_DEFAULT_BETTER``) so a verdict reads
1501
+ # the same in both places. Read off ``transit.summarize_transit_records``.
1502
+ AB_METRICS: dict[str, Better] = {
1503
+ "completeness_pct": Better.HIGHER,
1504
+ "loss_pct": Better.LOWER,
1505
+ "latency_p50_ms": Better.LOWER,
1506
+ "latency_p95_ms": Better.LOWER,
1507
+ "jitter_p50_ms": Better.LOWER,
1508
+ "jitter_p95_ms": Better.LOWER,
1509
+ }
1510
+
1511
+ # The default watched set: the completeness/loss pair (bottleneck-dominated —
1512
+ # counted off exact sequence numbers, so trustworthy even on a shared runner)
1513
+ # plus the p95 tails (host-timing-dominated — directional hints, which is why the
1514
+ # spread is always reported, not just the median).
1515
+ DEFAULT_AB_METRICS: tuple[str, ...] = ("completeness_pct", "loss_pct", "latency_p95_ms", "jitter_p95_ms")
1516
+
1517
+ # A candidate median within this relative OR the metric's absolute half-width of
1518
+ # the baseline median is called unchanged (WITHIN). The absolute floors keep
1519
+ # sub-millisecond tail noise on a tiny baseline from reading as a change; loss
1520
+ # and completeness are exact percentages and get no floor (a real 1% delta is
1521
+ # signal).
1522
+ DEFAULT_AB_REL_TOLERANCE = 0.10
1523
+ DEFAULT_AB_ABS_TOLERANCE: dict[str, float] = {
1524
+ "latency_p50_ms": 1.0,
1525
+ "latency_p95_ms": 2.0,
1526
+ "jitter_p50_ms": 1.0,
1527
+ "jitter_p95_ms": 2.0,
1528
+ }
1529
+
1530
+
1531
+ def ab_better(metric: str) -> Better:
1532
+ """The better-direction of an A/B-watched metric (raises on an unknown one)."""
1533
+ better = AB_METRICS.get(metric)
1534
+ if better is None:
1535
+ raise BandError(f"metric {metric!r} is not an A/B-watched metric; known: {sorted(AB_METRICS)}")
1536
+ return better
1537
+
1538
+
1539
+ def _bare_topic(label: str) -> str:
1540
+ """``a->b:/topic`` -> ``/topic``; a plain topic label is returned unchanged."""
1541
+ return label.split(":", 1)[1] if ":" in label else label
1542
+
1543
+
1544
+ def ab_metrics_from_topic(topic_summary: Mapping[str, Any]) -> dict[str, float | None]:
1545
+ """Pull the A/B-watched metrics out of one topic's transit-summary block.
1546
+
1547
+ A metric is ``None`` when the run produced no sample for it (e.g. latency
1548
+ with 100% loss); the aggregator drops ``None`` holes from a spread.
1549
+ """
1550
+ expected = topic_summary.get("expected")
1551
+ delivered = topic_summary.get("delivered")
1552
+ completeness: float | None = None
1553
+ if expected and delivered is not None:
1554
+ completeness = round(100.0 * float(delivered) / float(expected), 3)
1555
+ ota = topic_summary.get("ota_hop_ms") or {}
1556
+ jitter = topic_summary.get("jitter_ms") or {}
1557
+
1558
+ def opt(value: Any) -> float | None:
1559
+ return None if value is None else float(value)
1560
+
1561
+ return {
1562
+ "completeness_pct": completeness,
1563
+ "loss_pct": opt(topic_summary.get("loss_pct")),
1564
+ "latency_p50_ms": opt(ota.get("p50")),
1565
+ "latency_p95_ms": opt(ota.get("p95")),
1566
+ "jitter_p50_ms": opt(jitter.get("p50")),
1567
+ "jitter_p95_ms": opt(jitter.get("p95")),
1568
+ }
1569
+
1570
+
1571
+ @dataclass(frozen=True)
1572
+ class AbTolerance:
1573
+ """Half-width of the unchanged band around a baseline value: ``max(abs, rel*|baseline|)``."""
1574
+
1575
+ rel: float = DEFAULT_AB_REL_TOLERANCE
1576
+ abs: float = 0.0
1577
+
1578
+ def half_width(self, baseline: float) -> float:
1579
+ return max(self.abs, abs(baseline) * self.rel)
1580
+
1581
+
1582
+ def default_ab_tolerance(metric: str, *, rel: float = DEFAULT_AB_REL_TOLERANCE) -> AbTolerance:
1583
+ """The default tolerance for a metric: ``rel`` plus the metric's absolute floor."""
1584
+ return AbTolerance(rel=rel, abs=DEFAULT_AB_ABS_TOLERANCE.get(metric, 0.0))
1585
+
1586
+
1587
+ def classify_change(baseline: float, candidate: float, *, better: Better, tol: AbTolerance) -> Verdict:
1588
+ """Candidate vs baseline as WITHIN / IMPROVED / REGRESSED.
1589
+
1590
+ ``_two_sided_verdict`` on the ephemeral ``[baseline +/- tol]`` envelope, so an
1591
+ A/B cell classifies identically to a committed-band gate cell.
1592
+ """
1593
+ half = tol.half_width(baseline)
1594
+ return _two_sided_verdict(candidate, lo=baseline - half, hi=baseline + half, better=better)
1595
+
1596
+
1597
+ def ab_schedule(configs: Sequence[str], repeats: int) -> list[tuple[str, int]]:
1598
+ """Deterministic interleaving of ``configs`` over ``repeats`` as ``(config, repeat)``.
1599
+
1600
+ Every repeat runs each config exactly once; the starting config rotates each
1601
+ repeat so no config is always measured first. Interleaving (not "all of A,
1602
+ then all of B") spreads any slow host-drift across every config, and the
1603
+ rotation removes the residual first-slot warm-up bias — both guard the verdict
1604
+ against drift being mistaken for a config effect. Deterministic by
1605
+ construction, so a verdict is reproducible from the same runs.
1606
+ """
1607
+ if repeats < 1:
1608
+ raise ValueError("repeats must be >= 1.")
1609
+ if not configs:
1610
+ raise ValueError("need at least one config.")
1611
+ order: list[tuple[str, int]] = []
1612
+ count = len(configs)
1613
+ for repeat in range(1, repeats + 1):
1614
+ rotation = (repeat - 1) % count
1615
+ rotated = [*configs[rotation:], *configs[:rotation]]
1616
+ order.extend((config, repeat) for config in rotated)
1617
+ return order
1618
+
1619
+
1620
+ @dataclass(frozen=True)
1621
+ class MetricSpread:
1622
+ """Repeat-to-repeat spread of one metric for one config+topic."""
1623
+
1624
+ n: int # repeats that produced a value
1625
+ min: float
1626
+ median: float
1627
+ max: float
1628
+ values: list[float] # per-repeat, in execution order
1629
+
1630
+
1631
+ def _metric_spread(values: Sequence[float]) -> MetricSpread | None:
1632
+ vals = [float(v) for v in values]
1633
+ if not vals:
1634
+ return None
1635
+ return MetricSpread(
1636
+ n=len(vals),
1637
+ min=round(min(vals), 3),
1638
+ median=round(_median(vals), 3),
1639
+ max=round(max(vals), 3),
1640
+ values=[round(v, 3) for v in vals],
1641
+ )
1642
+
1643
+
1644
+ @dataclass(frozen=True)
1645
+ class AbRun:
1646
+ """One executed run in an experiment: which config, which 1-based repeat, its summary."""
1647
+
1648
+ config: str
1649
+ repeat: int
1650
+ summary: Mapping[str, Any] # transit.summarize_transit_records output
1651
+
1652
+
1653
+ @dataclass(frozen=True)
1654
+ class AbCell:
1655
+ """One (topic, metric) candidate-vs-baseline verdict, both spreads carried."""
1656
+
1657
+ topic: str
1658
+ metric: str
1659
+ better: str # Better value
1660
+ baseline: MetricSpread | None
1661
+ candidate: MetricSpread | None
1662
+ verdict: str | None # Verdict value; None when a side has no sample
1663
+ delta: float | None # candidate.median - baseline.median
1664
+ separated: bool | None # spreads disjoint => effect separable at this N
1665
+
1666
+
1667
+ @dataclass(frozen=True)
1668
+ class CandidateReport:
1669
+ """A candidate config's full comparison against the baseline."""
1670
+
1671
+ config: str
1672
+ cells: list[AbCell]
1673
+ regressed: list[list[str]] # [topic, metric] pairs that regressed beyond tolerance
1674
+ improved: list[list[str]] # [topic, metric] pairs that improved beyond tolerance
1675
+ dropped_topics: list[str] # topics the baseline delivered that this config did not
1676
+ passed: bool # no regression and no dropped topic
1677
+
1678
+
1679
+ @dataclass(frozen=True)
1680
+ class AbReport:
1681
+ baseline: str
1682
+ metrics: list[str]
1683
+ repeats: int
1684
+ candidates: list[CandidateReport]
1685
+ passed: bool # every candidate passed
1686
+
1687
+
1688
+ def _collect_ab_metrics(
1689
+ runs: Iterable[AbRun], *, metrics: Sequence[str], topic_filter: str
1690
+ ) -> dict[str, dict[str, dict[str, list[float]]]]:
1691
+ """``{config: {topic: {metric: [per-repeat values]}}}`` in execution order."""
1692
+ collected: dict[str, dict[str, dict[str, list[float]]]] = {}
1693
+ for run in runs:
1694
+ topics = run.summary.get("topics") or {}
1695
+ for topic_label, topic_summary in topics.items():
1696
+ if topic_filter and topic_filter not in {topic_label, _bare_topic(topic_label)}:
1697
+ continue
1698
+ values = ab_metrics_from_topic(topic_summary)
1699
+ per_topic = collected.setdefault(run.config, {}).setdefault(topic_label, {})
1700
+ for metric in metrics:
1701
+ value = values.get(metric)
1702
+ if value is not None:
1703
+ per_topic.setdefault(metric, []).append(value)
1704
+ return collected
1705
+
1706
+
1707
+ def ab_verdict(
1708
+ runs: Sequence[AbRun],
1709
+ *,
1710
+ baseline: str,
1711
+ metrics: Sequence[str] = DEFAULT_AB_METRICS,
1712
+ tolerances: Mapping[str, AbTolerance] | None = None,
1713
+ topic: str = "",
1714
+ ) -> AbReport:
1715
+ """Classify every candidate config against the baseline, per topic and metric.
1716
+
1717
+ Comparison is on the per-config **median** of each metric across repeats (one
1718
+ unlucky repeat cannot flip a verdict), and both spreads travel with the cell
1719
+ so a reader sees whether the medians are actually separable at this N
1720
+ (``separated``). A candidate ``passed`` iff nothing regressed beyond tolerance
1721
+ and it did not drop a topic the baseline delivered.
1722
+ """
1723
+ metric_list = list(metrics)
1724
+ tol_map = {metric: (tolerances or {}).get(metric) or default_ab_tolerance(metric) for metric in metric_list}
1725
+ ordered = sorted(runs, key=lambda run: (run.config, run.repeat))
1726
+ collected = _collect_ab_metrics(ordered, metrics=metric_list, topic_filter=topic)
1727
+
1728
+ if baseline not in collected:
1729
+ raise BandError(f"baseline config {baseline!r} produced no runs to compare against.")
1730
+
1731
+ configs_in_order: list[str] = []
1732
+ for run in ordered:
1733
+ if run.config not in configs_in_order:
1734
+ configs_in_order.append(run.config)
1735
+ repeats = max((run.repeat for run in ordered), default=0)
1736
+ baseline_topics = collected[baseline]
1737
+
1738
+ candidates: list[CandidateReport] = []
1739
+ for config in configs_in_order:
1740
+ if config == baseline:
1741
+ continue
1742
+ candidate_topics = collected.get(config, {})
1743
+ cells: list[AbCell] = []
1744
+ regressed: list[list[str]] = []
1745
+ improved: list[list[str]] = []
1746
+ dropped: list[str] = []
1747
+ for topic_label in sorted(baseline_topics):
1748
+ candidate_metrics = candidate_topics.get(topic_label)
1749
+ if candidate_metrics is None:
1750
+ dropped.append(topic_label)
1751
+ continue
1752
+ baseline_metrics = baseline_topics[topic_label]
1753
+ for metric in metric_list:
1754
+ better = ab_better(metric)
1755
+ baseline_spread = _metric_spread(baseline_metrics.get(metric, []))
1756
+ candidate_spread = _metric_spread(candidate_metrics.get(metric, []))
1757
+ verdict: Verdict | None = None
1758
+ delta: float | None = None
1759
+ separated: bool | None = None
1760
+ if baseline_spread is not None and candidate_spread is not None:
1761
+ verdict = classify_change(
1762
+ baseline_spread.median, candidate_spread.median, better=better, tol=tol_map[metric]
1763
+ )
1764
+ delta = round(candidate_spread.median - baseline_spread.median, 3)
1765
+ separated = candidate_spread.max < baseline_spread.min or baseline_spread.max < candidate_spread.min
1766
+ if verdict is Verdict.REGRESSED:
1767
+ regressed.append([topic_label, metric])
1768
+ elif verdict is Verdict.IMPROVED:
1769
+ improved.append([topic_label, metric])
1770
+ cells.append(
1771
+ AbCell(
1772
+ topic=topic_label,
1773
+ metric=metric,
1774
+ better=better.value,
1775
+ baseline=baseline_spread,
1776
+ candidate=candidate_spread,
1777
+ verdict=None if verdict is None else verdict.value,
1778
+ delta=delta,
1779
+ separated=separated,
1780
+ )
1781
+ )
1782
+ candidates.append(
1783
+ CandidateReport(
1784
+ config=config,
1785
+ cells=cells,
1786
+ regressed=regressed,
1787
+ improved=improved,
1788
+ dropped_topics=dropped,
1789
+ passed=not regressed and not dropped,
1790
+ )
1791
+ )
1792
+
1793
+ return AbReport(
1794
+ baseline=baseline,
1795
+ metrics=metric_list,
1796
+ repeats=repeats,
1797
+ candidates=candidates,
1798
+ passed=all(candidate.passed for candidate in candidates),
1799
+ )
1800
+
1801
+
1802
+ def _fmt_spread(spread: MetricSpread | None) -> str:
1803
+ if spread is None:
1804
+ return "n/a"
1805
+ if spread.min == spread.max:
1806
+ return f"{spread.median:g}"
1807
+ return f"{spread.median:g} ({spread.min:g}–{spread.max:g})"
1808
+
1809
+
1810
+ def render_ab_markdown(report: AbReport) -> str:
1811
+ """A self-describing markdown table per candidate: the paper-grade byproduct."""
1812
+ lines: list[str] = []
1813
+ verdict_word = "PASS" if report.passed else "FAIL"
1814
+ lines.append(f"# A/B verdict: **{verdict_word}** (baseline `{report.baseline}`, {report.repeats} repeats)")
1815
+ lines.append("")
1816
+ lines.append(
1817
+ "Cells compare the candidate median against the baseline median; "
1818
+ "`baseline`/`candidate` show `median (min–max)` across repeats. "
1819
+ "`sep?` is yes when the two spreads do not overlap (the effect is "
1820
+ "separable at this repeat count)."
1821
+ )
1822
+ for candidate in report.candidates:
1823
+ lines.append("")
1824
+ status = "PASS" if candidate.passed else "FAIL"
1825
+ lines.append(f"## `{candidate.config}` vs `{report.baseline}` — **{status}**")
1826
+ if candidate.dropped_topics:
1827
+ lines.append("")
1828
+ lines.append(
1829
+ f"Dropped topics (baseline delivered, candidate did not): {', '.join(candidate.dropped_topics)}"
1830
+ )
1831
+ lines.append("")
1832
+ lines.append("| topic | metric | better | baseline | candidate | Δ | sep? | verdict |")
1833
+ lines.append("|---|---|---|---|---|---|---|---|")
1834
+ for cell in candidate.cells:
1835
+ delta = "" if cell.delta is None else f"{cell.delta:+g}"
1836
+ sep = "" if cell.separated is None else ("yes" if cell.separated else "no")
1837
+ verdict = cell.verdict or "n/a"
1838
+ lines.append(
1839
+ f"| {cell.topic} | {cell.metric} | {cell.better} | "
1840
+ f"{_fmt_spread(cell.baseline)} | {_fmt_spread(cell.candidate)} | {delta} | {sep} | {verdict} |"
1841
+ )
1842
+ return "\n".join(lines)