rosotacom-dev 2.5.dev33__py3-none-any.whl → 2.5.dev36__py3-none-any.whl

rosotacom/cli.py

@@ -1083,10 +1083,30 @@ def _mark_interactive_smoke_stopped(instance_dir: Path, target_type: str, target
 
 
 def _write_docker_log(container_name: str, instance: SessionInstance, identity: str) -> None:
+    """Snapshot the peer container's own stdout into ``logs/<peer>/docker.log``.
+
+    In `attach` mode the container is `--rm` and already gone by the time this
+    runs, so `docker logs` answers "No such container". That answer used to be
+    written verbatim, which produced a 75-byte file that reads like the record
+    of the run and says nothing -- it was the whole of `logs/<peer>/` besides
+    `launcher.log` when a field session had to be diagnosed after the fact
+    (#266). The catmux per-pane logs under `catmux/` are the actual record, so
+    say that instead of pretending this file is one.
+    """
     try:
         result = subprocess.run(["docker", "logs", container_name], text=True, capture_output=True, check=False)
     except Exception:
         return
+    if result.returncode != 0:
+        stderr = (result.stderr or "").strip()
+        _append_log(
+            _peer_docker_log(instance, identity),
+            f"[rosotacom] docker logs {container_name}: {stderr or 'unavailable'}\n"
+            "[rosotacom] The container is not (or no longer) present -- in attach mode it is\n"
+            "[rosotacom] `--rm` and exits with the foreground command. What each pane inside\n"
+            f"[rosotacom] it printed is under {_peer_docker_log(instance, identity).parent / 'catmux'}.",
+        )
+        return
     logs = (result.stdout or "") + (result.stderr or "")
     if logs:
         _append_log(_peer_docker_log(instance, identity), logs)

rosotacom/resources/examples/ros2docker.json

@@ -11,7 +11,6 @@
   "run_args": [
     "-e", "ROS_DOMAIN_ID=48",
     "-e", "BUILD_ROS2WS=1",
-    "-e", "CHECK_ROS2WS_DEPENDENCIES=1",
     "--network", "host"
   ],
 

rosotacom/resources/ros2docker.json.example

@@ -11,7 +11,6 @@
   "run_args": [
     "-e", "ROS_DOMAIN_ID=47",
     "-e", "BUILD_ROS2WS=1",
-    "-e", "CHECK_ROS2WS_DEPENDENCIES=1",
     "--network", "host"
   ],
 

rosotacom/resources/ws/ota_configs/cyclonedds_local_participants.xml.template

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Local-domain CycloneDDS configuration: default discovery, more participants.
+
+Set it as `shared.rmw.local.cyclone.config` (or a peer's `local_config_template`)
+on a machine where many processes share the communication peer's local domain.
+
+Why: Cyclone gives each process one participant index per domain, and its default
+MaxAutoParticipantIndex is 32 - 33 slots per domain per host. A control centre
+spends them faster than that sounds. The processes that start last then lose the
+race, and the error names the domain rather than the cause:
+
+  [46] heartbeat_: Failed to find a free participant index for domain 46
+  [ERROR] [rmw_cyclonedds_cpp]: rmw_create_node: failed to create domain, error Error
+
+On 2026-08-13 that killed the centre's `heartbeat_echo` and one `topic_monitor`
+two seconds after start. catmux left both panes at a shell prompt, so the session
+ran for two hours with no heartbeat publisher: no RTT, no clock-offset estimate,
+and every EchoHeartbeat the far side received carried `echo_valid=false`.
+
+119, not higher: Cyclone's unicast discovery ports are
+7400 + 250*domain + 10 + 2*index, so 119 is the last index whose ports stay
+inside this domain's 250-port block. Index 120 would answer on the next
+domain's ports.
+
+Nothing else is set. This is the *local* domain -- intra-host discovery must keep
+working, so unlike the OTA templates it pins no interface, disables no multicast,
+and sets no peer list.
+-->
+<CycloneDDS xmlns="https://cdds.io/config">
+  <Domain Id="any">
+    <Discovery>
+      <ParticipantIndex>auto</ParticipantIndex>
+      <MaxAutoParticipantIndex>119</MaxAutoParticipantIndex>
+    </Discovery>
+  </Domain>
+</CycloneDDS>

rosotacom/resources/ws/ros2src/com_py/com_py/link_bytes.py

@@ -78,6 +78,19 @@ def parse_ip_addr_for_ip(text: str, ip: str) -> Optional[str]:
     return None
 
 
+#: Interface names that carry intra-host traffic only. Sampling one of these and
+#: calling the result "link bandwidth" is the failure of issue #267: the number
+#: is real, it is just not the link. On a machine that also runs a `-a` recorder
+#: it reads ~28 Gbit/s next to a 6 Mbit/s OTA link.
+LOOPBACK_INTERFACES = frozenset({"lo", "lo0"})
+
+
+def is_loopback_address(ip: str) -> bool:
+    """Is ``ip`` a loopback address (so a loopback interface would be correct)?"""
+    ip = (ip or "").strip()
+    return ip.startswith("127.") or ip in {"::1", "0:0:0:0:0:0:0:1"}
+
+
 def find_interface_for_ip(ip: str) -> Optional[str]:
     """The interface owning ``ip``, via `ip -o addr show` (None if not found)."""
     if not ip:
@@ -91,6 +104,40 @@ def find_interface_for_ip(ip: str) -> Optional[str]:
     return parse_ip_addr_for_ip(out, ip)
 
 
+def resolve_link_interface(
+    explicit: str,
+    host_ip: str,
+    finder=find_interface_for_ip,
+) -> Tuple[str, str]:
+    """The OTA link interface and where it came from, or raise.
+
+    ``explicit`` wins when given; otherwise the interface is the one owning
+    ``host_ip`` (the peer's own OTA address). A loopback interface for a
+    non-loopback address is refused rather than reported: loopback carries this
+    host's own ROS traffic, which on a machine running an ``-a`` recorder is
+    three orders of magnitude above the link and looks like a working
+    measurement. That mistake is issue #267; it survived a field session and an
+    offline analysis before anyone doubted the number.
+    """
+    interface = (explicit or "").strip()
+    provenance = "given explicitly"
+    if not interface:
+        interface = (finder(host_ip) or "").strip()
+        provenance = f"resolved from {host_ip}"
+    if not interface:
+        raise ValueError(
+            f"no interface owns the OTA address '{host_ip}'; "
+            "pass ip_local (preferred) or interface"
+        )
+    if interface in LOOPBACK_INTERFACES and not is_loopback_address(host_ip):
+        raise ValueError(
+            f"refusing to report loopback interface '{interface}' as the link for OTA "
+            f"address '{host_ip}' ({provenance}). Loopback carries this host's own ROS "
+            "traffic, not the link."
+        )
+    return interface, provenance
+
+
 def read_iface_counters(iface: str, proc_path: str = PROC_NET_DEV) -> Optional[Tuple[int, int]]:
     """Return (rx_bytes, tx_bytes) for ``iface`` now, or None if unavailable."""
     row = read_iface_counter_row(iface, proc_path)

rosotacom/resources/ws/ros2src/com_py/com_py/status_overview.py

@@ -67,9 +67,10 @@ from rclpy.qos import DurabilityPolicy, HistoryPolicy, QoSProfile, ReliabilityPo
 from rclpy.serialization import serialize_message
 from rosidl_runtime_py.utilities import get_message
 
-from com_py.link_bytes import LinkByteSampler, find_interface_for_ip
+from com_py.link_bytes import LinkByteSampler, resolve_link_interface
 from com_py.link_trace import LinkTraceRecorder
 from com_py.status_overview_core import (
+    STARTUP_GRACE_S,
     ClockOffsetEstimator,
     StageObservation,
     StatusAggregator,
@@ -334,6 +335,8 @@ class StatusOverview(Node):
         self.declare_parameter("link_trace_interval_s", 1.0)
         self.declare_parameter("link_trace_modem_command", "")
         self.declare_parameter("link_trace_modem_timeout_s", 2.0)
+        # Grace before the one-shot startup verdict (#266). 0 disables it.
+        self.declare_parameter("startup_grace_s", STARTUP_GRACE_S)
 
         spec_file = str(self.get_parameter("status_spec_file").value or "").strip()
         output_dir = str(self.get_parameter("output_dir").value or "").strip()
@@ -410,20 +413,23 @@ class StatusOverview(Node):
 
         ota_interface = str(self.get_parameter("ota_interface").value or "").strip()
         ota_local_ip = str(self.get_parameter("ota_local_ip").value or "").strip()
-        if not ota_interface and ota_local_ip:
-            ota_interface = find_interface_for_ip(ota_local_ip) or ""
-            if not ota_interface:
+        link_sampler = None
+        if ota_interface or ota_local_ip:
+            # Same resolution and the same loopback refusal as topic_monitor: a
+            # link number nobody can tell apart from loopback traffic is worse
+            # than no link number, so this reports nothing rather than that.
+            try:
+                ota_interface, provenance = resolve_link_interface(ota_interface, ota_local_ip)
+            except ValueError as exc:
+                self.get_logger().warning(
+                    f"status_overview: {exc}; link-overhead measurement disabled"
+                )
+            else:
+                link_sampler = LinkByteSampler(ota_interface)
                 self.get_logger().info(
-                    f"status_overview: no interface found for OTA address '{ota_local_ip}'; "
-                    "link-overhead measurement disabled"
+                    f"status_overview: link-overhead sampling on interface "
+                    f"'{ota_interface}' ({provenance})"
                 )
-        link_sampler = None
-        if ota_interface:
-            link_sampler = LinkByteSampler(ota_interface)
-            self.get_logger().info(
-                f"status_overview: link-overhead sampling on interface '{ota_interface}'"
-                + (f" (resolved from {ota_local_ip})" if ota_local_ip else "")
-            )
 
         link_trace_recorder = None
         if bool(self.get_parameter("link_trace").value):
@@ -450,6 +456,7 @@ class StatusOverview(Node):
             link_sampler=link_sampler,
             clock_estimator=clock_estimator,
             link_trace_recorder=link_trace_recorder,
+            startup_grace_s=float(self.get_parameter("startup_grace_s").value),
         )
 
         self.create_timer(self.write_interval_s, self._on_write)

rosotacom/resources/ws/ros2src/com_py/com_py/status_overview_core.py

@@ -175,6 +175,128 @@ def stamp_delay(raw_delay_s: float, clock_offset_s: Optional[float] = None) -> O
     return None
 
 
+#: How long a peer is given to bring its own publishers up before their absence
+#: is called a defect. Node start, DDS discovery and the first `refresh_interval`
+#: of the observers all happen inside it; 20 s is roughly four times what a
+#: healthy bring-up needs.
+STARTUP_GRACE_S = 20.0
+
+
+#: Stage producers rosotacom starts itself, and can therefore promise. The
+#: application that feeds a link is the user's, and a session that declares a
+#: topic says nothing about whether its publisher is running right now -- a
+#: smoke run legitimately exercises the peers without the data source.
+ROSOTACOM_STAGE_PRODUCERS = frozenset({"heartbeat_echo", "preprocessing", "relay_out", "bridge_out"})
+
+
+def outbound_startup_gaps(snapshot: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Where each outbound topic first has no publisher at all, and whose fault it is.
+
+    Only outbound topics: their stages are produced on this machine. Inbound
+    stages depend on the peer and the link, which the per-topic rollup already
+    diagnoses.
+
+    Only the *first* absent stage per topic, because the ones behind it are
+    absent for the trivial reason that nothing reached them, and reporting a
+    cascade as four findings buries the one that matters.
+
+    `owner` is what makes the finding actionable, and it takes three values that
+    look identical in the graph:
+
+    * ``rosotacom`` -- a stage this peer starts itself, whose input is arriving.
+      It had something to forward and forwarded nothing. This is the defect.
+    * ``application`` -- the native stage of a topic the user's application (or a
+      replay) publishes. Not this peer's promise.
+    * ``upstream`` -- a stage this peer starts whose input has not delivered a
+      message yet. Its publisher appears on discovery of the input, so there is
+      nothing wrong: `universal_ota_wrapper` has no `…/ota_stamped` publisher
+      until something publishes the topic it wraps.
+
+    This exists because of a failure that produced no error anywhere. On
+    2026-08-13 the centre's `heartbeat_echo` and one `topic_monitor` died two
+    seconds after start -- Cyclone had no free participant index left on the
+    local domain -- and catmux left both panes sitting at a shell prompt. The
+    session ran for two hours with no heartbeat publisher, so the run has no RTT
+    and no clock-offset measurement, and the far side reported `status=LOST
+    reason=age hz=0.00` about a peer that was otherwise healthy. The heartbeat is
+    the one outbound topic rosotacom publishes itself, which is why the
+    generator marks its native stage `heartbeat_echo` rather than `application`.
+    """
+    gaps: List[Dict[str, Any]] = []
+    for topic in snapshot.get("topics", []):
+        if topic.get("direction") != "outbound":
+            continue
+        upstream: Optional[Dict[str, Any]] = None
+        for stage in topic.get("stages", []):
+            if stage.get("state") != ABSENT:
+                upstream = stage
+                continue
+            producer = stage.get("produced_by")
+            if producer not in ROSOTACOM_STAGE_PRODUCERS:
+                owner = "application"
+                reason = "the application that publishes this topic is not running"
+            elif upstream is not None and upstream.get("state") not in (FLOWING, STALE):
+                owner = "upstream"
+                reason = (
+                    f"nothing has arrived on {upstream.get('topic')} "
+                    f"(stage {upstream.get('stage')}), so there is nothing to publish yet"
+                )
+            else:
+                owner = "rosotacom"
+                reason = f"{producer} did not come up"
+            gaps.append(
+                {
+                    "base": topic.get("base"),
+                    "stage": stage.get("stage"),
+                    "topic": stage.get("topic"),
+                    "domain": stage.get("domain"),
+                    "produced_by": producer,
+                    "owner": owner,
+                    "reason": reason,
+                }
+            )
+            break
+    return gaps
+
+
+#: How far back a sequence number may jump and still be called reordering.
+#: `universal_ota_wrapper` counts per topic from zero and increments by one, so
+#: a receiver sees a backward jump only when the wire reordered a message or
+#: when the publishing peer restarted. Reordering is bounded by the transport's
+#: history depth (single digits for the QoS this stack uses); a restart lands
+#: thousands of sequence numbers back. 64 sits between the two by two orders of
+#: magnitude on both sides.
+SEQUENCE_REORDER_TOLERANCE = 64
+
+
+def is_sequence_epoch_reset(
+    seq: int,
+    expected_next_seq: Optional[int],
+    tolerance: int = SEQUENCE_REORDER_TOLERANCE,
+) -> bool:
+    """Does ``seq`` begin a new sequence epoch (the peer's wrapper restarted)?
+
+    Zero is unambiguous and was the only case handled before. It is also the
+    case that almost never arrives: DDS discovery takes long enough that the
+    first tens of messages of a fresh publisher are written before the reader
+    has matched. In the 2026-08-13 field instance the peer restarted four times
+    and the first arrival after each restart carried seq 37, 39, 39 and 41 --
+    never 0. Every one of them was therefore counted as `reordered`, and
+    because the baseline was never reset, so was every message after it: 26,453
+    of 55,437 transits on one 10 Hz topic, for the rest of the instance.
+
+    So a large backward jump counts as a restart too. It is corroborated by an
+    arrival gap (105 s, 32 s, 44 s and 195 s in that instance), but the gap is
+    not required: a fast restart is still a restart, and the jump alone already
+    separates the two causes by orders of magnitude.
+    """
+    if expected_next_seq is None or seq >= expected_next_seq:
+        return False
+    if seq == 0:
+        return True
+    return (expected_next_seq - seq) > tolerance
+
+
 # --- Link overhead (session-level) -----------------------------------------
 
 def _payload_kbps(stage: Optional[Dict[str, Any]]) -> float:
@@ -258,6 +380,8 @@ class StageObservation:
     expected_next_seq: Optional[int] = None
     last_seq: Optional[int] = None
     tracks_sequence: bool = False
+    epoch: int = 0
+    epoch_resets: int = 0
     total_expected: int = 0
     total_missing: int = 0
     total_reordered: int = 0
@@ -302,11 +426,15 @@ class StageObservation:
                 if self.expected_next_seq is None:
                     expected_delta = 1
                     self.expected_next_seq = seq + 1
-                elif seq == 0 and self.expected_next_seq > 1:
-                    # A publisher restart begins a new sequence epoch. Without
-                    # an explicit boot id, zero is the only unambiguous reset.
+                elif is_sequence_epoch_reset(seq, self.expected_next_seq):
+                    # A publisher restart begins a new sequence epoch: the
+                    # wrapper counts from zero again. See
+                    # `is_sequence_epoch_reset` for why the first arrival after
+                    # a restart is almost never seq 0, and what that cost.
+                    self.epoch += 1
+                    self.epoch_resets += 1
                     expected_delta = 1
-                    self.expected_next_seq = 1
+                    self.expected_next_seq = seq + 1
                 elif seq >= self.expected_next_seq:
                     missing = seq - self.expected_next_seq
                     missing_start = self.expected_next_seq if missing else None
@@ -331,6 +459,10 @@ class StageObservation:
                     "topic": transit.get("topic"),
                     "direction": transit.get("direction"),
                     "stage": transit.get("stage"),
+                    # Which run of the peer's wrapper produced this sequence
+                    # number. Without it the offline join collapses seq 0..N of
+                    # every epoch after the first onto the first epoch's rows.
+                    "epoch": self.epoch,
                 }
                 if missing_start is not None:
                     for missing_seq in range(missing_start, int(seq)):
@@ -421,6 +553,8 @@ class StageObservation:
             total_missing = self.total_missing
             total_reordered = self.total_reordered
             total_max_burst = self.max_burst_missing
+            epoch = self.epoch
+            epoch_resets = self.epoch_resets
         hz = count / window_s if window_s > 0 else 0.0
         mean_size = (total_bytes / count) if count > 0 else 0.0
         age = (now_mono - last_recv_mono) if last_recv_mono is not None else None
@@ -441,6 +575,8 @@ class StageObservation:
             "loss_total_pct": _pct(total_missing, total_expected) if tracks_sequence else None,
             "reordered_total": total_reordered if tracks_sequence else None,
             "max_burst_missing_total": total_max_burst if tracks_sequence else None,
+            "epoch": epoch if tracks_sequence else None,
+            "epoch_resets": epoch_resets if tracks_sequence else None,
         }
 
     def drain_transit_records(self) -> List[Dict[str, Any]]:
@@ -464,7 +600,9 @@ class StatusAggregator:
                  *, liveness_window_s: float = 3.0, stale_after_s: float = 3.0,
                  delay_good_ms: float = 100.0, delay_bad_ms: float = 200.0,
                  link_sampler: Any = None, clock_estimator: Any = None,
-                 link_trace_recorder: Any = None):
+                 link_trace_recorder: Any = None,
+                 startup_grace_s: float = STARTUP_GRACE_S,
+                 started_mono: Optional[float] = None):
         self._log = logger
         self._spec = spec
         self._output_dir = output_dir
@@ -484,6 +622,21 @@ class StatusAggregator:
         self._json_path = os.path.join(self._output_dir, "status.json")
         self._txt_path = os.path.join(self._output_dir, "status.txt")
         self._events_path = os.path.join(self._output_dir, "events.jsonl")
+        self._startup_path = os.path.join(self._output_dir, "startup_check.json")
+        # `pane_failures.log` is written by catmux_log_setup.sh one level up, in
+        # logs/<peer>/. The verdict quotes it because a pane that exited is the
+        # most common reason a promised publisher never appears, and the two
+        # facts are useless apart.
+        self._pane_failure_path = os.path.join(
+            os.path.dirname(os.path.abspath(self._output_dir)), "pane_failures.log"
+        )
+        self._startup_grace_s = startup_grace_s
+        self._started_mono = time.monotonic() if started_mono is None else started_mono
+        self._startup_check: Optional[Dict[str, Any]] = None
+        #: (when, topics) of the first evaluation that found a defect. A verdict
+        #: is only committed once the same defect survives a second look; see
+        #: `_maybe_run_startup_check`.
+        self._startup_pending: Optional[Tuple[float, set]] = None
 
     # -- observation lookup --
     def _obs_for(self, stage: Dict[str, Any]) -> Optional[StageObservation]:
@@ -516,6 +669,8 @@ class StatusAggregator:
             "reordered_total": None,
             "max_burst_missing": None,
             "max_burst_missing_total": None,
+            "epoch": None,
+            "epoch_resets": None,
             "age_s": None,
             "last_message_wall": None,
             "messages_total": 0,
@@ -551,6 +706,8 @@ class StatusAggregator:
             "reordered_total",
             "max_burst_missing",
             "max_burst_missing_total",
+            "epoch",
+            "epoch_resets",
         ):
             result[key] = m[key]
         if m["last_recv_wall"] is not None:
@@ -589,6 +746,8 @@ class StatusAggregator:
             "reordered_total",
             "max_burst_missing",
             "max_burst_missing_total",
+            "epoch",
+            "epoch_resets",
             "age_s",
             "last_message_wall",
             "messages_total",
@@ -838,6 +997,7 @@ class StatusAggregator:
             "summary": counts,
             "link": compute_link_overview(topics_out, link_sample),
             "clock_sync": self._clock_snapshot(now_mono),
+            "startup_check": self._startup_check,
             "topics": topics_out,
         }
 
@@ -915,6 +1075,20 @@ class StatusAggregator:
             f"summary: OK={s.get('OK', 0)} PARTIAL={s.get('PARTIAL', 0)} "
             f"STALLED={s.get('STALLED', 0)} ABSENT={s.get('ABSENT', 0)}   (Phase 1: local observation)"
         )
+        # The STAT watcher pane shows this file, so a failed startup check has to
+        # be readable here or it is not surfaced where anybody looks.
+        check = snapshot.get("startup_check")
+        if check and check.get("verdict") != "ok":
+            gaps = check.get("missing_outbound_stages") or []
+            panes = check.get("pane_failures") or []
+            lines.append(
+                f"STARTUP CHECK FAILED after {check.get('checked_after_s')}s: "
+                f"{len(gaps)} outbound stage(s) without a publisher, {len(panes)} pane(s) exited"
+            )
+            for gap in gaps:
+                lines.append(f"    no publisher: {gap.get('topic')} ({gap.get('produced_by')})")
+            for line in panes:
+                lines.append(f"    pane exited: {line}")
         lines.append("")
         for t in snapshot["topics"]:
             arrow = "->" if t["direction"] == "outbound" else "<-"
@@ -962,6 +1136,101 @@ class StatusAggregator:
             fp.write(text)
         os.replace(tmp, path)
 
+    def _read_pane_failures(self) -> List[str]:
+        try:
+            with open(self._pane_failure_path, "r", encoding="utf-8", errors="replace") as fp:
+                return [line.rstrip("\n") for line in fp if line.strip()]
+        except OSError:
+            return []
+
+    def _maybe_run_startup_check(
+        self, snapshot: Dict[str, Any], now_mono: Optional[float]
+    ) -> None:
+        """Once, after the grace period: did this peer bring up what it promised?
+
+        Written as a verdict at a defined moment rather than left implicit in a
+        table that is refreshed every two seconds. `status.json` already carried
+        the evidence on 2026-08-13 -- 13 stages STALLED, `clock_sync: null` --
+        and nobody read it, because nothing said "this is wrong now".
+        """
+        if self._startup_check is not None or self._startup_grace_s <= 0.0:
+            return
+        now = time.monotonic() if now_mono is None else now_mono
+        if now - self._started_mono < self._startup_grace_s:
+            return
+
+        gaps = outbound_startup_gaps(snapshot)
+        mine = [gap for gap in gaps if gap["owner"] == "rosotacom"]
+        theirs = [gap for gap in gaps if gap["owner"] != "rosotacom"]
+        pane_failures = self._read_pane_failures()
+
+        # A stage can be late rather than dead. `relay_out` and the wrapper
+        # create their publisher on *discovering* their input, on a refresh
+        # interval, so an input that started publishing shortly before this
+        # moment leaves a gap that is about to close on its own. Waiting a
+        # second grace period and reporting only what is still missing
+        # separates the two without weakening either: a node that exited never
+        # recovers. CI found this the way it should be found -- the smoke that
+        # starts its payload publisher last failed on a topic whose relay was
+        # working seconds later.
+        if mine and self._startup_pending is None:
+            self._startup_pending = (now, {gap["topic"] for gap in mine})
+            return
+        if mine and self._startup_pending is not None:
+            first_seen_at, first_seen = self._startup_pending
+            if now - first_seen_at < self._startup_grace_s:
+                return
+            mine = [gap for gap in mine if gap["topic"] in first_seen]
+        verdict = "ok" if not mine and not pane_failures else "incomplete"
+        self._startup_check = {
+            "verdict": verdict,
+            "checked_after_s": round(now - self._started_mono, 1),
+            "generated_at": snapshot.get("generated_at"),
+            "peer": snapshot.get("peer"),
+            "missing_outbound_stages": mine,
+            "waiting_for_input": theirs,
+            "pane_failures": pane_failures,
+        }
+        snapshot["startup_check"] = self._startup_check
+        try:
+            self._atomic_write(
+                self._startup_path, json.dumps(self._startup_check, indent=2) + "\n"
+            )
+        except Exception as exc:  # pragma: no cover - defensive
+            if self._log is not None:
+                self._log.warning(f"status_overview: failed to write startup check: {exc}")
+        if self._log is None:
+            return
+        if theirs:
+            # Not a defect and deliberately not an error: whether the data source
+            # is publishing is the application's business, not this peer's, and a
+            # stage with no input has nothing to publish.
+            self._log.info(
+                f"status_overview: startup check -- {len(theirs)} outbound topic(s) are "
+                "waiting for input: "
+                + ", ".join(gap["topic"] for gap in theirs[:8])
+                + (" ..." if len(theirs) > 8 else "")
+            )
+        if verdict == "ok":
+            self._log.info(
+                "status_overview: startup check OK -- every stage this peer produces "
+                f"for {snapshot.get('peer')} has a publisher"
+            )
+            return
+        self._log.error(
+            f"status_overview: STARTUP CHECK FAILED for peer {snapshot.get('peer')} "
+            f"after {self._startup_check['checked_after_s']}s -- "
+            f"{len(mine)} stage(s) this peer produces have no publisher, "
+            f"{len(pane_failures)} pane(s) exited. See {self._startup_path}"
+        )
+        for gap in mine:
+            self._log.error(
+                f"status_overview:   no publisher on {gap['topic']} "
+                f"(stage {gap['stage']}, produced_by {gap['produced_by']})"
+            )
+        for line in pane_failures:
+            self._log.error(f"status_overview:   pane exited: {line}")
+
     def write(self, now_mono: Optional[float] = None) -> int:
         """Build a snapshot, write artifacts, and return number of transition events."""
         link_sample = None
@@ -972,6 +1241,7 @@ class StatusAggregator:
                 if self._log is not None:
                     self._log.warning(f"status_overview: link sampling failed: {exc}")
         snapshot = self.build_snapshot(now_mono, link_sample=link_sample)
+        self._maybe_run_startup_check(snapshot, now_mono)
         if self._link_trace_recorder is not None:
             try:
                 self._link_trace_recorder.maybe_write(snapshot, link_sample)

rosotacom/resources/ws/ros2src/com_py/com_py/topic_monitor.py

@@ -47,7 +47,7 @@ from rosidl_runtime_py.utilities import get_message
 from rclpy.serialization import serialize_message
 from std_msgs.msg import Float64
 
-from com_py.link_bytes import LinkByteSampler
+from com_py.link_bytes import LinkByteSampler, resolve_link_interface
 
 
 def is_internal_transport_topic(topic_name: str) -> bool:
@@ -99,7 +99,9 @@ class TopicMonitor(Node):
         self.declare_parameter('to_adressant', '')  # optional, e.g., 'shuttle_ella' for /to_shuttle_ella
         self.declare_parameter("ip_local", "")
         self.declare_parameter("ip_remote", "")
-        self.declare_parameter("interface", "")      # OTA link interface (e.g. the VPN tunnel)
+        # Explicit override for the OTA link interface. Normally left empty:
+        # it is resolved from ip_local below.
+        self.declare_parameter("interface", "")
         # Accepted for launch compatibility; the /proc/net/dev sampler needs no
         # capture window, so this tshark-era buffer is no longer used.
         self.declare_parameter("link_duration_buffer_s", 1.0)
@@ -109,9 +111,9 @@ class TopicMonitor(Node):
         self.sourcename = self.get_parameter('sourcename').value
         self.direction = self.get_parameter('direction').value
         self.to_adressant = self.get_parameter('to_adressant').value
-        self.host_ip = self.get_parameter("ip_local").value
-        self.peer_ip = self.get_parameter("ip_remote").value
-        self.interface = self.get_parameter("interface").value
+        self.host_ip = str(self.get_parameter("ip_local").value or "").strip()
+        self.peer_ip = str(self.get_parameter("ip_remote").value or "").strip()
+        self.interface = str(self.get_parameter("interface").value or "")
 
         # Track actual elapsed time between prints (avoid timer drift issues)
         self._last_print_t = time.monotonic()
@@ -131,8 +133,22 @@ class TopicMonitor(Node):
         else:
             self.topic_prefix = f"/com/{self.direction}/{self.sourcename}/"
 
-        if not self.interface:
-            raise ValueError("Parameter 'interface' is required for link bandwidth measurement")
+        # The OTA link interface. `ip_local` is the peer's own OTA address, so
+        # the interface follows from it -- the same resolution status_overview
+        # does, in the same place, from the same helper. `interface` stays an
+        # explicit override for a setup the address cannot describe.
+        #
+        # It used to be resolved in the session template instead, by a shell
+        # line that expanded a catmux parameter with bash's substring-replace
+        # syntax. catmux substitutes `${name}` and nothing else, so the pattern
+        # stayed literal, bash expanded an unset variable to nothing, and the
+        # awk program collapsed to "first address line with a prefix length" --
+        # `lo`, on every machine. See docs/link-bandwidth.md.
+        try:
+            self.interface, provenance = resolve_link_interface(self.interface, self.host_ip)
+        except ValueError as exc:
+            raise ValueError(f"topic_monitor: {exc}") from exc
+        self.get_logger().info(f"OTA link interface '{self.interface}' ({provenance})")
 
         # Link bandwidth is measured from the kernel's own interface byte counters
         # (/proc/net/dev) via the shared link_bytes sampler -- no tshark, no sudo,
@@ -163,8 +179,14 @@ class TopicMonitor(Node):
         )
 
         self.get_logger().info(f"topic_monitor node started. Searching for topics with prefix: {self.topic_prefix}")
-        self.get_logger().info(f"Publishing ROS topic bandwidth to: {self.ros_topic_bandwidth_topic}")
-        self.get_logger().info(f"Publishing link bandwidth to: {self.link_bandwidth_topic}")
+        self.get_logger().info(
+            f"Publishing ROS topic bandwidth to: {self.ros_topic_bandwidth_topic} "
+            f"(serialized payload of {self.topic_prefix}*, Kibit/s)"
+        )
+        self.get_logger().info(
+            f"Publishing link bandwidth to: {self.link_bandwidth_topic} "
+            f"(wire bytes on '{self.interface}', {'tx' if self.direction == 'out' else 'rx'}, Kibit/s)"
+        )
 
         # Prime the link sampler so the first print reports a real delta.
         self._link_sampler.sample()

rosotacom/resources/ws/session/content/base/session_plugin_base.yaml

@@ -62,6 +62,9 @@ parameters:
   status_write_interval_s: 2.0
   status_liveness_window_s: 3.0
   status_stale_after_s: 3.0
+  # One-shot verdict N seconds after start: does every outbound stage this peer
+  # promises actually have a publisher, and did any pane exit? 0 disables it.
+  status_startup_grace_s: 20.0
   link_trace: false
   link_trace_interval_s: 1.0
   link_trace_modem_command: ""
@@ -330,12 +333,16 @@ windows:
   - name: TM
     if: topic_monitor
     splits:
+      # No `interface=` line: the node resolves the OTA interface from ip_local
+      # itself (com_py.link_bytes.find_interface_for_ip), exactly as the STAT
+      # window's status_overview does. The line that used to stand here expanded
+      # a catmux parameter with bash substring-replace syntax, which catmux does
+      # not substitute -- so it resolved `lo` on every machine and reported this
+      # host's own ROS traffic as link bandwidth (#267).
       - commands:
-        - interface=$(ip -o addr | awk '/'${ip_local//./\\.}'\/[0-9]+/ {print $2; exit}')
-        - ros2 run com_py topic_monitor --ros-args -p sourcename:=${remote_name} -p direction:=in -p to_adressant:='${tm_in_to_adressant}' -p ip_local:=${ip_local} -p ip_remote:=${ip_remote} -p interface:=${interface} -p link_duration_buffer_s:=${tm_link_duration_buffer_s}
+        - ros2 run com_py topic_monitor --ros-args -p sourcename:=${remote_name} -p direction:=in -p to_adressant:='${tm_in_to_adressant}' -p ip_local:=${ip_local} -p ip_remote:=${ip_remote} -p link_duration_buffer_s:=${tm_link_duration_buffer_s}
       - commands:
-        - interface=$(ip -o addr | awk '/'${ip_local//./\\.}'\/[0-9]+/ {print $2; exit}')
-        - ros2 run com_py topic_monitor --ros-args -p sourcename:=${local_name} -p direction:=out -p to_adressant:='${tm_out_to_adressant}' -p ip_local:=${ip_local} -p ip_remote:=${ip_remote} -p interface:=${interface} -p link_duration_buffer_s:=${tm_link_duration_buffer_s}
+        - ros2 run com_py topic_monitor --ros-args -p sourcename:=${local_name} -p direction:=out -p to_adressant:='${tm_out_to_adressant}' -p ip_local:=${ip_local} -p ip_remote:=${ip_remote} -p link_duration_buffer_s:=${tm_link_duration_buffer_s}
   - name: STAT
     if: status_overview
     layout: even-vertical
@@ -352,6 +359,7 @@ windows:
             -p write_interval_s:=${status_write_interval_s}
             -p liveness_window_s:=${status_liveness_window_s}
             -p stale_after_s:=${status_stale_after_s}
+            -p startup_grace_s:=${status_startup_grace_s}
             -p ota_local_ip:=${ip_local}
             -p link_trace:=${link_trace}
             -p link_trace_interval_s:=${link_trace_interval_s}
@@ -360,12 +368,15 @@ windows:
       - commands:
         - status_out_dir="$(dirname "${ROSOTACOM_CATMUX_LOG_DIR:-/tmp/rosotacom_logs/catmux}")/status"
         - status_txt="$status_out_dir/status.txt"
-        - 'while true; do clear; echo "[INFO] watching status overview file: $status_txt"; if [ -f "$status_txt" ]; then cat "$status_txt"; else echo "[INFO] waiting for status_overview to write status.txt"; fi; sleep "${status_write_interval_s:-2.0}"; done'
+        - 'while true; do clear; echo "[INFO] watching status overview file: $status_txt"; if [ -f "$status_txt" ]; then cat "$status_txt"; else echo "[INFO] waiting for status_overview to write status.txt"; fi; sleep "${status_write_interval_s}"; done'
   - name: NET
     if: network_monitor
     splits:
       - commands:
-        - interface=$(ip -o addr | awk '/'${ip_local//./\\.}'\/[0-9]+/ {print $2; exit}')
+        # Exact match on the address column, with the address passed in through
+        # awk -v: no shell pattern expansion, so nothing here depends on catmux
+        # substituting a form it does not know (#267).
+        - interface=$(ip -o -4 addr show | awk -v ip='${ip_local}' '{split($4, a, "/"); if (a[1] == ip) {print $2; exit}}')
         - sudo iftop -i "$interface" -f "host ${ip_local} and host ${ip_remote}"
   - name: METRIC_BAG
     if: metric_stage_bag

rosotacom/resources/ws/session/creation/catmux_log_setup.sh

@@ -52,6 +52,50 @@ fi
 tmux pipe-pane -o -t "${TMUX_PANE}" \
   "${__rosotacom_pipe_filter} >> '${__rosotacom_pane_log_file//\'/\'\\\'\'}'"
 
+# --- a pane whose command dies must say so -----------------------------------
+#
+# A pane log is only read by somebody who already suspects the pane. On
+# 2026-08-13 the centre's `heartbeat_echo` and one `topic_monitor` exited two
+# seconds after start (Cyclone had no free participant index left on domain 46),
+# catmux left both panes at a shell prompt, and the session ran for two hours
+# with no heartbeat publisher. Nothing anywhere said so: `docker ps` showed a
+# healthy container, the other panes worked, and the far side blamed the link.
+#
+# So every non-zero exit of a pane command is appended to one file per peer,
+# which the status overview's startup check reads and quotes. 130 (Ctrl-C) is
+# excluded: that is an operator stopping a pane on purpose.
+ROSOTACOM_PANE_FAILURE_LOG="$(dirname "${ROSOTACOM_CATMUX_LOG_DIR}")/pane_failures.log"
+ROSOTACOM_PANE_LABEL="${__rosotacom_window_prefix}-${__rosotacom_window_safe}/${__rosotacom_pane}"
+export ROSOTACOM_PANE_FAILURE_LOG ROSOTACOM_PANE_LABEL
+
+__rosotacom_note_exit() {
+  local __status=$?
+  if [ "${__status}" -ne 0 ] && [ "${__status}" -ne 130 ]; then
+    local __cmd
+    __cmd="$(HISTTIMEFORMAT='' builtin history 1 2>/dev/null | sed 's/^[[:space:]]*[0-9]*[[:space:]]*//')"
+    printf '%s\tpane=%s\texit=%s\tcommand=%s\n' \
+      "$(date -Iseconds)" "${ROSOTACOM_PANE_LABEL}" "${__status}" "${__cmd}" \
+      >> "${ROSOTACOM_PANE_FAILURE_LOG}" 2>/dev/null
+    printf '\n[rosotacom] pane %s: command exited %s -- this pane is no longer doing its job.\n' \
+      "${ROSOTACOM_PANE_LABEL}" "${__status}" >&2
+    printf '[rosotacom] recorded in %s\n\n' "${ROSOTACOM_PANE_FAILURE_LOG}" >&2
+  fi
+  return "${__status}"
+}
+
+case "$-" in
+  *i*)
+    # Deliberately not exported: a child shell would inherit the hook without
+    # the function and complain at every prompt.
+    if [[ ";${PROMPT_COMMAND:-};" != *";__rosotacom_note_exit;"* ]]; then
+      PROMPT_COMMAND="__rosotacom_note_exit;${PROMPT_COMMAND:+${PROMPT_COMMAND}}"
+    fi
+    ;;
+  *)
+    echo "[rosotacom_catmux_log] non-interactive shell; pane-exit reporting disabled" >&2
+    ;;
+esac
+
 {
   printf '\n--- rosotacom catmux pipe-pane started %s ---\n' "$(date -Iseconds)"
   printf '    window=%s (#%s) pane=%s\n' \

rosotacom/resources/ws/session/creation/generate_session_files.py

@@ -1413,6 +1413,12 @@ def _build_status_pipeline_spec(
             outbound_items.insert(0, (hb_entry, hb_pipe))
 
         for e, p in outbound_items:
+            # Who publishes the native stage decides who is answerable when it is
+            # missing. The heartbeat is the one outbound topic rosotacom publishes
+            # itself (`heartbeat_echo`), so its absence is a defect here rather
+            # than an application that has not started -- which is exactly the
+            # 2026-08-13 failure the startup check exists for.
+            native_produced_by = "heartbeat_echo" if e.index == -1 else "application"
             base = e.base
             final = p.get("final", base)
             forward_final = f"/to_{remote_name}{final}" if use_target_prefix else final
@@ -1441,7 +1447,7 @@ def _build_status_pipeline_spec(
                     "topic": native_topic,
                     "type": native_type,
                     "domain": "local",
-                    "produced_by": "application",
+                    "produced_by": native_produced_by,
                 },
             ]
             if final != base and not is_g2l:

rosotacom/transit.py

@@ -32,15 +32,34 @@ def load_transit_records(paths: Iterable[Path]) -> list[dict[str, Any]]:
     return records
 
 
+def record_epoch(record: dict[str, Any]) -> int:
+    """Which run of the publishing peer's wrapper produced this record.
+
+    A peer restart resets ``seq`` to zero mid-instance, so ``(source, topic,
+    seq)`` is not unique over an instance that outlived one. Records written
+    before the field learned this carry no ``epoch``; they are all epoch 0,
+    which is what a single-epoch instance produces anyway.
+    """
+    value = record.get("epoch")
+    return 0 if value is None else int(value)
+
+
 def join_transit_records(records: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
-    """Join duplicate observations by ``(topic, seq)`` and keep the richest row."""
-    joined: dict[tuple[str, str, int], dict[str, Any]] = {}
+    """Join duplicate observations by ``(source, topic, epoch, seq)``.
+
+    Keeps the richest row per key. ``epoch`` is part of the key because it is
+    what makes the key unique: on 2026-08-13 the vehicle restarted four times
+    inside one centre instance, and joining on ``(source, topic, seq)`` alone
+    silently dropped 123,253 of 304,876 raw transit lines -- 40% of the run,
+    concentrated in exactly the mission window the analysis was about.
+    """
+    joined: dict[tuple[str, str, int, int], dict[str, Any]] = {}
     status_rank = {"lost": 0, "reordered": 1, "delivered": 2}
     for record in records:
         topic = str(record.get("topic") or "")
         source = str(record.get("source") or "")
         seq = int(record["seq"])
-        key = (source, topic, seq)
+        key = (source, topic, record_epoch(record), seq)
         current = joined.get(key)
         if current is None:
             joined[key] = dict(record)
@@ -67,19 +86,26 @@ def filter_transit_records_by_publish_window(
 ) -> list[dict[str, Any]]:
     """Keep records whose publish time falls inside a measurement window.
 
-    Lost records do not carry timestamps. For each source/topic/target stream,
-    keep lost sequence records only when they are bounded by delivered/reordered
-    records inside the window. This excludes startup and teardown sequence gaps
-    without hiding losses that happened during the measured interval.
+    Lost records do not carry timestamps. For each source/target/topic/epoch
+    stream, keep lost sequence records only when they are bounded by
+    delivered/reordered records inside the window. This excludes startup and
+    teardown sequence gaps without hiding losses that happened during the
+    measured interval.
+
+    The epoch belongs in the grouping key for the same reason it belongs in the
+    join key: sequence numbers only order messages within one run of the
+    publishing wrapper, so a post-restart seq 40 must not bound a pre-restart
+    seq 40's loss.
     """
     if end_s < start_s:
         raise ValueError("end_s must be >= start_s")
-    grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = {}
+    grouped: dict[tuple[str, str, str, int], list[dict[str, Any]]] = {}
     for record in join_transit_records(records):
         key = (
             str(record.get("source") or ""),
             str(record.get("target") or ""),
             str(record.get("topic") or ""),
+            record_epoch(record),
         )
         grouped.setdefault(key, []).append(record)
 
@@ -111,6 +137,7 @@ def filter_transit_records_by_publish_window(
             str(record.get("source") or ""),
             str(record.get("target") or ""),
             str(record.get("topic") or ""),
+            record_epoch(record),
             int(record["seq"]),
         ),
     )
@@ -173,15 +200,24 @@ def _latency_trend_ms(
 
 
 def _nominal_send_period_ms(topic_records: list[dict[str, Any]], *, time_field: str = "t_wrap") -> float | None:
-    """Median publish spacing per sequence step — the send cadence of the stream."""
-    stamped = sorted(
-        (int(record["seq"]), float(record[time_field]))
-        for record in topic_records
-        if record.get(time_field) is not None
-    )
-    periods = [
-        (t1 - t0) / (s1 - s0) for (s0, t0), (s1, t1) in zip(stamped, stamped[1:], strict=False) if s1 > s0 and t1 >= t0
-    ]
+    """Median publish spacing per sequence step — the send cadence of the stream.
+
+    Per epoch: a sequence step only means "one message" inside one run of the
+    publishing wrapper. Across a restart the step is a reset, and dividing the
+    wall-clock gap by it produces a period that describes nothing.
+    """
+    periods: list[float] = []
+    for epoch in {record_epoch(record) for record in topic_records}:
+        stamped = sorted(
+            (int(record["seq"]), float(record[time_field]))
+            for record in topic_records
+            if record.get(time_field) is not None and record_epoch(record) == epoch
+        )
+        periods.extend(
+            (t1 - t0) / (s1 - s0)
+            for (s0, t0), (s1, t1) in zip(stamped, stamped[1:], strict=False)
+            if s1 > s0 and t1 >= t0
+        )
     if not periods:
         return None
     ordered = sorted(periods)
@@ -239,6 +275,10 @@ def summarize_transit_records(records: Iterable[dict[str, Any]]) -> dict[str, An
             "expected": len(topic_records),
             "delivered": delivered,
             "lost": lost,
+            # >1 means the publishing peer restarted inside this instance. Kept
+            # in the summary because it changes how every rate-like number
+            # below should be read, and because it is otherwise invisible.
+            "epochs": len({record_epoch(record) for record in topic_records}),
             "loss_pct": round(100.0 * lost / len(topic_records), 3) if topic_records else 0.0,
             "reordered": reordered,
             "ota_hop_ms": _distribution_ms(ota),

{rosotacom_dev-2.5.dev33.dist-info → rosotacom_dev-2.5.dev36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosotacom-dev
-Version: 2.5.dev33
+Version: 2.5.dev36
 Summary: Host CLI for ROS OTA communication sessions.
 Author: develNor
 License-Expression: BSD-3-Clause
@@ -311,6 +311,25 @@ rosotacom start 1_heartbeat --identity b \
 
 `rosotacom` reads the static session input and creates generated files under `session-instances/<date>/<session>_<timestamp>_<id>/config/`, including per-peer plugin/session specs, topic lists, optional QoS, and optional `domain_bridge.yaml`. Catmux pane output is logged under the same instance in `logs/<peer>/catmux/`.
 
+What else a run leaves under `logs/<peer>/`, and which file to open first when a
+peer looked healthy but did nothing:
+
+| File | Written by | Read it for |
+|---|---|---|
+| `catmux/<NN>-<WINDOW>/<pane>.log` | `catmux_log_setup.sh` (`tmux pipe-pane`) | everything a node printed, including the traceback it died with |
+| `pane_failures.log` | the same script's prompt hook | one line per non-zero pane exit — **the file that answers "is any pane dead?"** |
+| `status/startup_check.json` | `status_overview`, once, after `status_startup_grace_s` and one confirming look a grace later | the verdict: which outbound stages this peer produces and never published, plus the pane failures above |
+| `status/status.{json,txt}` and `status/events.jsonl` | `status_overview`, continuously | live per-stage state and the transit records |
+| `scenario/<component>.log` | `rosotacom start <scenario>` | each scenario application's own output |
+| `launcher.log`, `docker.log` | `rosotacom start` | the exact command, and the container's own stdout where it has one |
+
+A pane whose command exits drops back to a shell prompt and keeps looking alive
+— `docker ps` shows a healthy container and the other panes keep working. That
+is how a control centre ran for two hours on 2026-08-13 with no heartbeat
+publisher, because Cyclone had no free participant index left on the local
+domain and `heartbeat_echo` died two seconds after start. The last two rows of
+that table exist so the next one announces itself.
+
 ### Complete use cases with scenarios
 
 A session intentionally describes only the communication contract. When a use
@@ -577,9 +596,15 @@ The running session writes, under
 - `status.json` — machine-readable snapshot (source of truth) for tools/agents,
   refreshed on a short interval and on every state transition,
 - `status.txt` — a human-rendered table, and
-- `events.jsonl` — state transitions plus per-`(topic, seq)` transit records for
-  wrapped topics (delivered/lost/reordered, section latency, size, inter-arrival,
-  and jitter).
+- `events.jsonl` — state transitions plus per-`(topic, epoch, seq)` transit
+  records for wrapped topics (delivered/lost/reordered, section latency, size,
+  inter-arrival, and jitter). `epoch` counts the restarts of the sending peer's
+  wrapper: sequence numbers start at zero again after one, so it is what keeps
+  the numbering unique over an instance that outlived a restart.
+
+Which of the bandwidth numbers a session publishes answers which question —
+per-stream payload, whole-interface wire traffic, and the overhead ratio
+between them — is in [link bandwidth](docs/link-bandwidth.md).
 
 For fixed-interval network-condition samples alongside the same session, enable
 the [link trace recorder](docs/link-trace.md). It writes

{rosotacom_dev-2.5.dev33.dist-info → rosotacom_dev-2.5.dev36.dist-info}/RECORD

@@ -6,7 +6,7 @@ rosotacom/benched_set.py,sha256=lBA1Rj04dbr2e-lIvYTWoL55_LdUpCAC-y7eIvJZ8lc,2721
 rosotacom/benchmark.py,sha256=pd8GMr_ENRIO4orKBAXDNPFVrgDtSjiqKZHW9icqGeg,76184
 rosotacom/benchmark_delta.py,sha256=qXzriBkbXOeJdlLKM4ziov1F-JfwlseYHE48S9hjFsQ,10005
 rosotacom/bundle_check.py,sha256=l_3BSuYk6juXQmsNU6jkxRbH8ayH2GkiW9ONKMQ1oJY,18270
-rosotacom/cli.py,sha256=PAVho_WTTqrdULGj2aLuXthrXO4lSCO36CTQ2HKC_UQ,398761
+rosotacom/cli.py,sha256=Z22DGj14_EJ6sgqNRPliOrKTeHyOkeAtHWpSrv2YEOI,399930
 rosotacom/cli_benchmark.py,sha256=vbGAxO7haTQx5RL7p150nlfKMaAFlm8chZ7VN4cQPC4,349881
 rosotacom/deployment.py,sha256=XVhB8JjxhAsglSEEPtiVSeV6fCG0rgGmDRulh14oIYQ,7318
 rosotacom/ffmpeg_packet.py,sha256=zB1BHgY1kexr5sETVMkm4bkVnRytj9k2t0uOYiHUZ6M,4267
@@ -20,16 +20,16 @@ rosotacom/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 rosotacom/status_eval.py,sha256=V9KVho5DEBYCD_iG1X2Yg6qd6ZEzopdKSY5QJ6K1sP4,23115
 rosotacom/stream_stats.py,sha256=ey09g4B7y0E-NnsOBW8Ng9kqyMbo2xuabNut9op6uM4,23744
 rosotacom/trace_profiles.py,sha256=KNM1RExHDXkppWZdqbSPXTjwJ8f7io8IU48HnKzmHKY,20567
-rosotacom/transit.py,sha256=0BqDO3nfYJ1PMXRz8b3084sDOrCjhO5zwdqC_-qU6bo,10531
+rosotacom/transit.py,sha256=3eM9EC8N9JAHFqEHBRqqubQDR5FOQmrQXREWfDeaNSo,12501
 rosotacom/video_quality.py,sha256=9NqWA3MgHDvjGmJoHHXQCqb17cyIkHYTrkwNxBoVqMA,33845
 rosotacom/resources/benched-set.yaml,sha256=wbWsoPHRQ5ji_pjM-rPe0dOSXv5kfD31FJoTn9YGsO0,14978
-rosotacom/resources/ros2docker.json.example,sha256=ARbtWmVNWCcAMcFhaHIPBocefDlc9Zx8gHS47uD5D_Q,2582
+rosotacom/resources/ros2docker.json.example,sha256=DwJYoqN6M0-Don4nRHtM2h96k1oEmi7UrEfZuEjfrz0,2541
 rosotacom/resources/examples/.gitignore,sha256=9FFIV42CR379XRSDOC9UdXsfJP8cGACLjHxiU9TJG6E,19
 rosotacom/resources/examples/README.md,sha256=RclnxBhlKcl9nRp2PvllVy6akKMZ82cOLv29et-SPyQ,5889
 rosotacom/resources/examples/__init__.py,sha256=IKLiqdO5jeBX3rMkWyb_Uc3SrQZp5vkJtqoTD_NZ90E,52
 rosotacom/resources/examples/deployment.example.yaml,sha256=E3HMJhv-Ft5mA6PC8d7yVZz1uwG3OQhCD_MBnhxtvAg,450
 rosotacom/resources/examples/profiles.yaml,sha256=KhmbIvO7GAWEQEm8QmIr_VCP14eioh_nSohxFf9gouw,3375
-rosotacom/resources/examples/ros2docker.json,sha256=jX9YYAD1plb9oJsjtRcJuPA5t542x6iRZ6eInUG5IQE,2540
+rosotacom/resources/examples/ros2docker.json,sha256=Gd_g1P_OA-_7-nM8-KmTTX2cRUyoNCpaPj9MCgxld3M,2499
 rosotacom/resources/examples/rosotacom.yaml,sha256=lpFnRNJv9kjS3Pd6dJAG1PXF-UP-YbSOK8dNcUe68hY,351
 rosotacom/resources/examples/bags/8_drop_reference/metadata.yaml,sha256=esIzUWq2anN89egMyZcHDCs5eIuVyruSU0eZPtfo2-c,445
 rosotacom/resources/examples/scenarios/2_native_chatter/scenario-definition.yaml,sha256=rxDLMqCyX-WR0Q4moGgmDhxpO4ffvf9pSSihJhXAsNQ,313
@@ -104,6 +104,7 @@ rosotacom/resources/examples/sessions/bench_1_5_camera/session-definition.yaml,s
 rosotacom/resources/ws/.vscode/c_cpp_properties.json,sha256=tnO19diLYiK_OoyLXyQqADJkXgfeswy95r4qMCTBeT8,422
 rosotacom/resources/ws/.vscode/settings.json,sha256=xNEQ9qGmUhxkLXYDJj4QVt4l3BtgiwMECuS16qnpdt0,201
 rosotacom/resources/ws/ota_configs/create_zenoh_json5.py,sha256=um2xU1SfNVo_FvlS8iYeAauu67I1N1Z8x-Ckw7KUAAk,2356
+rosotacom/resources/ws/ota_configs/cyclonedds_local_participants.xml.template,sha256=_PAy5sj9_pk2If7xQ_iHF2Oi1tC6HaZe3dDyl8fRSoo,1708
 rosotacom/resources/ws/ota_configs/cyclonedds_minimal.xml.template,sha256=YspSB4xbk2C_CODoowBQUkqsehdGYEd9I0YCjdGN1dc,197
 rosotacom/resources/ws/ota_configs/cyclonedds_tuned.xml.template,sha256=Tb8l9FDEhmfuGp53hTcEqmS6SckdEHjDQ8zQdWPUI6A,1165
 rosotacom/resources/ws/ota_configs/fastdds_unicast.xml.template,sha256=j_BF3E9Dub1l-lVreZxmUa0plGRssCWoNkaIy_6jFVQ,1972
@@ -145,7 +146,7 @@ rosotacom/resources/ws/ros2src/com_py/com_py/image_bandwidth_reducer.py,sha256=F
 rosotacom/resources/ws/ros2src/com_py/com_py/image_pixel_capper.py,sha256=8nc456sfoL7BdDTikpYJmF3lty0DjV1XdJUTvawTWkc,14382
 rosotacom/resources/ws/ros2src/com_py/com_py/lanelet_saver.py,sha256=KtzrNw2rYQ_0TAxO4k73bCnAy7J2WnBHVxgYFxVNELg,2902
 rosotacom/resources/ws/ros2src/com_py/com_py/latch_relay.py,sha256=jkt8C-voAAqZfI6qNIKvTD8-jzQyEoHmZyrLyR1aqhs,8019
-rosotacom/resources/ws/ros2src/com_py/com_py/link_bytes.py,sha256=uDl_XvPHZISEt72fSOML3fgONUXyVKQZh6VHvsNen-Q,6977
+rosotacom/resources/ws/ros2src/com_py/com_py/link_bytes.py,sha256=SB-M4_EIihb6lS6hlaV1b4SuunXje9y6RBxYOpMQB0Q,8985
 rosotacom/resources/ws/ros2src/com_py/com_py/link_trace.py,sha256=7zwRYQcbsGwd9jLWozWIQ6u_mI-HeMPRxp6BiPVY6zw,7055
 rosotacom/resources/ws/ros2src/com_py/com_py/local_global_frame_bridge.py,sha256=BCuq_njJ_sXgdSGJ1J9FtH6sxqvELTU0UdXAw13XqhA,18724
 rosotacom/resources/ws/ros2src/com_py/com_py/localization_map_split_reduce_saver.py,sha256=m6lMb51q5pUYTyHVWiss8vPlsbYRpKMDwHG9gG_fY4E,9088
@@ -159,10 +160,10 @@ rosotacom/resources/ws/ros2src/com_py/com_py/restamp.py,sha256=S8LjtZs_WBdnvL5oN
 rosotacom/resources/ws/ros2src/com_py/com_py/sized_publisher.py,sha256=prpmQf_cXp3KMYw3FlBZfNhq0Rrz56YUjWOunVF6xL8,9259
 rosotacom/resources/ws/ros2src/com_py/com_py/stage_latency.py,sha256=su_oWQBeDeLHdCy08YzzsASPgYoWID-syqqVkrjSg4g,2768
 rosotacom/resources/ws/ros2src/com_py/com_py/stage_latency_core.py,sha256=VXR0EN8EdExxLSb8UbY4mTv94khLZi6weqJpQtRMbd8,2600
-rosotacom/resources/ws/ros2src/com_py/com_py/status_overview.py,sha256=C9WK6bhW-8rKPmqcmtC3Wl8kTluEiS4lR-sGwg9Dh-k,22245
-rosotacom/resources/ws/ros2src/com_py/com_py/status_overview_core.py,sha256=evMMHhuOggQ_W28z12aJLH0KAayxfQJJmeYC13bUthU,45407
+rosotacom/resources/ws/ros2src/com_py/com_py/status_overview.py,sha256=uEG2Rr8Ck7DyaYQn9AtZA3AhQJ3-mAsGQyrc9WzkMKI,22669
+rosotacom/resources/ws/ros2src/com_py/com_py/status_overview_core.py,sha256=i7_U52zML9CPY0oBg6toGHhMETSer3ObqAJThCDzFA8,58877
 rosotacom/resources/ws/ros2src/com_py/com_py/tf_map_to_cart_saver.py,sha256=eWq0D-pVDadj186e2ZC4kZpxTBkXMzi24BfBj3NjGXc,3777
-rosotacom/resources/ws/ros2src/com_py/com_py/topic_monitor.py,sha256=tS_7gLtEVwyPBE2kk_khFBvJSiHaUPSK8w2OtJRSJRc,15019
+rosotacom/resources/ws/ros2src/com_py/com_py/topic_monitor.py,sha256=5iNoHU3SiiDwzsOj7tudh5d801ELVxaHiGKmTCP3JFw,16320
 rosotacom/resources/ws/ros2src/com_py/com_py/topic_resolution.py,sha256=WA34WSuY4jpzqsJ9S3HgETTWP5tZHGsslF7FAqE8oik,7092
 rosotacom/resources/ws/ros2src/com_py/com_py/trickle.py,sha256=gp2qb55dEuF_7LsqDSdnBw0lO-zJb9ZyuhlkGKxxqzs,6788
 rosotacom/resources/ws/ros2src/com_py/com_py/universal_compressor.py,sha256=b_teazIo8ymguKq-f9Xpwe3Q4uRFBhCqb1qB5EC0NP0,25730
@@ -179,19 +180,19 @@ rosotacom/resources/ws/ros2src/com_py/resource/com_py,sha256=47DEQpj8HBSa-_TImW-
 rosotacom/resources/ws/ros2src/com_py/test/test_copyright.py,sha256=6nG7vGJHtcFNF27QnznuJ01WPMmy7DVYKOpFHO_zF1c,962
 rosotacom/resources/ws/ros2src/com_py/test/test_flake8.py,sha256=N6gToFB_DOUaLQtXMS0swHigMHo8qFWA6oAbDS-XVdI,884
 rosotacom/resources/ws/ros2src/com_py/test/test_pep257.py,sha256=6LiNSe4paYextABp-BAZtfoJsaob_umh-q5Fu1z1Pto,803
-rosotacom/resources/ws/session/content/base/session_plugin_base.yaml,sha256=2Cu37dvvYvYrpMxz2MY0k4nn6YMQiJi6_gzt99xiIZ4,39002
+rosotacom/resources/ws/session/content/base/session_plugin_base.yaml,sha256=fcs7G5jlOkTUT3lSOxvoWBkuMokJ3xLr5n15KiYY32U,39717
 rosotacom/resources/ws/session/content/network/up_connectivity.py,sha256=-a6E97Uw159DfRXxYA0MBWRUPA6mQhuAA7pSjASM9qU,13593
 rosotacom/resources/ws/session/content/network/wifi_network_analysis_2.py,sha256=UVRtGdbHYayXM0OFkXs2BV2lW0CsY0m_NTlbPO1J9oE,8682
 rosotacom/resources/ws/session/creation/anonymize_bag.py,sha256=jxOTKFjmde-pyoVWE9-9icpSjf4nh97-q2Eqx-BDUl8,6154
-rosotacom/resources/ws/session/creation/catmux_log_setup.sh,sha256=d6slqefxfCUhBlGWw3fkC50RbntuQR93e8h54hTww7c,3135
+rosotacom/resources/ws/session/creation/catmux_log_setup.sh,sha256=VyBAFqmhQcIziSnDCRDNUsT5QpbLzKWU-DqRzaj_WQs,5243
 rosotacom/resources/ws/session/creation/create_session_yaml.py,sha256=8cfzkUxK7IebIBC36Vu7lcWtogM8fG0zveBCUGeiVdU,5727
-rosotacom/resources/ws/session/creation/generate_session_files.py,sha256=Iw5spNwJrO7FLRGMk2pxnj5CBVkmkuQKLOulF-jUzQs,141798
+rosotacom/resources/ws/session/creation/generate_session_files.py,sha256=biJheNo1NnERwYCcuZw0ktGYofBR9l2zeF6BVpiCuMM,142276
 rosotacom/resources/ws/session/creation/run_session.py,sha256=G2hJhrmLp11gm3JZ6_C-zZcqu_reXlE_wmobB7BiCfU,16849
 rosotacom/resources/ws/session/creation/run_session_in_external_terminal.py,sha256=Ub_5f-ttUUc-rAPrZmg_Op34s8Z4C91hTyEn8fgyt-E,2633
 rosotacom/resources/ws/session/creation/strip_ansi.py,sha256=vNjpemcq4xHr0VG4x0bZqvgwEBEfOgN-b4PzbrM5nfM,851
-rosotacom_dev-2.5.dev33.dist-info/licenses/LICENSE.txt,sha256=FDB3-fao3lqZ94wsJHKAH7U88kAnfvZhZC8Hm5rN__g,1459
-rosotacom_dev-2.5.dev33.dist-info/METADATA,sha256=aNZPP_sF9HgAcaTI9RFmiBnUfN9k5NiI2vd8ERmvimI,52217
-rosotacom_dev-2.5.dev33.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-rosotacom_dev-2.5.dev33.dist-info/entry_points.txt,sha256=-Hl9WKAfkLjMHUonDTxvz86EgSfZf6oyatRMxypS5iA,147
-rosotacom_dev-2.5.dev33.dist-info/top_level.txt,sha256=BZrtQcIVDWn_pQZR3Mlssqy5SaRXHjOa6B-1D9Fy4Ao,10
-rosotacom_dev-2.5.dev33.dist-info/RECORD,,
+rosotacom_dev-2.5.dev36.dist-info/licenses/LICENSE.txt,sha256=FDB3-fao3lqZ94wsJHKAH7U88kAnfvZhZC8Hm5rN__g,1459
+rosotacom_dev-2.5.dev36.dist-info/METADATA,sha256=Yzy1-og2TcINwvCW7w0NSf7QNFtqtqGbto5dkqZjgdA,54130
+rosotacom_dev-2.5.dev36.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rosotacom_dev-2.5.dev36.dist-info/entry_points.txt,sha256=-Hl9WKAfkLjMHUonDTxvz86EgSfZf6oyatRMxypS5iA,147
+rosotacom_dev-2.5.dev36.dist-info/top_level.txt,sha256=BZrtQcIVDWn_pQZR3Mlssqy5SaRXHjOa6B-1D9Fy4Ao,10
+rosotacom_dev-2.5.dev36.dist-info/RECORD,,