rosotacom-dev 2.5.dev55__tar.gz → 2.5.dev57__tar.gz

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosotacom-dev
-Version: 2.5.dev55
+Version: 2.5.dev57
 Summary: Host CLI for ROS OTA communication sessions.
 Author: develNor
 License-Expression: BSD-3-Clause

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/session-configuration.md

@@ -720,8 +720,35 @@ ros2 run com_py playout_pacer --ros-args \
   -p target_ms:=350.0 -p adaptive:=true
 ```
 
-It republishes on `<topic>/paced` plus `.../paced/budget_ms` and
-`.../paced/queue_depth` debug topics.
+It republishes on `<topic>/paced` plus three topics next to it:
+`.../paced/budget_ms` and `.../paced/queue_depth` at 1 Hz, and
+`.../paced/hold_ms` with every release, carrying how long *that* message was
+actually held.
+
+##### What the status overview does with it
+
+A paced stage's age is the link plus a delay this session asked for, and one
+number cannot be judged against a threshold meant for the link. So the
+generator names the pacer's report on the stage it explains
+(`paced_hold_topic` in `pipeline_spec.yaml`), and `status_overview` takes the
+hold back out before classifying:
+
+```
+native_in  FLOWING  .../ffmpeg/compressed  10.0Hz 286ms(link 46+paced 240) ...
+```
+
+`latency_ms` stays the age on screen; `link_latency_ms` and `pacing_hold_ms`
+are the split, and the quality verdict uses the former. Without this the
+default 200 ms bad-threshold — set before anything on a link was held on
+purpose — marks every healthy paced stream BAD, which is how an operator learns
+to stop reading the panel.
+
+It is the *applied* hold that is subtracted, never the configured budget: a
+message that arrived later than its release time passes straight through, is
+reported as held for 0 ms, and still flags. A budget-based correction would
+credit that message a buffer it never received and quietly stop reporting a slow
+link. Before the pacer has reported at all, nothing is subtracted — neither zero
+nor the budget is a safe guess.
 
 `type: foxglove` supports (CompressedVideo):
 - `gop_size` (int)

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/src/rosotacom/cli.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 import argparse
 import base64
 import binascii
+import contextlib
 import getpass
 import hashlib
 import importlib
@@ -23,12 +24,13 @@ import os
 import re
 import shlex
 import shutil
+import signal
 import subprocess
 import sys
 import tarfile
 import tempfile
 import time
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass, replace
 from datetime import datetime
 from pathlib import Path
@@ -2761,12 +2763,49 @@ def _passwordless_watchdog_command(argv: Sequence[str]) -> str:
     return f"nohup sudo -n {shlex.join(argv)} >/dev/null 2>&1 &"
 
 
+@contextlib.contextmanager
+def _terminate_via_exception() -> Iterator[None]:
+    """Make SIGTERM unwind the stack instead of ending the process outright.
+
+    The OTA run already has a `finally` that reverts shaping, collects logs and
+    stops both peers. SIGINT reaches it, because Python raises KeyboardInterrupt.
+    SIGTERM does not: the default disposition ends the process immediately, so
+    `finally` never runs and every peer keeps its containers -- and, worse, any
+    netem qdisc stays armed on the peer's own interface, quietly degrading a
+    machine nobody is looking at.
+
+    SIGTERM is not the exotic case. `timeout` sends it, systemd sends it, and a
+    CI job cancelling a step sends it. On 2026-08-19 a run killed by `timeout`
+    left containers behind on both peers, which then blocked the next run's
+    exclusivity check.
+
+    Raising SystemExit gives the same exit status while letting the cleanup run.
+    The handler is restored on the way out so this does not leak into a caller.
+    """
+
+    def handler(signum: int, _frame: object) -> None:
+        raise SystemExit(128 + signum)
+
+    try:
+        previous = signal.signal(signal.SIGTERM, handler)
+    except ValueError:
+        # Not the main thread; the parent already owns signal disposition.
+        yield
+        return
+    try:
+        yield
+    finally:
+        signal.signal(signal.SIGTERM, previous)
+
+
 def _ota_conflict_check(plan: OtaSmokePlan, *, dry_run: bool) -> None:
     """Fail-safe preflight: OTA runs require exclusive control of the peers.
 
-    Aborts when a peer already runs rosotacom containers or has active network
-    shaping (netem/tbf/htb) on the data interface toward another peer — both
-    signal an OTA run that is still active or was not cleaned up.
+    Aborts when a peer already runs rosotacom containers, has active network
+    shaping (netem/tbf/htb) on the data interface toward another peer, or is
+    carrying so much unrelated load that a measurement taken on it would not
+    mean anything. The first two signal an OTA run that is still active or was
+    not cleaned up; the third signals a machine somebody else is using.
     """
     from .network_shaper import ota_interface_from_route
 
@@ -2820,6 +2859,59 @@ def _ota_conflict_check(plan: OtaSmokePlan, *, dry_run: bool) -> None:
                 )
 
 
+# A run on a machine somebody else is saturating does not fail — it succeeds
+# and reports numbers that describe the contention rather than the link. That
+# is the expensive kind of wrong, because nothing in the artefact says so
+# afterwards.
+#
+# The thresholds are per-CPU, so they read the same on an 8-core laptop and a
+# 32-thread desktop. At 2.0 the run stops: with twice as many runnable threads
+# as CPUs, a thread waits on average as long as it runs, which lands directly
+# in the scheduling latency a transport measurement is trying to resolve.
+_LOAD_WARN_PER_CPU = 0.7
+_LOAD_REFUSE_PER_CPU = 2.0
+
+
+def _ota_load_check(plan: OtaSmokePlan, *, dry_run: bool) -> None:
+    """Refuse to measure on a peer that is busy with someone else's work."""
+    for peer in plan.peers.values():
+        result = _ota_run(
+            peer,
+            "cut -d' ' -f1-3 /proc/loadavg; nproc",
+            label=f"{peer.name}: host load",
+            dry_run=dry_run,
+            batch=True,
+            check=False,
+        )
+        if dry_run or result.returncode != 0:
+            continue
+        lines = (result.stdout or "").split()
+        if len(lines) < 4:
+            continue
+        try:
+            load1, load5, load15 = (float(value) for value in lines[:3])
+            cpus = max(1, int(lines[3]))
+        except ValueError:
+            continue
+        # load5 as well as load1: a run takes minutes, and a machine that was
+        # busy a moment ago is about to be busy again.
+        ratio = max(load1, load5) / cpus
+        # Two decimals to match /proc/loadavg and uptime, so the number in an
+        # artefact can be compared against what the machine reports itself.
+        summary = f"load {load1:.2f}/{load5:.2f}/{load15:.2f} on {cpus} CPUs ({ratio:.2f} per CPU)"
+        if ratio >= _LOAD_REFUSE_PER_CPU:
+            raise RuntimeError(
+                f"Peer {peer.name} is carrying unrelated load: {summary}.\n"
+                "A measurement taken here would describe the contention, not the link.\n"
+                "Wait for the machine to go quiet, measure on another pair, or — if this "
+                "load is yours and expected — pass --skip-conflict-check."
+            )
+        if ratio >= _LOAD_WARN_PER_CPU:
+            print(f"! {peer.name}: {summary} — comparisons survive if runs are interleaved, absolute latencies do not")
+        else:
+            print(f"+ {peer.name}: {summary}")
+
+
 def _ota_preflight(
     plan: OtaSmokePlan,
     *,
@@ -2924,6 +3016,7 @@ def _ota_preflight(
 
     if check_conflicts:
         _ota_conflict_check(plan, dry_run=dry_run)
+        _ota_load_check(plan, dry_run=dry_run)
 
 
 _OTA_STAGE_EXCLUDES = {
@@ -3137,6 +3230,7 @@ def _ota_verify_checkouts(plan: OtaSmokePlan, *, dry_run: bool) -> None:
     distinct one-line failure here, before any container starts, rather than a
     confusing one an hour later.
     """
+    peer_profiles: dict[str, list[str]] = {}
     for peer in plan.peers.values():
         checkout = shlex.quote(_ota_peer_workdir(plan, peer))
         project = shlex.quote(plan.project)
@@ -3174,6 +3268,46 @@ def _ota_verify_checkouts(plan: OtaSmokePlan, *, dry_run: bool) -> None:
             # a real hazard, but it is also a legitimate mid-migration state,
             # and a check that refuses it would be the wrong place to find out.
             print(f"+ {peer.name}: rosotacom {version.stdout.strip() or 'unknown'}")
+        profiles = _ota_run(
+            peer,
+            f"{OTA_CHECKOUT_PATH_PREFIX}"
+            'ls "$(rosotacom resources path ws)"/ota_configs/*.template '
+            "2>/dev/null | xargs -r -n1 basename | sort | tr '\\n' ' '",
+            label=f"{peer.name}: OTA profile templates",
+            dry_run=dry_run,
+            batch=True,
+            check=False,
+        )
+        if not dry_run and profiles.returncode == 0:
+            peer_profiles[peer.name] = profiles.stdout.split()
+
+    # Unlike the version, this one refuses. A peer whose installed stack lacks
+    # the template a session names does not fail loudly: the renderer errors,
+    # the profile file is left unusable, and the DDS stack silently falls back
+    # to its own defaults — multicast discovery, which no tunnelled OTA link
+    # carries. The run then completes and reports zero delivered messages, which
+    # reads as a middleware that does not work. That is exactly what happened on
+    # 2026-08-19, when #299 made fastdds_tuned.xml the Fast DDS default while
+    # the deployed stack predated the template: eight runs, all recorded as
+    # evidence about Fast DDS, none of which had ever configured it.
+    if len(peer_profiles) > 1:
+        names = sorted(peer_profiles)
+        reference = set(peer_profiles[names[0]])
+        for name in names[1:]:
+            missing = reference - set(peer_profiles[name])
+            extra = set(peer_profiles[name]) - reference
+            if missing or extra:
+                detail = []
+                if missing:
+                    detail.append(f"absent on {name}: {', '.join(sorted(missing))}")
+                if extra:
+                    detail.append(f"only on {name}: {', '.join(sorted(extra))}")
+                raise RuntimeError(
+                    f"Peers disagree about their OTA profile templates ({'; '.join(detail)}). "
+                    "A missing template renders nothing and the DDS stack falls back to "
+                    "defaults, so the link would come up and carry no data. Bring the peers "
+                    "to the same rosotacom before measuring."
+                )
 
 
 def _ota_prepare_hosts(args: argparse.Namespace, runtime: RuntimeConfig, plan: OtaSmokePlan) -> None:
@@ -4159,6 +4293,7 @@ def _start_interactive_ota_smoke(args: argparse.Namespace) -> int:
 
     if not getattr(args, "skip_preflight", False) and not getattr(args, "skip_conflict_check", False):
         _ota_conflict_check(plan, dry_run=dry_run)
+        _ota_load_check(plan, dry_run=dry_run)
     _ota_prepare_hosts(args, runtime, plan)
     instance = _resolve_session_instance(
         runtime,
@@ -4355,7 +4490,8 @@ def ota_smoke(args: argparse.Namespace) -> int:
                 "session). Run the non-interactive ota-smoke to shape the link, or apply the profile manually."
             )
         return _start_interactive_ota_smoke(args)
-    return _start_noninteractive_ota_smoke(args)
+    with _terminate_via_exception():
+        return _start_noninteractive_ota_smoke(args)
 
 
 def _scenario_name_completer(

rosotacom_dev-2.5.dev57/src/rosotacom/resources/ws/ota_configs/fastdds_local.xml.template

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Local-domain Fast DDS configuration: pin the loopback locators, change nothing else.
+
+  This is the Fast DDS counterpart to cyclonedds_local_participants.xml, and it
+  exists because the two stacks fail differently when nothing is configured.
+  CycloneDDS talks between containers on a host out of the box. Fast DDS does
+  not, and the way it fails is quiet: the publisher reports a matched
+  subscriber, the subscriber reports zero messages, and the session's status
+  shows the last stage IDLE with `pub=1` next to it — a pipeline that looks
+  connected and moves nothing.
+
+  Measured on lamborghini, 2026-08-19, two containers on `--network host`,
+  84 B at 10 Hz, ~80 samples per run:
+
+    no profile at all .................................. 0 of 80
+    + shared memory off (useBuiltinTransports=false) ... 0 of 80
+    + a unicast initial peer on 127.0.0.1 .............. 0 of 80
+    + explicit unicast locator lists ................... 79 of 80
+    locator lists ALONE, shared memory left on ......... 79 of 80
+    (control) CycloneDDS, no profile ................... 79 of 80
+
+  So the locator lists are the whole fix, and shared memory is a red herring —
+  worth stating because turning SHM off is the obvious first guess and it
+  changes nothing here.
+
+  Why they are load-bearing: with no locator list Fast DDS binds every interface
+  it can enumerate and announces all of them. A developer machine running Docker
+  has dozens of `br-*` bridges, most of them DOWN, and a peer that walks that
+  list can settle on one that carries nothing. Naming 127.0.0.1 makes the
+  announcement match the one path that is certain to work between processes on
+  a shared network namespace.
+
+  Scope, and its one limit: this pins the LOCAL domain, which is host-local by
+  definition, so loopback is enough — and it is more robust than the host's
+  routable address, which depends on which interface is up. It assumes the
+  session's containers share a network namespace (`--network host`, which the
+  ros2docker profiles set). A container on its own bridge network has its own
+  127.0.0.1 and would need its address named here instead.
+
+  Nothing else is set: no interface allowlist, no transport override, no
+  discovery tuning. Unlike the OTA templates this profile is not trying to cross
+  a tunnel.
+-->
+<dds xmlns="http://www.eprosima.com/XMLSchemas/fastRTPS_Profiles">
+  <profiles>
+    <participant profile_name="rosotacom_fastdds_local" is_default_profile="true">
+      <rtps>
+        <defaultUnicastLocatorList>
+          <locator>
+            <udpv4>
+              <address>127.0.0.1</address>
+              <port>0</port>
+            </udpv4>
+          </locator>
+        </defaultUnicastLocatorList>
+
+        <builtin>
+          <metatrafficUnicastLocatorList>
+            <locator>
+              <udpv4>
+                <address>127.0.0.1</address>
+                <port>0</port>
+              </udpv4>
+            </locator>
+          </metatrafficUnicastLocatorList>
+
+          <initialPeersList>
+            <locator>
+              <udpv4>
+                <address>127.0.0.1</address>
+                <port>0</port>
+              </udpv4>
+            </locator>
+          </initialPeersList>
+        </builtin>
+      </rtps>
+    </participant>
+  </profiles>
+</dds>

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/src/rosotacom/resources/ws/ota_configs/get_ota_xml.py

@@ -12,6 +12,8 @@ get_fastdds_easy_mode_xml.py, get_cyclonedds_xml.py).
 import argparse
 import os
 import re
+import sys
+import tempfile
 
 
 KNOWN_PLACEHOLDERS = {"#host_ip", "#peer", "#easy_mode_ip", "#spdp_interval"}
@@ -204,6 +206,16 @@ if __name__ == "__main__":
         "-e", "--easy-mode-ip", dest="easy_mode_ip",
         help="Resolved Fast DDS Easy Mode address (#easy_mode_ip).",
     )
+    parser.add_argument(
+        "-o", "--output", dest="output",
+        help=(
+            "Write the rendered XML to this path, atomically. Prefer this over a shell "
+            "redirect: `> file` truncates the target before this script runs, so a "
+            "failure here leaves a zero-byte profile behind, and a zero-byte profile is "
+            "not an error to a DDS stack — it falls back to its defaults and the link "
+            "comes up carrying nothing."
+        ),
+    )
     parser.add_argument(
         "--spdp-interval",
         dest="spdp_interval",
@@ -211,4 +223,25 @@ if __name__ == "__main__":
         help=f"CycloneDDS SPDP interval duration (#spdp_interval, default: {DEFAULT_SPDP_INTERVAL}).",
     )
     args = parser.parse_args()
-    print(main(**{k: v for k, v in vars(args).items() if v is not None}))
+    output = args.output
+    rendered = main(**{k: v for k, v in vars(args).items()
+                       if v is not None and k != "output"})
+    if not output:
+        print(rendered)
+        raise SystemExit(0)
+    if not rendered.strip():
+        raise SystemExit(f"Refusing to write an empty profile to {output}.")
+    directory = os.path.dirname(os.path.abspath(output)) or "."
+    os.makedirs(directory, exist_ok=True)
+    handle, temporary = tempfile.mkstemp(dir=directory, suffix=".partial")
+    try:
+        with os.fdopen(handle, "w", encoding="utf-8") as stream:
+            stream.write(rendered)
+        os.replace(temporary, output)
+    except BaseException:
+        try:
+            os.unlink(temporary)
+        except OSError:
+            pass
+        raise
+    print(f"Wrote {output} ({len(rendered)} bytes).", file=sys.stderr)

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/src/rosotacom/resources/ws/ros2src/com_py/com_py/playout_pacer.py

@@ -8,7 +8,16 @@ Inserted between the OTA unwrapper and a decoder chain it turns network jitter
 into a constant, bounded age instead of visible stutter.
 
 Debug topics next to the output: ``.../paced/budget_ms`` (current delay
-budget) and ``.../paced/queue_depth``.
+budget) and ``.../paced/queue_depth``, both at 1 Hz, and ``.../paced/hold_ms``
+— published with *every* release, carrying how long that message was actually
+held.
+
+``hold_ms`` exists so the status overview can tell the link's contribution from
+this node's. Deliberately the applied hold rather than the budget: a message
+that arrived later than its release time passes straight through and is held
+for 0 ms, so a genuinely slow link cannot be credited with a buffer it never
+used. Subtracting the *budget* would do exactly that, and would quietly stop
+reporting the problem it exists to make visible.
 """
 
 from __future__ import annotations
@@ -62,6 +71,7 @@ class PlayoutPacerNode(Node):
         self._pub = self.create_publisher(msg_cls, out_topic, qos)
         self._budget_pub = self.create_publisher(Float64, out_topic + "/budget_ms", 10)
         self._depth_pub = self.create_publisher(Int32, out_topic + "/queue_depth", 10)
+        self._hold_pub = self.create_publisher(Float64, out_topic + "/hold_ms", 10)
         self.create_subscription(msg_cls, topic, self._on_message, qos)
         # 2 ms release tick: bounds added jitter to well below frame cadence.
         self.create_timer(0.002, self._release_due)
@@ -72,12 +82,16 @@ class PlayoutPacerNode(Node):
         now = time.time()
         stamp = msg.header.stamp.sec + msg.header.stamp.nanosec * 1e-9
         release = self.schedule.on_message(stamp, now)
-        self._queue.append((release, msg))
+        self._queue.append((release, msg, now))
 
     def _release_due(self) -> None:
         now = time.time()
         while self._queue and self._queue[0][0] <= now:
-            self._pub.publish(self._queue.popleft()[1])
+            release, msg, arrival = self._queue.popleft()
+            self._pub.publish(msg)
+            # What this node added, not what it was allowed to add. Clamped
+            # because a late message's release time is already in the past.
+            self._hold_pub.publish(Float64(data=max(0.0, (release - arrival) * 1000.0)))
 
     def _publish_debug(self) -> None:
         self._budget_pub.publish(Float64(data=float(self.schedule.budget_ms)))

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/src/rosotacom/resources/ws/ros2src/com_py/com_py/status_overview.py

@@ -71,6 +71,7 @@ from com_py.ffmpeg_flags import FFMPEG_PACKET_TYPE, parse_keyframe_flag
 from com_py.link_bytes import LinkByteSampler, resolve_link_interface
 from com_py.link_trace import LinkTraceRecorder
 from com_py.status_overview_core import (
+    SCALAR_MSG_TYPE,
     STARTUP_GRACE_S,
     ClockOffsetEstimator,
     StageObservation,
@@ -309,6 +310,11 @@ class StageObserver(Node):
                         "t_com_out": now_ros_s,
                         "keyframe": keyframe,
                     }
+            elif obs.type_str == SCALAR_MSG_TYPE:
+                # Not a stage: a number this node reads to interpret one. The
+                # pacing hold arrives here (`.../paced/hold_ms`); it carries no
+                # header, so there is no latency to record, only the value.
+                obs.last_value = float(msg.data)
             else:
                 header = getattr(msg, "header", None)
                 stamp = getattr(header, "stamp", None) if header is not None else None

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/src/rosotacom/resources/ws/ros2src/com_py/com_py/status_overview_core.py

@@ -61,6 +61,12 @@ GOOD = "GOOD"
 DEGRADED = "DEGRADED"
 BAD = "BAD"
 
+#: A stage whose age includes a delay this session asked for names the topic on
+#: which the node responsible reports what it actually held. Set by the
+#: generator for a decode that reads a playout pacer's output.
+PACED_HOLD_KEY = "paced_hold_topic"
+SCALAR_MSG_TYPE = "std_msgs/msg/Float64"
+
 # Topic-level rollup
 OK = "OK"
 PARTIAL = "PARTIAL"
@@ -371,6 +377,9 @@ class StageObservation:
     last_recv_wall: Optional[float] = None
     last_delay_s: Optional[float] = None
     last_raw_delay_s: Optional[float] = None
+    #: Payload of the last message, for the scalar topics this node reads as
+    #: numbers rather than observes as stages (the pacing hold).
+    last_value: Optional[float] = None
     last_rtt_s: Optional[float] = None
     last_clock_offset_s: Optional[float] = None
     # (t_mono, size_bytes, delay_s_or_None)
@@ -673,6 +682,21 @@ class StatusAggregator:
         self._startup_pending: Optional[Tuple[float, set]] = None
 
     # -- observation lookup --
+    def _pacing_hold_ms(self, stage: Dict[str, Any]) -> Optional[float]:
+        """How long this session deliberately held the last message of a stage.
+
+        None whenever the answer is not actually known -- no pacer named, or the
+        pacer has not reported yet. Guessing zero would read a paced picture as
+        a slow link; guessing the configured budget would do the opposite and
+        credit a hold that a late message never received.
+        """
+        hold_topic = stage.get(PACED_HOLD_KEY)
+        if not hold_topic:
+            return None
+        observer = self._observers.get("local")
+        observation = observer.observations.get(str(hold_topic)) if observer else None
+        return None if observation is None else observation.last_value
+
     def _obs_for(self, stage: Dict[str, Any]) -> Optional[StageObservation]:
         domain = stage.get("domain", "local")
         observer = self._observers.get(domain) or self._observers.get("local")
@@ -714,6 +738,9 @@ class StatusAggregator:
             "observation": "graph" if obs is not None and obs.graph_only else "payload",
             "inferred_from": None,
             "latched": str((expect or {}).get("mode", "")).strip().lower() == "latched",
+            # Split of `latency_ms` for a stage this session delays on purpose.
+            "pacing_hold_ms": None,
+            "link_latency_ms": None,
         }
         if obs is None:
             return result
@@ -727,6 +754,10 @@ class StatusAggregator:
         result["age_s"] = round(m["age_s"], 3) if m["age_s"] is not None else None
         if m["last_delay_s"] is not None:
             result["latency_ms"] = round(m["last_delay_s"] * 1000.0, 1)
+            hold_ms = self._pacing_hold_ms(stage)
+            if hold_ms is not None:
+                result["pacing_hold_ms"] = round(hold_ms, 1)
+                result["link_latency_ms"] = round(max(0.0, result["latency_ms"] - hold_ms), 1)
         if m["last_raw_delay_s"] is not None:
             result["latency_uncorrected_ms"] = round(m["last_raw_delay_s"] * 1000.0, 1)
         if m["last_rtt_s"] is not None:
@@ -759,7 +790,9 @@ class StatusAggregator:
             return result
 
         result["state"] = FLOWING
-        quality, reason = self._classify_quality(m, expected_hz, expect)
+        quality, reason = self._classify_quality(
+            m, expected_hz, expect, pacing_hold_ms=result["pacing_hold_ms"]
+        )
         result["quality"] = quality
         result["quality_reason"] = reason
         return result
@@ -823,7 +856,12 @@ class StatusAggregator:
                 self._copy_inferred_activity(stage_result, source)
 
     def _classify_quality(
-        self, m: Dict[str, Any], expected_hz: Optional[float], expect: Optional[Dict[str, Any]] = None
+        self,
+        m: Dict[str, Any],
+        expected_hz: Optional[float],
+        expect: Optional[Dict[str, Any]] = None,
+        *,
+        pacing_hold_ms: Optional[float] = None,
     ) -> Tuple[str, Optional[str]]:
         expect = expect or {}
         hz_exp = expect.get("hz") or {}
@@ -833,8 +871,17 @@ class StatusAggregator:
         # Latency: a declared `expect.latency_ms.max` overrides the global bad
         # threshold; the good band stays the default so a tight contract still
         # surfaces a DEGRADED before BAD where it makes sense.
+        #
+        # A stage the session delays on purpose is judged on what is left after
+        # that delay is taken back out. Otherwise a threshold calibrated for a
+        # link measures a buffer, which reads as a fault on a healthy stream and
+        # teaches an operator to stop believing the panel. The subtracted value
+        # is the hold actually applied, so a late message -- which the pacer
+        # passes straight through -- is credited nothing and still flags.
         bad_lat = float(lat_exp["max"]) if "max" in lat_exp else self._delay_bad_ms
         delay_ms = (m["last_delay_s"] * 1000.0) if m["last_delay_s"] is not None else None
+        if delay_ms is not None and pacing_hold_ms is not None:
+            delay_ms = max(0.0, delay_ms - pacing_hold_ms)
         if delay_ms is not None:
             if delay_ms >= bad_lat:
                 return BAD, "latency"
@@ -1142,7 +1189,16 @@ class StatusAggregator:
                 if st["state"] in (FLOWING, STALE):
                     parts = [f"{st['hz']:.1f}Hz"]
                     if st["latency_ms"] is not None:
-                        parts.append(f"{st['latency_ms']:.0f}ms")
+                        # A paced stage reads as one big number unless the split
+                        # is on the same line: what the link cost, and what this
+                        # session chose to add on top of it.
+                        if st.get("pacing_hold_ms") is not None:
+                            parts.append(
+                                f"{st['latency_ms']:.0f}ms"
+                                f"(link {st['link_latency_ms']:.0f}+paced {st['pacing_hold_ms']:.0f})"
+                            )
+                        else:
+                            parts.append(f"{st['latency_ms']:.0f}ms")
                     elif st["latency_uncorrected_ms"] is not None:
                         parts.append(f"{st['latency_uncorrected_ms']:.0f}ms(raw)")
                     if st["loss_pct"] is not None:
@@ -1298,7 +1354,12 @@ class StatusAggregator:
 
 
 def collect_stage_topics(spec: Dict[str, Any]) -> Dict[str, Dict[str, Optional[str]]]:
-    """Return {domain: {topic: type_hint}} from the pipeline spec."""
+    """Return {domain: {topic: type_hint}} from the pipeline spec.
+
+    Includes the pacing-hold topics a stage names: they are not stages of the
+    link -- nothing is delivered on them -- but the aggregator has to read them
+    to tell this session's deliberate delay from the network's.
+    """
     by_domain: Dict[str, Dict[str, Optional[str]]] = {"local": {}, "ota": {}}
     for topic_spec in spec.get("topics", []):
         type_hint = topic_spec.get("type")
@@ -1307,6 +1368,9 @@ def collect_stage_topics(spec: Dict[str, Any]) -> Dict[str, Dict[str, Optional[s
             by_domain.setdefault(domain, {})
             existing = by_domain[domain].get(stage["topic"])
             by_domain[domain][stage["topic"]] = existing or stage.get("type") or type_hint
+            hold_topic = stage.get(PACED_HOLD_KEY)
+            if hold_topic:
+                by_domain["local"].setdefault(str(hold_topic), SCALAR_MSG_TYPE)
     return by_domain
 
 

{rosotacom_dev-2.5.dev55 → rosotacom_dev-2.5.dev57}/src/rosotacom/resources/ws/session/content/base/session_plugin_base.yaml

@@ -292,7 +292,7 @@ common:
   before_commands:
     - if [ '${local_domain_id}' != "None" ] && [ -n '${local_domain_id}' ]; then export ROS_DOMAIN_ID='${local_domain_id}'; fi
     - if [ '${rmw_local}' != "None" ]; then case ${rmw_local} in cyclone) export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp;; fastdds) export RMW_IMPLEMENTATION=rmw_fastrtps_cpp; export FASTDDS_BUILTIN_TRANSPORTS=${FASTDDS_BUILTIN_TRANSPORTS:-UDPv4};; zenoh) export RMW_IMPLEMENTATION=rmw_zenoh_cpp;; *) export RMW_IMPLEMENTATION='${rmw_local}';; esac; fi
-    - if [ '${local_config_template}' != "None" ] && [ -n '${local_config_template}' ]; then local_args=(--config "${local_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${local_easy_mode_ip}' != "None" ] && [ -n '${local_easy_mode_ip}' ] && local_args+=(--easy-mode-ip "${local_easy_mode_ip}"); [ '${local_spdp_interval}' != "None" ] && [ -n '${local_spdp_interval}' ] && local_args+=(--spdp-interval "${local_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${local_args[@]}" > "${local_config_file}"; case ${rmw_local} in cyclone) export CYCLONEDDS_URI="file://${local_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${local_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
+    - if [ '${local_config_template}' != "None" ] && [ -n '${local_config_template}' ]; then local_args=(--config "${local_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${local_easy_mode_ip}' != "None" ] && [ -n '${local_easy_mode_ip}' ] && local_args+=(--easy-mode-ip "${local_easy_mode_ip}"); [ '${local_spdp_interval}' != "None" ] && [ -n '${local_spdp_interval}' ] && local_args+=(--spdp-interval "${local_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${local_args[@]}" --output "${local_config_file}" || { echo "FATAL - could not render the local DDS profile ${local_config_template}; refusing to start on defaults." >&2; exit 1; }; case ${rmw_local} in cyclone) export CYCLONEDDS_URI="file://${local_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${local_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
     - if [ '${before_command}' != "None" ]; then ${before_command}; fi
 
 windows:
@@ -338,7 +338,7 @@ windows:
     if: use_domain_bridge
     splits:
       - commands:
-        - if [ '${ota_config_template}' != "None" ] && [ -n '${ota_config_template}' ]; then ota_args=(--config "${ota_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${ota_easy_mode_ip}' != "None" ] && [ -n '${ota_easy_mode_ip}' ] && ota_args+=(--easy-mode-ip "${ota_easy_mode_ip}"); [ '${ota_spdp_interval}' != "None" ] && [ -n '${ota_spdp_interval}' ] && ota_args+=(--spdp-interval "${ota_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${ota_args[@]}" > "${ota_config_file}"; case ${rmw_ota} in cyclone) export CYCLONEDDS_URI="file://${ota_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${ota_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
+        - if [ '${ota_config_template}' != "None" ] && [ -n '${ota_config_template}' ]; then ota_args=(--config "${ota_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${ota_easy_mode_ip}' != "None" ] && [ -n '${ota_easy_mode_ip}' ] && ota_args+=(--easy-mode-ip "${ota_easy_mode_ip}"); [ '${ota_spdp_interval}' != "None" ] && [ -n '${ota_spdp_interval}' ] && ota_args+=(--spdp-interval "${ota_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${ota_args[@]}" --output "${ota_config_file}" || { echo "FATAL - could not render the OTA DDS profile ${ota_config_template}; refusing to start on defaults." >&2; exit 1; }; case ${rmw_ota} in cyclone) export CYCLONEDDS_URI="file://${ota_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${ota_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
         - ros2 run domain_bridge domain_bridge --wait-for-publisher false ${domain_bridge_config_file}
   - name: IN
     if: in
@@ -346,7 +346,7 @@ windows:
       - commands:
         - if [ '${ota_domain_id}' != "None" ] && [ -n '${ota_domain_id}' ]; then export ROS_DOMAIN_ID='${ota_domain_id}'; fi
         - case ${rmw_ota} in cyclone) export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp;; fastdds) export RMW_IMPLEMENTATION=rmw_fastrtps_cpp; export FASTDDS_BUILTIN_TRANSPORTS=${FASTDDS_BUILTIN_TRANSPORTS:-UDPv4};; zenoh) export RMW_IMPLEMENTATION=rmw_zenoh_cpp;; esac
-        - if [ '${ota_config_template}' != "None" ] && [ -n '${ota_config_template}' ]; then ota_args=(--config "${ota_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${ota_easy_mode_ip}' != "None" ] && [ -n '${ota_easy_mode_ip}' ] && ota_args+=(--easy-mode-ip "${ota_easy_mode_ip}"); [ '${ota_spdp_interval}' != "None" ] && [ -n '${ota_spdp_interval}' ] && ota_args+=(--spdp-interval "${ota_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${ota_args[@]}" > "${ota_config_file}"; case ${rmw_ota} in cyclone) export CYCLONEDDS_URI="file://${ota_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${ota_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
+        - if [ '${ota_config_template}' != "None" ] && [ -n '${ota_config_template}' ]; then ota_args=(--config "${ota_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${ota_easy_mode_ip}' != "None" ] && [ -n '${ota_easy_mode_ip}' ] && ota_args+=(--easy-mode-ip "${ota_easy_mode_ip}"); [ '${ota_spdp_interval}' != "None" ] && [ -n '${ota_spdp_interval}' ] && ota_args+=(--spdp-interval "${ota_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${ota_args[@]}" --output "${ota_config_file}" || { echo "FATAL - could not render the OTA DDS profile ${ota_config_template}; refusing to start on defaults." >&2; exit 1; }; case ${rmw_ota} in cyclone) export CYCLONEDDS_URI="file://${ota_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${ota_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
         # For zenoh_ros2dds the OTA domain is loopback-only (zenoh is the inter-host
         # transport), so confine discovery to localhost instead of statically peering
         # the remote host — otherwise the OTA-domain DDS double-delivers alongside zenoh.
@@ -373,7 +373,7 @@ windows:
       - commands:
         - if [ '${ota_domain_id}' != "None" ] && [ -n '${ota_domain_id}' ]; then export ROS_DOMAIN_ID='${ota_domain_id}'; fi
         - case ${rmw_ota} in cyclone) export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp;; fastdds) export RMW_IMPLEMENTATION=rmw_fastrtps_cpp; export FASTDDS_BUILTIN_TRANSPORTS=${FASTDDS_BUILTIN_TRANSPORTS:-UDPv4};; zenoh) export RMW_IMPLEMENTATION=rmw_zenoh_cpp;; esac
-        - if [ '${ota_config_template}' != "None" ] && [ -n '${ota_config_template}' ]; then ota_args=(--config "${ota_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${ota_easy_mode_ip}' != "None" ] && [ -n '${ota_easy_mode_ip}' ] && ota_args+=(--easy-mode-ip "${ota_easy_mode_ip}"); [ '${ota_spdp_interval}' != "None" ] && [ -n '${ota_spdp_interval}' ] && ota_args+=(--spdp-interval "${ota_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${ota_args[@]}" > "${ota_config_file}"; case ${rmw_ota} in cyclone) export CYCLONEDDS_URI="file://${ota_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${ota_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
+        - if [ '${ota_config_template}' != "None" ] && [ -n '${ota_config_template}' ]; then ota_args=(--config "${ota_config_template}" --host-ip "${ip_local}" --peer "${ip_remote}"); [ '${ota_easy_mode_ip}' != "None" ] && [ -n '${ota_easy_mode_ip}' ] && ota_args+=(--easy-mode-ip "${ota_easy_mode_ip}"); [ '${ota_spdp_interval}' != "None" ] && [ -n '${ota_spdp_interval}' ] && ota_args+=(--spdp-interval "${ota_spdp_interval}"); python3 /ws/ota_configs/get_ota_xml.py "${ota_args[@]}" --output "${ota_config_file}" || { echo "FATAL - could not render the OTA DDS profile ${ota_config_template}; refusing to start on defaults." >&2; exit 1; }; case ${rmw_ota} in cyclone) export CYCLONEDDS_URI="file://${ota_config_file}";; fastdds) export FASTDDS_DEFAULT_PROFILES_FILE="${ota_config_file}"; export RMW_FASTRTPS_USE_QOS_FROM_XML=1;; esac; fi
         # For zenoh_ros2dds the OTA domain is loopback-only (zenoh is the inter-host
         # transport), so confine discovery to localhost instead of statically peering
         # the remote host — otherwise the OTA-domain DDS double-delivers alongside zenoh.