pingerplot 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pingerplot/monitor.py ADDED
@@ -0,0 +1,1404 @@
1
+ """Monitoring engine: discover the route, then probe every hop continuously.
2
+
3
+ Mirrors the MTR model — each round re-probes the destination with an
4
+ increasing TTL (1..route_len). The router at each TTL answers "TTL exceeded",
5
+ so one round is effectively a full traceroute, and we accumulate latency/loss
6
+ per hop over time. The final hop (TTL == route_len) is the destination itself.
7
+
8
+ Because every round is a full retrace, route changes are detected continuously:
9
+ * a hop whose responding address changes -> reroute (logged)
10
+ * the destination answering at a lower TTL -> route shortened
11
+ * the destination no longer answering -> probe deeper for a longer route
12
+
13
+ Sustained-degradation alerts are evaluated on the *destination* hop only — the
14
+ one measurement that isn't muddied by routers that deprioritise/ignore ICMP.
15
+ An alert is gated on the destination having actually been reached at least once,
16
+ so a target that simply blocks ICMP does not raise a false alarm.
17
+
18
+ Threading model:
19
+ * One worker thread runs :meth:`_run` (trace, then the round loop).
20
+ * Probes within a round fan out across a small thread pool.
21
+ * Reverse-DNS lookups run on their own pool; they never block probing.
22
+ * All shared state is guarded by ``self._lock`` (re-entrant). The GUI never
23
+ touches Hop objects directly — it calls the snapshot accessors, which copy.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import ipaddress
28
+ import json
29
+ import math
30
+ import os
31
+ import re
32
+ import socket
33
+ import sys
34
+ import threading
35
+ import time
36
+ import urllib.parse
37
+ import urllib.request
38
+ from collections import deque
39
+ from concurrent.futures import ThreadPoolExecutor
40
+ from typing import Callable, Deque, Dict, List, Optional, Tuple
41
+
42
+ try:
43
+ import winsound # Windows stdlib; used for the alert beep
44
+ except ImportError: # pragma: no cover - non-Windows
45
+ winsound = None # type: ignore[assignment]
46
+
47
+ from . import __version__, history, icmp, tcpudp
48
+ from .routes import RouteTracker
49
+ from .model import (DEFAULT_HISTORY, Event, Hop, HopView, Sample, hop_from_session, mos,
50
+ host_name, mos_label, opt_str, plausible_time)
51
+
52
+ GROW_PROBE_SPAN = 8 # extra TTLs to probe when looking for a longer route
53
+ UNREACHED_BEFORE_GROW = 3 # consecutive dest-miss rounds before probing deeper
54
+ MAX_HOPS_CEILING = 64 # hard cap on max_hops, and so on the probe pool width
55
+ FINAL_HOP_TTL = 255 # TTL used to ping the destination directly
56
+ # Windows does not hand a connecting socket the RST the instant it arrives — it
57
+ # finishes retransmitting the SYN first, then surfaces WSAECONNREFUSED. Measured
58
+ # at ~2.0 s on Windows 11. Below that a *closed* port is indistinguishable from
59
+ # an unreachable host, which is the wrong answer in the most common diagnostic
60
+ # case ("the service is down but the box is fine"). Shortening the window with
61
+ # TCP_MAXRT does not help: it replaces WSAECONNREFUSED with WSAETIMEDOUT and
62
+ # destroys the very distinction the probe exists to draw. So we wait it out.
63
+ MAX_TIMEOUT_MS = 10_000 # the Settings field's own ceiling
64
+ TCP_REFUSAL_FLOOR_MS = 3000
65
+ MAX_LOAD_HISTORY = 50_000 # cap a loaded session's per-hop ring buffer (DoS guard)
66
+ MAX_LOG_BYTES = 25 * 1024 * 1024 # roll the probe CSV past ~25 MB (one backup kept)
67
+ LOG_ROTATE_RETRY_S = 60.0 # after a failed rollover, keep appending this long before retrying
68
+ MAX_LOAD_HOPS = 1024 # cap hops loaded from a session file (defense-in-depth)
69
+ # A hostname target is looked up again this often. Big services (Microsoft,
70
+ # CDNs) move clients between front ends by DNS; without this a run kept
71
+ # probing the address it started with and never saw the move.
72
+ DNS_RECHECK_S = 300.0
73
+
74
+
75
+ # start() keyword -> the attribute it lands in. One table, so saving a target's
76
+ # options and validating them on restore cannot disagree about what exists.
77
+ _START_OPTIONS = {
78
+ "interval": "interval", "timeout_ms": "timeout_ms", "max_hops": "max_hops",
79
+ "resolve_names": "resolve_names", "packet_size": "packet_size",
80
+ "send_delay_ms": "send_delay_ms", "final_hop_only": "final_hop_only",
81
+ "packet_type": "packet_type", "port": "port", "dscp": "dscp",
82
+ "source_ip": "source_ip", "log_path": "log_path",
83
+ "alert_enabled": "alert_enabled", "alert_loss_pct": "alert_loss_pct",
84
+ "alert_latency_ms": "alert_latency_ms", "alert_window": "alert_window",
85
+ "alert_sound": "alert_sound", "alert_mos": "alert_mos",
86
+ "webhook_url": "webhook_url",
87
+ }
88
+ START_OPTION_KEYS = frozenset(_START_OPTIONS)
89
+
90
+
91
+ def _log_suffix(target: str) -> str:
92
+ """``target`` reduced to characters that are safe in a file name on every
93
+ platform. Hostnames and IPv4 literals pass through unchanged."""
94
+ safe = re.sub(r"[^A-Za-z0-9.-]", "_", target.strip()).strip(".")
95
+ return "_" + (safe or "target")
96
+
97
+
98
+ def per_target_log_path(path: str, target: str) -> str:
99
+ """``logs/probe.csv`` -> ``logs/probe_<target>.csv``.
100
+
101
+ The probe CSV has no target column, so two monitors sharing one file
102
+ interleave rows that cannot be told apart (the first hops are the same
103
+ gateways for every target), and one monitor's rollover renames the file
104
+ out from under the others. Idempotent, so a path that already carries the
105
+ suffix -- as Edit settings shows it -- is not suffixed twice.
106
+ """
107
+ if not path:
108
+ return path
109
+ root, ext = os.path.splitext(path)
110
+ suffix = _log_suffix(target)
111
+ if root.endswith(suffix):
112
+ return path
113
+ return f"{root}{suffix}{ext}"
114
+
115
+
116
+ def log_path_template(path: str, target: str) -> str:
117
+ """Inverse of :func:`per_target_log_path`, for showing the configured path
118
+ back in the Engine dialog."""
119
+ root, ext = os.path.splitext(path)
120
+ suffix = _log_suffix(target)
121
+ return f"{root[:-len(suffix)]}{ext}" if root.endswith(suffix) else path
122
+
123
+
124
+ def _addresses_of(name: str) -> List[str]:
125
+ """Every IPv4 address ``name`` resolves to, in the resolver's order. The
126
+ whole set, not one: round-robin DNS rotates the order on every lookup,
127
+ and only an address leaving the set is a move."""
128
+ seen: List[str] = []
129
+ for info in socket.getaddrinfo(name, None, socket.AF_INET, socket.SOCK_STREAM):
130
+ addr = info[4][0]
131
+ if addr not in seen:
132
+ seen.append(addr)
133
+ return seen
134
+
135
+
136
+ def _is_address(text: str) -> bool:
137
+ try:
138
+ ipaddress.ip_address(text)
139
+ return True
140
+ except ValueError:
141
+ return False
142
+
143
+
144
+ def text_option(value, name: str) -> str:
145
+ """A text option: a string, or nothing. Anything else (a hand-edited
146
+ settings.json with ``"source_ip": 5``) is refused like a bad number, with
147
+ TypeError, so the caller's fall-back to defaults runs; ``.strip()`` on it
148
+ raised AttributeError, which nothing caught, and the window never opened
149
+ (2026-11 review)."""
150
+ if value is None:
151
+ return ""
152
+ if not isinstance(value, str):
153
+ raise TypeError(f"{name} must be text, not {type(value).__name__}")
154
+ return value.strip()
155
+
156
+
157
+ def _build_payload(size: int) -> bytes:
158
+ """An ICMP data payload of ``size`` bytes (repeating, recognisable text)."""
159
+ if size <= 0:
160
+ return b""
161
+ base = b"PingerPlot probe "
162
+ return (base * (size // len(base) + 1))[:size]
163
+
164
+
165
+ def _send_webhook(url: str, payload: dict) -> Optional[str]:
166
+ """POST ``payload`` as JSON to ``url`` (http/https only). Never raises.
167
+ Returns None when delivered (2xx/3xx), otherwise why not -- so a mistyped
168
+ URL is reported instead of discovered when the alert you needed never
169
+ arrives. Lets the monitor *tell* you when the destination degrades instead
170
+ of only beeping."""
171
+ if not url.lower().startswith(("http://", "https://")):
172
+ return "not an http(s) URL"
173
+ if any(c.isspace() or ord(c) < 0x20 or c == "" for c in url):
174
+ # http.client would refuse it with an InvalidURL that quotes the
175
+ # path, and for Slack, Teams and ntfy the path is the secret.
176
+ return "the URL contains a space or a control character"
177
+ try:
178
+ data = json.dumps(payload).encode("utf-8")
179
+ req = urllib.request.Request(
180
+ url, data=data, method="POST",
181
+ headers={"Content-Type": "application/json", "User-Agent": "PingerPlot"},
182
+ )
183
+ with urllib.request.urlopen(req, timeout=8) as resp:
184
+ status = getattr(resp, "status", 200)
185
+ return None if 200 <= status < 400 else f"HTTP {status}"
186
+ except Exception as exc: # any failure is reported, never raised
187
+ return _scrub_url(str(exc) or type(exc).__name__, url)
188
+
189
+
190
+ def _scrub_url(text: str, url: str) -> str:
191
+ """``text`` without the parts of ``url`` past the host: an error message
192
+ that quotes the URL goes into events, the console and saved sessions."""
193
+ parts = urllib.parse.urlsplit(url)
194
+ secrets = [url, parts.path + ("?" + parts.query if parts.query else ""),
195
+ parts.path, parts.query]
196
+ for secret in sorted({s for s in secrets if len(s) > 1}, key=len, reverse=True):
197
+ text = text.replace(secret, "/...")
198
+ return text
199
+
200
+
201
+ def _webhook_host(url: str) -> str:
202
+ """Where a webhook points, for messages. Never the whole URL: Slack, Teams
203
+ and ntfy put the secret in the path, and these messages are printed."""
204
+ try:
205
+ host = urllib.parse.urlsplit(url).hostname
206
+ except ValueError:
207
+ host = None
208
+ return host or "(unparseable URL)"
209
+
210
+
211
+ class Monitor:
212
+ def __init__(self, on_update: Optional[Callable[[], None]] = None,
213
+ recorder: "Optional[history.Recorder]" = None) -> None:
214
+ self.on_update = on_update
215
+ # The history database, shared by every target; None records nothing
216
+ # (a loaded session, the self-test, most tests).
217
+ self.recorder = recorder
218
+ # generation -> run handle. Per generation, not one slot: a worker
219
+ # that outlives stop() must still close its own run, after a new
220
+ # start() has already opened the next one.
221
+ self._rec_runs: Dict[int, Optional[int]] = {}
222
+ self._rec_problem_seen = 0
223
+ # Which routers answer at each hop, so a reroute is reported once and
224
+ # a load-balanced hop isn't reported at all (see routes.py).
225
+ self._routes = RouteTracker()
226
+ self._recorded_route: Optional[list] = None
227
+ self._dns_check = None # the pending re-lookup (a Future), if any
228
+ self._dns_next = 0.0 # monotonic time of the next one
229
+ self._dns_failing = False
230
+
231
+ # configuration (set on start)
232
+ self.target_input = ""
233
+ self.target_ip: Optional[str] = None
234
+ self.interval = 2.5
235
+ self.timeout_ms = 1000
236
+ self.max_hops = 30
237
+ self.resolve_names = True
238
+ self.packet_size = 32
239
+ self.send_delay_ms = 0
240
+ self.final_hop_only = False
241
+ self.packet_type = "icmp" # "icmp" | "tcp" | "udp"
242
+ self.port = 443
243
+ self.dscp = 0 # DiffServ code point (0-63); 0 = best effort
244
+ self.source_ip = "" # pin the outgoing interface; "" = let the stack pick
245
+ self._local_ip = "0.0.0.0"
246
+ self._tos = 0 # derived from dscp; the byte that goes on the wire
247
+ self._payload = icmp.DEFAULT_PAYLOAD
248
+ self.log_path = ""
249
+ self.timeout_note = "" # set when start() raises a too-short timeout
250
+ self.needs_admin = False # stopped because full TCP/UDP needs Administrator
251
+ self.interval_note = "" # set when the interval is shorter than a round can be
252
+ self.log_note = "" # set when the probe log could not be opened
253
+ self._log_fh = None
254
+ self._rotate_retry_at = 0.0 # monotonic time before which not to retry a rollover
255
+ self._rotate_failing = False
256
+ self._round = 0 # 0 = initial trace; 1,2,3… = monitoring rounds
257
+ self.alert_enabled = True
258
+ self.alert_loss_pct = 20.0
259
+ self.alert_latency_ms = 250.0
260
+ self.alert_window = 20
261
+ self.alert_sound = True
262
+ self.alert_mos = 0.0 # 0 disables; MOS alerts BELOW this score
263
+ self.webhook_url = ""
264
+
265
+ # live state
266
+ self.status = "Idle"
267
+ self.route_len = 0
268
+ self.reached_target = False
269
+ self.running = False
270
+ self.paused = False
271
+
272
+ self._hops: List[Hop] = []
273
+ self._events: Deque[Event] = deque(maxlen=1000)
274
+ self._event_seq = 0
275
+ self._active_alerts: Dict[Tuple[int, str], str] = {}
276
+ self._unreached = 0
277
+
278
+ self._lock = threading.RLock()
279
+ self._thread: Optional[threading.Thread] = None
280
+ self._generation = 0 # bumped each start(); a worker ignores stale gens
281
+ # Pools are created on start() and torn down on shutdown(); a Monitor
282
+ # that never starts (e.g. the GUI's placeholder) allocates none.
283
+ self._ping_pool: Optional[ThreadPoolExecutor] = None
284
+ self._ping_workers = 0 # width of the live probe pool (see _ensure_pools)
285
+ self._dns_pool: Optional[ThreadPoolExecutor] = None
286
+
287
+ def _ensure_pools(self) -> None:
288
+ """(Re)create the probe and DNS thread pools if absent. Lets a Monitor
289
+ be reused after shutdown(), and keeps a never-started one pool-free.
290
+
291
+ The probe pool is sized to ``max_hops`` so a whole route fits in ONE
292
+ wave. It used to be a flat 16 while max_hops defaults to 30 and may go
293
+ to 64, so any route past 16 hops silently split into two or more waves
294
+ — and a wave costs a full reply timeout, because a silent router never
295
+ answers. That turned the engine's core promise (a round costs about one
296
+ timeout, not the sum) into a 2x-4x overrun on ordinary internet paths.
297
+
298
+ Sizing to the ceiling is free: ThreadPoolExecutor spawns its threads
299
+ lazily as work is submitted, so a 6-hop route still only ever runs six.
300
+ """
301
+ want = max(1, min(int(self.max_hops), MAX_HOPS_CEILING))
302
+ if self._ping_pool is not None and self._ping_workers < want:
303
+ # A later start() asked for a longer route than the live pool can
304
+ # probe at once. Replace it rather than quietly serialising; the
305
+ # old one drains its in-flight probes and exits.
306
+ self._ping_pool.shutdown(wait=False)
307
+ self._ping_pool = None
308
+ if self._ping_pool is None:
309
+ self._ping_pool = ThreadPoolExecutor(max_workers=want, thread_name_prefix="ping")
310
+ self._ping_workers = want
311
+ if self._dns_pool is None:
312
+ self._dns_pool = ThreadPoolExecutor(max_workers=8, thread_name_prefix="dns")
313
+
314
+ # --- lifecycle ---------------------------------------------------------
315
+ def start(
316
+ self,
317
+ target: str,
318
+ interval: float = 2.5,
319
+ timeout_ms: int = 1000,
320
+ max_hops: int = 30,
321
+ resolve_names: bool = True,
322
+ packet_size: int = 32,
323
+ send_delay_ms: int = 0,
324
+ final_hop_only: bool = False,
325
+ packet_type: str = "icmp",
326
+ port: int = 443,
327
+ dscp: int = 0,
328
+ source_ip: str = "",
329
+ log_path: str = "",
330
+ alert_enabled: bool = True,
331
+ alert_loss_pct: float = 20.0,
332
+ alert_latency_ms: float = 250.0,
333
+ alert_window: int = 20,
334
+ alert_sound: bool = True,
335
+ alert_mos: float = 0.0,
336
+ webhook_url: str = "",
337
+ ) -> None:
338
+ self.stop()
339
+ # Every numeric option, before any is used: int(inf) raises
340
+ # OverflowError, which nothing expects, and an infinite interval or
341
+ # threshold would quietly do nothing. A settings file can hold
342
+ # Infinity or 1e999 (json writes and reads both).
343
+ for name, value in list(locals().items()):
344
+ if isinstance(value, float) and not math.isfinite(value):
345
+ raise ValueError(f"{name} must be a finite number, not {value}")
346
+ self.target_input = target.strip()
347
+ self.interval = max(0.25, float(interval))
348
+ # Ceilings as well as floors. A timeout of 100 s made every round, and
349
+ # closing the window, take that long; an alert window longer than the
350
+ # samples a hop keeps (DEFAULT_HISTORY) could never fill, so it
351
+ # silently turned every alert off.
352
+ self.timeout_ms = max(100, min(int(timeout_ms), MAX_TIMEOUT_MS))
353
+ self.max_hops = max(1, min(int(max_hops), MAX_HOPS_CEILING))
354
+ self.resolve_names = bool(resolve_names)
355
+ self.packet_size = max(0, min(int(packet_size), 1472))
356
+ self.send_delay_ms = max(0, min(int(send_delay_ms), 1000))
357
+ self.final_hop_only = bool(final_hop_only)
358
+ self.packet_type = packet_type if packet_type in ("icmp", "tcp", "udp") else "icmp"
359
+ self.port = max(1, min(int(port), 65535))
360
+ self.dscp = max(0, min(int(dscp), 63))
361
+ self._tos = icmp.dscp_to_tos(self.dscp)
362
+ self.source_ip = text_option(source_ip, "source_ip")
363
+ self._payload = _build_payload(self.packet_size)
364
+ self.timeout_note = ""
365
+ self.needs_admin = False
366
+ if self.packet_type == "tcp" and self.timeout_ms < TCP_REFUSAL_FLOOR_MS:
367
+ # Raise it rather than let the probe report a closed port as an
368
+ # unreachable host. Silently overriding a setting the user typed
369
+ # would be its own bug, so record why for the status line.
370
+ self.timeout_note = (
371
+ f"reply timeout raised {self.timeout_ms} -> {TCP_REFUSAL_FLOOR_MS} ms: "
372
+ f"below that, Windows has not yet surfaced a TCP reset and a closed "
373
+ f"port is indistinguishable from an unreachable host"
374
+ )
375
+ self.timeout_ms = TCP_REFUSAL_FLOOR_MS
376
+ self.interval_note = ""
377
+ if self.interval * 1000 < self.timeout_ms:
378
+ # A round waits for every hop, so a silent one holds it to the reply
379
+ # timeout and the next starts straight after. Silent hops are the
380
+ # norm on real paths, and in TCP mode the floor above sets that
381
+ # timeout to 3 s whatever the user typed: say why "every 1 s" isn't.
382
+ self.interval_note = (
383
+ f"probing every {self.interval:g} s, but each round waits up to "
384
+ f"{self.timeout_ms} ms for hops that don't answer, so while one is "
385
+ f"silent the rounds start about every {self.timeout_ms / 1000:g} s"
386
+ )
387
+ self.log_path = text_option(log_path, "log_path")
388
+ self.alert_enabled = bool(alert_enabled)
389
+ self.alert_loss_pct = max(0.0, float(alert_loss_pct))
390
+ self.alert_latency_ms = max(0.0, float(alert_latency_ms))
391
+ self.alert_window = max(1, min(int(alert_window), DEFAULT_HISTORY))
392
+ self.alert_sound = bool(alert_sound)
393
+ # MOS runs 1.0-5.0; anything above 5 would alert permanently.
394
+ self.alert_mos = max(0.0, min(float(alert_mos), 5.0))
395
+ self.webhook_url = text_option(webhook_url, "webhook_url")
396
+ self._rotate_retry_at = 0.0
397
+ self._rotate_failing = False
398
+ with self._lock:
399
+ self._generation += 1
400
+ gen = self._generation
401
+ self._hops = []
402
+ self._routes = RouteTracker()
403
+ self._recorded_route = None
404
+ self._dns_check = None
405
+ self._dns_next = time.monotonic() + DNS_RECHECK_S
406
+ self._dns_failing = False
407
+ self._events.clear()
408
+ self._active_alerts.clear()
409
+ self._event_seq = 0
410
+ self._unreached = 0
411
+ self.route_len = 0
412
+ self.reached_target = False
413
+ self.target_ip = None
414
+ self.paused = False
415
+ self._open_log()
416
+ self._ensure_pools()
417
+ self.running = True
418
+ self._thread = threading.Thread(target=self._run, args=(gen,), name="monitor", daemon=True)
419
+ self._thread.start()
420
+
421
+ def request_stop(self) -> None:
422
+ """Tell the worker to stop, without waiting for it. To stop several
423
+ monitors, call this on all of them before stop() on each: a worker
424
+ mid-round takes up to a reply timeout to wind down, and signalling
425
+ them together makes those waits overlap instead of adding up."""
426
+ self.running = False
427
+
428
+ def stop(self) -> None:
429
+ self.running = False
430
+ t = self._thread
431
+ if t and t.is_alive() and t is not threading.current_thread():
432
+ t.join(timeout=self.timeout_ms / 1000 + 2)
433
+ self._thread = None
434
+ self._close_log()
435
+
436
+ def shutdown(self) -> None:
437
+ self.stop()
438
+ for pool in (self._ping_pool, self._dns_pool):
439
+ if pool is not None:
440
+ pool.shutdown(wait=False, cancel_futures=True)
441
+ self._ping_pool = None
442
+ self._ping_workers = 0
443
+ self._dns_pool = None
444
+
445
+ def pause(self) -> None:
446
+ """Halt probing but keep the worker thread and accumulated history.
447
+ resume() continues without the reset a fresh start() would do."""
448
+ self.paused = True
449
+
450
+ def resume(self) -> None:
451
+ self.paused = False
452
+
453
+ # --- snapshots for the UI thread --------------------------------------
454
+ def snapshot(self) -> Tuple[List[HopView], str, Optional[str], str]:
455
+ with self._lock:
456
+ views = [HopView.of(h) for h in self._hops]
457
+ return views, self.status, self.target_ip, self.target_input
458
+
459
+ def summary(self) -> Tuple[int, Optional[float], Optional[float], Optional[float], bool]:
460
+ """Cheap sidebar snapshot: only the destination hop's stats, with no
461
+ per-hop HopView allocation (the grid doesn't need the intermediate hops).
462
+ Returns ``(n_hops, dest_loss, dest_avg, dest_jitter, reached)``; the
463
+ dest values are None when there are no hops yet."""
464
+ with self._lock:
465
+ n = len(self._hops)
466
+ if n == 0:
467
+ return (0, None, None, None, self.reached_target)
468
+ _, _, loss, _, avg, _, _, jitter = self._hops[-1].compute()
469
+ return (n, loss, avg, jitter, self.reached_target)
470
+
471
+ def hop_addresses(self) -> List[str]:
472
+ """The routers answering at each hop now (for looking them up)."""
473
+ with self._lock:
474
+ return [h.address for h in self._hops if h.address]
475
+
476
+ def routers(self) -> List[Tuple[Optional[str], Optional[float]]]:
477
+ """Each hop's current router as (name, its own best round trip), for
478
+ checking locations (locating.anchors). Cheap: no samples are read."""
479
+ with self._lock:
480
+ return [(h.hostname, h.best_by_addr.get(h.address)) for h in self._hops if h.address]
481
+
482
+ def samples_for(self, ttl: int) -> List[Sample]:
483
+ with self._lock:
484
+ for h in self._hops:
485
+ if h.ttl == ttl:
486
+ return list(h.samples)
487
+ return []
488
+
489
+ def all_samples(self) -> Dict[int, List[Sample]]:
490
+ with self._lock:
491
+ return {h.ttl: list(h.samples) for h in self._hops}
492
+
493
+ def events_after(self, seq: int) -> Tuple[List[Event], int]:
494
+ """Events with id > ``seq``, plus the newest id seen (for the caller)."""
495
+ with self._lock:
496
+ new = [e for e in self._events if e.seq > seq]
497
+ last = self._event_seq - 1 if self._event_seq else seq
498
+ return new, last
499
+
500
+ def active_alerts(self) -> List[str]:
501
+ with self._lock:
502
+ return list(self._active_alerts.values())
503
+
504
+ def start_options(self) -> dict:
505
+ """The keyword arguments that reproduce this monitor's configured run
506
+ through :meth:`start` -- what the GUI saves per target, so a relaunch
507
+ restores each target as configured rather than from the toolbar."""
508
+ return {key: getattr(self, attr) for key, attr in _START_OPTIONS.items()}
509
+
510
+ def run_summary(self) -> dict:
511
+ """Run conditions + the wall-clock window the current samples cover, so
512
+ an export can document exactly how it was produced (the missing context
513
+ that made two runs impossible to compare after the fact)."""
514
+ with self._lock:
515
+ times = [s.t for h in self._hops for s in h.samples]
516
+ return {
517
+ "version": __version__,
518
+ "target_input": self.target_input,
519
+ "target_ip": self.target_ip,
520
+ "packet_type": self.packet_type,
521
+ "port": self.port,
522
+ "interval": self.interval,
523
+ "timeout_ms": self.timeout_ms,
524
+ "packet_size": self.packet_size,
525
+ "send_delay_ms": self.send_delay_ms,
526
+ "max_hops": self.max_hops,
527
+ "dscp": self.dscp,
528
+ "source_ip": self.source_ip,
529
+ "final_hop_only": self.final_hop_only,
530
+ "samples": max((h.sent for h in self._hops), default=0),
531
+ "window_start": min(times) if times else None,
532
+ "window_end": max(times) if times else None,
533
+ }
534
+
535
+ # --- session persistence ----------------------------------------------
536
+ def to_dict(self) -> dict:
537
+ """Serialisable snapshot of the whole session (config + per-hop samples
538
+ + events) for Save. The GUI json-dumps this."""
539
+ with self._lock:
540
+ return {
541
+ "app": "PingerPlot",
542
+ "version": __version__,
543
+ "saved_at": time.time(),
544
+ "target_input": self.target_input,
545
+ "target_ip": self.target_ip,
546
+ "route_len": self.route_len,
547
+ "reached_target": self.reached_target,
548
+ "final_hop_only": self.final_hop_only,
549
+ "config": {
550
+ "interval": self.interval,
551
+ "timeout_ms": self.timeout_ms,
552
+ "max_hops": self.max_hops,
553
+ "packet_size": self.packet_size,
554
+ "send_delay_ms": self.send_delay_ms,
555
+ "packet_type": self.packet_type,
556
+ "port": self.port,
557
+ "dscp": self.dscp,
558
+ "source_ip": self.source_ip,
559
+ },
560
+ "hops": [
561
+ {
562
+ "ttl": h.ttl,
563
+ "address": h.address,
564
+ "hostname": h.hostname,
565
+ "last_status": h.last_status,
566
+ "samples": [[round(s.t, 3), s.rtt] for s in h.samples],
567
+ }
568
+ for h in self._hops
569
+ ],
570
+ "events": [[e.seq, e.t, e.kind, e.text] for e in self._events],
571
+ }
572
+
573
+ def load_dict(self, data: dict) -> None:
574
+ """Replace live state with a saved session (for offline review). Stops
575
+ any running monitor first; the result renders through the normal
576
+ snapshot path, just with ``running == False``.
577
+
578
+ The file is not necessarily ours, so every field is type-checked and a
579
+ bad one falls back to its default. Everything is parsed before any
580
+ state changes: one non-numeric config value used to raise after the
581
+ target, address and route length had already been replaced, leaving a
582
+ half-loaded monitor. Raises ValueError only for something that is not
583
+ a session object at all.
584
+ """
585
+ if not isinstance(data, dict):
586
+ raise ValueError("not a PingerPlot session (expected a JSON object)")
587
+ cfg = data.get("config")
588
+ cfg = cfg if isinstance(cfg, dict) else {}
589
+
590
+ def _cfg(key, conv, current, lo=None, hi=None):
591
+ # Within the same floors and ceilings start() applies: shown and
592
+ # exported as the session's settings, so an interval of Infinity
593
+ # or a port of 99999 would be a header that can't be true.
594
+ try:
595
+ value = conv(cfg[key]) if key in cfg else current
596
+ except (TypeError, ValueError, OverflowError):
597
+ return current
598
+ if isinstance(value, float) and not math.isfinite(value):
599
+ return current
600
+ if lo is not None:
601
+ value = max(lo, min(value, hi))
602
+ return value
603
+
604
+ def _int(value) -> int:
605
+ try:
606
+ return int(value or 0)
607
+ except (TypeError, ValueError, OverflowError):
608
+ return 0
609
+
610
+ raw_hops = data.get("hops")
611
+ raw_hops = raw_hops if isinstance(raw_hops, list) else []
612
+ # Clamp the ring buffers so a crafted/corrupt session can't blow up
613
+ # memory; the most recent samples are kept.
614
+ hops = [h for h in (hop_from_session(hd, MAX_LOAD_HISTORY)
615
+ for hd in raw_hops[:MAX_LOAD_HOPS]) if h is not None]
616
+ raw_events = data.get("events")
617
+ events: List[Event] = []
618
+ for ev in (raw_events if isinstance(raw_events, list) else []):
619
+ try:
620
+ event = Event(int(ev[0]), float(ev[1]), str(ev[2]), str(ev[3]))
621
+ except (IndexError, KeyError, ValueError, TypeError, OverflowError):
622
+ continue
623
+ if plausible_time(event.t):
624
+ events.append(event)
625
+ target = data.get("target_input")
626
+
627
+ self.stop()
628
+ with self._lock:
629
+ self.target_input = "" if target is None else str(target)
630
+ self.target_ip = opt_str(data.get("target_ip"))
631
+ self.route_len = _int(data.get("route_len"))
632
+ self.reached_target = bool(data.get("reached_target", False))
633
+ self.final_hop_only = bool(data.get("final_hop_only", False))
634
+ self.interval = _cfg("interval", float, self.interval, 0.25, float("inf"))
635
+ self.timeout_ms = _cfg("timeout_ms", int, self.timeout_ms, 100, MAX_TIMEOUT_MS)
636
+ self.max_hops = _cfg("max_hops", int, self.max_hops, 1, MAX_HOPS_CEILING)
637
+ self.packet_size = _cfg("packet_size", int, self.packet_size, 0, 1472)
638
+ self.send_delay_ms = _cfg("send_delay_ms", int, self.send_delay_ms, 0, 1000)
639
+ packet_type = _cfg("packet_type", str, self.packet_type)
640
+ self.packet_type = packet_type if packet_type in ("icmp", "tcp", "udp") else "icmp"
641
+ self.port = _cfg("port", int, self.port, 1, 65535)
642
+ self.dscp = _cfg("dscp", int, self.dscp, 0, 63)
643
+ self.source_ip = opt_str(cfg.get("source_ip")) or ""
644
+ self._hops = hops
645
+ self._events.clear()
646
+ self._events.extend(events)
647
+ self._event_seq = max((e.seq + 1 for e in events), default=0)
648
+ self._active_alerts.clear()
649
+ self.status = f"Loaded session: {self.target_input} [{self.target_ip}] - {len(self._hops)} hops"
650
+ self._notify()
651
+
652
+ # --- crash-safe probe log ---------------------------------------------
653
+ def _open_log(self) -> None:
654
+ self._close_log()
655
+ self.log_note = ""
656
+ if not self.log_path:
657
+ return
658
+ try:
659
+ fresh = (not os.path.exists(self.log_path)) or os.path.getsize(self.log_path) == 0
660
+ self._log_fh = open(self.log_path, "a", encoding="utf-8", newline="")
661
+ if fresh:
662
+ self._log_fh.write("epoch,iso_time,round,hop,ip,rtt_ms,status\n")
663
+ self._log_fh.flush()
664
+ except OSError as exc:
665
+ self._log_fh = None
666
+ # NOT self.status: _open_log runs inside start(), and the first
667
+ # thing the worker does is set "Resolving ...", so this survived a
668
+ # few milliseconds and was gone. The run then looked entirely
669
+ # normal while writing nothing -- which is the failure the
670
+ # unattended mode exists to avoid. Same treatment as timeout_note:
671
+ # timestamped in the Events tab, where it stays.
672
+ self.log_note = (f"Probe logging DISABLED - cannot open "
673
+ f"{self.log_path}: {exc}")
674
+
675
+ def _close_log(self) -> None:
676
+ fh = self._log_fh
677
+ self._log_fh = None
678
+ if fh is not None:
679
+ try:
680
+ fh.close()
681
+ except OSError:
682
+ pass
683
+
684
+ def _log_probe(self, ttl: int, r: icmp.PingResult) -> None:
685
+ fh = self._log_fh
686
+ if fh is None:
687
+ return
688
+ now = time.time()
689
+ iso = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(now))
690
+ # Microsecond precision, trailing zeros dropped: the POSIX and TCP/UDP
691
+ # timings are sub-millisecond (".0f" logged a 0.17 ms LAN hop as 0),
692
+ # while Windows ICMP's whole milliseconds still log exactly as before.
693
+ rtt = "" if r.rtt_ms is None else f"{r.rtt_ms:.3f}".rstrip("0").rstrip(".")
694
+ try:
695
+ fh.write(f"{now:.3f},{iso},{self._round},{ttl},{r.address or ''},{rtt},{r.status}\n")
696
+ fh.flush()
697
+ except (OSError, ValueError):
698
+ # ValueError is "I/O operation on closed file". stop() joins the
699
+ # worker with a bounded timeout and then closes the log regardless,
700
+ # so a probe that outlived the join can still be holding this
701
+ # handle. Losing one line to a shutdown race is fine; letting it
702
+ # escape is not — it aborts the round and surfaces as
703
+ # "Monitor error" for what is a benign teardown.
704
+ return
705
+ self._rotate_log_if_needed()
706
+
707
+ def _rotate_log_if_needed(self) -> None:
708
+ """Roll the probe CSV over once it passes MAX_LOG_BYTES so an unattended
709
+ multi-week run can't fill the disk. Keeps a single backup
710
+ (``log.csv`` -> ``log.1.csv``); best-effort, skipped on any error."""
711
+ fh = self._log_fh
712
+ if fh is None:
713
+ return
714
+ try:
715
+ if fh.tell() < MAX_LOG_BYTES:
716
+ return
717
+ except (OSError, ValueError):
718
+ return # same shutdown race as _log_probe: tell() on a closed file
719
+ now = time.monotonic()
720
+ if now < self._rotate_retry_at:
721
+ # A rollover just failed. Keep appending rather than close, fail
722
+ # and reopen on every probe until whatever holds the file lets go.
723
+ return
724
+ self._close_log()
725
+ root, ext = os.path.splitext(self.log_path)
726
+ try:
727
+ # os.replace overwrites an existing backup atomically, so the old
728
+ # backup is only lost if the rollover actually happens.
729
+ os.replace(self.log_path, f"{root}.1{ext}")
730
+ except OSError as exc:
731
+ # On Windows this is the CSV (or its backup) being open in Excel.
732
+ self._rotate_retry_at = now + LOG_ROTATE_RETRY_S
733
+ if not self._rotate_failing:
734
+ self._rotate_failing = True
735
+ self._log_event(
736
+ "warn",
737
+ f"Probe log rollover failed ({exc}); still appending to "
738
+ f"{self.log_path} past the {MAX_LOG_BYTES // (1024 * 1024)} MB "
739
+ f"cap, retrying every {LOG_ROTATE_RETRY_S:.0f}s")
740
+ else:
741
+ self._rotate_retry_at = 0.0
742
+ if self._rotate_failing:
743
+ self._rotate_failing = False
744
+ self._log_event("info", f"Probe log rolled over to {root}.1{ext}")
745
+ self._open_log() # reopen (writes a new header if the rollover happened)
746
+ if self._log_fh is None:
747
+ # _open_log's note is otherwise only read at start-up, so logging
748
+ # would stop here mid-run with nothing saying so.
749
+ self._log_event("warn", self.log_note)
750
+
751
+ # --- worker ------------------------------------------------------------
752
+ def _notify(self) -> None:
753
+ if self.on_update is not None:
754
+ try:
755
+ self.on_update()
756
+ except Exception:
757
+ pass
758
+
759
+ def _set_status(self, text: str, gen: Optional[int] = None) -> None:
760
+ """``gen`` is the caller's run generation: a worker that outlived
761
+ stop()'s join must not overwrite the status of the run replacing it."""
762
+ if gen is not None and gen != self._generation:
763
+ return
764
+ self.status = text
765
+ self._notify()
766
+
767
+ def _log_event(self, kind: str, text: str) -> None:
768
+ with self._lock:
769
+ ev = Event(self._event_seq, time.time(), kind, text)
770
+ self._event_seq += 1
771
+ self._events.append(ev)
772
+ if self.recorder is not None:
773
+ self.recorder.event(self._rec_runs.get(self._generation), ev.t, kind, text)
774
+ if kind == "alert":
775
+ self._beep()
776
+ if kind in ("alert", "clear"):
777
+ self._fire_webhook(kind, text)
778
+
779
+ def _beep(self) -> None:
780
+ if self.alert_sound and winsound is not None:
781
+ try:
782
+ winsound.MessageBeep(winsound.MB_ICONHAND)
783
+ except Exception:
784
+ pass
785
+
786
+ def _fire_webhook(self, kind: str, text: str) -> None:
787
+ """Notify a configured webhook (off the worker thread) when a
788
+ destination alert raises or clears."""
789
+ url = self.webhook_url
790
+ if not url:
791
+ return
792
+ payload = {
793
+ "app": "PingerPlot",
794
+ "event": kind, # "alert" | "clear"
795
+ "target": self.target_input,
796
+ "target_ip": self.target_ip,
797
+ "text": text,
798
+ "time": time.time(),
799
+ }
800
+ threading.Thread(target=self._deliver_webhook, args=(url, payload),
801
+ name="webhook", daemon=True).start()
802
+
803
+ def _deliver_webhook(self, url: str, payload: dict) -> None:
804
+ error = _send_webhook(url, payload)
805
+ if error:
806
+ # "warn", not "alert": it must not beep, and it must not fire the
807
+ # webhook that just failed.
808
+ self._log_event("warn", f"Webhook to {_webhook_host(url)} failed "
809
+ f"({payload.get('event')}): {error}")
810
+
811
+ def _alive(self, gen: int) -> bool:
812
+ """True while ``gen`` is still the active run (and we haven't stopped).
813
+ Workers test this instead of ``running`` so a slow worker from a prior
814
+ start() can't keep mutating state once a new run has begun."""
815
+ return self.running and gen == self._generation
816
+
817
+ def _run(self, gen: int) -> None:
818
+ try:
819
+ self._begin_recording(gen)
820
+ self._set_status(f"Resolving {self.target_input}...", gen)
821
+ try:
822
+ ip = socket.gethostbyname(self.target_input)
823
+ except OSError as exc:
824
+ why = f"Cannot resolve '{self.target_input}': {exc.strerror or exc}"
825
+ # An event as well as the status: a name that stopped
826
+ # resolving at 3 a.m. belongs in the history.
827
+ self._log_event("warn", why)
828
+ self._set_status(why, gen)
829
+ if gen == self._generation:
830
+ self.running = False
831
+ return
832
+ # Resolution is the one step stop()'s join cannot bound: with DNS
833
+ # down it can outlast the join, and a new start() may have begun
834
+ # in the meantime. Its answer is not this run's to record.
835
+ with self._lock:
836
+ if gen != self._generation:
837
+ return
838
+ self.target_ip = ip
839
+ if self.recorder is not None:
840
+ self.recorder.set_target_ip(self._rec_runs.get(gen), ip)
841
+
842
+ if self.timeout_note:
843
+ # Timestamped in the Events tab rather than appended to the
844
+ # status line, which would repeat it on every round.
845
+ self._log_event("info", self.timeout_note)
846
+ if self.interval_note:
847
+ self._log_event("info", self.interval_note)
848
+ if self.log_note:
849
+ self._log_event("warn", self.log_note)
850
+
851
+ # An explicit source pins the outgoing interface -- the point of
852
+ # the setting on a multi-homed box, where the route table would
853
+ # otherwise decide for you and you could not ask "what does this
854
+ # path look like from the other VLAN".
855
+ if self.source_ip:
856
+ # Checked here, in every mode, because each mode otherwise
857
+ # failed with the wrong explanation: ICMP traced nothing and
858
+ # blamed the target, TCP/UDP's capture bind failed and blamed
859
+ # a lack of Administrator.
860
+ problem = tcpudp.source_problem(self.source_ip)
861
+ if problem:
862
+ self._set_status(
863
+ f"Source IP {self.source_ip} is not an address on this machine "
864
+ f"({problem}). Fix the source address in its settings, or leave it blank.", gen)
865
+ if gen == self._generation:
866
+ self.running = False
867
+ return
868
+ self._local_ip = self.source_ip or tcpudp.local_ip_for(self.target_ip)
869
+ needs_capture = self.packet_type != "icmp" and not (
870
+ self.final_hop_only and self.packet_type == "tcp"
871
+ )
872
+ if needs_capture and not tcpudp.capture_supported(self._local_ip):
873
+ kind = self.packet_type.upper()
874
+ # The option names are the Settings page's. Elevation is the
875
+ # fix only where the capture exists: SIO_RCVALL is Windows'.
876
+ if sys.platform == "win32":
877
+ self.needs_admin = True # the window offers to restart elevated
878
+ why = (f"Full {kind} traceroute needs Administrator (raw capture). Run "
879
+ "PingerPlot as administrator, switch")
880
+ else:
881
+ why = (f"Full {kind} traceroute is Windows-only for now: the raw capture it "
882
+ "uses doesn't exist here. Switch")
883
+ self._set_status(why + " to ICMP, or use TCP with Destination only.", gen)
884
+ if gen == self._generation:
885
+ self.running = False
886
+ return
887
+
888
+ if self.final_hop_only:
889
+ with self._lock:
890
+ self._ensure_hop(1)
891
+ self.route_len = 1
892
+ route_len = 1
893
+ else:
894
+ self._set_status(f"Tracing route to {self.target_input} [{self.target_ip}]...", gen)
895
+ route_len = self._trace(gen)
896
+ if not self._alive(gen):
897
+ return
898
+ if route_len == 0:
899
+ self._set_status(f"No reply from {self.target_input} - target may block ICMP or be down.", gen)
900
+ if gen == self._generation:
901
+ self.running = False
902
+ return
903
+
904
+ while self._alive(gen):
905
+ t0 = time.perf_counter()
906
+ if self.paused:
907
+ self._set_status(f"Paused - {self.target_input} [{self.target_ip}] "
908
+ f"({len(self._hops)} hops, history kept)", gen)
909
+ self._notify()
910
+ self._interruptible_sleep(self.interval, gen)
911
+ continue
912
+ route_len = self._check_dns(route_len, gen)
913
+ if not self._alive(gen):
914
+ break
915
+ route_len = self._probe_round(route_len, gen)
916
+ if not self._alive(gen):
917
+ break # stopped or superseded mid-round: judge nothing
918
+ self._record_route(route_len, gen)
919
+ self._evaluate_alerts(route_len)
920
+ self._check_recorder()
921
+ self._set_status(self._monitor_status(route_len), gen)
922
+ self._notify()
923
+ self._interruptible_sleep(self.interval - (time.perf_counter() - t0), gen)
924
+ except Exception as exc: # never let the worker die silently
925
+ self._set_status(f"Monitor error: {exc!r}", gen)
926
+ finally:
927
+ if gen == self._generation: # don't clobber a newer run's flag
928
+ self.running = False
929
+ self._end_recording(gen)
930
+ self._notify()
931
+
932
+ # --- history -------------------------------------------------------------
933
+ def _begin_recording(self, gen: int) -> None:
934
+ if self.recorder is None:
935
+ return
936
+ options = self.start_options()
937
+ # Never into the history file: a webhook URL can carry a secret
938
+ # (Slack, Teams and ntfy put it in the path). Whether one was set is
939
+ # all an investigation needs.
940
+ options["webhook_url"] = bool(options.get("webhook_url"))
941
+ self._rec_runs[gen] = self.recorder.begin_run(
942
+ self.target_input, self.packet_type,
943
+ None if self.packet_type == "icmp" else self.port, options)
944
+ self._check_recorder()
945
+
946
+ # --- following the target's name -----------------------------------------
947
+ def _check_dns(self, route_len: int, gen: int) -> int:
948
+ """Every DNS_RECHECK_S, look a hostname target up again (on the DNS
949
+ pool, so a slow resolver never holds up a round) and follow it if the
950
+ address being probed is no longer one it resolves to."""
951
+ if _is_address(self.target_input) or self._dns_pool is None:
952
+ return route_len
953
+ fut = self._dns_check
954
+ if fut is None:
955
+ if time.monotonic() >= self._dns_next:
956
+ self._dns_check = self._dns_pool.submit(_addresses_of, self.target_input)
957
+ return route_len
958
+ if not fut.done():
959
+ return route_len
960
+ self._dns_check = None
961
+ self._dns_next = time.monotonic() + DNS_RECHECK_S
962
+ try:
963
+ addrs = fut.result()
964
+ except OSError as exc:
965
+ if not self._dns_failing:
966
+ self._dns_failing = True
967
+ self._log_event("warn", f"Can't look up {self.target_input} again "
968
+ f"({exc.strerror or exc}); still probing {self.target_ip}")
969
+ return route_len
970
+ if self._dns_failing:
971
+ self._dns_failing = False
972
+ self._log_event("info", f"{self.target_input} can be looked up again")
973
+ if not addrs or self.target_ip in addrs:
974
+ return route_len
975
+ return self._follow(addrs, gen)
976
+
977
+ def _follow(self, addrs: List[str], gen: int) -> int:
978
+ """The name moved: say so, start a new recorded run for the new
979
+ address, and trace the route to it from scratch."""
980
+ old, new = self.target_ip, addrs[0]
981
+ self._log_event("route", f"{self.target_input} now resolves to {', '.join(addrs)} "
982
+ f"(was {old}); following it to {new}")
983
+ # A run per destination, so every run in the history has one address.
984
+ self._end_recording(gen)
985
+ self._begin_recording(gen)
986
+ if self.recorder is not None:
987
+ self.recorder.set_target_ip(self._rec_runs.get(gen), new)
988
+ with self._lock:
989
+ if gen != self._generation:
990
+ return 0
991
+ self.target_ip = new
992
+ self._hops = []
993
+ self._routes = RouteTracker()
994
+ self._recorded_route = None
995
+ self.reached_target = False
996
+ self._unreached = 0
997
+ self.route_len = 0
998
+ self._retire_alerts(keep_ttl=None, reason=f"{self.target_input} moved to {new}")
999
+ if self.final_hop_only:
1000
+ with self._lock:
1001
+ self._ensure_hop(1)
1002
+ self.route_len = 1
1003
+ return 1
1004
+ self._set_status(f"Tracing route to {self.target_input} [{new}]...", gen)
1005
+ route_len = self._trace(gen)
1006
+ # Nothing answered yet: keep probing every hop, and the round logic
1007
+ # settles the length once the destination replies.
1008
+ return route_len or self.max_hops
1009
+
1010
+ def _record_route(self, route_len: int, gen: int) -> None:
1011
+ """Tell the history when the route differs from the one it has: the
1012
+ first round, a reroute, a newly seen load-balanced router, a route
1013
+ that got longer or shorter."""
1014
+ if self.final_hop_only:
1015
+ return
1016
+ self._routes.trim(route_len)
1017
+ route = self._routes.signature(route_len)
1018
+ if route != self._recorded_route and any(route):
1019
+ self._recorded_route = route
1020
+ if self.recorder is not None:
1021
+ self.recorder.route(self._rec_runs.get(gen), time.time(), route)
1022
+
1023
+ def set_recorder(self, recorder: "Optional[history.Recorder]") -> None:
1024
+ """Start or stop recording a target that may already be running (the
1025
+ Settings page's switch): the run in progress closes in the old
1026
+ recorder and carries on in the new one."""
1027
+ if recorder is self.recorder:
1028
+ return
1029
+ gen = self._generation
1030
+ self._end_recording(gen)
1031
+ self.recorder = recorder
1032
+ self._rec_problem_seen = 0 # say what's wrong with the new one, if anything
1033
+ self._recorded_route = None # so the new one learns the current route
1034
+ if recorder is not None and self.running:
1035
+ self._begin_recording(gen)
1036
+ if self.target_ip:
1037
+ recorder.set_target_ip(self._rec_runs.get(gen), self.target_ip)
1038
+
1039
+ def _end_recording(self, gen: int) -> None:
1040
+ handle = self._rec_runs.pop(gen, None)
1041
+ if self.recorder is not None and handle is not None:
1042
+ self.recorder.end_run(handle)
1043
+
1044
+ def _check_recorder(self) -> None:
1045
+ """Put a new recording problem (or its recovery) in this target's
1046
+ Events, once."""
1047
+ rec = self.recorder
1048
+ if rec is not None and rec.problem_seq != self._rec_problem_seen:
1049
+ self._rec_problem_seen = rec.problem_seq
1050
+ self._log_event(rec.problem_kind, rec.problem)
1051
+
1052
+ def _monitor_status(self, route_len: int) -> str:
1053
+ proto = self.packet_type.upper()
1054
+ if self.packet_type != "icmp":
1055
+ proto += f":{self.port}"
1056
+ if self.dscp:
1057
+ proto += f" DSCP {self.dscp}"
1058
+ if self.source_ip:
1059
+ proto += f" from {self.source_ip}"
1060
+ scope = "destination only" if self.final_hop_only else f"{route_len} hops"
1061
+ base = f"Monitoring {self.target_input} [{self.target_ip}] via {proto} - {scope}"
1062
+ if not self.reached_target:
1063
+ base += " (destination not answering)"
1064
+ n_alerts = len(self._active_alerts)
1065
+ if n_alerts:
1066
+ base += f" - {n_alerts} active alert{'s' if n_alerts != 1 else ''}"
1067
+ return base
1068
+
1069
+ def _ensure_hop(self, ttl: int) -> Hop:
1070
+ # Caller holds the lock. Hops are stored densely, index == ttl-1.
1071
+ while len(self._hops) < ttl:
1072
+ self._hops.append(Hop(len(self._hops) + 1))
1073
+ return self._hops[ttl - 1]
1074
+
1075
+ def _apply_probe(self, ttl: int, r: icmp.PingResult, gen: int) -> None:
1076
+ """Record one probe result and react to any route change at this hop.
1077
+ A result carrying a superseded generation is dropped, so a slow worker
1078
+ from a previous start() cannot corrupt the freshly-reset new run."""
1079
+ with self._lock:
1080
+ if gen != self._generation:
1081
+ return
1082
+ hop = self._ensure_hop(ttl)
1083
+ hop.record(r.rtt_ms, r.address, r.status)
1084
+ handle = self._rec_runs.get(gen)
1085
+ if self.recorder is not None:
1086
+ self.recorder.probe(handle, time.time(), self._round, ttl, r.address, r.rtt_ms, r.status)
1087
+ self._log_probe(ttl, r)
1088
+ if r.reached:
1089
+ self.reached_target = True
1090
+ if r.address:
1091
+ self._maybe_resolve(ttl, r.address)
1092
+
1093
+ def _trace(self, gen: int) -> int:
1094
+ """Walk TTLs upward until the destination answers. Returns route length
1095
+ (the TTL that reached the target), the deepest responding TTL if the
1096
+ destination never replies, or 0 if nothing answered at all."""
1097
+ self._round = 0
1098
+ reached_ttl = 0
1099
+ deepest = 0
1100
+ for ttl in range(1, self.max_hops + 1):
1101
+ if not self._alive(gen):
1102
+ break
1103
+ r = self._do_probe(ttl)
1104
+ self._apply_probe(ttl, r, gen)
1105
+ if r.address:
1106
+ deepest = ttl
1107
+ self._set_status(f"Tracing route... hop {ttl}: {r.address or '*'}", gen)
1108
+ if r.reached:
1109
+ reached_ttl = ttl
1110
+ break
1111
+
1112
+ route_len = reached_ttl or deepest
1113
+ with self._lock:
1114
+ if gen != self._generation:
1115
+ # Superseded: the hop list now belongs to the new run, and an
1116
+ # aborted trace's route_len of 0 would delete all of it.
1117
+ return 0
1118
+ del self._hops[route_len:] # trim trailing all-timeout hops
1119
+ self.route_len = route_len
1120
+ return route_len
1121
+
1122
+ def _do_probe(self, ttl: int, ip_ttl: Optional[int] = None) -> icmp.PingResult:
1123
+ """One probe via the configured backend. ``ip_ttl`` overrides the IP TTL
1124
+ (used for final-hop-only, which pings the destination at full TTL)."""
1125
+ actual_ttl = ttl if ip_ttl is None else ip_ttl
1126
+ if self.packet_type == "icmp":
1127
+ return icmp.ping(self.target_ip, actual_ttl, self.timeout_ms,
1128
+ self._payload, self._tos, self.source_ip or None)
1129
+ return tcpudp.probe(
1130
+ self.target_ip, actual_ttl, self.timeout_ms,
1131
+ self.packet_type, self.port, self._local_ip, self._payload, self._tos,
1132
+ strict_source=bool(self.source_ip),
1133
+ )
1134
+
1135
+ def _gather_range(self, lo: int, hi: int,
1136
+ gen: Optional[int] = None) -> Dict[int, icmp.PingResult]:
1137
+ """Probe TTLs lo..hi. ICMP fans out across the pool (honouring send
1138
+ delay); TCP/UDP go through one parallel round on a shared capture socket
1139
+ (~one timeout total instead of the sum).
1140
+
1141
+ ``gen`` is the caller's run generation, so the submit loop can stop for
1142
+ the same reason the rest of the worker does. Optional only because a
1143
+ test may call this directly with no run in progress."""
1144
+ if self.packet_type != "icmp":
1145
+ return tcpudp.probe_path(self.target_ip, range(lo, hi + 1), self.timeout_ms,
1146
+ self.packet_type, self.port, self._local_ip,
1147
+ self._payload, self._tos,
1148
+ strict_source=bool(self.source_ip))
1149
+ results: Dict[int, icmp.PingResult] = {}
1150
+ if self._ping_pool is None: # pools torn down (e.g. mid-shutdown)
1151
+ return {ttl: icmp.PingResult(icmp.IP_REQ_TIMED_OUT, None, None, False)
1152
+ for ttl in range(lo, hi + 1)}
1153
+ futures = {}
1154
+ for ttl in range(lo, hi + 1):
1155
+ # _alive(gen), not running: a superseded worker sees running ==
1156
+ # True again the moment a new start() flips it, and would keep
1157
+ # submitting probes for a run that is over. _apply_probe's
1158
+ # generation check discards the results, so this only ever cost a
1159
+ # few wasted probes -- but every other loop in the worker uses
1160
+ # _alive, and one that does not is the kind of inconsistency
1161
+ # somebody later reads as significant.
1162
+ if not (self._alive(gen) if gen is not None else self.running):
1163
+ break
1164
+ futures[ttl] = self._ping_pool.submit(self._do_probe, ttl)
1165
+ if self.send_delay_ms and ttl < hi:
1166
+ time.sleep(self.send_delay_ms / 1000.0)
1167
+ for ttl, fut in futures.items():
1168
+ try:
1169
+ results[ttl] = fut.result()
1170
+ except Exception:
1171
+ results[ttl] = icmp.PingResult(icmp.IP_REQ_TIMED_OUT, None, None, False)
1172
+ return results
1173
+
1174
+ def _probe_round(self, route_len: int, gen: int) -> int:
1175
+ """One monitoring round across all hops. Returns the (possibly adjusted)
1176
+ route length so the caller tracks grow/shrink continuously."""
1177
+ self._round += 1
1178
+ if self.final_hop_only:
1179
+ r = self._do_probe(1, ip_ttl=FINAL_HOP_TTL)
1180
+ self._apply_probe(1, r, gen)
1181
+ return 1
1182
+
1183
+ results = self._gather_range(1, route_len, gen)
1184
+ min_reached: Optional[int] = None
1185
+ for ttl in range(1, route_len + 1):
1186
+ if not self._alive(gen): # stop()/superseded during the round:
1187
+ break # don't keep recording a stale round
1188
+ r = results.get(ttl)
1189
+ if r is None:
1190
+ r = icmp.PingResult(icmp.IP_REQ_TIMED_OUT, None, None, False)
1191
+ self._apply_probe(ttl, r, gen)
1192
+ if r.reached and min_reached is None:
1193
+ min_reached = ttl
1194
+
1195
+ if self._alive(gen):
1196
+ for change in self._routes.observe(
1197
+ {ttl: (results[ttl].address if ttl in results else None)
1198
+ for ttl in range(1, route_len + 1)}):
1199
+ self._log_event("route" if change.kind == "reroute" else "info", change.text)
1200
+
1201
+ if min_reached is not None:
1202
+ self._unreached = 0
1203
+ if min_reached < route_len:
1204
+ return self._shrink_route(route_len, min_reached, gen)
1205
+ return route_len
1206
+
1207
+ # Destination missed this round. If we have ever reached it, the route
1208
+ # may have grown; otherwise it is ordinary loss and the alert path deals
1209
+ # with it.
1210
+ if self.reached_target:
1211
+ self._unreached += 1
1212
+ if self._unreached >= UNREACHED_BEFORE_GROW:
1213
+ grown = self._grow_route(route_len, gen)
1214
+ if grown:
1215
+ self._unreached = 0
1216
+ return grown
1217
+ return route_len
1218
+
1219
+ def _shrink_route(self, old_len: int, new_len: int, gen: int) -> int:
1220
+ with self._lock:
1221
+ if gen != self._generation:
1222
+ return old_len
1223
+ del self._hops[new_len:]
1224
+ self.route_len = new_len
1225
+ self._log_event("route", f"Route shortened {old_len} -> {new_len} hops (destination now closer)")
1226
+ return new_len
1227
+
1228
+ def _grow_route(self, route_len: int, gen: int) -> Optional[int]:
1229
+ """Probe past the current end to find the destination's new distance."""
1230
+ upper = min(self.max_hops, route_len + GROW_PROBE_SPAN)
1231
+ if upper <= route_len:
1232
+ return None
1233
+ results = self._gather_range(route_len + 1, upper, gen)
1234
+ reached = [ttl for ttl, r in results.items() if r.reached]
1235
+ if not reached and upper < self.max_hops:
1236
+ results.update(self._gather_range(upper + 1, self.max_hops, gen))
1237
+ reached = [ttl for ttl, r in results.items() if r.reached]
1238
+ if not reached:
1239
+ return None
1240
+ new_len = min(reached)
1241
+ for ttl in range(route_len + 1, new_len + 1):
1242
+ r = results.get(ttl)
1243
+ if r is None:
1244
+ r = self._do_probe(ttl)
1245
+ self._apply_probe(ttl, r, gen)
1246
+ with self._lock:
1247
+ if gen != self._generation:
1248
+ return None
1249
+ del self._hops[new_len:]
1250
+ self.route_len = new_len
1251
+ self._log_event("route", f"Route lengthened {route_len} -> {new_len} hops (destination now farther)")
1252
+ return new_len
1253
+
1254
+ # --- alerts ------------------------------------------------------------
1255
+ def _evaluate_alerts(self, route_len: int) -> None:
1256
+ if not self.alert_enabled or not self.reached_target or route_len < 1:
1257
+ self._clear_all_alerts()
1258
+ return
1259
+ with self._lock:
1260
+ if route_len > len(self._hops):
1261
+ # Returning before _retire_alerts is deliberate. route_len and
1262
+ # _hops are only ever mutated by this same worker thread, so
1263
+ # this is a defensive guard against a state that should not
1264
+ # occur -- and in it there is no destination TTL to keep, so
1265
+ # retiring would have to guess which alerts are stale. Doing
1266
+ # nothing costs at most one extra round of a stale banner and
1267
+ # the next round resolves it, which is strictly better than
1268
+ # guessing wrong. (Raised as F14 by an external audit; kept.)
1269
+ return
1270
+ dest = self._hops[route_len - 1]
1271
+ # Only samples since the destination took this TTL. After a
1272
+ # reroute the TTL's history can belong to a router that held it,
1273
+ # and that router's rate-limited replies are not the destination's
1274
+ # loss. A responder change therefore restarts the "enough history
1275
+ # to call it sustained" wait below.
1276
+ sent_window = min(self.alert_window, dest.sent, dest.since_change)
1277
+ ever = dest.received
1278
+ # One pass for all three: the MOS alert needs jitter as well, and
1279
+ # the E-model wants latency, jitter and loss from the same window.
1280
+ r_loss, r_avg, r_jitter = dest.recent_stats(self.alert_window)
1281
+ ttl = dest.ttl
1282
+ name = dest.hostname or dest.address or "?"
1283
+
1284
+ # Alerts are keyed by TTL, and only the destination's TTL is ever
1285
+ # evaluated. A reroute that changes the path length moves the
1286
+ # destination to a different TTL, orphaning the previous key: nothing
1287
+ # would revisit it, so the banner kept reporting loss on a hop that no
1288
+ # longer exists. Retire those explicitly, before the history check
1289
+ # below — the new destination not having enough history yet is a reason
1290
+ # to say nothing about it, not a reason to keep showing the old one.
1291
+ self._retire_alerts(keep_ttl=ttl, reason="route changed")
1292
+
1293
+ if sent_window < self.alert_window:
1294
+ return # not enough history yet to call anything "sustained"
1295
+
1296
+ loss_active = ever > 0 and self.alert_loss_pct > 0 and r_loss >= self.alert_loss_pct
1297
+ self._set_alert(
1298
+ (ttl, "loss"),
1299
+ loss_active,
1300
+ f"Hop {ttl} ({name}): {r_loss:.0f}% packet loss over last {self.alert_window} probes",
1301
+ )
1302
+
1303
+ lat_active = self.alert_latency_ms > 0 and r_avg is not None and r_avg >= self.alert_latency_ms
1304
+ self._set_alert(
1305
+ (ttl, "lat"),
1306
+ lat_active,
1307
+ f"Hop {ttl} ({name}): avg {r_avg:.0f} ms over last {self.alert_window} probes"
1308
+ if r_avg is not None else "",
1309
+ )
1310
+
1311
+ # MOS folds latency, jitter and loss into one number, so it catches the
1312
+ # combination that ruins a call while each ingredient sits under its own
1313
+ # threshold. Note the direction: 1.0-5.0, and *lower* is worse, so this
1314
+ # one fires at or BELOW its threshold. Gated on ever > 0 like the loss
1315
+ # alert — a destination that has never answered has no score to judge.
1316
+ r_mos = mos(r_avg, r_jitter, r_loss) if ever > 0 else None
1317
+ mos_active = self.alert_mos > 0 and r_mos is not None and r_mos <= self.alert_mos
1318
+ self._set_alert(
1319
+ (ttl, "mos"),
1320
+ mos_active,
1321
+ f"Hop {ttl} ({name}): MOS {r_mos:.1f} ({mos_label(r_mos)}) "
1322
+ f"over last {self.alert_window} probes" if r_mos is not None else "",
1323
+ )
1324
+
1325
+ def _set_alert(self, key: Tuple[int, str], active: bool, text: str) -> None:
1326
+ fire = False
1327
+ cleared: Optional[str] = None
1328
+ with self._lock:
1329
+ present = key in self._active_alerts
1330
+ if active:
1331
+ self._active_alerts[key] = text # set, or refresh the live value
1332
+ fire = not present
1333
+ elif present:
1334
+ cleared = self._active_alerts.pop(key)
1335
+ else:
1336
+ return
1337
+ # Outside the lock, both ways. _log_event beeps and starts a webhook
1338
+ # thread, and neither should happen holding it -- the rule
1339
+ # _retire_alerts states in its own docstring. The clear branch used to
1340
+ # break it, which was not a deadlock (the lock is an RLock and neither
1341
+ # call blocks) but left the next person to add work to _log_event
1342
+ # reasonably assuming an invariant that did not hold everywhere.
1343
+ if cleared is not None:
1344
+ self._log_event("clear", f"Cleared: {cleared}")
1345
+ elif fire:
1346
+ self._log_event("alert", f"ALERT: {text}")
1347
+
1348
+ def _clear_all_alerts(self) -> None:
1349
+ self._retire_alerts(keep_ttl=None)
1350
+
1351
+ def _retire_alerts(self, keep_ttl: Optional[int], reason: str = "") -> None:
1352
+ """Clear active alerts that will never be evaluated again.
1353
+
1354
+ ``keep_ttl=None`` clears everything (alerts switched off, destination
1355
+ no longer reachable). Otherwise only ``keep_ttl`` survives — every
1356
+ other key belongs to a hop that is no longer the destination.
1357
+
1358
+ The lock is released between the pop and the log, because _log_event
1359
+ beeps and fires the webhook and neither should happen holding it.
1360
+ """
1361
+ with self._lock:
1362
+ stale = [k for k in self._active_alerts
1363
+ if keep_ttl is None or k[0] != keep_ttl]
1364
+ suffix = f" ({reason})" if reason else ""
1365
+ for key in stale:
1366
+ with self._lock:
1367
+ cleared = self._active_alerts.pop(key, None)
1368
+ if cleared:
1369
+ self._log_event("clear", f"Cleared{suffix}: {cleared}")
1370
+
1371
+ # --- reverse DNS (best effort, off the probe path) --------------------
1372
+ def _maybe_resolve(self, ttl: int, address: str) -> None:
1373
+ if not self.resolve_names or self._dns_pool is None:
1374
+ return
1375
+ with self._lock:
1376
+ if ttl > len(self._hops):
1377
+ return
1378
+ hop = self._hops[ttl - 1]
1379
+ if hop.resolving or (hop.hostname is not None and hop.address == address):
1380
+ return
1381
+ hop.resolving = True
1382
+ self._dns_pool.submit(self._resolve_worker, ttl, address)
1383
+
1384
+ def _resolve_worker(self, ttl: int, address: str) -> None:
1385
+ try:
1386
+ name = host_name(socket.gethostbyaddr(address)[0]) or ""
1387
+ except OSError:
1388
+ name = "" # no PTR record; remember that so we don't retry forever
1389
+ with self._lock:
1390
+ if ttl <= len(self._hops):
1391
+ hop = self._hops[ttl - 1]
1392
+ if hop.address == address:
1393
+ hop.hostname = name
1394
+ hop.resolving = False
1395
+ if self.recorder is not None:
1396
+ self.recorder.name(address, name)
1397
+
1398
+ def _interruptible_sleep(self, seconds: float, gen: int) -> None:
1399
+ end = time.perf_counter() + max(0.0, seconds)
1400
+ while self._alive(gen):
1401
+ remaining = end - time.perf_counter()
1402
+ if remaining <= 0:
1403
+ break
1404
+ time.sleep(min(0.1, remaining))