pingerplot 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pingerplot/__init__.py +8 -0
- pingerplot/__main__.py +5 -0
- pingerplot/appicon.py +144 -0
- pingerplot/compare.py +257 -0
- pingerplot/ed25519.py +145 -0
- pingerplot/geoip.py +337 -0
- pingerplot/gfx.py +203 -0
- pingerplot/gui.py +1792 -0
- pingerplot/gui_charts.py +409 -0
- pingerplot/gui_history.py +684 -0
- pingerplot/gui_hopinfo.py +125 -0
- pingerplot/gui_recording.py +176 -0
- pingerplot/gui_settings.py +322 -0
- pingerplot/gui_updates.py +270 -0
- pingerplot/gui_widgets.py +248 -0
- pingerplot/headless.py +661 -0
- pingerplot/history.py +823 -0
- pingerplot/icmp.py +305 -0
- pingerplot/icmp_posix.py +456 -0
- pingerplot/investigate.py +205 -0
- pingerplot/launch.py +37 -0
- pingerplot/locating.py +248 -0
- pingerplot/model.py +410 -0
- pingerplot/monitor.py +1404 -0
- pingerplot/places.py +259 -0
- pingerplot/routes.py +112 -0
- pingerplot/selfcheck.py +97 -0
- pingerplot/selftest.py +76 -0
- pingerplot/settings.py +98 -0
- pingerplot/tcpudp.py +479 -0
- pingerplot/theme.py +576 -0
- pingerplot/tkthreads.py +68 -0
- pingerplot/tray.py +373 -0
- pingerplot/trouble.py +129 -0
- pingerplot/update_key.py +8 -0
- pingerplot/updater.py +499 -0
- pingerplot/winplatform.py +366 -0
- pingerplot/worldmap.py +141 -0
- pingerplot/xdg.py +198 -0
- pingerplot-1.6.0.dist-info/METADATA +583 -0
- pingerplot-1.6.0.dist-info/RECORD +45 -0
- pingerplot-1.6.0.dist-info/WHEEL +5 -0
- pingerplot-1.6.0.dist-info/entry_points.txt +5 -0
- pingerplot-1.6.0.dist-info/licenses/LICENSE +21 -0
- pingerplot-1.6.0.dist-info/top_level.txt +1 -0
pingerplot/monitor.py
ADDED
|
@@ -0,0 +1,1404 @@
|
|
|
1
|
+
"""Monitoring engine: discover the route, then probe every hop continuously.
|
|
2
|
+
|
|
3
|
+
Mirrors the MTR model — each round re-probes the destination with an
|
|
4
|
+
increasing TTL (1..route_len). The router at each TTL answers "TTL exceeded",
|
|
5
|
+
so one round is effectively a full traceroute, and we accumulate latency/loss
|
|
6
|
+
per hop over time. The final hop (TTL == route_len) is the destination itself.
|
|
7
|
+
|
|
8
|
+
Because every round is a full retrace, route changes are detected continuously:
|
|
9
|
+
* a hop whose responding address changes -> reroute (logged)
|
|
10
|
+
* the destination answering at a lower TTL -> route shortened
|
|
11
|
+
* the destination no longer answering -> probe deeper for a longer route
|
|
12
|
+
|
|
13
|
+
Sustained-degradation alerts are evaluated on the *destination* hop only — the
|
|
14
|
+
one measurement that isn't muddied by routers that deprioritise/ignore ICMP.
|
|
15
|
+
An alert is gated on the destination having actually been reached at least once,
|
|
16
|
+
so a target that simply blocks ICMP does not raise a false alarm.
|
|
17
|
+
|
|
18
|
+
Threading model:
|
|
19
|
+
* One worker thread runs :meth:`_run` (trace, then the round loop).
|
|
20
|
+
* Probes within a round fan out across a small thread pool.
|
|
21
|
+
* Reverse-DNS lookups run on their own pool; they never block probing.
|
|
22
|
+
* All shared state is guarded by ``self._lock`` (re-entrant). The GUI never
|
|
23
|
+
touches Hop objects directly — it calls the snapshot accessors, which copy.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import ipaddress
|
|
28
|
+
import json
|
|
29
|
+
import math
|
|
30
|
+
import os
|
|
31
|
+
import re
|
|
32
|
+
import socket
|
|
33
|
+
import sys
|
|
34
|
+
import threading
|
|
35
|
+
import time
|
|
36
|
+
import urllib.parse
|
|
37
|
+
import urllib.request
|
|
38
|
+
from collections import deque
|
|
39
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
40
|
+
from typing import Callable, Deque, Dict, List, Optional, Tuple
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
import winsound # Windows stdlib; used for the alert beep
|
|
44
|
+
except ImportError: # pragma: no cover - non-Windows
|
|
45
|
+
winsound = None # type: ignore[assignment]
|
|
46
|
+
|
|
47
|
+
from . import __version__, history, icmp, tcpudp
|
|
48
|
+
from .routes import RouteTracker
|
|
49
|
+
from .model import (DEFAULT_HISTORY, Event, Hop, HopView, Sample, hop_from_session, mos,
|
|
50
|
+
host_name, mos_label, opt_str, plausible_time)
|
|
51
|
+
|
|
52
|
+
GROW_PROBE_SPAN = 8 # extra TTLs to probe when looking for a longer route
|
|
53
|
+
UNREACHED_BEFORE_GROW = 3 # consecutive dest-miss rounds before probing deeper
|
|
54
|
+
MAX_HOPS_CEILING = 64 # hard cap on max_hops, and so on the probe pool width
|
|
55
|
+
FINAL_HOP_TTL = 255 # TTL used to ping the destination directly
|
|
56
|
+
# Windows does not hand a connecting socket the RST the instant it arrives — it
|
|
57
|
+
# finishes retransmitting the SYN first, then surfaces WSAECONNREFUSED. Measured
|
|
58
|
+
# at ~2.0 s on Windows 11. Below that a *closed* port is indistinguishable from
|
|
59
|
+
# an unreachable host, which is the wrong answer in the most common diagnostic
|
|
60
|
+
# case ("the service is down but the box is fine"). Shortening the window with
|
|
61
|
+
# TCP_MAXRT does not help: it replaces WSAECONNREFUSED with WSAETIMEDOUT and
|
|
62
|
+
# destroys the very distinction the probe exists to draw. So we wait it out.
|
|
63
|
+
MAX_TIMEOUT_MS = 10_000 # the Settings field's own ceiling
|
|
64
|
+
TCP_REFUSAL_FLOOR_MS = 3000
|
|
65
|
+
MAX_LOAD_HISTORY = 50_000 # cap a loaded session's per-hop ring buffer (DoS guard)
|
|
66
|
+
MAX_LOG_BYTES = 25 * 1024 * 1024 # roll the probe CSV past ~25 MB (one backup kept)
|
|
67
|
+
LOG_ROTATE_RETRY_S = 60.0 # after a failed rollover, keep appending this long before retrying
|
|
68
|
+
MAX_LOAD_HOPS = 1024 # cap hops loaded from a session file (defense-in-depth)
|
|
69
|
+
# A hostname target is looked up again this often. Big services (Microsoft,
|
|
70
|
+
# CDNs) move clients between front ends by DNS; without this a run kept
|
|
71
|
+
# probing the address it started with and never saw the move.
|
|
72
|
+
DNS_RECHECK_S = 300.0
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
# start() keyword -> the attribute it lands in. One table, so saving a target's
|
|
76
|
+
# options and validating them on restore cannot disagree about what exists.
|
|
77
|
+
_START_OPTIONS = {
|
|
78
|
+
"interval": "interval", "timeout_ms": "timeout_ms", "max_hops": "max_hops",
|
|
79
|
+
"resolve_names": "resolve_names", "packet_size": "packet_size",
|
|
80
|
+
"send_delay_ms": "send_delay_ms", "final_hop_only": "final_hop_only",
|
|
81
|
+
"packet_type": "packet_type", "port": "port", "dscp": "dscp",
|
|
82
|
+
"source_ip": "source_ip", "log_path": "log_path",
|
|
83
|
+
"alert_enabled": "alert_enabled", "alert_loss_pct": "alert_loss_pct",
|
|
84
|
+
"alert_latency_ms": "alert_latency_ms", "alert_window": "alert_window",
|
|
85
|
+
"alert_sound": "alert_sound", "alert_mos": "alert_mos",
|
|
86
|
+
"webhook_url": "webhook_url",
|
|
87
|
+
}
|
|
88
|
+
START_OPTION_KEYS = frozenset(_START_OPTIONS)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _log_suffix(target: str) -> str:
|
|
92
|
+
"""``target`` reduced to characters that are safe in a file name on every
|
|
93
|
+
platform. Hostnames and IPv4 literals pass through unchanged."""
|
|
94
|
+
safe = re.sub(r"[^A-Za-z0-9.-]", "_", target.strip()).strip(".")
|
|
95
|
+
return "_" + (safe or "target")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def per_target_log_path(path: str, target: str) -> str:
|
|
99
|
+
"""``logs/probe.csv`` -> ``logs/probe_<target>.csv``.
|
|
100
|
+
|
|
101
|
+
The probe CSV has no target column, so two monitors sharing one file
|
|
102
|
+
interleave rows that cannot be told apart (the first hops are the same
|
|
103
|
+
gateways for every target), and one monitor's rollover renames the file
|
|
104
|
+
out from under the others. Idempotent, so a path that already carries the
|
|
105
|
+
suffix -- as Edit settings shows it -- is not suffixed twice.
|
|
106
|
+
"""
|
|
107
|
+
if not path:
|
|
108
|
+
return path
|
|
109
|
+
root, ext = os.path.splitext(path)
|
|
110
|
+
suffix = _log_suffix(target)
|
|
111
|
+
if root.endswith(suffix):
|
|
112
|
+
return path
|
|
113
|
+
return f"{root}{suffix}{ext}"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def log_path_template(path: str, target: str) -> str:
|
|
117
|
+
"""Inverse of :func:`per_target_log_path`, for showing the configured path
|
|
118
|
+
back in the Engine dialog."""
|
|
119
|
+
root, ext = os.path.splitext(path)
|
|
120
|
+
suffix = _log_suffix(target)
|
|
121
|
+
return f"{root[:-len(suffix)]}{ext}" if root.endswith(suffix) else path
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _addresses_of(name: str) -> List[str]:
|
|
125
|
+
"""Every IPv4 address ``name`` resolves to, in the resolver's order. The
|
|
126
|
+
whole set, not one: round-robin DNS rotates the order on every lookup,
|
|
127
|
+
and only an address leaving the set is a move."""
|
|
128
|
+
seen: List[str] = []
|
|
129
|
+
for info in socket.getaddrinfo(name, None, socket.AF_INET, socket.SOCK_STREAM):
|
|
130
|
+
addr = info[4][0]
|
|
131
|
+
if addr not in seen:
|
|
132
|
+
seen.append(addr)
|
|
133
|
+
return seen
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _is_address(text: str) -> bool:
|
|
137
|
+
try:
|
|
138
|
+
ipaddress.ip_address(text)
|
|
139
|
+
return True
|
|
140
|
+
except ValueError:
|
|
141
|
+
return False
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def text_option(value, name: str) -> str:
|
|
145
|
+
"""A text option: a string, or nothing. Anything else (a hand-edited
|
|
146
|
+
settings.json with ``"source_ip": 5``) is refused like a bad number, with
|
|
147
|
+
TypeError, so the caller's fall-back to defaults runs; ``.strip()`` on it
|
|
148
|
+
raised AttributeError, which nothing caught, and the window never opened
|
|
149
|
+
(2026-11 review)."""
|
|
150
|
+
if value is None:
|
|
151
|
+
return ""
|
|
152
|
+
if not isinstance(value, str):
|
|
153
|
+
raise TypeError(f"{name} must be text, not {type(value).__name__}")
|
|
154
|
+
return value.strip()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _build_payload(size: int) -> bytes:
|
|
158
|
+
"""An ICMP data payload of ``size`` bytes (repeating, recognisable text)."""
|
|
159
|
+
if size <= 0:
|
|
160
|
+
return b""
|
|
161
|
+
base = b"PingerPlot probe "
|
|
162
|
+
return (base * (size // len(base) + 1))[:size]
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _send_webhook(url: str, payload: dict) -> Optional[str]:
|
|
166
|
+
"""POST ``payload`` as JSON to ``url`` (http/https only). Never raises.
|
|
167
|
+
Returns None when delivered (2xx/3xx), otherwise why not -- so a mistyped
|
|
168
|
+
URL is reported instead of discovered when the alert you needed never
|
|
169
|
+
arrives. Lets the monitor *tell* you when the destination degrades instead
|
|
170
|
+
of only beeping."""
|
|
171
|
+
if not url.lower().startswith(("http://", "https://")):
|
|
172
|
+
return "not an http(s) URL"
|
|
173
|
+
if any(c.isspace() or ord(c) < 0x20 or c == "" for c in url):
|
|
174
|
+
# http.client would refuse it with an InvalidURL that quotes the
|
|
175
|
+
# path, and for Slack, Teams and ntfy the path is the secret.
|
|
176
|
+
return "the URL contains a space or a control character"
|
|
177
|
+
try:
|
|
178
|
+
data = json.dumps(payload).encode("utf-8")
|
|
179
|
+
req = urllib.request.Request(
|
|
180
|
+
url, data=data, method="POST",
|
|
181
|
+
headers={"Content-Type": "application/json", "User-Agent": "PingerPlot"},
|
|
182
|
+
)
|
|
183
|
+
with urllib.request.urlopen(req, timeout=8) as resp:
|
|
184
|
+
status = getattr(resp, "status", 200)
|
|
185
|
+
return None if 200 <= status < 400 else f"HTTP {status}"
|
|
186
|
+
except Exception as exc: # any failure is reported, never raised
|
|
187
|
+
return _scrub_url(str(exc) or type(exc).__name__, url)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _scrub_url(text: str, url: str) -> str:
|
|
191
|
+
"""``text`` without the parts of ``url`` past the host: an error message
|
|
192
|
+
that quotes the URL goes into events, the console and saved sessions."""
|
|
193
|
+
parts = urllib.parse.urlsplit(url)
|
|
194
|
+
secrets = [url, parts.path + ("?" + parts.query if parts.query else ""),
|
|
195
|
+
parts.path, parts.query]
|
|
196
|
+
for secret in sorted({s for s in secrets if len(s) > 1}, key=len, reverse=True):
|
|
197
|
+
text = text.replace(secret, "/...")
|
|
198
|
+
return text
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _webhook_host(url: str) -> str:
|
|
202
|
+
"""Where a webhook points, for messages. Never the whole URL: Slack, Teams
|
|
203
|
+
and ntfy put the secret in the path, and these messages are printed."""
|
|
204
|
+
try:
|
|
205
|
+
host = urllib.parse.urlsplit(url).hostname
|
|
206
|
+
except ValueError:
|
|
207
|
+
host = None
|
|
208
|
+
return host or "(unparseable URL)"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class Monitor:
|
|
212
|
+
def __init__(self, on_update: Optional[Callable[[], None]] = None,
|
|
213
|
+
recorder: "Optional[history.Recorder]" = None) -> None:
|
|
214
|
+
self.on_update = on_update
|
|
215
|
+
# The history database, shared by every target; None records nothing
|
|
216
|
+
# (a loaded session, the self-test, most tests).
|
|
217
|
+
self.recorder = recorder
|
|
218
|
+
# generation -> run handle. Per generation, not one slot: a worker
|
|
219
|
+
# that outlives stop() must still close its own run, after a new
|
|
220
|
+
# start() has already opened the next one.
|
|
221
|
+
self._rec_runs: Dict[int, Optional[int]] = {}
|
|
222
|
+
self._rec_problem_seen = 0
|
|
223
|
+
# Which routers answer at each hop, so a reroute is reported once and
|
|
224
|
+
# a load-balanced hop isn't reported at all (see routes.py).
|
|
225
|
+
self._routes = RouteTracker()
|
|
226
|
+
self._recorded_route: Optional[list] = None
|
|
227
|
+
self._dns_check = None # the pending re-lookup (a Future), if any
|
|
228
|
+
self._dns_next = 0.0 # monotonic time of the next one
|
|
229
|
+
self._dns_failing = False
|
|
230
|
+
|
|
231
|
+
# configuration (set on start)
|
|
232
|
+
self.target_input = ""
|
|
233
|
+
self.target_ip: Optional[str] = None
|
|
234
|
+
self.interval = 2.5
|
|
235
|
+
self.timeout_ms = 1000
|
|
236
|
+
self.max_hops = 30
|
|
237
|
+
self.resolve_names = True
|
|
238
|
+
self.packet_size = 32
|
|
239
|
+
self.send_delay_ms = 0
|
|
240
|
+
self.final_hop_only = False
|
|
241
|
+
self.packet_type = "icmp" # "icmp" | "tcp" | "udp"
|
|
242
|
+
self.port = 443
|
|
243
|
+
self.dscp = 0 # DiffServ code point (0-63); 0 = best effort
|
|
244
|
+
self.source_ip = "" # pin the outgoing interface; "" = let the stack pick
|
|
245
|
+
self._local_ip = "0.0.0.0"
|
|
246
|
+
self._tos = 0 # derived from dscp; the byte that goes on the wire
|
|
247
|
+
self._payload = icmp.DEFAULT_PAYLOAD
|
|
248
|
+
self.log_path = ""
|
|
249
|
+
self.timeout_note = "" # set when start() raises a too-short timeout
|
|
250
|
+
self.needs_admin = False # stopped because full TCP/UDP needs Administrator
|
|
251
|
+
self.interval_note = "" # set when the interval is shorter than a round can be
|
|
252
|
+
self.log_note = "" # set when the probe log could not be opened
|
|
253
|
+
self._log_fh = None
|
|
254
|
+
self._rotate_retry_at = 0.0 # monotonic time before which not to retry a rollover
|
|
255
|
+
self._rotate_failing = False
|
|
256
|
+
self._round = 0 # 0 = initial trace; 1,2,3… = monitoring rounds
|
|
257
|
+
self.alert_enabled = True
|
|
258
|
+
self.alert_loss_pct = 20.0
|
|
259
|
+
self.alert_latency_ms = 250.0
|
|
260
|
+
self.alert_window = 20
|
|
261
|
+
self.alert_sound = True
|
|
262
|
+
self.alert_mos = 0.0 # 0 disables; MOS alerts BELOW this score
|
|
263
|
+
self.webhook_url = ""
|
|
264
|
+
|
|
265
|
+
# live state
|
|
266
|
+
self.status = "Idle"
|
|
267
|
+
self.route_len = 0
|
|
268
|
+
self.reached_target = False
|
|
269
|
+
self.running = False
|
|
270
|
+
self.paused = False
|
|
271
|
+
|
|
272
|
+
self._hops: List[Hop] = []
|
|
273
|
+
self._events: Deque[Event] = deque(maxlen=1000)
|
|
274
|
+
self._event_seq = 0
|
|
275
|
+
self._active_alerts: Dict[Tuple[int, str], str] = {}
|
|
276
|
+
self._unreached = 0
|
|
277
|
+
|
|
278
|
+
self._lock = threading.RLock()
|
|
279
|
+
self._thread: Optional[threading.Thread] = None
|
|
280
|
+
self._generation = 0 # bumped each start(); a worker ignores stale gens
|
|
281
|
+
# Pools are created on start() and torn down on shutdown(); a Monitor
|
|
282
|
+
# that never starts (e.g. the GUI's placeholder) allocates none.
|
|
283
|
+
self._ping_pool: Optional[ThreadPoolExecutor] = None
|
|
284
|
+
self._ping_workers = 0 # width of the live probe pool (see _ensure_pools)
|
|
285
|
+
self._dns_pool: Optional[ThreadPoolExecutor] = None
|
|
286
|
+
|
|
287
|
+
def _ensure_pools(self) -> None:
|
|
288
|
+
"""(Re)create the probe and DNS thread pools if absent. Lets a Monitor
|
|
289
|
+
be reused after shutdown(), and keeps a never-started one pool-free.
|
|
290
|
+
|
|
291
|
+
The probe pool is sized to ``max_hops`` so a whole route fits in ONE
|
|
292
|
+
wave. It used to be a flat 16 while max_hops defaults to 30 and may go
|
|
293
|
+
to 64, so any route past 16 hops silently split into two or more waves
|
|
294
|
+
— and a wave costs a full reply timeout, because a silent router never
|
|
295
|
+
answers. That turned the engine's core promise (a round costs about one
|
|
296
|
+
timeout, not the sum) into a 2x-4x overrun on ordinary internet paths.
|
|
297
|
+
|
|
298
|
+
Sizing to the ceiling is free: ThreadPoolExecutor spawns its threads
|
|
299
|
+
lazily as work is submitted, so a 6-hop route still only ever runs six.
|
|
300
|
+
"""
|
|
301
|
+
want = max(1, min(int(self.max_hops), MAX_HOPS_CEILING))
|
|
302
|
+
if self._ping_pool is not None and self._ping_workers < want:
|
|
303
|
+
# A later start() asked for a longer route than the live pool can
|
|
304
|
+
# probe at once. Replace it rather than quietly serialising; the
|
|
305
|
+
# old one drains its in-flight probes and exits.
|
|
306
|
+
self._ping_pool.shutdown(wait=False)
|
|
307
|
+
self._ping_pool = None
|
|
308
|
+
if self._ping_pool is None:
|
|
309
|
+
self._ping_pool = ThreadPoolExecutor(max_workers=want, thread_name_prefix="ping")
|
|
310
|
+
self._ping_workers = want
|
|
311
|
+
if self._dns_pool is None:
|
|
312
|
+
self._dns_pool = ThreadPoolExecutor(max_workers=8, thread_name_prefix="dns")
|
|
313
|
+
|
|
314
|
+
# --- lifecycle ---------------------------------------------------------
|
|
315
|
+
def start(
|
|
316
|
+
self,
|
|
317
|
+
target: str,
|
|
318
|
+
interval: float = 2.5,
|
|
319
|
+
timeout_ms: int = 1000,
|
|
320
|
+
max_hops: int = 30,
|
|
321
|
+
resolve_names: bool = True,
|
|
322
|
+
packet_size: int = 32,
|
|
323
|
+
send_delay_ms: int = 0,
|
|
324
|
+
final_hop_only: bool = False,
|
|
325
|
+
packet_type: str = "icmp",
|
|
326
|
+
port: int = 443,
|
|
327
|
+
dscp: int = 0,
|
|
328
|
+
source_ip: str = "",
|
|
329
|
+
log_path: str = "",
|
|
330
|
+
alert_enabled: bool = True,
|
|
331
|
+
alert_loss_pct: float = 20.0,
|
|
332
|
+
alert_latency_ms: float = 250.0,
|
|
333
|
+
alert_window: int = 20,
|
|
334
|
+
alert_sound: bool = True,
|
|
335
|
+
alert_mos: float = 0.0,
|
|
336
|
+
webhook_url: str = "",
|
|
337
|
+
) -> None:
|
|
338
|
+
self.stop()
|
|
339
|
+
# Every numeric option, before any is used: int(inf) raises
|
|
340
|
+
# OverflowError, which nothing expects, and an infinite interval or
|
|
341
|
+
# threshold would quietly do nothing. A settings file can hold
|
|
342
|
+
# Infinity or 1e999 (json writes and reads both).
|
|
343
|
+
for name, value in list(locals().items()):
|
|
344
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
345
|
+
raise ValueError(f"{name} must be a finite number, not {value}")
|
|
346
|
+
self.target_input = target.strip()
|
|
347
|
+
self.interval = max(0.25, float(interval))
|
|
348
|
+
# Ceilings as well as floors. A timeout of 100 s made every round, and
|
|
349
|
+
# closing the window, take that long; an alert window longer than the
|
|
350
|
+
# samples a hop keeps (DEFAULT_HISTORY) could never fill, so it
|
|
351
|
+
# silently turned every alert off.
|
|
352
|
+
self.timeout_ms = max(100, min(int(timeout_ms), MAX_TIMEOUT_MS))
|
|
353
|
+
self.max_hops = max(1, min(int(max_hops), MAX_HOPS_CEILING))
|
|
354
|
+
self.resolve_names = bool(resolve_names)
|
|
355
|
+
self.packet_size = max(0, min(int(packet_size), 1472))
|
|
356
|
+
self.send_delay_ms = max(0, min(int(send_delay_ms), 1000))
|
|
357
|
+
self.final_hop_only = bool(final_hop_only)
|
|
358
|
+
self.packet_type = packet_type if packet_type in ("icmp", "tcp", "udp") else "icmp"
|
|
359
|
+
self.port = max(1, min(int(port), 65535))
|
|
360
|
+
self.dscp = max(0, min(int(dscp), 63))
|
|
361
|
+
self._tos = icmp.dscp_to_tos(self.dscp)
|
|
362
|
+
self.source_ip = text_option(source_ip, "source_ip")
|
|
363
|
+
self._payload = _build_payload(self.packet_size)
|
|
364
|
+
self.timeout_note = ""
|
|
365
|
+
self.needs_admin = False
|
|
366
|
+
if self.packet_type == "tcp" and self.timeout_ms < TCP_REFUSAL_FLOOR_MS:
|
|
367
|
+
# Raise it rather than let the probe report a closed port as an
|
|
368
|
+
# unreachable host. Silently overriding a setting the user typed
|
|
369
|
+
# would be its own bug, so record why for the status line.
|
|
370
|
+
self.timeout_note = (
|
|
371
|
+
f"reply timeout raised {self.timeout_ms} -> {TCP_REFUSAL_FLOOR_MS} ms: "
|
|
372
|
+
f"below that, Windows has not yet surfaced a TCP reset and a closed "
|
|
373
|
+
f"port is indistinguishable from an unreachable host"
|
|
374
|
+
)
|
|
375
|
+
self.timeout_ms = TCP_REFUSAL_FLOOR_MS
|
|
376
|
+
self.interval_note = ""
|
|
377
|
+
if self.interval * 1000 < self.timeout_ms:
|
|
378
|
+
# A round waits for every hop, so a silent one holds it to the reply
|
|
379
|
+
# timeout and the next starts straight after. Silent hops are the
|
|
380
|
+
# norm on real paths, and in TCP mode the floor above sets that
|
|
381
|
+
# timeout to 3 s whatever the user typed: say why "every 1 s" isn't.
|
|
382
|
+
self.interval_note = (
|
|
383
|
+
f"probing every {self.interval:g} s, but each round waits up to "
|
|
384
|
+
f"{self.timeout_ms} ms for hops that don't answer, so while one is "
|
|
385
|
+
f"silent the rounds start about every {self.timeout_ms / 1000:g} s"
|
|
386
|
+
)
|
|
387
|
+
self.log_path = text_option(log_path, "log_path")
|
|
388
|
+
self.alert_enabled = bool(alert_enabled)
|
|
389
|
+
self.alert_loss_pct = max(0.0, float(alert_loss_pct))
|
|
390
|
+
self.alert_latency_ms = max(0.0, float(alert_latency_ms))
|
|
391
|
+
self.alert_window = max(1, min(int(alert_window), DEFAULT_HISTORY))
|
|
392
|
+
self.alert_sound = bool(alert_sound)
|
|
393
|
+
# MOS runs 1.0-5.0; anything above 5 would alert permanently.
|
|
394
|
+
self.alert_mos = max(0.0, min(float(alert_mos), 5.0))
|
|
395
|
+
self.webhook_url = text_option(webhook_url, "webhook_url")
|
|
396
|
+
self._rotate_retry_at = 0.0
|
|
397
|
+
self._rotate_failing = False
|
|
398
|
+
with self._lock:
|
|
399
|
+
self._generation += 1
|
|
400
|
+
gen = self._generation
|
|
401
|
+
self._hops = []
|
|
402
|
+
self._routes = RouteTracker()
|
|
403
|
+
self._recorded_route = None
|
|
404
|
+
self._dns_check = None
|
|
405
|
+
self._dns_next = time.monotonic() + DNS_RECHECK_S
|
|
406
|
+
self._dns_failing = False
|
|
407
|
+
self._events.clear()
|
|
408
|
+
self._active_alerts.clear()
|
|
409
|
+
self._event_seq = 0
|
|
410
|
+
self._unreached = 0
|
|
411
|
+
self.route_len = 0
|
|
412
|
+
self.reached_target = False
|
|
413
|
+
self.target_ip = None
|
|
414
|
+
self.paused = False
|
|
415
|
+
self._open_log()
|
|
416
|
+
self._ensure_pools()
|
|
417
|
+
self.running = True
|
|
418
|
+
self._thread = threading.Thread(target=self._run, args=(gen,), name="monitor", daemon=True)
|
|
419
|
+
self._thread.start()
|
|
420
|
+
|
|
421
|
+
def request_stop(self) -> None:
|
|
422
|
+
"""Tell the worker to stop, without waiting for it. To stop several
|
|
423
|
+
monitors, call this on all of them before stop() on each: a worker
|
|
424
|
+
mid-round takes up to a reply timeout to wind down, and signalling
|
|
425
|
+
them together makes those waits overlap instead of adding up."""
|
|
426
|
+
self.running = False
|
|
427
|
+
|
|
428
|
+
def stop(self) -> None:
|
|
429
|
+
self.running = False
|
|
430
|
+
t = self._thread
|
|
431
|
+
if t and t.is_alive() and t is not threading.current_thread():
|
|
432
|
+
t.join(timeout=self.timeout_ms / 1000 + 2)
|
|
433
|
+
self._thread = None
|
|
434
|
+
self._close_log()
|
|
435
|
+
|
|
436
|
+
def shutdown(self) -> None:
|
|
437
|
+
self.stop()
|
|
438
|
+
for pool in (self._ping_pool, self._dns_pool):
|
|
439
|
+
if pool is not None:
|
|
440
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
441
|
+
self._ping_pool = None
|
|
442
|
+
self._ping_workers = 0
|
|
443
|
+
self._dns_pool = None
|
|
444
|
+
|
|
445
|
+
def pause(self) -> None:
|
|
446
|
+
"""Halt probing but keep the worker thread and accumulated history.
|
|
447
|
+
resume() continues without the reset a fresh start() would do."""
|
|
448
|
+
self.paused = True
|
|
449
|
+
|
|
450
|
+
def resume(self) -> None:
|
|
451
|
+
self.paused = False
|
|
452
|
+
|
|
453
|
+
# --- snapshots for the UI thread --------------------------------------
|
|
454
|
+
def snapshot(self) -> Tuple[List[HopView], str, Optional[str], str]:
|
|
455
|
+
with self._lock:
|
|
456
|
+
views = [HopView.of(h) for h in self._hops]
|
|
457
|
+
return views, self.status, self.target_ip, self.target_input
|
|
458
|
+
|
|
459
|
+
def summary(self) -> Tuple[int, Optional[float], Optional[float], Optional[float], bool]:
|
|
460
|
+
"""Cheap sidebar snapshot: only the destination hop's stats, with no
|
|
461
|
+
per-hop HopView allocation (the grid doesn't need the intermediate hops).
|
|
462
|
+
Returns ``(n_hops, dest_loss, dest_avg, dest_jitter, reached)``; the
|
|
463
|
+
dest values are None when there are no hops yet."""
|
|
464
|
+
with self._lock:
|
|
465
|
+
n = len(self._hops)
|
|
466
|
+
if n == 0:
|
|
467
|
+
return (0, None, None, None, self.reached_target)
|
|
468
|
+
_, _, loss, _, avg, _, _, jitter = self._hops[-1].compute()
|
|
469
|
+
return (n, loss, avg, jitter, self.reached_target)
|
|
470
|
+
|
|
471
|
+
def hop_addresses(self) -> List[str]:
|
|
472
|
+
"""The routers answering at each hop now (for looking them up)."""
|
|
473
|
+
with self._lock:
|
|
474
|
+
return [h.address for h in self._hops if h.address]
|
|
475
|
+
|
|
476
|
+
def routers(self) -> List[Tuple[Optional[str], Optional[float]]]:
|
|
477
|
+
"""Each hop's current router as (name, its own best round trip), for
|
|
478
|
+
checking locations (locating.anchors). Cheap: no samples are read."""
|
|
479
|
+
with self._lock:
|
|
480
|
+
return [(h.hostname, h.best_by_addr.get(h.address)) for h in self._hops if h.address]
|
|
481
|
+
|
|
482
|
+
def samples_for(self, ttl: int) -> List[Sample]:
|
|
483
|
+
with self._lock:
|
|
484
|
+
for h in self._hops:
|
|
485
|
+
if h.ttl == ttl:
|
|
486
|
+
return list(h.samples)
|
|
487
|
+
return []
|
|
488
|
+
|
|
489
|
+
def all_samples(self) -> Dict[int, List[Sample]]:
|
|
490
|
+
with self._lock:
|
|
491
|
+
return {h.ttl: list(h.samples) for h in self._hops}
|
|
492
|
+
|
|
493
|
+
def events_after(self, seq: int) -> Tuple[List[Event], int]:
|
|
494
|
+
"""Events with id > ``seq``, plus the newest id seen (for the caller)."""
|
|
495
|
+
with self._lock:
|
|
496
|
+
new = [e for e in self._events if e.seq > seq]
|
|
497
|
+
last = self._event_seq - 1 if self._event_seq else seq
|
|
498
|
+
return new, last
|
|
499
|
+
|
|
500
|
+
def active_alerts(self) -> List[str]:
|
|
501
|
+
with self._lock:
|
|
502
|
+
return list(self._active_alerts.values())
|
|
503
|
+
|
|
504
|
+
def start_options(self) -> dict:
|
|
505
|
+
"""The keyword arguments that reproduce this monitor's configured run
|
|
506
|
+
through :meth:`start` -- what the GUI saves per target, so a relaunch
|
|
507
|
+
restores each target as configured rather than from the toolbar."""
|
|
508
|
+
return {key: getattr(self, attr) for key, attr in _START_OPTIONS.items()}
|
|
509
|
+
|
|
510
|
+
def run_summary(self) -> dict:
|
|
511
|
+
"""Run conditions + the wall-clock window the current samples cover, so
|
|
512
|
+
an export can document exactly how it was produced (the missing context
|
|
513
|
+
that made two runs impossible to compare after the fact)."""
|
|
514
|
+
with self._lock:
|
|
515
|
+
times = [s.t for h in self._hops for s in h.samples]
|
|
516
|
+
return {
|
|
517
|
+
"version": __version__,
|
|
518
|
+
"target_input": self.target_input,
|
|
519
|
+
"target_ip": self.target_ip,
|
|
520
|
+
"packet_type": self.packet_type,
|
|
521
|
+
"port": self.port,
|
|
522
|
+
"interval": self.interval,
|
|
523
|
+
"timeout_ms": self.timeout_ms,
|
|
524
|
+
"packet_size": self.packet_size,
|
|
525
|
+
"send_delay_ms": self.send_delay_ms,
|
|
526
|
+
"max_hops": self.max_hops,
|
|
527
|
+
"dscp": self.dscp,
|
|
528
|
+
"source_ip": self.source_ip,
|
|
529
|
+
"final_hop_only": self.final_hop_only,
|
|
530
|
+
"samples": max((h.sent for h in self._hops), default=0),
|
|
531
|
+
"window_start": min(times) if times else None,
|
|
532
|
+
"window_end": max(times) if times else None,
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
# --- session persistence ----------------------------------------------
|
|
536
|
+
def to_dict(self) -> dict:
|
|
537
|
+
"""Serialisable snapshot of the whole session (config + per-hop samples
|
|
538
|
+
+ events) for Save. The GUI json-dumps this."""
|
|
539
|
+
with self._lock:
|
|
540
|
+
return {
|
|
541
|
+
"app": "PingerPlot",
|
|
542
|
+
"version": __version__,
|
|
543
|
+
"saved_at": time.time(),
|
|
544
|
+
"target_input": self.target_input,
|
|
545
|
+
"target_ip": self.target_ip,
|
|
546
|
+
"route_len": self.route_len,
|
|
547
|
+
"reached_target": self.reached_target,
|
|
548
|
+
"final_hop_only": self.final_hop_only,
|
|
549
|
+
"config": {
|
|
550
|
+
"interval": self.interval,
|
|
551
|
+
"timeout_ms": self.timeout_ms,
|
|
552
|
+
"max_hops": self.max_hops,
|
|
553
|
+
"packet_size": self.packet_size,
|
|
554
|
+
"send_delay_ms": self.send_delay_ms,
|
|
555
|
+
"packet_type": self.packet_type,
|
|
556
|
+
"port": self.port,
|
|
557
|
+
"dscp": self.dscp,
|
|
558
|
+
"source_ip": self.source_ip,
|
|
559
|
+
},
|
|
560
|
+
"hops": [
|
|
561
|
+
{
|
|
562
|
+
"ttl": h.ttl,
|
|
563
|
+
"address": h.address,
|
|
564
|
+
"hostname": h.hostname,
|
|
565
|
+
"last_status": h.last_status,
|
|
566
|
+
"samples": [[round(s.t, 3), s.rtt] for s in h.samples],
|
|
567
|
+
}
|
|
568
|
+
for h in self._hops
|
|
569
|
+
],
|
|
570
|
+
"events": [[e.seq, e.t, e.kind, e.text] for e in self._events],
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
def load_dict(self, data: dict) -> None:
|
|
574
|
+
"""Replace live state with a saved session (for offline review). Stops
|
|
575
|
+
any running monitor first; the result renders through the normal
|
|
576
|
+
snapshot path, just with ``running == False``.
|
|
577
|
+
|
|
578
|
+
The file is not necessarily ours, so every field is type-checked and a
|
|
579
|
+
bad one falls back to its default. Everything is parsed before any
|
|
580
|
+
state changes: one non-numeric config value used to raise after the
|
|
581
|
+
target, address and route length had already been replaced, leaving a
|
|
582
|
+
half-loaded monitor. Raises ValueError only for something that is not
|
|
583
|
+
a session object at all.
|
|
584
|
+
"""
|
|
585
|
+
if not isinstance(data, dict):
|
|
586
|
+
raise ValueError("not a PingerPlot session (expected a JSON object)")
|
|
587
|
+
cfg = data.get("config")
|
|
588
|
+
cfg = cfg if isinstance(cfg, dict) else {}
|
|
589
|
+
|
|
590
|
+
def _cfg(key, conv, current, lo=None, hi=None):
|
|
591
|
+
# Within the same floors and ceilings start() applies: shown and
|
|
592
|
+
# exported as the session's settings, so an interval of Infinity
|
|
593
|
+
# or a port of 99999 would be a header that can't be true.
|
|
594
|
+
try:
|
|
595
|
+
value = conv(cfg[key]) if key in cfg else current
|
|
596
|
+
except (TypeError, ValueError, OverflowError):
|
|
597
|
+
return current
|
|
598
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
599
|
+
return current
|
|
600
|
+
if lo is not None:
|
|
601
|
+
value = max(lo, min(value, hi))
|
|
602
|
+
return value
|
|
603
|
+
|
|
604
|
+
def _int(value) -> int:
|
|
605
|
+
try:
|
|
606
|
+
return int(value or 0)
|
|
607
|
+
except (TypeError, ValueError, OverflowError):
|
|
608
|
+
return 0
|
|
609
|
+
|
|
610
|
+
raw_hops = data.get("hops")
|
|
611
|
+
raw_hops = raw_hops if isinstance(raw_hops, list) else []
|
|
612
|
+
# Clamp the ring buffers so a crafted/corrupt session can't blow up
|
|
613
|
+
# memory; the most recent samples are kept.
|
|
614
|
+
hops = [h for h in (hop_from_session(hd, MAX_LOAD_HISTORY)
|
|
615
|
+
for hd in raw_hops[:MAX_LOAD_HOPS]) if h is not None]
|
|
616
|
+
raw_events = data.get("events")
|
|
617
|
+
events: List[Event] = []
|
|
618
|
+
for ev in (raw_events if isinstance(raw_events, list) else []):
|
|
619
|
+
try:
|
|
620
|
+
event = Event(int(ev[0]), float(ev[1]), str(ev[2]), str(ev[3]))
|
|
621
|
+
except (IndexError, KeyError, ValueError, TypeError, OverflowError):
|
|
622
|
+
continue
|
|
623
|
+
if plausible_time(event.t):
|
|
624
|
+
events.append(event)
|
|
625
|
+
target = data.get("target_input")
|
|
626
|
+
|
|
627
|
+
self.stop()
|
|
628
|
+
with self._lock:
|
|
629
|
+
self.target_input = "" if target is None else str(target)
|
|
630
|
+
self.target_ip = opt_str(data.get("target_ip"))
|
|
631
|
+
self.route_len = _int(data.get("route_len"))
|
|
632
|
+
self.reached_target = bool(data.get("reached_target", False))
|
|
633
|
+
self.final_hop_only = bool(data.get("final_hop_only", False))
|
|
634
|
+
self.interval = _cfg("interval", float, self.interval, 0.25, float("inf"))
|
|
635
|
+
self.timeout_ms = _cfg("timeout_ms", int, self.timeout_ms, 100, MAX_TIMEOUT_MS)
|
|
636
|
+
self.max_hops = _cfg("max_hops", int, self.max_hops, 1, MAX_HOPS_CEILING)
|
|
637
|
+
self.packet_size = _cfg("packet_size", int, self.packet_size, 0, 1472)
|
|
638
|
+
self.send_delay_ms = _cfg("send_delay_ms", int, self.send_delay_ms, 0, 1000)
|
|
639
|
+
packet_type = _cfg("packet_type", str, self.packet_type)
|
|
640
|
+
self.packet_type = packet_type if packet_type in ("icmp", "tcp", "udp") else "icmp"
|
|
641
|
+
self.port = _cfg("port", int, self.port, 1, 65535)
|
|
642
|
+
self.dscp = _cfg("dscp", int, self.dscp, 0, 63)
|
|
643
|
+
self.source_ip = opt_str(cfg.get("source_ip")) or ""
|
|
644
|
+
self._hops = hops
|
|
645
|
+
self._events.clear()
|
|
646
|
+
self._events.extend(events)
|
|
647
|
+
self._event_seq = max((e.seq + 1 for e in events), default=0)
|
|
648
|
+
self._active_alerts.clear()
|
|
649
|
+
self.status = f"Loaded session: {self.target_input} [{self.target_ip}] - {len(self._hops)} hops"
|
|
650
|
+
self._notify()
|
|
651
|
+
|
|
652
|
+
# --- crash-safe probe log ---------------------------------------------
|
|
653
|
+
def _open_log(self) -> None:
|
|
654
|
+
self._close_log()
|
|
655
|
+
self.log_note = ""
|
|
656
|
+
if not self.log_path:
|
|
657
|
+
return
|
|
658
|
+
try:
|
|
659
|
+
fresh = (not os.path.exists(self.log_path)) or os.path.getsize(self.log_path) == 0
|
|
660
|
+
self._log_fh = open(self.log_path, "a", encoding="utf-8", newline="")
|
|
661
|
+
if fresh:
|
|
662
|
+
self._log_fh.write("epoch,iso_time,round,hop,ip,rtt_ms,status\n")
|
|
663
|
+
self._log_fh.flush()
|
|
664
|
+
except OSError as exc:
|
|
665
|
+
self._log_fh = None
|
|
666
|
+
# NOT self.status: _open_log runs inside start(), and the first
|
|
667
|
+
# thing the worker does is set "Resolving ...", so this survived a
|
|
668
|
+
# few milliseconds and was gone. The run then looked entirely
|
|
669
|
+
# normal while writing nothing -- which is the failure the
|
|
670
|
+
# unattended mode exists to avoid. Same treatment as timeout_note:
|
|
671
|
+
# timestamped in the Events tab, where it stays.
|
|
672
|
+
self.log_note = (f"Probe logging DISABLED - cannot open "
|
|
673
|
+
f"{self.log_path}: {exc}")
|
|
674
|
+
|
|
675
|
+
def _close_log(self) -> None:
|
|
676
|
+
fh = self._log_fh
|
|
677
|
+
self._log_fh = None
|
|
678
|
+
if fh is not None:
|
|
679
|
+
try:
|
|
680
|
+
fh.close()
|
|
681
|
+
except OSError:
|
|
682
|
+
pass
|
|
683
|
+
|
|
684
|
+
def _log_probe(self, ttl: int, r: icmp.PingResult) -> None:
|
|
685
|
+
fh = self._log_fh
|
|
686
|
+
if fh is None:
|
|
687
|
+
return
|
|
688
|
+
now = time.time()
|
|
689
|
+
iso = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(now))
|
|
690
|
+
# Microsecond precision, trailing zeros dropped: the POSIX and TCP/UDP
|
|
691
|
+
# timings are sub-millisecond (".0f" logged a 0.17 ms LAN hop as 0),
|
|
692
|
+
# while Windows ICMP's whole milliseconds still log exactly as before.
|
|
693
|
+
rtt = "" if r.rtt_ms is None else f"{r.rtt_ms:.3f}".rstrip("0").rstrip(".")
|
|
694
|
+
try:
|
|
695
|
+
fh.write(f"{now:.3f},{iso},{self._round},{ttl},{r.address or ''},{rtt},{r.status}\n")
|
|
696
|
+
fh.flush()
|
|
697
|
+
except (OSError, ValueError):
|
|
698
|
+
# ValueError is "I/O operation on closed file". stop() joins the
|
|
699
|
+
# worker with a bounded timeout and then closes the log regardless,
|
|
700
|
+
# so a probe that outlived the join can still be holding this
|
|
701
|
+
# handle. Losing one line to a shutdown race is fine; letting it
|
|
702
|
+
# escape is not — it aborts the round and surfaces as
|
|
703
|
+
# "Monitor error" for what is a benign teardown.
|
|
704
|
+
return
|
|
705
|
+
self._rotate_log_if_needed()
|
|
706
|
+
|
|
707
|
+
def _rotate_log_if_needed(self) -> None:
|
|
708
|
+
"""Roll the probe CSV over once it passes MAX_LOG_BYTES so an unattended
|
|
709
|
+
multi-week run can't fill the disk. Keeps a single backup
|
|
710
|
+
(``log.csv`` -> ``log.1.csv``); best-effort, skipped on any error."""
|
|
711
|
+
fh = self._log_fh
|
|
712
|
+
if fh is None:
|
|
713
|
+
return
|
|
714
|
+
try:
|
|
715
|
+
if fh.tell() < MAX_LOG_BYTES:
|
|
716
|
+
return
|
|
717
|
+
except (OSError, ValueError):
|
|
718
|
+
return # same shutdown race as _log_probe: tell() on a closed file
|
|
719
|
+
now = time.monotonic()
|
|
720
|
+
if now < self._rotate_retry_at:
|
|
721
|
+
# A rollover just failed. Keep appending rather than close, fail
|
|
722
|
+
# and reopen on every probe until whatever holds the file lets go.
|
|
723
|
+
return
|
|
724
|
+
self._close_log()
|
|
725
|
+
root, ext = os.path.splitext(self.log_path)
|
|
726
|
+
try:
|
|
727
|
+
# os.replace overwrites an existing backup atomically, so the old
|
|
728
|
+
# backup is only lost if the rollover actually happens.
|
|
729
|
+
os.replace(self.log_path, f"{root}.1{ext}")
|
|
730
|
+
except OSError as exc:
|
|
731
|
+
# On Windows this is the CSV (or its backup) being open in Excel.
|
|
732
|
+
self._rotate_retry_at = now + LOG_ROTATE_RETRY_S
|
|
733
|
+
if not self._rotate_failing:
|
|
734
|
+
self._rotate_failing = True
|
|
735
|
+
self._log_event(
|
|
736
|
+
"warn",
|
|
737
|
+
f"Probe log rollover failed ({exc}); still appending to "
|
|
738
|
+
f"{self.log_path} past the {MAX_LOG_BYTES // (1024 * 1024)} MB "
|
|
739
|
+
f"cap, retrying every {LOG_ROTATE_RETRY_S:.0f}s")
|
|
740
|
+
else:
|
|
741
|
+
self._rotate_retry_at = 0.0
|
|
742
|
+
if self._rotate_failing:
|
|
743
|
+
self._rotate_failing = False
|
|
744
|
+
self._log_event("info", f"Probe log rolled over to {root}.1{ext}")
|
|
745
|
+
self._open_log() # reopen (writes a new header if the rollover happened)
|
|
746
|
+
if self._log_fh is None:
|
|
747
|
+
# _open_log's note is otherwise only read at start-up, so logging
|
|
748
|
+
# would stop here mid-run with nothing saying so.
|
|
749
|
+
self._log_event("warn", self.log_note)
|
|
750
|
+
|
|
751
|
+
# --- worker ------------------------------------------------------------
|
|
752
|
+
def _notify(self) -> None:
|
|
753
|
+
if self.on_update is not None:
|
|
754
|
+
try:
|
|
755
|
+
self.on_update()
|
|
756
|
+
except Exception:
|
|
757
|
+
pass
|
|
758
|
+
|
|
759
|
+
def _set_status(self, text: str, gen: Optional[int] = None) -> None:
|
|
760
|
+
"""``gen`` is the caller's run generation: a worker that outlived
|
|
761
|
+
stop()'s join must not overwrite the status of the run replacing it."""
|
|
762
|
+
if gen is not None and gen != self._generation:
|
|
763
|
+
return
|
|
764
|
+
self.status = text
|
|
765
|
+
self._notify()
|
|
766
|
+
|
|
767
|
+
def _log_event(self, kind: str, text: str) -> None:
|
|
768
|
+
with self._lock:
|
|
769
|
+
ev = Event(self._event_seq, time.time(), kind, text)
|
|
770
|
+
self._event_seq += 1
|
|
771
|
+
self._events.append(ev)
|
|
772
|
+
if self.recorder is not None:
|
|
773
|
+
self.recorder.event(self._rec_runs.get(self._generation), ev.t, kind, text)
|
|
774
|
+
if kind == "alert":
|
|
775
|
+
self._beep()
|
|
776
|
+
if kind in ("alert", "clear"):
|
|
777
|
+
self._fire_webhook(kind, text)
|
|
778
|
+
|
|
779
|
+
def _beep(self) -> None:
|
|
780
|
+
if self.alert_sound and winsound is not None:
|
|
781
|
+
try:
|
|
782
|
+
winsound.MessageBeep(winsound.MB_ICONHAND)
|
|
783
|
+
except Exception:
|
|
784
|
+
pass
|
|
785
|
+
|
|
786
|
+
def _fire_webhook(self, kind: str, text: str) -> None:
|
|
787
|
+
"""Notify a configured webhook (off the worker thread) when a
|
|
788
|
+
destination alert raises or clears."""
|
|
789
|
+
url = self.webhook_url
|
|
790
|
+
if not url:
|
|
791
|
+
return
|
|
792
|
+
payload = {
|
|
793
|
+
"app": "PingerPlot",
|
|
794
|
+
"event": kind, # "alert" | "clear"
|
|
795
|
+
"target": self.target_input,
|
|
796
|
+
"target_ip": self.target_ip,
|
|
797
|
+
"text": text,
|
|
798
|
+
"time": time.time(),
|
|
799
|
+
}
|
|
800
|
+
threading.Thread(target=self._deliver_webhook, args=(url, payload),
|
|
801
|
+
name="webhook", daemon=True).start()
|
|
802
|
+
|
|
803
|
+
def _deliver_webhook(self, url: str, payload: dict) -> None:
|
|
804
|
+
error = _send_webhook(url, payload)
|
|
805
|
+
if error:
|
|
806
|
+
# "warn", not "alert": it must not beep, and it must not fire the
|
|
807
|
+
# webhook that just failed.
|
|
808
|
+
self._log_event("warn", f"Webhook to {_webhook_host(url)} failed "
|
|
809
|
+
f"({payload.get('event')}): {error}")
|
|
810
|
+
|
|
811
|
+
def _alive(self, gen: int) -> bool:
|
|
812
|
+
"""True while ``gen`` is still the active run (and we haven't stopped).
|
|
813
|
+
Workers test this instead of ``running`` so a slow worker from a prior
|
|
814
|
+
start() can't keep mutating state once a new run has begun."""
|
|
815
|
+
return self.running and gen == self._generation
|
|
816
|
+
|
|
817
|
+
def _run(self, gen: int) -> None:
|
|
818
|
+
try:
|
|
819
|
+
self._begin_recording(gen)
|
|
820
|
+
self._set_status(f"Resolving {self.target_input}...", gen)
|
|
821
|
+
try:
|
|
822
|
+
ip = socket.gethostbyname(self.target_input)
|
|
823
|
+
except OSError as exc:
|
|
824
|
+
why = f"Cannot resolve '{self.target_input}': {exc.strerror or exc}"
|
|
825
|
+
# An event as well as the status: a name that stopped
|
|
826
|
+
# resolving at 3 a.m. belongs in the history.
|
|
827
|
+
self._log_event("warn", why)
|
|
828
|
+
self._set_status(why, gen)
|
|
829
|
+
if gen == self._generation:
|
|
830
|
+
self.running = False
|
|
831
|
+
return
|
|
832
|
+
# Resolution is the one step stop()'s join cannot bound: with DNS
|
|
833
|
+
# down it can outlast the join, and a new start() may have begun
|
|
834
|
+
# in the meantime. Its answer is not this run's to record.
|
|
835
|
+
with self._lock:
|
|
836
|
+
if gen != self._generation:
|
|
837
|
+
return
|
|
838
|
+
self.target_ip = ip
|
|
839
|
+
if self.recorder is not None:
|
|
840
|
+
self.recorder.set_target_ip(self._rec_runs.get(gen), ip)
|
|
841
|
+
|
|
842
|
+
if self.timeout_note:
|
|
843
|
+
# Timestamped in the Events tab rather than appended to the
|
|
844
|
+
# status line, which would repeat it on every round.
|
|
845
|
+
self._log_event("info", self.timeout_note)
|
|
846
|
+
if self.interval_note:
|
|
847
|
+
self._log_event("info", self.interval_note)
|
|
848
|
+
if self.log_note:
|
|
849
|
+
self._log_event("warn", self.log_note)
|
|
850
|
+
|
|
851
|
+
# An explicit source pins the outgoing interface -- the point of
|
|
852
|
+
# the setting on a multi-homed box, where the route table would
|
|
853
|
+
# otherwise decide for you and you could not ask "what does this
|
|
854
|
+
# path look like from the other VLAN".
|
|
855
|
+
if self.source_ip:
|
|
856
|
+
# Checked here, in every mode, because each mode otherwise
|
|
857
|
+
# failed with the wrong explanation: ICMP traced nothing and
|
|
858
|
+
# blamed the target, TCP/UDP's capture bind failed and blamed
|
|
859
|
+
# a lack of Administrator.
|
|
860
|
+
problem = tcpudp.source_problem(self.source_ip)
|
|
861
|
+
if problem:
|
|
862
|
+
self._set_status(
|
|
863
|
+
f"Source IP {self.source_ip} is not an address on this machine "
|
|
864
|
+
f"({problem}). Fix the source address in its settings, or leave it blank.", gen)
|
|
865
|
+
if gen == self._generation:
|
|
866
|
+
self.running = False
|
|
867
|
+
return
|
|
868
|
+
self._local_ip = self.source_ip or tcpudp.local_ip_for(self.target_ip)
|
|
869
|
+
needs_capture = self.packet_type != "icmp" and not (
|
|
870
|
+
self.final_hop_only and self.packet_type == "tcp"
|
|
871
|
+
)
|
|
872
|
+
if needs_capture and not tcpudp.capture_supported(self._local_ip):
|
|
873
|
+
kind = self.packet_type.upper()
|
|
874
|
+
# The option names are the Settings page's. Elevation is the
|
|
875
|
+
# fix only where the capture exists: SIO_RCVALL is Windows'.
|
|
876
|
+
if sys.platform == "win32":
|
|
877
|
+
self.needs_admin = True # the window offers to restart elevated
|
|
878
|
+
why = (f"Full {kind} traceroute needs Administrator (raw capture). Run "
|
|
879
|
+
"PingerPlot as administrator, switch")
|
|
880
|
+
else:
|
|
881
|
+
why = (f"Full {kind} traceroute is Windows-only for now: the raw capture it "
|
|
882
|
+
"uses doesn't exist here. Switch")
|
|
883
|
+
self._set_status(why + " to ICMP, or use TCP with Destination only.", gen)
|
|
884
|
+
if gen == self._generation:
|
|
885
|
+
self.running = False
|
|
886
|
+
return
|
|
887
|
+
|
|
888
|
+
if self.final_hop_only:
|
|
889
|
+
with self._lock:
|
|
890
|
+
self._ensure_hop(1)
|
|
891
|
+
self.route_len = 1
|
|
892
|
+
route_len = 1
|
|
893
|
+
else:
|
|
894
|
+
self._set_status(f"Tracing route to {self.target_input} [{self.target_ip}]...", gen)
|
|
895
|
+
route_len = self._trace(gen)
|
|
896
|
+
if not self._alive(gen):
|
|
897
|
+
return
|
|
898
|
+
if route_len == 0:
|
|
899
|
+
self._set_status(f"No reply from {self.target_input} - target may block ICMP or be down.", gen)
|
|
900
|
+
if gen == self._generation:
|
|
901
|
+
self.running = False
|
|
902
|
+
return
|
|
903
|
+
|
|
904
|
+
while self._alive(gen):
|
|
905
|
+
t0 = time.perf_counter()
|
|
906
|
+
if self.paused:
|
|
907
|
+
self._set_status(f"Paused - {self.target_input} [{self.target_ip}] "
|
|
908
|
+
f"({len(self._hops)} hops, history kept)", gen)
|
|
909
|
+
self._notify()
|
|
910
|
+
self._interruptible_sleep(self.interval, gen)
|
|
911
|
+
continue
|
|
912
|
+
route_len = self._check_dns(route_len, gen)
|
|
913
|
+
if not self._alive(gen):
|
|
914
|
+
break
|
|
915
|
+
route_len = self._probe_round(route_len, gen)
|
|
916
|
+
if not self._alive(gen):
|
|
917
|
+
break # stopped or superseded mid-round: judge nothing
|
|
918
|
+
self._record_route(route_len, gen)
|
|
919
|
+
self._evaluate_alerts(route_len)
|
|
920
|
+
self._check_recorder()
|
|
921
|
+
self._set_status(self._monitor_status(route_len), gen)
|
|
922
|
+
self._notify()
|
|
923
|
+
self._interruptible_sleep(self.interval - (time.perf_counter() - t0), gen)
|
|
924
|
+
except Exception as exc: # never let the worker die silently
|
|
925
|
+
self._set_status(f"Monitor error: {exc!r}", gen)
|
|
926
|
+
finally:
|
|
927
|
+
if gen == self._generation: # don't clobber a newer run's flag
|
|
928
|
+
self.running = False
|
|
929
|
+
self._end_recording(gen)
|
|
930
|
+
self._notify()
|
|
931
|
+
|
|
932
|
+
# --- history -------------------------------------------------------------
|
|
933
|
+
def _begin_recording(self, gen: int) -> None:
|
|
934
|
+
if self.recorder is None:
|
|
935
|
+
return
|
|
936
|
+
options = self.start_options()
|
|
937
|
+
# Never into the history file: a webhook URL can carry a secret
|
|
938
|
+
# (Slack, Teams and ntfy put it in the path). Whether one was set is
|
|
939
|
+
# all an investigation needs.
|
|
940
|
+
options["webhook_url"] = bool(options.get("webhook_url"))
|
|
941
|
+
self._rec_runs[gen] = self.recorder.begin_run(
|
|
942
|
+
self.target_input, self.packet_type,
|
|
943
|
+
None if self.packet_type == "icmp" else self.port, options)
|
|
944
|
+
self._check_recorder()
|
|
945
|
+
|
|
946
|
+
# --- following the target's name -----------------------------------------
|
|
947
|
+
def _check_dns(self, route_len: int, gen: int) -> int:
|
|
948
|
+
"""Every DNS_RECHECK_S, look a hostname target up again (on the DNS
|
|
949
|
+
pool, so a slow resolver never holds up a round) and follow it if the
|
|
950
|
+
address being probed is no longer one it resolves to."""
|
|
951
|
+
if _is_address(self.target_input) or self._dns_pool is None:
|
|
952
|
+
return route_len
|
|
953
|
+
fut = self._dns_check
|
|
954
|
+
if fut is None:
|
|
955
|
+
if time.monotonic() >= self._dns_next:
|
|
956
|
+
self._dns_check = self._dns_pool.submit(_addresses_of, self.target_input)
|
|
957
|
+
return route_len
|
|
958
|
+
if not fut.done():
|
|
959
|
+
return route_len
|
|
960
|
+
self._dns_check = None
|
|
961
|
+
self._dns_next = time.monotonic() + DNS_RECHECK_S
|
|
962
|
+
try:
|
|
963
|
+
addrs = fut.result()
|
|
964
|
+
except OSError as exc:
|
|
965
|
+
if not self._dns_failing:
|
|
966
|
+
self._dns_failing = True
|
|
967
|
+
self._log_event("warn", f"Can't look up {self.target_input} again "
|
|
968
|
+
f"({exc.strerror or exc}); still probing {self.target_ip}")
|
|
969
|
+
return route_len
|
|
970
|
+
if self._dns_failing:
|
|
971
|
+
self._dns_failing = False
|
|
972
|
+
self._log_event("info", f"{self.target_input} can be looked up again")
|
|
973
|
+
if not addrs or self.target_ip in addrs:
|
|
974
|
+
return route_len
|
|
975
|
+
return self._follow(addrs, gen)
|
|
976
|
+
|
|
977
|
+
def _follow(self, addrs: List[str], gen: int) -> int:
|
|
978
|
+
"""The name moved: say so, start a new recorded run for the new
|
|
979
|
+
address, and trace the route to it from scratch."""
|
|
980
|
+
old, new = self.target_ip, addrs[0]
|
|
981
|
+
self._log_event("route", f"{self.target_input} now resolves to {', '.join(addrs)} "
|
|
982
|
+
f"(was {old}); following it to {new}")
|
|
983
|
+
# A run per destination, so every run in the history has one address.
|
|
984
|
+
self._end_recording(gen)
|
|
985
|
+
self._begin_recording(gen)
|
|
986
|
+
if self.recorder is not None:
|
|
987
|
+
self.recorder.set_target_ip(self._rec_runs.get(gen), new)
|
|
988
|
+
with self._lock:
|
|
989
|
+
if gen != self._generation:
|
|
990
|
+
return 0
|
|
991
|
+
self.target_ip = new
|
|
992
|
+
self._hops = []
|
|
993
|
+
self._routes = RouteTracker()
|
|
994
|
+
self._recorded_route = None
|
|
995
|
+
self.reached_target = False
|
|
996
|
+
self._unreached = 0
|
|
997
|
+
self.route_len = 0
|
|
998
|
+
self._retire_alerts(keep_ttl=None, reason=f"{self.target_input} moved to {new}")
|
|
999
|
+
if self.final_hop_only:
|
|
1000
|
+
with self._lock:
|
|
1001
|
+
self._ensure_hop(1)
|
|
1002
|
+
self.route_len = 1
|
|
1003
|
+
return 1
|
|
1004
|
+
self._set_status(f"Tracing route to {self.target_input} [{new}]...", gen)
|
|
1005
|
+
route_len = self._trace(gen)
|
|
1006
|
+
# Nothing answered yet: keep probing every hop, and the round logic
|
|
1007
|
+
# settles the length once the destination replies.
|
|
1008
|
+
return route_len or self.max_hops
|
|
1009
|
+
|
|
1010
|
+
def _record_route(self, route_len: int, gen: int) -> None:
|
|
1011
|
+
"""Tell the history when the route differs from the one it has: the
|
|
1012
|
+
first round, a reroute, a newly seen load-balanced router, a route
|
|
1013
|
+
that got longer or shorter."""
|
|
1014
|
+
if self.final_hop_only:
|
|
1015
|
+
return
|
|
1016
|
+
self._routes.trim(route_len)
|
|
1017
|
+
route = self._routes.signature(route_len)
|
|
1018
|
+
if route != self._recorded_route and any(route):
|
|
1019
|
+
self._recorded_route = route
|
|
1020
|
+
if self.recorder is not None:
|
|
1021
|
+
self.recorder.route(self._rec_runs.get(gen), time.time(), route)
|
|
1022
|
+
|
|
1023
|
+
def set_recorder(self, recorder: "Optional[history.Recorder]") -> None:
|
|
1024
|
+
"""Start or stop recording a target that may already be running (the
|
|
1025
|
+
Settings page's switch): the run in progress closes in the old
|
|
1026
|
+
recorder and carries on in the new one."""
|
|
1027
|
+
if recorder is self.recorder:
|
|
1028
|
+
return
|
|
1029
|
+
gen = self._generation
|
|
1030
|
+
self._end_recording(gen)
|
|
1031
|
+
self.recorder = recorder
|
|
1032
|
+
self._rec_problem_seen = 0 # say what's wrong with the new one, if anything
|
|
1033
|
+
self._recorded_route = None # so the new one learns the current route
|
|
1034
|
+
if recorder is not None and self.running:
|
|
1035
|
+
self._begin_recording(gen)
|
|
1036
|
+
if self.target_ip:
|
|
1037
|
+
recorder.set_target_ip(self._rec_runs.get(gen), self.target_ip)
|
|
1038
|
+
|
|
1039
|
+
def _end_recording(self, gen: int) -> None:
|
|
1040
|
+
handle = self._rec_runs.pop(gen, None)
|
|
1041
|
+
if self.recorder is not None and handle is not None:
|
|
1042
|
+
self.recorder.end_run(handle)
|
|
1043
|
+
|
|
1044
|
+
def _check_recorder(self) -> None:
|
|
1045
|
+
"""Put a new recording problem (or its recovery) in this target's
|
|
1046
|
+
Events, once."""
|
|
1047
|
+
rec = self.recorder
|
|
1048
|
+
if rec is not None and rec.problem_seq != self._rec_problem_seen:
|
|
1049
|
+
self._rec_problem_seen = rec.problem_seq
|
|
1050
|
+
self._log_event(rec.problem_kind, rec.problem)
|
|
1051
|
+
|
|
1052
|
+
def _monitor_status(self, route_len: int) -> str:
|
|
1053
|
+
proto = self.packet_type.upper()
|
|
1054
|
+
if self.packet_type != "icmp":
|
|
1055
|
+
proto += f":{self.port}"
|
|
1056
|
+
if self.dscp:
|
|
1057
|
+
proto += f" DSCP {self.dscp}"
|
|
1058
|
+
if self.source_ip:
|
|
1059
|
+
proto += f" from {self.source_ip}"
|
|
1060
|
+
scope = "destination only" if self.final_hop_only else f"{route_len} hops"
|
|
1061
|
+
base = f"Monitoring {self.target_input} [{self.target_ip}] via {proto} - {scope}"
|
|
1062
|
+
if not self.reached_target:
|
|
1063
|
+
base += " (destination not answering)"
|
|
1064
|
+
n_alerts = len(self._active_alerts)
|
|
1065
|
+
if n_alerts:
|
|
1066
|
+
base += f" - {n_alerts} active alert{'s' if n_alerts != 1 else ''}"
|
|
1067
|
+
return base
|
|
1068
|
+
|
|
1069
|
+
def _ensure_hop(self, ttl: int) -> Hop:
|
|
1070
|
+
# Caller holds the lock. Hops are stored densely, index == ttl-1.
|
|
1071
|
+
while len(self._hops) < ttl:
|
|
1072
|
+
self._hops.append(Hop(len(self._hops) + 1))
|
|
1073
|
+
return self._hops[ttl - 1]
|
|
1074
|
+
|
|
1075
|
+
def _apply_probe(self, ttl: int, r: icmp.PingResult, gen: int) -> None:
|
|
1076
|
+
"""Record one probe result and react to any route change at this hop.
|
|
1077
|
+
A result carrying a superseded generation is dropped, so a slow worker
|
|
1078
|
+
from a previous start() cannot corrupt the freshly-reset new run."""
|
|
1079
|
+
with self._lock:
|
|
1080
|
+
if gen != self._generation:
|
|
1081
|
+
return
|
|
1082
|
+
hop = self._ensure_hop(ttl)
|
|
1083
|
+
hop.record(r.rtt_ms, r.address, r.status)
|
|
1084
|
+
handle = self._rec_runs.get(gen)
|
|
1085
|
+
if self.recorder is not None:
|
|
1086
|
+
self.recorder.probe(handle, time.time(), self._round, ttl, r.address, r.rtt_ms, r.status)
|
|
1087
|
+
self._log_probe(ttl, r)
|
|
1088
|
+
if r.reached:
|
|
1089
|
+
self.reached_target = True
|
|
1090
|
+
if r.address:
|
|
1091
|
+
self._maybe_resolve(ttl, r.address)
|
|
1092
|
+
|
|
1093
|
+
def _trace(self, gen: int) -> int:
|
|
1094
|
+
"""Walk TTLs upward until the destination answers. Returns route length
|
|
1095
|
+
(the TTL that reached the target), the deepest responding TTL if the
|
|
1096
|
+
destination never replies, or 0 if nothing answered at all."""
|
|
1097
|
+
self._round = 0
|
|
1098
|
+
reached_ttl = 0
|
|
1099
|
+
deepest = 0
|
|
1100
|
+
for ttl in range(1, self.max_hops + 1):
|
|
1101
|
+
if not self._alive(gen):
|
|
1102
|
+
break
|
|
1103
|
+
r = self._do_probe(ttl)
|
|
1104
|
+
self._apply_probe(ttl, r, gen)
|
|
1105
|
+
if r.address:
|
|
1106
|
+
deepest = ttl
|
|
1107
|
+
self._set_status(f"Tracing route... hop {ttl}: {r.address or '*'}", gen)
|
|
1108
|
+
if r.reached:
|
|
1109
|
+
reached_ttl = ttl
|
|
1110
|
+
break
|
|
1111
|
+
|
|
1112
|
+
route_len = reached_ttl or deepest
|
|
1113
|
+
with self._lock:
|
|
1114
|
+
if gen != self._generation:
|
|
1115
|
+
# Superseded: the hop list now belongs to the new run, and an
|
|
1116
|
+
# aborted trace's route_len of 0 would delete all of it.
|
|
1117
|
+
return 0
|
|
1118
|
+
del self._hops[route_len:] # trim trailing all-timeout hops
|
|
1119
|
+
self.route_len = route_len
|
|
1120
|
+
return route_len
|
|
1121
|
+
|
|
1122
|
+
def _do_probe(self, ttl: int, ip_ttl: Optional[int] = None) -> icmp.PingResult:
|
|
1123
|
+
"""One probe via the configured backend. ``ip_ttl`` overrides the IP TTL
|
|
1124
|
+
(used for final-hop-only, which pings the destination at full TTL)."""
|
|
1125
|
+
actual_ttl = ttl if ip_ttl is None else ip_ttl
|
|
1126
|
+
if self.packet_type == "icmp":
|
|
1127
|
+
return icmp.ping(self.target_ip, actual_ttl, self.timeout_ms,
|
|
1128
|
+
self._payload, self._tos, self.source_ip or None)
|
|
1129
|
+
return tcpudp.probe(
|
|
1130
|
+
self.target_ip, actual_ttl, self.timeout_ms,
|
|
1131
|
+
self.packet_type, self.port, self._local_ip, self._payload, self._tos,
|
|
1132
|
+
strict_source=bool(self.source_ip),
|
|
1133
|
+
)
|
|
1134
|
+
|
|
1135
|
+
def _gather_range(self, lo: int, hi: int,
|
|
1136
|
+
gen: Optional[int] = None) -> Dict[int, icmp.PingResult]:
|
|
1137
|
+
"""Probe TTLs lo..hi. ICMP fans out across the pool (honouring send
|
|
1138
|
+
delay); TCP/UDP go through one parallel round on a shared capture socket
|
|
1139
|
+
(~one timeout total instead of the sum).
|
|
1140
|
+
|
|
1141
|
+
``gen`` is the caller's run generation, so the submit loop can stop for
|
|
1142
|
+
the same reason the rest of the worker does. Optional only because a
|
|
1143
|
+
test may call this directly with no run in progress."""
|
|
1144
|
+
if self.packet_type != "icmp":
|
|
1145
|
+
return tcpudp.probe_path(self.target_ip, range(lo, hi + 1), self.timeout_ms,
|
|
1146
|
+
self.packet_type, self.port, self._local_ip,
|
|
1147
|
+
self._payload, self._tos,
|
|
1148
|
+
strict_source=bool(self.source_ip))
|
|
1149
|
+
results: Dict[int, icmp.PingResult] = {}
|
|
1150
|
+
if self._ping_pool is None: # pools torn down (e.g. mid-shutdown)
|
|
1151
|
+
return {ttl: icmp.PingResult(icmp.IP_REQ_TIMED_OUT, None, None, False)
|
|
1152
|
+
for ttl in range(lo, hi + 1)}
|
|
1153
|
+
futures = {}
|
|
1154
|
+
for ttl in range(lo, hi + 1):
|
|
1155
|
+
# _alive(gen), not running: a superseded worker sees running ==
|
|
1156
|
+
# True again the moment a new start() flips it, and would keep
|
|
1157
|
+
# submitting probes for a run that is over. _apply_probe's
|
|
1158
|
+
# generation check discards the results, so this only ever cost a
|
|
1159
|
+
# few wasted probes -- but every other loop in the worker uses
|
|
1160
|
+
# _alive, and one that does not is the kind of inconsistency
|
|
1161
|
+
# somebody later reads as significant.
|
|
1162
|
+
if not (self._alive(gen) if gen is not None else self.running):
|
|
1163
|
+
break
|
|
1164
|
+
futures[ttl] = self._ping_pool.submit(self._do_probe, ttl)
|
|
1165
|
+
if self.send_delay_ms and ttl < hi:
|
|
1166
|
+
time.sleep(self.send_delay_ms / 1000.0)
|
|
1167
|
+
for ttl, fut in futures.items():
|
|
1168
|
+
try:
|
|
1169
|
+
results[ttl] = fut.result()
|
|
1170
|
+
except Exception:
|
|
1171
|
+
results[ttl] = icmp.PingResult(icmp.IP_REQ_TIMED_OUT, None, None, False)
|
|
1172
|
+
return results
|
|
1173
|
+
|
|
1174
|
+
def _probe_round(self, route_len: int, gen: int) -> int:
|
|
1175
|
+
"""One monitoring round across all hops. Returns the (possibly adjusted)
|
|
1176
|
+
route length so the caller tracks grow/shrink continuously."""
|
|
1177
|
+
self._round += 1
|
|
1178
|
+
if self.final_hop_only:
|
|
1179
|
+
r = self._do_probe(1, ip_ttl=FINAL_HOP_TTL)
|
|
1180
|
+
self._apply_probe(1, r, gen)
|
|
1181
|
+
return 1
|
|
1182
|
+
|
|
1183
|
+
results = self._gather_range(1, route_len, gen)
|
|
1184
|
+
min_reached: Optional[int] = None
|
|
1185
|
+
for ttl in range(1, route_len + 1):
|
|
1186
|
+
if not self._alive(gen): # stop()/superseded during the round:
|
|
1187
|
+
break # don't keep recording a stale round
|
|
1188
|
+
r = results.get(ttl)
|
|
1189
|
+
if r is None:
|
|
1190
|
+
r = icmp.PingResult(icmp.IP_REQ_TIMED_OUT, None, None, False)
|
|
1191
|
+
self._apply_probe(ttl, r, gen)
|
|
1192
|
+
if r.reached and min_reached is None:
|
|
1193
|
+
min_reached = ttl
|
|
1194
|
+
|
|
1195
|
+
if self._alive(gen):
|
|
1196
|
+
for change in self._routes.observe(
|
|
1197
|
+
{ttl: (results[ttl].address if ttl in results else None)
|
|
1198
|
+
for ttl in range(1, route_len + 1)}):
|
|
1199
|
+
self._log_event("route" if change.kind == "reroute" else "info", change.text)
|
|
1200
|
+
|
|
1201
|
+
if min_reached is not None:
|
|
1202
|
+
self._unreached = 0
|
|
1203
|
+
if min_reached < route_len:
|
|
1204
|
+
return self._shrink_route(route_len, min_reached, gen)
|
|
1205
|
+
return route_len
|
|
1206
|
+
|
|
1207
|
+
# Destination missed this round. If we have ever reached it, the route
|
|
1208
|
+
# may have grown; otherwise it is ordinary loss and the alert path deals
|
|
1209
|
+
# with it.
|
|
1210
|
+
if self.reached_target:
|
|
1211
|
+
self._unreached += 1
|
|
1212
|
+
if self._unreached >= UNREACHED_BEFORE_GROW:
|
|
1213
|
+
grown = self._grow_route(route_len, gen)
|
|
1214
|
+
if grown:
|
|
1215
|
+
self._unreached = 0
|
|
1216
|
+
return grown
|
|
1217
|
+
return route_len
|
|
1218
|
+
|
|
1219
|
+
def _shrink_route(self, old_len: int, new_len: int, gen: int) -> int:
|
|
1220
|
+
with self._lock:
|
|
1221
|
+
if gen != self._generation:
|
|
1222
|
+
return old_len
|
|
1223
|
+
del self._hops[new_len:]
|
|
1224
|
+
self.route_len = new_len
|
|
1225
|
+
self._log_event("route", f"Route shortened {old_len} -> {new_len} hops (destination now closer)")
|
|
1226
|
+
return new_len
|
|
1227
|
+
|
|
1228
|
+
def _grow_route(self, route_len: int, gen: int) -> Optional[int]:
|
|
1229
|
+
"""Probe past the current end to find the destination's new distance."""
|
|
1230
|
+
upper = min(self.max_hops, route_len + GROW_PROBE_SPAN)
|
|
1231
|
+
if upper <= route_len:
|
|
1232
|
+
return None
|
|
1233
|
+
results = self._gather_range(route_len + 1, upper, gen)
|
|
1234
|
+
reached = [ttl for ttl, r in results.items() if r.reached]
|
|
1235
|
+
if not reached and upper < self.max_hops:
|
|
1236
|
+
results.update(self._gather_range(upper + 1, self.max_hops, gen))
|
|
1237
|
+
reached = [ttl for ttl, r in results.items() if r.reached]
|
|
1238
|
+
if not reached:
|
|
1239
|
+
return None
|
|
1240
|
+
new_len = min(reached)
|
|
1241
|
+
for ttl in range(route_len + 1, new_len + 1):
|
|
1242
|
+
r = results.get(ttl)
|
|
1243
|
+
if r is None:
|
|
1244
|
+
r = self._do_probe(ttl)
|
|
1245
|
+
self._apply_probe(ttl, r, gen)
|
|
1246
|
+
with self._lock:
|
|
1247
|
+
if gen != self._generation:
|
|
1248
|
+
return None
|
|
1249
|
+
del self._hops[new_len:]
|
|
1250
|
+
self.route_len = new_len
|
|
1251
|
+
self._log_event("route", f"Route lengthened {route_len} -> {new_len} hops (destination now farther)")
|
|
1252
|
+
return new_len
|
|
1253
|
+
|
|
1254
|
+
# --- alerts ------------------------------------------------------------
|
|
1255
|
+
def _evaluate_alerts(self, route_len: int) -> None:
|
|
1256
|
+
if not self.alert_enabled or not self.reached_target or route_len < 1:
|
|
1257
|
+
self._clear_all_alerts()
|
|
1258
|
+
return
|
|
1259
|
+
with self._lock:
|
|
1260
|
+
if route_len > len(self._hops):
|
|
1261
|
+
# Returning before _retire_alerts is deliberate. route_len and
|
|
1262
|
+
# _hops are only ever mutated by this same worker thread, so
|
|
1263
|
+
# this is a defensive guard against a state that should not
|
|
1264
|
+
# occur -- and in it there is no destination TTL to keep, so
|
|
1265
|
+
# retiring would have to guess which alerts are stale. Doing
|
|
1266
|
+
# nothing costs at most one extra round of a stale banner and
|
|
1267
|
+
# the next round resolves it, which is strictly better than
|
|
1268
|
+
# guessing wrong. (Raised as F14 by an external audit; kept.)
|
|
1269
|
+
return
|
|
1270
|
+
dest = self._hops[route_len - 1]
|
|
1271
|
+
# Only samples since the destination took this TTL. After a
|
|
1272
|
+
# reroute the TTL's history can belong to a router that held it,
|
|
1273
|
+
# and that router's rate-limited replies are not the destination's
|
|
1274
|
+
# loss. A responder change therefore restarts the "enough history
|
|
1275
|
+
# to call it sustained" wait below.
|
|
1276
|
+
sent_window = min(self.alert_window, dest.sent, dest.since_change)
|
|
1277
|
+
ever = dest.received
|
|
1278
|
+
# One pass for all three: the MOS alert needs jitter as well, and
|
|
1279
|
+
# the E-model wants latency, jitter and loss from the same window.
|
|
1280
|
+
r_loss, r_avg, r_jitter = dest.recent_stats(self.alert_window)
|
|
1281
|
+
ttl = dest.ttl
|
|
1282
|
+
name = dest.hostname or dest.address or "?"
|
|
1283
|
+
|
|
1284
|
+
# Alerts are keyed by TTL, and only the destination's TTL is ever
|
|
1285
|
+
# evaluated. A reroute that changes the path length moves the
|
|
1286
|
+
# destination to a different TTL, orphaning the previous key: nothing
|
|
1287
|
+
# would revisit it, so the banner kept reporting loss on a hop that no
|
|
1288
|
+
# longer exists. Retire those explicitly, before the history check
|
|
1289
|
+
# below — the new destination not having enough history yet is a reason
|
|
1290
|
+
# to say nothing about it, not a reason to keep showing the old one.
|
|
1291
|
+
self._retire_alerts(keep_ttl=ttl, reason="route changed")
|
|
1292
|
+
|
|
1293
|
+
if sent_window < self.alert_window:
|
|
1294
|
+
return # not enough history yet to call anything "sustained"
|
|
1295
|
+
|
|
1296
|
+
loss_active = ever > 0 and self.alert_loss_pct > 0 and r_loss >= self.alert_loss_pct
|
|
1297
|
+
self._set_alert(
|
|
1298
|
+
(ttl, "loss"),
|
|
1299
|
+
loss_active,
|
|
1300
|
+
f"Hop {ttl} ({name}): {r_loss:.0f}% packet loss over last {self.alert_window} probes",
|
|
1301
|
+
)
|
|
1302
|
+
|
|
1303
|
+
lat_active = self.alert_latency_ms > 0 and r_avg is not None and r_avg >= self.alert_latency_ms
|
|
1304
|
+
self._set_alert(
|
|
1305
|
+
(ttl, "lat"),
|
|
1306
|
+
lat_active,
|
|
1307
|
+
f"Hop {ttl} ({name}): avg {r_avg:.0f} ms over last {self.alert_window} probes"
|
|
1308
|
+
if r_avg is not None else "",
|
|
1309
|
+
)
|
|
1310
|
+
|
|
1311
|
+
# MOS folds latency, jitter and loss into one number, so it catches the
|
|
1312
|
+
# combination that ruins a call while each ingredient sits under its own
|
|
1313
|
+
# threshold. Note the direction: 1.0-5.0, and *lower* is worse, so this
|
|
1314
|
+
# one fires at or BELOW its threshold. Gated on ever > 0 like the loss
|
|
1315
|
+
# alert — a destination that has never answered has no score to judge.
|
|
1316
|
+
r_mos = mos(r_avg, r_jitter, r_loss) if ever > 0 else None
|
|
1317
|
+
mos_active = self.alert_mos > 0 and r_mos is not None and r_mos <= self.alert_mos
|
|
1318
|
+
self._set_alert(
|
|
1319
|
+
(ttl, "mos"),
|
|
1320
|
+
mos_active,
|
|
1321
|
+
f"Hop {ttl} ({name}): MOS {r_mos:.1f} ({mos_label(r_mos)}) "
|
|
1322
|
+
f"over last {self.alert_window} probes" if r_mos is not None else "",
|
|
1323
|
+
)
|
|
1324
|
+
|
|
1325
|
+
def _set_alert(self, key: Tuple[int, str], active: bool, text: str) -> None:
|
|
1326
|
+
fire = False
|
|
1327
|
+
cleared: Optional[str] = None
|
|
1328
|
+
with self._lock:
|
|
1329
|
+
present = key in self._active_alerts
|
|
1330
|
+
if active:
|
|
1331
|
+
self._active_alerts[key] = text # set, or refresh the live value
|
|
1332
|
+
fire = not present
|
|
1333
|
+
elif present:
|
|
1334
|
+
cleared = self._active_alerts.pop(key)
|
|
1335
|
+
else:
|
|
1336
|
+
return
|
|
1337
|
+
# Outside the lock, both ways. _log_event beeps and starts a webhook
|
|
1338
|
+
# thread, and neither should happen holding it -- the rule
|
|
1339
|
+
# _retire_alerts states in its own docstring. The clear branch used to
|
|
1340
|
+
# break it, which was not a deadlock (the lock is an RLock and neither
|
|
1341
|
+
# call blocks) but left the next person to add work to _log_event
|
|
1342
|
+
# reasonably assuming an invariant that did not hold everywhere.
|
|
1343
|
+
if cleared is not None:
|
|
1344
|
+
self._log_event("clear", f"Cleared: {cleared}")
|
|
1345
|
+
elif fire:
|
|
1346
|
+
self._log_event("alert", f"ALERT: {text}")
|
|
1347
|
+
|
|
1348
|
+
def _clear_all_alerts(self) -> None:
|
|
1349
|
+
self._retire_alerts(keep_ttl=None)
|
|
1350
|
+
|
|
1351
|
+
def _retire_alerts(self, keep_ttl: Optional[int], reason: str = "") -> None:
|
|
1352
|
+
"""Clear active alerts that will never be evaluated again.
|
|
1353
|
+
|
|
1354
|
+
``keep_ttl=None`` clears everything (alerts switched off, destination
|
|
1355
|
+
no longer reachable). Otherwise only ``keep_ttl`` survives — every
|
|
1356
|
+
other key belongs to a hop that is no longer the destination.
|
|
1357
|
+
|
|
1358
|
+
The lock is released between the pop and the log, because _log_event
|
|
1359
|
+
beeps and fires the webhook and neither should happen holding it.
|
|
1360
|
+
"""
|
|
1361
|
+
with self._lock:
|
|
1362
|
+
stale = [k for k in self._active_alerts
|
|
1363
|
+
if keep_ttl is None or k[0] != keep_ttl]
|
|
1364
|
+
suffix = f" ({reason})" if reason else ""
|
|
1365
|
+
for key in stale:
|
|
1366
|
+
with self._lock:
|
|
1367
|
+
cleared = self._active_alerts.pop(key, None)
|
|
1368
|
+
if cleared:
|
|
1369
|
+
self._log_event("clear", f"Cleared{suffix}: {cleared}")
|
|
1370
|
+
|
|
1371
|
+
# --- reverse DNS (best effort, off the probe path) --------------------
|
|
1372
|
+
def _maybe_resolve(self, ttl: int, address: str) -> None:
|
|
1373
|
+
if not self.resolve_names or self._dns_pool is None:
|
|
1374
|
+
return
|
|
1375
|
+
with self._lock:
|
|
1376
|
+
if ttl > len(self._hops):
|
|
1377
|
+
return
|
|
1378
|
+
hop = self._hops[ttl - 1]
|
|
1379
|
+
if hop.resolving or (hop.hostname is not None and hop.address == address):
|
|
1380
|
+
return
|
|
1381
|
+
hop.resolving = True
|
|
1382
|
+
self._dns_pool.submit(self._resolve_worker, ttl, address)
|
|
1383
|
+
|
|
1384
|
+
def _resolve_worker(self, ttl: int, address: str) -> None:
|
|
1385
|
+
try:
|
|
1386
|
+
name = host_name(socket.gethostbyaddr(address)[0]) or ""
|
|
1387
|
+
except OSError:
|
|
1388
|
+
name = "" # no PTR record; remember that so we don't retry forever
|
|
1389
|
+
with self._lock:
|
|
1390
|
+
if ttl <= len(self._hops):
|
|
1391
|
+
hop = self._hops[ttl - 1]
|
|
1392
|
+
if hop.address == address:
|
|
1393
|
+
hop.hostname = name
|
|
1394
|
+
hop.resolving = False
|
|
1395
|
+
if self.recorder is not None:
|
|
1396
|
+
self.recorder.name(address, name)
|
|
1397
|
+
|
|
1398
|
+
def _interruptible_sleep(self, seconds: float, gen: int) -> None:
|
|
1399
|
+
end = time.perf_counter() + max(0.0, seconds)
|
|
1400
|
+
while self._alive(gen):
|
|
1401
|
+
remaining = end - time.perf_counter()
|
|
1402
|
+
if remaining <= 0:
|
|
1403
|
+
break
|
|
1404
|
+
time.sleep(min(0.1, remaining))
|