pingerplot 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pingerplot/__init__.py +8 -0
- pingerplot/__main__.py +5 -0
- pingerplot/appicon.py +144 -0
- pingerplot/compare.py +257 -0
- pingerplot/ed25519.py +145 -0
- pingerplot/geoip.py +337 -0
- pingerplot/gfx.py +203 -0
- pingerplot/gui.py +1792 -0
- pingerplot/gui_charts.py +409 -0
- pingerplot/gui_history.py +684 -0
- pingerplot/gui_hopinfo.py +125 -0
- pingerplot/gui_recording.py +176 -0
- pingerplot/gui_settings.py +322 -0
- pingerplot/gui_updates.py +270 -0
- pingerplot/gui_widgets.py +248 -0
- pingerplot/headless.py +661 -0
- pingerplot/history.py +823 -0
- pingerplot/icmp.py +305 -0
- pingerplot/icmp_posix.py +456 -0
- pingerplot/investigate.py +205 -0
- pingerplot/launch.py +37 -0
- pingerplot/locating.py +248 -0
- pingerplot/model.py +410 -0
- pingerplot/monitor.py +1404 -0
- pingerplot/places.py +259 -0
- pingerplot/routes.py +112 -0
- pingerplot/selfcheck.py +97 -0
- pingerplot/selftest.py +76 -0
- pingerplot/settings.py +98 -0
- pingerplot/tcpudp.py +479 -0
- pingerplot/theme.py +576 -0
- pingerplot/tkthreads.py +68 -0
- pingerplot/tray.py +373 -0
- pingerplot/trouble.py +129 -0
- pingerplot/update_key.py +8 -0
- pingerplot/updater.py +499 -0
- pingerplot/winplatform.py +366 -0
- pingerplot/worldmap.py +141 -0
- pingerplot/xdg.py +198 -0
- pingerplot-1.6.0.dist-info/METADATA +583 -0
- pingerplot-1.6.0.dist-info/RECORD +45 -0
- pingerplot-1.6.0.dist-info/WHEEL +5 -0
- pingerplot-1.6.0.dist-info/entry_points.txt +5 -0
- pingerplot-1.6.0.dist-info/licenses/LICENSE +21 -0
- pingerplot-1.6.0.dist-info/top_level.txt +1 -0
pingerplot/history.py
ADDED
|
@@ -0,0 +1,823 @@
|
|
|
1
|
+
"""The long-term record: everything a run measures, kept on disk.
|
|
2
|
+
|
|
3
|
+
Until 1.5 the only durable record was an opt-in CSV per target, set up by
|
|
4
|
+
typing a path, and everything else (the event list, which router answered at
|
|
5
|
+
which hop, the route a reroute replaced) lived in memory for about 25 minutes.
|
|
6
|
+
"What did the path do at 3 a.m. Tuesday?" had no answer.
|
|
7
|
+
|
|
8
|
+
This keeps one SQLite database (standard library ``sqlite3``) in the settings
|
|
9
|
+
folder, shared by every target and by the GUI and headless runner alike:
|
|
10
|
+
|
|
11
|
+
* ``probes`` -- every probe: when, which hop, which router answered, the RTT.
|
|
12
|
+
The heaviest table (about 25 MB a day for a 15-hop target at 2.5 s), so it
|
|
13
|
+
has the shortest retention.
|
|
14
|
+
* ``minutes`` -- one row per minute per hop per responding router: sent,
|
|
15
|
+
received, min/avg/max RTT and jitter. Small enough to keep for a year, and
|
|
16
|
+
enough to answer "which router, which minute" long after the probes are gone.
|
|
17
|
+
* ``events`` -- every event the run logged: alerts, clears, route changes,
|
|
18
|
+
warnings, notes.
|
|
19
|
+
* ``routes`` -- each distinct path to a target (the routers known at every
|
|
20
|
+
hop), and ``segments``: when a run was on which route. A route that comes
|
|
21
|
+
back is recognised as the same one, so "it flips between these two paths"
|
|
22
|
+
shows as two routes taking turns.
|
|
23
|
+
* ``runs`` -- what was monitored, how, and when.
|
|
24
|
+
|
|
25
|
+
Writes go through one background thread that batches them into transactions,
|
|
26
|
+
so a probe never waits on the disk. The database runs in WAL mode, so a crash
|
|
27
|
+
loses at most the last batch and readers (the History view) never block the
|
|
28
|
+
writer. Trouble (a damaged file, a full disk, a folder that can't be written)
|
|
29
|
+
is reported through :attr:`Recorder.problem` for the monitors to put in their
|
|
30
|
+
Events, never raised into the probing.
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import itertools
|
|
35
|
+
import json
|
|
36
|
+
import os
|
|
37
|
+
import queue
|
|
38
|
+
import sqlite3
|
|
39
|
+
import stat
|
|
40
|
+
import threading
|
|
41
|
+
import time
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
from typing import Dict, Iterable, List, Optional, Tuple
|
|
44
|
+
|
|
45
|
+
from .routes import compatible
|
|
46
|
+
|
|
47
|
+
SCHEMA_VERSION = 1
|
|
48
|
+
DB_NAME = "history.sqlite3"
|
|
49
|
+
|
|
50
|
+
DEFAULTS = {
|
|
51
|
+
"enabled": True,
|
|
52
|
+
"probes": True, "probe_days": 7,
|
|
53
|
+
"summaries": True, "summary_days": 365,
|
|
54
|
+
"events": True, "event_days": 365,
|
|
55
|
+
"routes": True,
|
|
56
|
+
"max_mb": 1024,
|
|
57
|
+
}
|
|
58
|
+
# (key, lowest, highest): the Settings page's own limits.
|
|
59
|
+
_LIMITS = {"probe_days": (1, 365), "summary_days": (1, 3650), "event_days": (1, 3650),
|
|
60
|
+
"max_mb": (50, 100_000)}
|
|
61
|
+
|
|
62
|
+
_BATCH = 2000 # items per transaction at most
|
|
63
|
+
_MAINTAIN_EVERY_S = 3600 # retention and the size cap, hourly (and at open)
|
|
64
|
+
NO_REPLY = 0 # addrs id for a probe nobody answered
|
|
65
|
+
MAX_QUEUED = 100_000 # probes/events waiting for the writer (~4 h of one 15-hop target)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _schema(value) -> int:
|
|
69
|
+
"""The file's schema number; anything else is a damaged file."""
|
|
70
|
+
try:
|
|
71
|
+
return int(value)
|
|
72
|
+
except (TypeError, ValueError) as exc:
|
|
73
|
+
raise sqlite3.DatabaseError(f"its schema version reads {value!r}") from exc
|
|
74
|
+
|
|
75
|
+
_SCHEMA = """
|
|
76
|
+
CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
77
|
+
CREATE TABLE IF NOT EXISTS runs(
|
|
78
|
+
id INTEGER PRIMARY KEY,
|
|
79
|
+
target TEXT NOT NULL,
|
|
80
|
+
target_ip TEXT,
|
|
81
|
+
mode TEXT NOT NULL,
|
|
82
|
+
port INTEGER,
|
|
83
|
+
options TEXT NOT NULL,
|
|
84
|
+
started REAL NOT NULL,
|
|
85
|
+
ended REAL,
|
|
86
|
+
program TEXT NOT NULL
|
|
87
|
+
);
|
|
88
|
+
CREATE INDEX IF NOT EXISTS runs_target ON runs(target, started);
|
|
89
|
+
CREATE TABLE IF NOT EXISTS addrs(id INTEGER PRIMARY KEY, addr TEXT NOT NULL UNIQUE, name TEXT);
|
|
90
|
+
CREATE TABLE IF NOT EXISTS probes(
|
|
91
|
+
run INTEGER NOT NULL,
|
|
92
|
+
t REAL NOT NULL,
|
|
93
|
+
round INTEGER NOT NULL,
|
|
94
|
+
ttl INTEGER NOT NULL,
|
|
95
|
+
addr INTEGER NOT NULL,
|
|
96
|
+
rtt REAL,
|
|
97
|
+
status INTEGER NOT NULL
|
|
98
|
+
);
|
|
99
|
+
CREATE INDEX IF NOT EXISTS probes_t ON probes(t);
|
|
100
|
+
CREATE TABLE IF NOT EXISTS minutes(
|
|
101
|
+
run INTEGER NOT NULL,
|
|
102
|
+
minute INTEGER NOT NULL,
|
|
103
|
+
ttl INTEGER NOT NULL,
|
|
104
|
+
addr INTEGER NOT NULL,
|
|
105
|
+
sent INTEGER NOT NULL,
|
|
106
|
+
recv INTEGER NOT NULL,
|
|
107
|
+
rtt_sum REAL NOT NULL,
|
|
108
|
+
rtt_min REAL,
|
|
109
|
+
rtt_max REAL,
|
|
110
|
+
jit_sum REAL NOT NULL,
|
|
111
|
+
jit_n INTEGER NOT NULL,
|
|
112
|
+
UNIQUE(run, minute, ttl, addr)
|
|
113
|
+
);
|
|
114
|
+
CREATE INDEX IF NOT EXISTS minutes_minute ON minutes(minute);
|
|
115
|
+
CREATE TABLE IF NOT EXISTS events(
|
|
116
|
+
run INTEGER,
|
|
117
|
+
t REAL NOT NULL,
|
|
118
|
+
kind TEXT NOT NULL,
|
|
119
|
+
text TEXT NOT NULL
|
|
120
|
+
);
|
|
121
|
+
CREATE INDEX IF NOT EXISTS events_t ON events(t);
|
|
122
|
+
CREATE TABLE IF NOT EXISTS routes(
|
|
123
|
+
id INTEGER PRIMARY KEY,
|
|
124
|
+
target TEXT NOT NULL,
|
|
125
|
+
dest TEXT,
|
|
126
|
+
hops TEXT NOT NULL,
|
|
127
|
+
first REAL NOT NULL
|
|
128
|
+
);
|
|
129
|
+
CREATE INDEX IF NOT EXISTS routes_target ON routes(target);
|
|
130
|
+
CREATE TABLE IF NOT EXISTS segments(
|
|
131
|
+
run INTEGER NOT NULL,
|
|
132
|
+
route INTEGER NOT NULL,
|
|
133
|
+
start REAL NOT NULL,
|
|
134
|
+
end REAL
|
|
135
|
+
);
|
|
136
|
+
CREATE INDEX IF NOT EXISTS segments_start ON segments(start);
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def normalize(raw) -> dict:
|
|
141
|
+
"""Recording options from settings.json, every value present and in range.
|
|
142
|
+
|
|
143
|
+
A hand-edited or older settings file must not be able to stop the app or
|
|
144
|
+
turn retention into "delete everything": anything unusable falls back to
|
|
145
|
+
its default."""
|
|
146
|
+
out = dict(DEFAULTS)
|
|
147
|
+
if not isinstance(raw, dict):
|
|
148
|
+
return out
|
|
149
|
+
for key, default in DEFAULTS.items():
|
|
150
|
+
value = raw.get(key, default)
|
|
151
|
+
if isinstance(default, bool):
|
|
152
|
+
out[key] = value if isinstance(value, bool) else default
|
|
153
|
+
continue
|
|
154
|
+
try:
|
|
155
|
+
value = int(value)
|
|
156
|
+
except (TypeError, ValueError, OverflowError):
|
|
157
|
+
continue
|
|
158
|
+
lo, hi = _LIMITS[key]
|
|
159
|
+
out[key] = max(lo, min(value, hi))
|
|
160
|
+
return out
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def default_dir() -> Path:
|
|
164
|
+
from . import settings
|
|
165
|
+
return settings.config_dir() / "history"
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _is_link(path: Path) -> bool:
|
|
169
|
+
"""A symlink, or on Windows any reparse point (a junction too, which
|
|
170
|
+
os.path.islink misses before Python 3.12)."""
|
|
171
|
+
try:
|
|
172
|
+
st = os.lstat(path)
|
|
173
|
+
except OSError:
|
|
174
|
+
return False
|
|
175
|
+
if stat.S_ISLNK(st.st_mode):
|
|
176
|
+
return True
|
|
177
|
+
return bool(getattr(st, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0))
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def disk_bytes(folder: Optional[os.PathLike] = None) -> int:
|
|
181
|
+
"""What the history takes on disk, its write-ahead log included."""
|
|
182
|
+
base = (Path(folder) if folder is not None else default_dir()) / DB_NAME
|
|
183
|
+
total = 0
|
|
184
|
+
for suffix in ("", "-wal", "-shm"):
|
|
185
|
+
try:
|
|
186
|
+
total += os.stat(str(base) + suffix).st_size
|
|
187
|
+
except OSError:
|
|
188
|
+
pass
|
|
189
|
+
return total
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def open_recorder(saved: dict, program: str) -> Optional["Recorder"]:
|
|
193
|
+
"""The recorder the GUI or headless runner should use, from its saved
|
|
194
|
+
settings' ``recording`` section, or None when recording is turned off."""
|
|
195
|
+
options = normalize(saved.get("recording") if isinstance(saved, dict) else None)
|
|
196
|
+
if not options["enabled"]:
|
|
197
|
+
return None
|
|
198
|
+
return Recorder(options=options, program=program).start()
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class _Minute:
|
|
202
|
+
"""Accumulates one (run, minute, ttl, responder) row."""
|
|
203
|
+
__slots__ = ("sent", "recv", "rtt_sum", "rtt_min", "rtt_max", "jit_sum", "jit_n")
|
|
204
|
+
|
|
205
|
+
def __init__(self) -> None:
|
|
206
|
+
self.sent = self.recv = self.jit_n = 0
|
|
207
|
+
self.rtt_sum = self.jit_sum = 0.0
|
|
208
|
+
self.rtt_min: Optional[float] = None
|
|
209
|
+
self.rtt_max: Optional[float] = None
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class Recorder:
|
|
213
|
+
"""Owns the database and the one thread that writes to it.
|
|
214
|
+
|
|
215
|
+
Every public method is safe to call from any thread and returns at once.
|
|
216
|
+
If the database cannot be opened the recorder stays usable and records
|
|
217
|
+
nothing; :attr:`problem` says why.
|
|
218
|
+
"""
|
|
219
|
+
|
|
220
|
+
def __init__(self, folder: Optional[os.PathLike] = None, options: Optional[dict] = None,
|
|
221
|
+
program: str = "gui", clock=time.time) -> None:
|
|
222
|
+
self.folder = Path(folder) if folder is not None else default_dir()
|
|
223
|
+
self.path = self.folder / DB_NAME
|
|
224
|
+
self.options = normalize(options)
|
|
225
|
+
self.program = program
|
|
226
|
+
self._clock = clock
|
|
227
|
+
self._q: "queue.Queue[tuple]" = queue.Queue()
|
|
228
|
+
self._thread: Optional[threading.Thread] = None
|
|
229
|
+
self._lock = threading.Lock()
|
|
230
|
+
self.problem = "" # the latest trouble, for the monitors' Events
|
|
231
|
+
self.problem_kind = "warn" # its event kind ("info" for a recovery)
|
|
232
|
+
self.problem_seq = 0 # bumped with each new problem
|
|
233
|
+
self._closed = False
|
|
234
|
+
self._dropping = False # probes are being dropped (see _put_bulk)
|
|
235
|
+
# A run's handle is issued here, not by the database: waiting for a
|
|
236
|
+
# row id would block the monitor whenever the writer is busy (an
|
|
237
|
+
# hourly clean-up of a large file), and a run that gave up waiting
|
|
238
|
+
# would go unrecorded without a word. The writer maps handles to ids.
|
|
239
|
+
self._handles = itertools.count(1)
|
|
240
|
+
|
|
241
|
+
# --- life cycle ---------------------------------------------------------
|
|
242
|
+
def start(self) -> "Recorder":
|
|
243
|
+
if self._thread is None:
|
|
244
|
+
self._thread = threading.Thread(target=self._writer, name="history", daemon=True)
|
|
245
|
+
self._thread.start()
|
|
246
|
+
return self
|
|
247
|
+
|
|
248
|
+
def close(self, timeout: float = 10.0) -> None:
|
|
249
|
+
"""Write everything queued, the open minutes included, then stop."""
|
|
250
|
+
if self._thread is None or self._closed:
|
|
251
|
+
return
|
|
252
|
+
self._closed = True
|
|
253
|
+
self._q.put(("close",))
|
|
254
|
+
self._thread.join(timeout)
|
|
255
|
+
|
|
256
|
+
def flush(self, timeout: float = 10.0) -> bool:
|
|
257
|
+
"""Wait until everything queued so far is on disk (for tests and for
|
|
258
|
+
reads that must see the latest data)."""
|
|
259
|
+
if self._thread is None or self._closed:
|
|
260
|
+
return False
|
|
261
|
+
done = threading.Event()
|
|
262
|
+
self._q.put(("flush", done))
|
|
263
|
+
return done.wait(timeout)
|
|
264
|
+
|
|
265
|
+
def _report(self, text: str, kind: str = "warn") -> None:
|
|
266
|
+
with self._lock:
|
|
267
|
+
self.problem = text
|
|
268
|
+
self.problem_kind = kind
|
|
269
|
+
self.problem_seq += 1
|
|
270
|
+
|
|
271
|
+
# --- what monitors call -------------------------------------------------
|
|
272
|
+
def set_options(self, options) -> None:
|
|
273
|
+
"""New recording options from the Settings page; they apply from the
|
|
274
|
+
next write (and retention from the next clean-up)."""
|
|
275
|
+
self.options = normalize(options)
|
|
276
|
+
|
|
277
|
+
def begin_run(self, target: str, mode: str, port: Optional[int],
|
|
278
|
+
options: dict) -> Optional[int]:
|
|
279
|
+
"""Register a run and return its handle for the calls below, or None
|
|
280
|
+
when nothing is being recorded."""
|
|
281
|
+
if self._thread is None or self._closed:
|
|
282
|
+
return None
|
|
283
|
+
run = next(self._handles)
|
|
284
|
+
self._q.put(("begin", run, target, mode, port, options, self._clock()))
|
|
285
|
+
return run
|
|
286
|
+
|
|
287
|
+
def set_target_ip(self, run: Optional[int], ip: Optional[str]) -> None:
|
|
288
|
+
if run is not None:
|
|
289
|
+
self._q.put(("target_ip", run, ip))
|
|
290
|
+
|
|
291
|
+
def probe(self, run: Optional[int], t: float, round_no: int, ttl: int,
|
|
292
|
+
addr: Optional[str], rtt: Optional[float], status: int) -> None:
|
|
293
|
+
if run is not None:
|
|
294
|
+
self._put_bulk(("probe", run, t, round_no, ttl, addr, rtt, status))
|
|
295
|
+
|
|
296
|
+
def event(self, run: Optional[int], t: float, kind: str, text: str) -> None:
|
|
297
|
+
if self.options["events"] and not self._closed:
|
|
298
|
+
self._put_bulk(("event", run, t, kind, text))
|
|
299
|
+
|
|
300
|
+
def _put_bulk(self, item: tuple) -> None:
|
|
301
|
+
"""Queue a probe or an event, unless MAX_QUEUED are already waiting:
|
|
302
|
+
then the writer has stopped or can't keep up, and an unbounded queue
|
|
303
|
+
would only grow (about half a million probes a day per target) until
|
|
304
|
+
memory ran out. Runs, routes and names are never dropped."""
|
|
305
|
+
waiting = self._q.qsize()
|
|
306
|
+
if waiting >= MAX_QUEUED:
|
|
307
|
+
if not self._dropping:
|
|
308
|
+
self._dropping = True
|
|
309
|
+
self._report("History recording is falling behind: some probes and events "
|
|
310
|
+
"aren't being recorded.")
|
|
311
|
+
return
|
|
312
|
+
if self._dropping and waiting < MAX_QUEUED // 2:
|
|
313
|
+
self._dropping = False
|
|
314
|
+
self._report("History recording has caught up.", kind="info")
|
|
315
|
+
self._q.put(item)
|
|
316
|
+
|
|
317
|
+
def route(self, run: Optional[int], t: float, hops: List[List[str]]) -> None:
|
|
318
|
+
"""The run is on this route from ``t`` (each hop's known routers)."""
|
|
319
|
+
if run is not None and self.options["routes"]:
|
|
320
|
+
self._q.put(("route", run, t, [list(h) for h in hops]))
|
|
321
|
+
|
|
322
|
+
def clear(self) -> None:
|
|
323
|
+
"""Delete everything recorded (the file stays, empty). Runs in
|
|
324
|
+
progress carry on into the emptied history."""
|
|
325
|
+
if self._thread is not None and not self._closed:
|
|
326
|
+
self._q.put(("clear",))
|
|
327
|
+
|
|
328
|
+
def name(self, addr: Optional[str], name: Optional[str]) -> None:
|
|
329
|
+
"""A router's DNS name ("" for none), so the history can say what
|
|
330
|
+
and where it was long after the run."""
|
|
331
|
+
if addr and name is not None and self._thread is not None and not self._closed:
|
|
332
|
+
self._q.put(("name", None, addr, name))
|
|
333
|
+
|
|
334
|
+
def end_run(self, run: Optional[int]) -> None:
|
|
335
|
+
if run is not None:
|
|
336
|
+
self._q.put(("end", run, self._clock()))
|
|
337
|
+
|
|
338
|
+
# --- the writer thread --------------------------------------------------
|
|
339
|
+
def _open(self) -> Optional[sqlite3.Connection]:
|
|
340
|
+
try:
|
|
341
|
+
self.folder.mkdir(parents=True, exist_ok=True)
|
|
342
|
+
except OSError as exc:
|
|
343
|
+
self._report(f"History recording is off: can't create {self.folder} ({exc})")
|
|
344
|
+
return None
|
|
345
|
+
# Never write through a link. Run as administrator (which full TCP/UDP
|
|
346
|
+
# traceroute needs), PingerPlot would otherwise create and rewrite
|
|
347
|
+
# files wherever a link that any program running as you can plant
|
|
348
|
+
# here points: the reason the elevated app ignores a saved log path.
|
|
349
|
+
for path in (self.folder, self.path):
|
|
350
|
+
if _is_link(path):
|
|
351
|
+
self._report(f"History recording is off: {path} is a link or junction, "
|
|
352
|
+
f"and PingerPlot won't write through one")
|
|
353
|
+
return None
|
|
354
|
+
for attempt in (1, 2):
|
|
355
|
+
try:
|
|
356
|
+
db = sqlite3.connect(self.path, timeout=10, isolation_level=None,
|
|
357
|
+
check_same_thread=False)
|
|
358
|
+
# No integrity check here: on a 1 GB file it takes seconds. A
|
|
359
|
+
# file that isn't a database fails on the first statement
|
|
360
|
+
# below; damage found later is handled where it's met.
|
|
361
|
+
# auto_vacuum only takes effect on a database with no tables
|
|
362
|
+
# yet, which is exactly when it's needed: retention deletes
|
|
363
|
+
# then give pages back instead of the file only ever growing.
|
|
364
|
+
# A database file is data, not code: no schema-defined
|
|
365
|
+
# function with side effects may run on our behalf.
|
|
366
|
+
db.execute("PRAGMA trusted_schema = OFF")
|
|
367
|
+
db.execute("PRAGMA auto_vacuum = INCREMENTAL")
|
|
368
|
+
db.execute("PRAGMA journal_mode = WAL")
|
|
369
|
+
db.execute("PRAGMA synchronous = NORMAL")
|
|
370
|
+
db.executescript(_SCHEMA)
|
|
371
|
+
version = db.execute("SELECT value FROM meta WHERE key='schema'").fetchone()
|
|
372
|
+
if version is None:
|
|
373
|
+
db.execute("INSERT INTO meta VALUES('schema', ?)", (str(SCHEMA_VERSION),))
|
|
374
|
+
elif _schema(version[0]) > SCHEMA_VERSION:
|
|
375
|
+
db.close()
|
|
376
|
+
self._report(f"History recording is off: {self.path} was written by a newer "
|
|
377
|
+
f"PingerPlot (schema {version[0]})")
|
|
378
|
+
return None
|
|
379
|
+
db.execute("INSERT OR IGNORE INTO addrs(id, addr) VALUES(?, '')", (NO_REPLY,))
|
|
380
|
+
# Router names came after the first files were made: add the
|
|
381
|
+
# column to a file that doesn't have it.
|
|
382
|
+
cols = {row[1] for row in db.execute("PRAGMA table_info(addrs)")}
|
|
383
|
+
if "name" not in cols:
|
|
384
|
+
db.execute("ALTER TABLE addrs ADD COLUMN name TEXT")
|
|
385
|
+
return db
|
|
386
|
+
except sqlite3.DatabaseError as exc:
|
|
387
|
+
try:
|
|
388
|
+
db.close()
|
|
389
|
+
except Exception:
|
|
390
|
+
pass
|
|
391
|
+
if attempt == 2 or not self.path.exists():
|
|
392
|
+
self._report(f"History recording is off: can't open {self.path} ({exc})")
|
|
393
|
+
return None
|
|
394
|
+
# Damaged: keep it for whoever wants to recover it, start afresh.
|
|
395
|
+
aside = self._set_aside()
|
|
396
|
+
self._report(f"The history database was damaged ({exc}). It was kept as "
|
|
397
|
+
f"{aside.name} and a new one started.")
|
|
398
|
+
return None
|
|
399
|
+
|
|
400
|
+
def _set_aside(self) -> Path:
|
|
401
|
+
stamp = time.strftime("%Y%m%d-%H%M%S")
|
|
402
|
+
aside = self.path.with_name(f"{DB_NAME}.damaged-{stamp}")
|
|
403
|
+
for suffix in ("", "-wal", "-shm"):
|
|
404
|
+
src = Path(str(self.path) + suffix)
|
|
405
|
+
if src.exists():
|
|
406
|
+
try:
|
|
407
|
+
os.replace(src, Path(str(aside) + suffix))
|
|
408
|
+
except OSError:
|
|
409
|
+
pass
|
|
410
|
+
return aside
|
|
411
|
+
|
|
412
|
+
def _writer(self) -> None:
|
|
413
|
+
try:
|
|
414
|
+
self._write_loop()
|
|
415
|
+
except Exception as exc:
|
|
416
|
+
self._report(f"History recording stopped ({type(exc).__name__}: {exc}); "
|
|
417
|
+
f"restart PingerPlot to record again.")
|
|
418
|
+
raise
|
|
419
|
+
|
|
420
|
+
def _write_loop(self) -> None:
|
|
421
|
+
db = self._open()
|
|
422
|
+
addr_ids: Dict[str, int] = {"": NO_REPLY}
|
|
423
|
+
run_ids: Dict[int, int] = {} # handle -> runs.id
|
|
424
|
+
open_seg: Dict[int, Tuple[int, int]] = {} # runs.id -> (segment rowid, route id)
|
|
425
|
+
minutes: Dict[Tuple[int, int, int, int], _Minute] = {}
|
|
426
|
+
last_rtt: Dict[Tuple[int, int], float] = {}
|
|
427
|
+
next_maint = 0.0
|
|
428
|
+
write_failing = False
|
|
429
|
+
|
|
430
|
+
def addr_id(addr: Optional[str]) -> int:
|
|
431
|
+
if not addr:
|
|
432
|
+
return NO_REPLY
|
|
433
|
+
aid = addr_ids.get(addr)
|
|
434
|
+
if aid is None:
|
|
435
|
+
db.execute("INSERT OR IGNORE INTO addrs(addr) VALUES(?)", (addr,))
|
|
436
|
+
aid = db.execute("SELECT id FROM addrs WHERE addr=?", (addr,)).fetchone()[0]
|
|
437
|
+
addr_ids[addr] = aid
|
|
438
|
+
return aid
|
|
439
|
+
|
|
440
|
+
def flush_minutes(before: Optional[int] = None, run: Optional[int] = None) -> None:
|
|
441
|
+
done = [k for k in minutes
|
|
442
|
+
if (before is None or k[1] < before) and (run is None or k[0] == run)]
|
|
443
|
+
rows = []
|
|
444
|
+
for k in done:
|
|
445
|
+
m = minutes.pop(k)
|
|
446
|
+
rows.append((*k, m.sent, m.recv, m.rtt_sum, m.rtt_min, m.rtt_max, m.jit_sum, m.jit_n))
|
|
447
|
+
if rows and self.options["summaries"]:
|
|
448
|
+
# Upsert: a minute flushed at end_run can be continued by a
|
|
449
|
+
# late probe of the same minute; add rather than overwrite.
|
|
450
|
+
db.executemany(
|
|
451
|
+
"INSERT INTO minutes VALUES(?,?,?,?,?,?,?,?,?,?,?) "
|
|
452
|
+
"ON CONFLICT(run, minute, ttl, addr) DO UPDATE SET "
|
|
453
|
+
"sent=sent+excluded.sent, recv=recv+excluded.recv, "
|
|
454
|
+
"rtt_sum=rtt_sum+excluded.rtt_sum, "
|
|
455
|
+
"rtt_min=min(coalesce(rtt_min, excluded.rtt_min), coalesce(excluded.rtt_min, rtt_min)), "
|
|
456
|
+
"rtt_max=max(coalesce(rtt_max, excluded.rtt_max), coalesce(excluded.rtt_max, rtt_max)), "
|
|
457
|
+
"jit_sum=jit_sum+excluded.jit_sum, jit_n=jit_n+excluded.jit_n", rows)
|
|
458
|
+
|
|
459
|
+
def apply(item: tuple) -> None:
|
|
460
|
+
op = item[0]
|
|
461
|
+
if op == "begin":
|
|
462
|
+
_, handle, target, mode, port, options, t = item
|
|
463
|
+
cur = db.execute(
|
|
464
|
+
"INSERT INTO runs(target, mode, port, options, started, program) "
|
|
465
|
+
"VALUES(?,?,?,?,?,?)",
|
|
466
|
+
(target, mode, port, json.dumps(options, sort_keys=True, default=str),
|
|
467
|
+
t, self.program))
|
|
468
|
+
run_ids[handle] = cur.lastrowid
|
|
469
|
+
return
|
|
470
|
+
if item[1] is not None and item[1] not in run_ids:
|
|
471
|
+
return # its run was never written (recording was failing)
|
|
472
|
+
run = run_ids.get(item[1])
|
|
473
|
+
if op == "probe":
|
|
474
|
+
_, _, t, round_no, ttl, addr, rtt, status = item
|
|
475
|
+
aid = addr_id(addr)
|
|
476
|
+
if self.options["probes"]:
|
|
477
|
+
db.execute("INSERT INTO probes VALUES(?,?,?,?,?,?,?)",
|
|
478
|
+
(run, t, round_no, ttl, aid, rtt, status))
|
|
479
|
+
minute = int(t // 60)
|
|
480
|
+
m = minutes.get((run, minute, ttl, aid))
|
|
481
|
+
if m is None:
|
|
482
|
+
m = minutes[(run, minute, ttl, aid)] = _Minute()
|
|
483
|
+
m.sent += 1
|
|
484
|
+
if rtt is not None:
|
|
485
|
+
m.recv += 1
|
|
486
|
+
m.rtt_sum += rtt
|
|
487
|
+
m.rtt_min = rtt if m.rtt_min is None else min(m.rtt_min, rtt)
|
|
488
|
+
m.rtt_max = rtt if m.rtt_max is None else max(m.rtt_max, rtt)
|
|
489
|
+
prev = last_rtt.get((run, ttl))
|
|
490
|
+
if prev is not None:
|
|
491
|
+
m.jit_sum += abs(rtt - prev)
|
|
492
|
+
m.jit_n += 1
|
|
493
|
+
last_rtt[(run, ttl)] = rtt
|
|
494
|
+
elif op == "event":
|
|
495
|
+
_, _, t, kind, text = item
|
|
496
|
+
db.execute("INSERT INTO events VALUES(?,?,?,?)", (run, t, kind, text))
|
|
497
|
+
elif op == "route":
|
|
498
|
+
_, _, t, hops = item
|
|
499
|
+
row = db.execute("SELECT target, target_ip FROM runs WHERE id=?", (run,)).fetchone()
|
|
500
|
+
if row is None:
|
|
501
|
+
return # its run row is gone (an edited file): nothing to attach it to
|
|
502
|
+
target, dest = row
|
|
503
|
+
route_id = None
|
|
504
|
+
for rid, stored in db.execute(
|
|
505
|
+
"SELECT id, hops FROM routes WHERE target=? AND dest IS ? ORDER BY id",
|
|
506
|
+
(target, dest)).fetchall():
|
|
507
|
+
old = json.loads(stored)
|
|
508
|
+
if compatible(old, hops):
|
|
509
|
+
route_id = rid
|
|
510
|
+
merged = [sorted(set(a) | set(b)) for a, b in zip(old, hops)]
|
|
511
|
+
if merged != old: # more load balancing seen: widen it
|
|
512
|
+
db.execute("UPDATE routes SET hops=? WHERE id=?",
|
|
513
|
+
(json.dumps(merged), rid))
|
|
514
|
+
break
|
|
515
|
+
if route_id is None:
|
|
516
|
+
route_id = db.execute(
|
|
517
|
+
"INSERT INTO routes(target, dest, hops, first) VALUES(?,?,?,?)",
|
|
518
|
+
(target, dest, json.dumps(hops), t)).lastrowid
|
|
519
|
+
current = open_seg.get(run)
|
|
520
|
+
if current is None or current[1] != route_id:
|
|
521
|
+
if current is not None:
|
|
522
|
+
db.execute("UPDATE segments SET end=? WHERE rowid=?", (t, current[0]))
|
|
523
|
+
seg = db.execute("INSERT INTO segments(run, route, start) VALUES(?,?,?)",
|
|
524
|
+
(run, route_id, t)).lastrowid
|
|
525
|
+
open_seg[run] = (seg, route_id)
|
|
526
|
+
elif op == "name":
|
|
527
|
+
aid = addr_id(item[2])
|
|
528
|
+
db.execute("UPDATE addrs SET name=? WHERE id=?", (item[3], aid))
|
|
529
|
+
elif op == "target_ip":
|
|
530
|
+
db.execute("UPDATE runs SET target_ip=? WHERE id=?", (item[2], run))
|
|
531
|
+
elif op == "end":
|
|
532
|
+
seg = open_seg.pop(run, None)
|
|
533
|
+
if seg is not None:
|
|
534
|
+
db.execute("UPDATE segments SET end=? WHERE rowid=?", (item[2], seg[0]))
|
|
535
|
+
flush_minutes(run=run)
|
|
536
|
+
for key in [k for k in last_rtt if k[0] == run]:
|
|
537
|
+
del last_rtt[key]
|
|
538
|
+
del run_ids[item[1]]
|
|
539
|
+
db.execute("UPDATE runs SET ended=? WHERE id=?", (item[2], run))
|
|
540
|
+
|
|
541
|
+
def release(items: Iterable[tuple]) -> None:
|
|
542
|
+
# Nobody may be left waiting on a write that won't happen.
|
|
543
|
+
for item in items:
|
|
544
|
+
if item[0] == "flush":
|
|
545
|
+
item[1].set()
|
|
546
|
+
|
|
547
|
+
closing = False
|
|
548
|
+
while not closing:
|
|
549
|
+
try:
|
|
550
|
+
batch = [self._q.get(timeout=1.0)]
|
|
551
|
+
except queue.Empty:
|
|
552
|
+
batch = []
|
|
553
|
+
while len(batch) < _BATCH:
|
|
554
|
+
try:
|
|
555
|
+
batch.append(self._q.get_nowait())
|
|
556
|
+
except queue.Empty:
|
|
557
|
+
break
|
|
558
|
+
if db is None:
|
|
559
|
+
release(batch)
|
|
560
|
+
closing = any(item[0] == "close" for item in batch)
|
|
561
|
+
continue
|
|
562
|
+
waiting = []
|
|
563
|
+
try:
|
|
564
|
+
db.execute("BEGIN")
|
|
565
|
+
for item in batch:
|
|
566
|
+
if item[0] == "close":
|
|
567
|
+
closing = True
|
|
568
|
+
elif item[0] == "clear":
|
|
569
|
+
# Everything goes except what keeps runs in progress
|
|
570
|
+
# recording: their run rows and the route each is on.
|
|
571
|
+
active = tuple(run_ids.values()) or (-1,)
|
|
572
|
+
marks = ",".join("?" * len(active))
|
|
573
|
+
for table in ("probes", "minutes", "events"):
|
|
574
|
+
db.execute(f"DELETE FROM {table}")
|
|
575
|
+
db.execute(f"DELETE FROM segments WHERE end IS NOT NULL "
|
|
576
|
+
f"OR run NOT IN ({marks})", active)
|
|
577
|
+
db.execute("DELETE FROM routes WHERE id NOT IN (SELECT route FROM segments)")
|
|
578
|
+
db.execute(f"DELETE FROM runs WHERE id NOT IN ({marks})", active)
|
|
579
|
+
minutes.clear()
|
|
580
|
+
last_rtt.clear()
|
|
581
|
+
elif item[0] == "flush":
|
|
582
|
+
waiting.append(item[1])
|
|
583
|
+
else:
|
|
584
|
+
apply(item)
|
|
585
|
+
# Minutes that have ended (by the clock, so a quiet target's
|
|
586
|
+
# last minute isn't held forever) go to disk with the batch.
|
|
587
|
+
flush_minutes(before=int(self._clock() // 60), run=None)
|
|
588
|
+
if closing:
|
|
589
|
+
flush_minutes()
|
|
590
|
+
db.execute("COMMIT")
|
|
591
|
+
if write_failing:
|
|
592
|
+
write_failing = False
|
|
593
|
+
self._report("History recording has recovered.", kind="info")
|
|
594
|
+
except Exception as exc: # any other type ended this thread, and all recording, unseen
|
|
595
|
+
try:
|
|
596
|
+
db.execute("ROLLBACK")
|
|
597
|
+
except sqlite3.Error:
|
|
598
|
+
pass
|
|
599
|
+
release(batch)
|
|
600
|
+
# Ids handed out inside the rolled-back transaction no longer
|
|
601
|
+
# exist; nothing later may point at them.
|
|
602
|
+
addr_ids.clear()
|
|
603
|
+
addr_ids[""] = NO_REPLY
|
|
604
|
+
open_seg.clear()
|
|
605
|
+
try:
|
|
606
|
+
for handle in [h for h, rid in run_ids.items() if not db.execute(
|
|
607
|
+
"SELECT 1 FROM runs WHERE id=?", (rid,)).fetchone()]:
|
|
608
|
+
del run_ids[handle]
|
|
609
|
+
except sqlite3.Error:
|
|
610
|
+
run_ids.clear()
|
|
611
|
+
if type(exc) is sqlite3.DatabaseError:
|
|
612
|
+
# The file itself is damaged ("malformed", "not a
|
|
613
|
+
# database"): every batch would fail from now on. Keep it
|
|
614
|
+
# aside for recovery and carry on in a new one.
|
|
615
|
+
try:
|
|
616
|
+
db.close()
|
|
617
|
+
except sqlite3.Error:
|
|
618
|
+
pass
|
|
619
|
+
aside = self._set_aside()
|
|
620
|
+
self._report(f"The history database was damaged ({exc}). It was kept as "
|
|
621
|
+
f"{aside.name} and a new one started.")
|
|
622
|
+
db = self._open()
|
|
623
|
+
run_ids.clear()
|
|
624
|
+
minutes.clear()
|
|
625
|
+
last_rtt.clear()
|
|
626
|
+
write_failing = False
|
|
627
|
+
continue
|
|
628
|
+
if not write_failing:
|
|
629
|
+
write_failing = True
|
|
630
|
+
why = exc if isinstance(exc, sqlite3.Error) else f"{type(exc).__name__}: {exc}"
|
|
631
|
+
self._report(f"History recording failed ({why}); what was measured in "
|
|
632
|
+
f"the meantime isn't recorded. Retrying.")
|
|
633
|
+
for done in waiting:
|
|
634
|
+
done.set()
|
|
635
|
+
now = time.monotonic()
|
|
636
|
+
if now >= next_maint:
|
|
637
|
+
next_maint = now + _MAINTAIN_EVERY_S
|
|
638
|
+
self._maintain(db)
|
|
639
|
+
if db is not None:
|
|
640
|
+
try:
|
|
641
|
+
db.close()
|
|
642
|
+
except sqlite3.Error:
|
|
643
|
+
pass
|
|
644
|
+
|
|
645
|
+
# --- retention and the size cap ----------------------------------------
|
|
646
|
+
def _maintain(self, db: sqlite3.Connection) -> None:
|
|
647
|
+
o = self.options
|
|
648
|
+
now = self._clock()
|
|
649
|
+
try:
|
|
650
|
+
db.execute("BEGIN")
|
|
651
|
+
db.execute("DELETE FROM probes WHERE t < ?", (now - o["probe_days"] * 86400,))
|
|
652
|
+
db.execute("DELETE FROM minutes WHERE minute < ?",
|
|
653
|
+
(int((now - o["summary_days"] * 86400) // 60),))
|
|
654
|
+
db.execute("DELETE FROM events WHERE t < ?", (now - o["event_days"] * 86400,))
|
|
655
|
+
# Route history is tiny; it lives as long as the events about it.
|
|
656
|
+
db.execute("DELETE FROM segments WHERE end IS NOT NULL AND end < ?",
|
|
657
|
+
(now - o["event_days"] * 86400,))
|
|
658
|
+
db.execute("DELETE FROM routes WHERE id NOT IN (SELECT route FROM segments)")
|
|
659
|
+
keep = max(o["probe_days"], o["summary_days"], o["event_days"]) * 86400
|
|
660
|
+
db.execute("DELETE FROM runs WHERE ended IS NOT NULL AND ended < ?", (now - keep,))
|
|
661
|
+
db.execute("COMMIT")
|
|
662
|
+
self._cap(db, o["max_mb"] * 1024 * 1024)
|
|
663
|
+
db.execute("PRAGMA incremental_vacuum")
|
|
664
|
+
except sqlite3.Error as exc:
|
|
665
|
+
try:
|
|
666
|
+
db.execute("ROLLBACK")
|
|
667
|
+
except sqlite3.Error:
|
|
668
|
+
pass
|
|
669
|
+
self._report(f"History clean-up failed ({exc}); it will try again in an hour.")
|
|
670
|
+
|
|
671
|
+
@staticmethod
|
|
672
|
+
def used_bytes(db: sqlite3.Connection) -> int:
|
|
673
|
+
pages = db.execute("PRAGMA page_count").fetchone()[0]
|
|
674
|
+
free = db.execute("PRAGMA freelist_count").fetchone()[0]
|
|
675
|
+
size = db.execute("PRAGMA page_size").fetchone()[0]
|
|
676
|
+
return (pages - free) * size
|
|
677
|
+
|
|
678
|
+
def _cap(self, db: sqlite3.Connection, limit: int) -> None:
|
|
679
|
+
"""Over the size limit, give up the oldest detail first: probes, then
|
|
680
|
+
minutes, then events, a tenth of the oldest at a time."""
|
|
681
|
+
for table, col in (("probes", "t"), ("minutes", "minute"), ("events", "t")):
|
|
682
|
+
for _ in range(20):
|
|
683
|
+
if self.used_bytes(db) <= limit:
|
|
684
|
+
return
|
|
685
|
+
n = db.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
|
|
686
|
+
if n == 0:
|
|
687
|
+
break
|
|
688
|
+
cut = db.execute(f"SELECT {col} FROM {table} ORDER BY {col} LIMIT 1 OFFSET ?",
|
|
689
|
+
(max(1, n // 10),)).fetchone()
|
|
690
|
+
if cut is None:
|
|
691
|
+
break
|
|
692
|
+
db.execute("BEGIN")
|
|
693
|
+
db.execute(f"DELETE FROM {table} WHERE {col} < ?", (cut[0],))
|
|
694
|
+
db.execute("COMMIT")
|
|
695
|
+
db.execute("PRAGMA incremental_vacuum")
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
# --- reading ----------------------------------------------------------------
|
|
699
|
+
class Reader:
|
|
700
|
+
"""Read-only queries for the History view, reports and exports.
|
|
701
|
+
|
|
702
|
+
Opens its own connection (WAL lets it read while the recorder writes) and
|
|
703
|
+
never creates the database: a missing file reads as empty."""
|
|
704
|
+
|
|
705
|
+
def __init__(self, folder: Optional[os.PathLike] = None) -> None:
|
|
706
|
+
self.path = (Path(folder) if folder is not None else default_dir()) / DB_NAME
|
|
707
|
+
self._db: Optional[sqlite3.Connection] = None
|
|
708
|
+
|
|
709
|
+
def _conn(self) -> Optional[sqlite3.Connection]:
|
|
710
|
+
if self._db is None:
|
|
711
|
+
if not self.path.exists():
|
|
712
|
+
return None
|
|
713
|
+
# Not mode=ro: a read-only connection to a WAL database fails when
|
|
714
|
+
# no writer has left its -shm file behind. query_only refuses writes.
|
|
715
|
+
db = sqlite3.connect(self.path, timeout=10, check_same_thread=False)
|
|
716
|
+
db.execute("PRAGMA query_only = ON")
|
|
717
|
+
# As the writer: the file is data, and no function its schema
|
|
718
|
+
# defines (a view's, a trigger's) may run on our behalf.
|
|
719
|
+
db.execute("PRAGMA trusted_schema = OFF")
|
|
720
|
+
# A recorder that has only just created the file may not have made
|
|
721
|
+
# its tables yet (a first run: the History page opens while the
|
|
722
|
+
# writer is still starting). Read that as empty and look again
|
|
723
|
+
# next time, rather than fail with "no such table". The last table
|
|
724
|
+
# the schema makes, so a half-made file counts as not made.
|
|
725
|
+
if not db.execute("SELECT 1 FROM sqlite_master WHERE type='table' "
|
|
726
|
+
"AND name='segments'").fetchone():
|
|
727
|
+
db.close()
|
|
728
|
+
return None
|
|
729
|
+
db.row_factory = sqlite3.Row
|
|
730
|
+
self._db = db
|
|
731
|
+
return self._db
|
|
732
|
+
|
|
733
|
+
def close(self) -> None:
|
|
734
|
+
if self._db is not None:
|
|
735
|
+
self._db.close()
|
|
736
|
+
self._db = None
|
|
737
|
+
|
|
738
|
+
def _rows(self, sql: str, args=(), lazy: bool = False):
|
|
739
|
+
"""The rows as a list, or with ``lazy`` the cursor itself, read as
|
|
740
|
+
it's iterated: an export of a week of every probe is millions of rows
|
|
741
|
+
and must not have to fit in memory."""
|
|
742
|
+
db = self._conn()
|
|
743
|
+
if db is None:
|
|
744
|
+
return iter(()) if lazy else []
|
|
745
|
+
cur = db.execute(sql, args)
|
|
746
|
+
return cur if lazy else cur.fetchall()
|
|
747
|
+
|
|
748
|
+
def targets(self) -> List[sqlite3.Row]:
|
|
749
|
+
"""Every target ever recorded, most recently monitored first."""
|
|
750
|
+
return self._rows("SELECT target, count(*) AS runs, min(started) AS first, "
|
|
751
|
+
"max(coalesce(ended, started)) AS last FROM runs "
|
|
752
|
+
"GROUP BY target ORDER BY last DESC")
|
|
753
|
+
|
|
754
|
+
def runs(self, target: Optional[str] = None) -> List[sqlite3.Row]:
|
|
755
|
+
if target is None:
|
|
756
|
+
return self._rows("SELECT * FROM runs ORDER BY started")
|
|
757
|
+
return self._rows("SELECT * FROM runs WHERE target=? ORDER BY started", (target,))
|
|
758
|
+
|
|
759
|
+
def events(self, since: float = 0.0, until: float = float("inf"),
|
|
760
|
+
target: Optional[str] = None, kinds: Optional[Iterable[str]] = None,
|
|
761
|
+
text: str = "", limit: Optional[int] = 100_000,
|
|
762
|
+
lazy: bool = False) -> Iterable[sqlite3.Row]:
|
|
763
|
+
sql = ("SELECT e.t, e.kind, e.text, r.target FROM events e LEFT JOIN runs r ON r.id = e.run "
|
|
764
|
+
"WHERE e.t >= ? AND e.t <= ?")
|
|
765
|
+
args: list = [since, until if until != float("inf") else 1e18]
|
|
766
|
+
if target is not None:
|
|
767
|
+
sql += " AND r.target = ?"
|
|
768
|
+
args.append(target)
|
|
769
|
+
kinds = list(kinds or [])
|
|
770
|
+
if kinds:
|
|
771
|
+
sql += f" AND e.kind IN ({','.join('?' * len(kinds))})"
|
|
772
|
+
args += kinds
|
|
773
|
+
if text:
|
|
774
|
+
sql += " AND e.text LIKE ? ESCAPE '\\'"
|
|
775
|
+
args.append("%" + text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%")
|
|
776
|
+
sql += " ORDER BY e.t"
|
|
777
|
+
if limit is not None:
|
|
778
|
+
sql += " LIMIT ?"
|
|
779
|
+
args.append(limit)
|
|
780
|
+
return self._rows(sql, args, lazy)
|
|
781
|
+
|
|
782
|
+
def probes(self, target: str, since: float, until: float,
|
|
783
|
+
lazy: bool = False) -> Iterable[sqlite3.Row]:
|
|
784
|
+
return self._rows(
|
|
785
|
+
"SELECT p.t, p.round, p.ttl, a.addr, p.rtt, p.status FROM probes p "
|
|
786
|
+
"JOIN runs r ON r.id = p.run JOIN addrs a ON a.id = p.addr "
|
|
787
|
+
"WHERE p.t >= ? AND p.t <= ? AND r.target = ? ORDER BY p.t, p.ttl",
|
|
788
|
+
(since, until, target), lazy)
|
|
789
|
+
|
|
790
|
+
def routes(self, target: str) -> List[sqlite3.Row]:
|
|
791
|
+
"""Every route recorded to ``target`` (``hops`` is JSON: each hop's
|
|
792
|
+
routers), oldest first."""
|
|
793
|
+
return self._rows("SELECT id, dest, hops, first FROM routes WHERE target=? ORDER BY id",
|
|
794
|
+
(target,))
|
|
795
|
+
|
|
796
|
+
def segments(self, target: str, since: float = 0.0,
|
|
797
|
+
until: float = 1e18) -> List[sqlite3.Row]:
|
|
798
|
+
"""When the target was on which route: start, end (None = still on
|
|
799
|
+
it, or the program stopped without closing it), route id."""
|
|
800
|
+
return self._rows(
|
|
801
|
+
"SELECT s.start, s.end, s.route, r.target_ip AS dest FROM segments s "
|
|
802
|
+
"JOIN runs r ON r.id = s.run WHERE r.target = ? AND s.start <= ? "
|
|
803
|
+
"AND coalesce(s.end, 1e18) >= ? ORDER BY s.start", (target, until, since))
|
|
804
|
+
|
|
805
|
+
def names(self, addrs: Iterable[str]) -> Dict[str, str]:
|
|
806
|
+
"""The DNS names recorded for these routers (those that have one)."""
|
|
807
|
+
addrs = [a for a in set(addrs) if a]
|
|
808
|
+
out: Dict[str, str] = {}
|
|
809
|
+
for i in range(0, len(addrs), 500):
|
|
810
|
+
chunk = addrs[i:i + 500]
|
|
811
|
+
for row in self._rows(f"SELECT addr, name FROM addrs WHERE name IS NOT NULL AND name != '' "
|
|
812
|
+
f"AND addr IN ({','.join('?' * len(chunk))})", chunk):
|
|
813
|
+
out[row["addr"]] = row["name"]
|
|
814
|
+
return out
|
|
815
|
+
|
|
816
|
+
def minutes(self, target: str, since: float, until: float,
|
|
817
|
+
lazy: bool = False) -> Iterable[sqlite3.Row]:
|
|
818
|
+
return self._rows(
|
|
819
|
+
"SELECT m.minute, m.ttl, a.addr, m.sent, m.recv, m.rtt_sum, m.rtt_min, m.rtt_max, "
|
|
820
|
+
"m.jit_sum, m.jit_n FROM minutes m JOIN runs r ON r.id = m.run "
|
|
821
|
+
"JOIN addrs a ON a.id = m.addr "
|
|
822
|
+
"WHERE m.minute >= ? AND m.minute <= ? AND r.target = ? ORDER BY m.minute, m.ttl",
|
|
823
|
+
(int(since // 60), int(until // 60), target), lazy)
|