pingerplot 1.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pingerplot/history.py ADDED
@@ -0,0 +1,823 @@
1
+ """The long-term record: everything a run measures, kept on disk.
2
+
3
+ Until 1.5 the only durable record was an opt-in CSV per target, set up by
4
+ typing a path, and everything else (the event list, which router answered at
5
+ which hop, the route a reroute replaced) lived in memory for about 25 minutes.
6
+ "What did the path do at 3 a.m. Tuesday?" had no answer.
7
+
8
+ This keeps one SQLite database (standard library ``sqlite3``) in the settings
9
+ folder, shared by every target and by the GUI and headless runner alike:
10
+
11
+ * ``probes`` -- every probe: when, which hop, which router answered, the RTT.
12
+ The heaviest table (about 25 MB a day for a 15-hop target at 2.5 s), so it
13
+ has the shortest retention.
14
+ * ``minutes`` -- one row per minute per hop per responding router: sent,
15
+ received, min/avg/max RTT and jitter. Small enough to keep for a year, and
16
+ enough to answer "which router, which minute" long after the probes are gone.
17
+ * ``events`` -- every event the run logged: alerts, clears, route changes,
18
+ warnings, notes.
19
+ * ``routes`` -- each distinct path to a target (the routers known at every
20
+ hop), and ``segments``: when a run was on which route. A route that comes
21
+ back is recognised as the same one, so "it flips between these two paths"
22
+ shows as two routes taking turns.
23
+ * ``runs`` -- what was monitored, how, and when.
24
+
25
+ Writes go through one background thread that batches them into transactions,
26
+ so a probe never waits on the disk. The database runs in WAL mode, so a crash
27
+ loses at most the last batch and readers (the History view) never block the
28
+ writer. Trouble (a damaged file, a full disk, a folder that can't be written)
29
+ is reported through :attr:`Recorder.problem` for the monitors to put in their
30
+ Events, never raised into the probing.
31
+ """
32
+ from __future__ import annotations
33
+
34
+ import itertools
35
+ import json
36
+ import os
37
+ import queue
38
+ import sqlite3
39
+ import stat
40
+ import threading
41
+ import time
42
+ from pathlib import Path
43
+ from typing import Dict, Iterable, List, Optional, Tuple
44
+
45
+ from .routes import compatible
46
+
47
+ SCHEMA_VERSION = 1
48
+ DB_NAME = "history.sqlite3"
49
+
50
+ DEFAULTS = {
51
+ "enabled": True,
52
+ "probes": True, "probe_days": 7,
53
+ "summaries": True, "summary_days": 365,
54
+ "events": True, "event_days": 365,
55
+ "routes": True,
56
+ "max_mb": 1024,
57
+ }
58
+ # (key, lowest, highest): the Settings page's own limits.
59
+ _LIMITS = {"probe_days": (1, 365), "summary_days": (1, 3650), "event_days": (1, 3650),
60
+ "max_mb": (50, 100_000)}
61
+
62
+ _BATCH = 2000 # items per transaction at most
63
+ _MAINTAIN_EVERY_S = 3600 # retention and the size cap, hourly (and at open)
64
+ NO_REPLY = 0 # addrs id for a probe nobody answered
65
+ MAX_QUEUED = 100_000 # probes/events waiting for the writer (~4 h of one 15-hop target)
66
+
67
+
68
+ def _schema(value) -> int:
69
+ """The file's schema number; anything else is a damaged file."""
70
+ try:
71
+ return int(value)
72
+ except (TypeError, ValueError) as exc:
73
+ raise sqlite3.DatabaseError(f"its schema version reads {value!r}") from exc
74
+
75
+ _SCHEMA = """
76
+ CREATE TABLE IF NOT EXISTS meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
77
+ CREATE TABLE IF NOT EXISTS runs(
78
+ id INTEGER PRIMARY KEY,
79
+ target TEXT NOT NULL,
80
+ target_ip TEXT,
81
+ mode TEXT NOT NULL,
82
+ port INTEGER,
83
+ options TEXT NOT NULL,
84
+ started REAL NOT NULL,
85
+ ended REAL,
86
+ program TEXT NOT NULL
87
+ );
88
+ CREATE INDEX IF NOT EXISTS runs_target ON runs(target, started);
89
+ CREATE TABLE IF NOT EXISTS addrs(id INTEGER PRIMARY KEY, addr TEXT NOT NULL UNIQUE, name TEXT);
90
+ CREATE TABLE IF NOT EXISTS probes(
91
+ run INTEGER NOT NULL,
92
+ t REAL NOT NULL,
93
+ round INTEGER NOT NULL,
94
+ ttl INTEGER NOT NULL,
95
+ addr INTEGER NOT NULL,
96
+ rtt REAL,
97
+ status INTEGER NOT NULL
98
+ );
99
+ CREATE INDEX IF NOT EXISTS probes_t ON probes(t);
100
+ CREATE TABLE IF NOT EXISTS minutes(
101
+ run INTEGER NOT NULL,
102
+ minute INTEGER NOT NULL,
103
+ ttl INTEGER NOT NULL,
104
+ addr INTEGER NOT NULL,
105
+ sent INTEGER NOT NULL,
106
+ recv INTEGER NOT NULL,
107
+ rtt_sum REAL NOT NULL,
108
+ rtt_min REAL,
109
+ rtt_max REAL,
110
+ jit_sum REAL NOT NULL,
111
+ jit_n INTEGER NOT NULL,
112
+ UNIQUE(run, minute, ttl, addr)
113
+ );
114
+ CREATE INDEX IF NOT EXISTS minutes_minute ON minutes(minute);
115
+ CREATE TABLE IF NOT EXISTS events(
116
+ run INTEGER,
117
+ t REAL NOT NULL,
118
+ kind TEXT NOT NULL,
119
+ text TEXT NOT NULL
120
+ );
121
+ CREATE INDEX IF NOT EXISTS events_t ON events(t);
122
+ CREATE TABLE IF NOT EXISTS routes(
123
+ id INTEGER PRIMARY KEY,
124
+ target TEXT NOT NULL,
125
+ dest TEXT,
126
+ hops TEXT NOT NULL,
127
+ first REAL NOT NULL
128
+ );
129
+ CREATE INDEX IF NOT EXISTS routes_target ON routes(target);
130
+ CREATE TABLE IF NOT EXISTS segments(
131
+ run INTEGER NOT NULL,
132
+ route INTEGER NOT NULL,
133
+ start REAL NOT NULL,
134
+ end REAL
135
+ );
136
+ CREATE INDEX IF NOT EXISTS segments_start ON segments(start);
137
+ """
138
+
139
+
140
+ def normalize(raw) -> dict:
141
+ """Recording options from settings.json, every value present and in range.
142
+
143
+ A hand-edited or older settings file must not be able to stop the app or
144
+ turn retention into "delete everything": anything unusable falls back to
145
+ its default."""
146
+ out = dict(DEFAULTS)
147
+ if not isinstance(raw, dict):
148
+ return out
149
+ for key, default in DEFAULTS.items():
150
+ value = raw.get(key, default)
151
+ if isinstance(default, bool):
152
+ out[key] = value if isinstance(value, bool) else default
153
+ continue
154
+ try:
155
+ value = int(value)
156
+ except (TypeError, ValueError, OverflowError):
157
+ continue
158
+ lo, hi = _LIMITS[key]
159
+ out[key] = max(lo, min(value, hi))
160
+ return out
161
+
162
+
163
+ def default_dir() -> Path:
164
+ from . import settings
165
+ return settings.config_dir() / "history"
166
+
167
+
168
+ def _is_link(path: Path) -> bool:
169
+ """A symlink, or on Windows any reparse point (a junction too, which
170
+ os.path.islink misses before Python 3.12)."""
171
+ try:
172
+ st = os.lstat(path)
173
+ except OSError:
174
+ return False
175
+ if stat.S_ISLNK(st.st_mode):
176
+ return True
177
+ return bool(getattr(st, "st_file_attributes", 0) & getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0))
178
+
179
+
180
+ def disk_bytes(folder: Optional[os.PathLike] = None) -> int:
181
+ """What the history takes on disk, its write-ahead log included."""
182
+ base = (Path(folder) if folder is not None else default_dir()) / DB_NAME
183
+ total = 0
184
+ for suffix in ("", "-wal", "-shm"):
185
+ try:
186
+ total += os.stat(str(base) + suffix).st_size
187
+ except OSError:
188
+ pass
189
+ return total
190
+
191
+
192
+ def open_recorder(saved: dict, program: str) -> Optional["Recorder"]:
193
+ """The recorder the GUI or headless runner should use, from its saved
194
+ settings' ``recording`` section, or None when recording is turned off."""
195
+ options = normalize(saved.get("recording") if isinstance(saved, dict) else None)
196
+ if not options["enabled"]:
197
+ return None
198
+ return Recorder(options=options, program=program).start()
199
+
200
+
201
+ class _Minute:
202
+ """Accumulates one (run, minute, ttl, responder) row."""
203
+ __slots__ = ("sent", "recv", "rtt_sum", "rtt_min", "rtt_max", "jit_sum", "jit_n")
204
+
205
+ def __init__(self) -> None:
206
+ self.sent = self.recv = self.jit_n = 0
207
+ self.rtt_sum = self.jit_sum = 0.0
208
+ self.rtt_min: Optional[float] = None
209
+ self.rtt_max: Optional[float] = None
210
+
211
+
212
+ class Recorder:
213
+ """Owns the database and the one thread that writes to it.
214
+
215
+ Every public method is safe to call from any thread and returns at once.
216
+ If the database cannot be opened the recorder stays usable and records
217
+ nothing; :attr:`problem` says why.
218
+ """
219
+
220
+ def __init__(self, folder: Optional[os.PathLike] = None, options: Optional[dict] = None,
221
+ program: str = "gui", clock=time.time) -> None:
222
+ self.folder = Path(folder) if folder is not None else default_dir()
223
+ self.path = self.folder / DB_NAME
224
+ self.options = normalize(options)
225
+ self.program = program
226
+ self._clock = clock
227
+ self._q: "queue.Queue[tuple]" = queue.Queue()
228
+ self._thread: Optional[threading.Thread] = None
229
+ self._lock = threading.Lock()
230
+ self.problem = "" # the latest trouble, for the monitors' Events
231
+ self.problem_kind = "warn" # its event kind ("info" for a recovery)
232
+ self.problem_seq = 0 # bumped with each new problem
233
+ self._closed = False
234
+ self._dropping = False # probes are being dropped (see _put_bulk)
235
+ # A run's handle is issued here, not by the database: waiting for a
236
+ # row id would block the monitor whenever the writer is busy (an
237
+ # hourly clean-up of a large file), and a run that gave up waiting
238
+ # would go unrecorded without a word. The writer maps handles to ids.
239
+ self._handles = itertools.count(1)
240
+
241
+ # --- life cycle ---------------------------------------------------------
242
+ def start(self) -> "Recorder":
243
+ if self._thread is None:
244
+ self._thread = threading.Thread(target=self._writer, name="history", daemon=True)
245
+ self._thread.start()
246
+ return self
247
+
248
+ def close(self, timeout: float = 10.0) -> None:
249
+ """Write everything queued, the open minutes included, then stop."""
250
+ if self._thread is None or self._closed:
251
+ return
252
+ self._closed = True
253
+ self._q.put(("close",))
254
+ self._thread.join(timeout)
255
+
256
+ def flush(self, timeout: float = 10.0) -> bool:
257
+ """Wait until everything queued so far is on disk (for tests and for
258
+ reads that must see the latest data)."""
259
+ if self._thread is None or self._closed:
260
+ return False
261
+ done = threading.Event()
262
+ self._q.put(("flush", done))
263
+ return done.wait(timeout)
264
+
265
+ def _report(self, text: str, kind: str = "warn") -> None:
266
+ with self._lock:
267
+ self.problem = text
268
+ self.problem_kind = kind
269
+ self.problem_seq += 1
270
+
271
+ # --- what monitors call -------------------------------------------------
272
+ def set_options(self, options) -> None:
273
+ """New recording options from the Settings page; they apply from the
274
+ next write (and retention from the next clean-up)."""
275
+ self.options = normalize(options)
276
+
277
+ def begin_run(self, target: str, mode: str, port: Optional[int],
278
+ options: dict) -> Optional[int]:
279
+ """Register a run and return its handle for the calls below, or None
280
+ when nothing is being recorded."""
281
+ if self._thread is None or self._closed:
282
+ return None
283
+ run = next(self._handles)
284
+ self._q.put(("begin", run, target, mode, port, options, self._clock()))
285
+ return run
286
+
287
+ def set_target_ip(self, run: Optional[int], ip: Optional[str]) -> None:
288
+ if run is not None:
289
+ self._q.put(("target_ip", run, ip))
290
+
291
+ def probe(self, run: Optional[int], t: float, round_no: int, ttl: int,
292
+ addr: Optional[str], rtt: Optional[float], status: int) -> None:
293
+ if run is not None:
294
+ self._put_bulk(("probe", run, t, round_no, ttl, addr, rtt, status))
295
+
296
+ def event(self, run: Optional[int], t: float, kind: str, text: str) -> None:
297
+ if self.options["events"] and not self._closed:
298
+ self._put_bulk(("event", run, t, kind, text))
299
+
300
+ def _put_bulk(self, item: tuple) -> None:
301
+ """Queue a probe or an event, unless MAX_QUEUED are already waiting:
302
+ then the writer has stopped or can't keep up, and an unbounded queue
303
+ would only grow (about half a million probes a day per target) until
304
+ memory ran out. Runs, routes and names are never dropped."""
305
+ waiting = self._q.qsize()
306
+ if waiting >= MAX_QUEUED:
307
+ if not self._dropping:
308
+ self._dropping = True
309
+ self._report("History recording is falling behind: some probes and events "
310
+ "aren't being recorded.")
311
+ return
312
+ if self._dropping and waiting < MAX_QUEUED // 2:
313
+ self._dropping = False
314
+ self._report("History recording has caught up.", kind="info")
315
+ self._q.put(item)
316
+
317
+ def route(self, run: Optional[int], t: float, hops: List[List[str]]) -> None:
318
+ """The run is on this route from ``t`` (each hop's known routers)."""
319
+ if run is not None and self.options["routes"]:
320
+ self._q.put(("route", run, t, [list(h) for h in hops]))
321
+
322
+ def clear(self) -> None:
323
+ """Delete everything recorded (the file stays, empty). Runs in
324
+ progress carry on into the emptied history."""
325
+ if self._thread is not None and not self._closed:
326
+ self._q.put(("clear",))
327
+
328
+ def name(self, addr: Optional[str], name: Optional[str]) -> None:
329
+ """A router's DNS name ("" for none), so the history can say what
330
+ and where it was long after the run."""
331
+ if addr and name is not None and self._thread is not None and not self._closed:
332
+ self._q.put(("name", None, addr, name))
333
+
334
+ def end_run(self, run: Optional[int]) -> None:
335
+ if run is not None:
336
+ self._q.put(("end", run, self._clock()))
337
+
338
+ # --- the writer thread --------------------------------------------------
339
+ def _open(self) -> Optional[sqlite3.Connection]:
340
+ try:
341
+ self.folder.mkdir(parents=True, exist_ok=True)
342
+ except OSError as exc:
343
+ self._report(f"History recording is off: can't create {self.folder} ({exc})")
344
+ return None
345
+ # Never write through a link. Run as administrator (which full TCP/UDP
346
+ # traceroute needs), PingerPlot would otherwise create and rewrite
347
+ # files wherever a link that any program running as you can plant
348
+ # here points: the reason the elevated app ignores a saved log path.
349
+ for path in (self.folder, self.path):
350
+ if _is_link(path):
351
+ self._report(f"History recording is off: {path} is a link or junction, "
352
+ f"and PingerPlot won't write through one")
353
+ return None
354
+ for attempt in (1, 2):
355
+ try:
356
+ db = sqlite3.connect(self.path, timeout=10, isolation_level=None,
357
+ check_same_thread=False)
358
+ # No integrity check here: on a 1 GB file it takes seconds. A
359
+ # file that isn't a database fails on the first statement
360
+ # below; damage found later is handled where it's met.
361
+ # auto_vacuum only takes effect on a database with no tables
362
+ # yet, which is exactly when it's needed: retention deletes
363
+ # then give pages back instead of the file only ever growing.
364
+ # A database file is data, not code: no schema-defined
365
+ # function with side effects may run on our behalf.
366
+ db.execute("PRAGMA trusted_schema = OFF")
367
+ db.execute("PRAGMA auto_vacuum = INCREMENTAL")
368
+ db.execute("PRAGMA journal_mode = WAL")
369
+ db.execute("PRAGMA synchronous = NORMAL")
370
+ db.executescript(_SCHEMA)
371
+ version = db.execute("SELECT value FROM meta WHERE key='schema'").fetchone()
372
+ if version is None:
373
+ db.execute("INSERT INTO meta VALUES('schema', ?)", (str(SCHEMA_VERSION),))
374
+ elif _schema(version[0]) > SCHEMA_VERSION:
375
+ db.close()
376
+ self._report(f"History recording is off: {self.path} was written by a newer "
377
+ f"PingerPlot (schema {version[0]})")
378
+ return None
379
+ db.execute("INSERT OR IGNORE INTO addrs(id, addr) VALUES(?, '')", (NO_REPLY,))
380
+ # Router names came after the first files were made: add the
381
+ # column to a file that doesn't have it.
382
+ cols = {row[1] for row in db.execute("PRAGMA table_info(addrs)")}
383
+ if "name" not in cols:
384
+ db.execute("ALTER TABLE addrs ADD COLUMN name TEXT")
385
+ return db
386
+ except sqlite3.DatabaseError as exc:
387
+ try:
388
+ db.close()
389
+ except Exception:
390
+ pass
391
+ if attempt == 2 or not self.path.exists():
392
+ self._report(f"History recording is off: can't open {self.path} ({exc})")
393
+ return None
394
+ # Damaged: keep it for whoever wants to recover it, start afresh.
395
+ aside = self._set_aside()
396
+ self._report(f"The history database was damaged ({exc}). It was kept as "
397
+ f"{aside.name} and a new one started.")
398
+ return None
399
+
400
+ def _set_aside(self) -> Path:
401
+ stamp = time.strftime("%Y%m%d-%H%M%S")
402
+ aside = self.path.with_name(f"{DB_NAME}.damaged-{stamp}")
403
+ for suffix in ("", "-wal", "-shm"):
404
+ src = Path(str(self.path) + suffix)
405
+ if src.exists():
406
+ try:
407
+ os.replace(src, Path(str(aside) + suffix))
408
+ except OSError:
409
+ pass
410
+ return aside
411
+
412
+ def _writer(self) -> None:
413
+ try:
414
+ self._write_loop()
415
+ except Exception as exc:
416
+ self._report(f"History recording stopped ({type(exc).__name__}: {exc}); "
417
+ f"restart PingerPlot to record again.")
418
+ raise
419
+
420
+ def _write_loop(self) -> None:
421
+ db = self._open()
422
+ addr_ids: Dict[str, int] = {"": NO_REPLY}
423
+ run_ids: Dict[int, int] = {} # handle -> runs.id
424
+ open_seg: Dict[int, Tuple[int, int]] = {} # runs.id -> (segment rowid, route id)
425
+ minutes: Dict[Tuple[int, int, int, int], _Minute] = {}
426
+ last_rtt: Dict[Tuple[int, int], float] = {}
427
+ next_maint = 0.0
428
+ write_failing = False
429
+
430
+ def addr_id(addr: Optional[str]) -> int:
431
+ if not addr:
432
+ return NO_REPLY
433
+ aid = addr_ids.get(addr)
434
+ if aid is None:
435
+ db.execute("INSERT OR IGNORE INTO addrs(addr) VALUES(?)", (addr,))
436
+ aid = db.execute("SELECT id FROM addrs WHERE addr=?", (addr,)).fetchone()[0]
437
+ addr_ids[addr] = aid
438
+ return aid
439
+
440
+ def flush_minutes(before: Optional[int] = None, run: Optional[int] = None) -> None:
441
+ done = [k for k in minutes
442
+ if (before is None or k[1] < before) and (run is None or k[0] == run)]
443
+ rows = []
444
+ for k in done:
445
+ m = minutes.pop(k)
446
+ rows.append((*k, m.sent, m.recv, m.rtt_sum, m.rtt_min, m.rtt_max, m.jit_sum, m.jit_n))
447
+ if rows and self.options["summaries"]:
448
+ # Upsert: a minute flushed at end_run can be continued by a
449
+ # late probe of the same minute; add rather than overwrite.
450
+ db.executemany(
451
+ "INSERT INTO minutes VALUES(?,?,?,?,?,?,?,?,?,?,?) "
452
+ "ON CONFLICT(run, minute, ttl, addr) DO UPDATE SET "
453
+ "sent=sent+excluded.sent, recv=recv+excluded.recv, "
454
+ "rtt_sum=rtt_sum+excluded.rtt_sum, "
455
+ "rtt_min=min(coalesce(rtt_min, excluded.rtt_min), coalesce(excluded.rtt_min, rtt_min)), "
456
+ "rtt_max=max(coalesce(rtt_max, excluded.rtt_max), coalesce(excluded.rtt_max, rtt_max)), "
457
+ "jit_sum=jit_sum+excluded.jit_sum, jit_n=jit_n+excluded.jit_n", rows)
458
+
459
+ def apply(item: tuple) -> None:
460
+ op = item[0]
461
+ if op == "begin":
462
+ _, handle, target, mode, port, options, t = item
463
+ cur = db.execute(
464
+ "INSERT INTO runs(target, mode, port, options, started, program) "
465
+ "VALUES(?,?,?,?,?,?)",
466
+ (target, mode, port, json.dumps(options, sort_keys=True, default=str),
467
+ t, self.program))
468
+ run_ids[handle] = cur.lastrowid
469
+ return
470
+ if item[1] is not None and item[1] not in run_ids:
471
+ return # its run was never written (recording was failing)
472
+ run = run_ids.get(item[1])
473
+ if op == "probe":
474
+ _, _, t, round_no, ttl, addr, rtt, status = item
475
+ aid = addr_id(addr)
476
+ if self.options["probes"]:
477
+ db.execute("INSERT INTO probes VALUES(?,?,?,?,?,?,?)",
478
+ (run, t, round_no, ttl, aid, rtt, status))
479
+ minute = int(t // 60)
480
+ m = minutes.get((run, minute, ttl, aid))
481
+ if m is None:
482
+ m = minutes[(run, minute, ttl, aid)] = _Minute()
483
+ m.sent += 1
484
+ if rtt is not None:
485
+ m.recv += 1
486
+ m.rtt_sum += rtt
487
+ m.rtt_min = rtt if m.rtt_min is None else min(m.rtt_min, rtt)
488
+ m.rtt_max = rtt if m.rtt_max is None else max(m.rtt_max, rtt)
489
+ prev = last_rtt.get((run, ttl))
490
+ if prev is not None:
491
+ m.jit_sum += abs(rtt - prev)
492
+ m.jit_n += 1
493
+ last_rtt[(run, ttl)] = rtt
494
+ elif op == "event":
495
+ _, _, t, kind, text = item
496
+ db.execute("INSERT INTO events VALUES(?,?,?,?)", (run, t, kind, text))
497
+ elif op == "route":
498
+ _, _, t, hops = item
499
+ row = db.execute("SELECT target, target_ip FROM runs WHERE id=?", (run,)).fetchone()
500
+ if row is None:
501
+ return # its run row is gone (an edited file): nothing to attach it to
502
+ target, dest = row
503
+ route_id = None
504
+ for rid, stored in db.execute(
505
+ "SELECT id, hops FROM routes WHERE target=? AND dest IS ? ORDER BY id",
506
+ (target, dest)).fetchall():
507
+ old = json.loads(stored)
508
+ if compatible(old, hops):
509
+ route_id = rid
510
+ merged = [sorted(set(a) | set(b)) for a, b in zip(old, hops)]
511
+ if merged != old: # more load balancing seen: widen it
512
+ db.execute("UPDATE routes SET hops=? WHERE id=?",
513
+ (json.dumps(merged), rid))
514
+ break
515
+ if route_id is None:
516
+ route_id = db.execute(
517
+ "INSERT INTO routes(target, dest, hops, first) VALUES(?,?,?,?)",
518
+ (target, dest, json.dumps(hops), t)).lastrowid
519
+ current = open_seg.get(run)
520
+ if current is None or current[1] != route_id:
521
+ if current is not None:
522
+ db.execute("UPDATE segments SET end=? WHERE rowid=?", (t, current[0]))
523
+ seg = db.execute("INSERT INTO segments(run, route, start) VALUES(?,?,?)",
524
+ (run, route_id, t)).lastrowid
525
+ open_seg[run] = (seg, route_id)
526
+ elif op == "name":
527
+ aid = addr_id(item[2])
528
+ db.execute("UPDATE addrs SET name=? WHERE id=?", (item[3], aid))
529
+ elif op == "target_ip":
530
+ db.execute("UPDATE runs SET target_ip=? WHERE id=?", (item[2], run))
531
+ elif op == "end":
532
+ seg = open_seg.pop(run, None)
533
+ if seg is not None:
534
+ db.execute("UPDATE segments SET end=? WHERE rowid=?", (item[2], seg[0]))
535
+ flush_minutes(run=run)
536
+ for key in [k for k in last_rtt if k[0] == run]:
537
+ del last_rtt[key]
538
+ del run_ids[item[1]]
539
+ db.execute("UPDATE runs SET ended=? WHERE id=?", (item[2], run))
540
+
541
+ def release(items: Iterable[tuple]) -> None:
542
+ # Nobody may be left waiting on a write that won't happen.
543
+ for item in items:
544
+ if item[0] == "flush":
545
+ item[1].set()
546
+
547
+ closing = False
548
+ while not closing:
549
+ try:
550
+ batch = [self._q.get(timeout=1.0)]
551
+ except queue.Empty:
552
+ batch = []
553
+ while len(batch) < _BATCH:
554
+ try:
555
+ batch.append(self._q.get_nowait())
556
+ except queue.Empty:
557
+ break
558
+ if db is None:
559
+ release(batch)
560
+ closing = any(item[0] == "close" for item in batch)
561
+ continue
562
+ waiting = []
563
+ try:
564
+ db.execute("BEGIN")
565
+ for item in batch:
566
+ if item[0] == "close":
567
+ closing = True
568
+ elif item[0] == "clear":
569
+ # Everything goes except what keeps runs in progress
570
+ # recording: their run rows and the route each is on.
571
+ active = tuple(run_ids.values()) or (-1,)
572
+ marks = ",".join("?" * len(active))
573
+ for table in ("probes", "minutes", "events"):
574
+ db.execute(f"DELETE FROM {table}")
575
+ db.execute(f"DELETE FROM segments WHERE end IS NOT NULL "
576
+ f"OR run NOT IN ({marks})", active)
577
+ db.execute("DELETE FROM routes WHERE id NOT IN (SELECT route FROM segments)")
578
+ db.execute(f"DELETE FROM runs WHERE id NOT IN ({marks})", active)
579
+ minutes.clear()
580
+ last_rtt.clear()
581
+ elif item[0] == "flush":
582
+ waiting.append(item[1])
583
+ else:
584
+ apply(item)
585
+ # Minutes that have ended (by the clock, so a quiet target's
586
+ # last minute isn't held forever) go to disk with the batch.
587
+ flush_minutes(before=int(self._clock() // 60), run=None)
588
+ if closing:
589
+ flush_minutes()
590
+ db.execute("COMMIT")
591
+ if write_failing:
592
+ write_failing = False
593
+ self._report("History recording has recovered.", kind="info")
594
+ except Exception as exc: # any other type ended this thread, and all recording, unseen
595
+ try:
596
+ db.execute("ROLLBACK")
597
+ except sqlite3.Error:
598
+ pass
599
+ release(batch)
600
+ # Ids handed out inside the rolled-back transaction no longer
601
+ # exist; nothing later may point at them.
602
+ addr_ids.clear()
603
+ addr_ids[""] = NO_REPLY
604
+ open_seg.clear()
605
+ try:
606
+ for handle in [h for h, rid in run_ids.items() if not db.execute(
607
+ "SELECT 1 FROM runs WHERE id=?", (rid,)).fetchone()]:
608
+ del run_ids[handle]
609
+ except sqlite3.Error:
610
+ run_ids.clear()
611
+ if type(exc) is sqlite3.DatabaseError:
612
+ # The file itself is damaged ("malformed", "not a
613
+ # database"): every batch would fail from now on. Keep it
614
+ # aside for recovery and carry on in a new one.
615
+ try:
616
+ db.close()
617
+ except sqlite3.Error:
618
+ pass
619
+ aside = self._set_aside()
620
+ self._report(f"The history database was damaged ({exc}). It was kept as "
621
+ f"{aside.name} and a new one started.")
622
+ db = self._open()
623
+ run_ids.clear()
624
+ minutes.clear()
625
+ last_rtt.clear()
626
+ write_failing = False
627
+ continue
628
+ if not write_failing:
629
+ write_failing = True
630
+ why = exc if isinstance(exc, sqlite3.Error) else f"{type(exc).__name__}: {exc}"
631
+ self._report(f"History recording failed ({why}); what was measured in "
632
+ f"the meantime isn't recorded. Retrying.")
633
+ for done in waiting:
634
+ done.set()
635
+ now = time.monotonic()
636
+ if now >= next_maint:
637
+ next_maint = now + _MAINTAIN_EVERY_S
638
+ self._maintain(db)
639
+ if db is not None:
640
+ try:
641
+ db.close()
642
+ except sqlite3.Error:
643
+ pass
644
+
645
+ # --- retention and the size cap ----------------------------------------
646
+ def _maintain(self, db: sqlite3.Connection) -> None:
647
+ o = self.options
648
+ now = self._clock()
649
+ try:
650
+ db.execute("BEGIN")
651
+ db.execute("DELETE FROM probes WHERE t < ?", (now - o["probe_days"] * 86400,))
652
+ db.execute("DELETE FROM minutes WHERE minute < ?",
653
+ (int((now - o["summary_days"] * 86400) // 60),))
654
+ db.execute("DELETE FROM events WHERE t < ?", (now - o["event_days"] * 86400,))
655
+ # Route history is tiny; it lives as long as the events about it.
656
+ db.execute("DELETE FROM segments WHERE end IS NOT NULL AND end < ?",
657
+ (now - o["event_days"] * 86400,))
658
+ db.execute("DELETE FROM routes WHERE id NOT IN (SELECT route FROM segments)")
659
+ keep = max(o["probe_days"], o["summary_days"], o["event_days"]) * 86400
660
+ db.execute("DELETE FROM runs WHERE ended IS NOT NULL AND ended < ?", (now - keep,))
661
+ db.execute("COMMIT")
662
+ self._cap(db, o["max_mb"] * 1024 * 1024)
663
+ db.execute("PRAGMA incremental_vacuum")
664
+ except sqlite3.Error as exc:
665
+ try:
666
+ db.execute("ROLLBACK")
667
+ except sqlite3.Error:
668
+ pass
669
+ self._report(f"History clean-up failed ({exc}); it will try again in an hour.")
670
+
671
+ @staticmethod
672
+ def used_bytes(db: sqlite3.Connection) -> int:
673
+ pages = db.execute("PRAGMA page_count").fetchone()[0]
674
+ free = db.execute("PRAGMA freelist_count").fetchone()[0]
675
+ size = db.execute("PRAGMA page_size").fetchone()[0]
676
+ return (pages - free) * size
677
+
678
+ def _cap(self, db: sqlite3.Connection, limit: int) -> None:
679
+ """Over the size limit, give up the oldest detail first: probes, then
680
+ minutes, then events, a tenth of the oldest at a time."""
681
+ for table, col in (("probes", "t"), ("minutes", "minute"), ("events", "t")):
682
+ for _ in range(20):
683
+ if self.used_bytes(db) <= limit:
684
+ return
685
+ n = db.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
686
+ if n == 0:
687
+ break
688
+ cut = db.execute(f"SELECT {col} FROM {table} ORDER BY {col} LIMIT 1 OFFSET ?",
689
+ (max(1, n // 10),)).fetchone()
690
+ if cut is None:
691
+ break
692
+ db.execute("BEGIN")
693
+ db.execute(f"DELETE FROM {table} WHERE {col} < ?", (cut[0],))
694
+ db.execute("COMMIT")
695
+ db.execute("PRAGMA incremental_vacuum")
696
+
697
+
698
+ # --- reading ----------------------------------------------------------------
699
+ class Reader:
700
+ """Read-only queries for the History view, reports and exports.
701
+
702
+ Opens its own connection (WAL lets it read while the recorder writes) and
703
+ never creates the database: a missing file reads as empty."""
704
+
705
+ def __init__(self, folder: Optional[os.PathLike] = None) -> None:
706
+ self.path = (Path(folder) if folder is not None else default_dir()) / DB_NAME
707
+ self._db: Optional[sqlite3.Connection] = None
708
+
709
+ def _conn(self) -> Optional[sqlite3.Connection]:
710
+ if self._db is None:
711
+ if not self.path.exists():
712
+ return None
713
+ # Not mode=ro: a read-only connection to a WAL database fails when
714
+ # no writer has left its -shm file behind. query_only refuses writes.
715
+ db = sqlite3.connect(self.path, timeout=10, check_same_thread=False)
716
+ db.execute("PRAGMA query_only = ON")
717
+ # As the writer: the file is data, and no function its schema
718
+ # defines (a view's, a trigger's) may run on our behalf.
719
+ db.execute("PRAGMA trusted_schema = OFF")
720
+ # A recorder that has only just created the file may not have made
721
+ # its tables yet (a first run: the History page opens while the
722
+ # writer is still starting). Read that as empty and look again
723
+ # next time, rather than fail with "no such table". The last table
724
+ # the schema makes, so a half-made file counts as not made.
725
+ if not db.execute("SELECT 1 FROM sqlite_master WHERE type='table' "
726
+ "AND name='segments'").fetchone():
727
+ db.close()
728
+ return None
729
+ db.row_factory = sqlite3.Row
730
+ self._db = db
731
+ return self._db
732
+
733
+ def close(self) -> None:
734
+ if self._db is not None:
735
+ self._db.close()
736
+ self._db = None
737
+
738
+ def _rows(self, sql: str, args=(), lazy: bool = False):
739
+ """The rows as a list, or with ``lazy`` the cursor itself, read as
740
+ it's iterated: an export of a week of every probe is millions of rows
741
+ and must not have to fit in memory."""
742
+ db = self._conn()
743
+ if db is None:
744
+ return iter(()) if lazy else []
745
+ cur = db.execute(sql, args)
746
+ return cur if lazy else cur.fetchall()
747
+
748
+ def targets(self) -> List[sqlite3.Row]:
749
+ """Every target ever recorded, most recently monitored first."""
750
+ return self._rows("SELECT target, count(*) AS runs, min(started) AS first, "
751
+ "max(coalesce(ended, started)) AS last FROM runs "
752
+ "GROUP BY target ORDER BY last DESC")
753
+
754
+ def runs(self, target: Optional[str] = None) -> List[sqlite3.Row]:
755
+ if target is None:
756
+ return self._rows("SELECT * FROM runs ORDER BY started")
757
+ return self._rows("SELECT * FROM runs WHERE target=? ORDER BY started", (target,))
758
+
759
+ def events(self, since: float = 0.0, until: float = float("inf"),
760
+ target: Optional[str] = None, kinds: Optional[Iterable[str]] = None,
761
+ text: str = "", limit: Optional[int] = 100_000,
762
+ lazy: bool = False) -> Iterable[sqlite3.Row]:
763
+ sql = ("SELECT e.t, e.kind, e.text, r.target FROM events e LEFT JOIN runs r ON r.id = e.run "
764
+ "WHERE e.t >= ? AND e.t <= ?")
765
+ args: list = [since, until if until != float("inf") else 1e18]
766
+ if target is not None:
767
+ sql += " AND r.target = ?"
768
+ args.append(target)
769
+ kinds = list(kinds or [])
770
+ if kinds:
771
+ sql += f" AND e.kind IN ({','.join('?' * len(kinds))})"
772
+ args += kinds
773
+ if text:
774
+ sql += " AND e.text LIKE ? ESCAPE '\\'"
775
+ args.append("%" + text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%")
776
+ sql += " ORDER BY e.t"
777
+ if limit is not None:
778
+ sql += " LIMIT ?"
779
+ args.append(limit)
780
+ return self._rows(sql, args, lazy)
781
+
782
+ def probes(self, target: str, since: float, until: float,
783
+ lazy: bool = False) -> Iterable[sqlite3.Row]:
784
+ return self._rows(
785
+ "SELECT p.t, p.round, p.ttl, a.addr, p.rtt, p.status FROM probes p "
786
+ "JOIN runs r ON r.id = p.run JOIN addrs a ON a.id = p.addr "
787
+ "WHERE p.t >= ? AND p.t <= ? AND r.target = ? ORDER BY p.t, p.ttl",
788
+ (since, until, target), lazy)
789
+
790
+ def routes(self, target: str) -> List[sqlite3.Row]:
791
+ """Every route recorded to ``target`` (``hops`` is JSON: each hop's
792
+ routers), oldest first."""
793
+ return self._rows("SELECT id, dest, hops, first FROM routes WHERE target=? ORDER BY id",
794
+ (target,))
795
+
796
+ def segments(self, target: str, since: float = 0.0,
797
+ until: float = 1e18) -> List[sqlite3.Row]:
798
+ """When the target was on which route: start, end (None = still on
799
+ it, or the program stopped without closing it), route id."""
800
+ return self._rows(
801
+ "SELECT s.start, s.end, s.route, r.target_ip AS dest FROM segments s "
802
+ "JOIN runs r ON r.id = s.run WHERE r.target = ? AND s.start <= ? "
803
+ "AND coalesce(s.end, 1e18) >= ? ORDER BY s.start", (target, until, since))
804
+
805
+ def names(self, addrs: Iterable[str]) -> Dict[str, str]:
806
+ """The DNS names recorded for these routers (those that have one)."""
807
+ addrs = [a for a in set(addrs) if a]
808
+ out: Dict[str, str] = {}
809
+ for i in range(0, len(addrs), 500):
810
+ chunk = addrs[i:i + 500]
811
+ for row in self._rows(f"SELECT addr, name FROM addrs WHERE name IS NOT NULL AND name != '' "
812
+ f"AND addr IN ({','.join('?' * len(chunk))})", chunk):
813
+ out[row["addr"]] = row["name"]
814
+ return out
815
+
816
+ def minutes(self, target: str, since: float, until: float,
817
+ lazy: bool = False) -> Iterable[sqlite3.Row]:
818
+ return self._rows(
819
+ "SELECT m.minute, m.ttl, a.addr, m.sent, m.recv, m.rtt_sum, m.rtt_min, m.rtt_max, "
820
+ "m.jit_sum, m.jit_n FROM minutes m JOIN runs r ON r.id = m.run "
821
+ "JOIN addrs a ON a.id = m.addr "
822
+ "WHERE m.minute >= ? AND m.minute <= ? AND r.target = ? ORDER BY m.minute, m.ttl",
823
+ (int(since // 60), int(until // 60), target), lazy)