mod-wsgi-telemetry 1.0.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1536 @@
1
+ """Terminal monitor for mod_wsgi telemetry.
2
+
3
+ Connects to a running ``mod_wsgi-telemetry serve`` ingester's WebSocket and
4
+ renders the live state as a top-style terminal UI. Multiple views switchable
5
+ by keystroke; the persistent header stays visible across views.
6
+
7
+ Usage:
8
+ mod_wsgi-telemetry top # connect to localhost
9
+ mod_wsgi-telemetry top --url ws://host:8888/ws
10
+ mod_wsgi-telemetry top --view slow # start on slow-requests view
11
+ mod_wsgi-telemetry top --once # render one frame and exit
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import argparse
17
+ import asyncio
18
+ import curses
19
+ import json
20
+ import logging
21
+ import math
22
+ import os
23
+ import signal
24
+ import sys
25
+ import time
26
+ from collections import deque
27
+ from dataclasses import dataclass, field
28
+ from typing import Any
29
+
30
+ import aiohttp
31
+
32
+
33
+ PHASES = ["server", "queue", "daemon", "application", "request"]
34
+
35
+ # HDR histogram layout. Mirrors the daemon-side encoder: 16 octaves, each
36
+ # split into 4 sub-buckets, plus 1 overflow bucket = 65 entries. Bucket 0
37
+ # covers [1ms, 1.25ms), bucket 1 [1.25ms, 1.5ms), and so on; bucket 64 is
38
+ # >65.5s.
39
+ HDR_OCTAVES = 16
40
+ HDR_SUBS = 4
41
+ HDR_TOTAL = HDR_OCTAVES * HDR_SUBS + 1
42
+ HDR_BASE_SECONDS = 0.001 # bucket 0 lower bound: 1 ms expressed as seconds
43
+
44
+ DEFAULT_URL = "ws://127.0.0.1:8888/ws"
45
+ SAMPLE_RETENTION = 600 # max samples per pid we keep client-side
46
+ REFRESH_RATES = [0.5, 1.0, 2.0, 5.0]
47
+ WINDOW_CHOICES = [10, 60, 600] # rolling window in seconds for overview/latency
48
+
49
+ VIEWS = ["overview", "processes", "workers", "latency", "slow"]
50
+ VIEW_KEYS = {
51
+ ord("o"): "overview", ord("1"): "overview",
52
+ ord("p"): "processes", ord("2"): "processes",
53
+ ord("w"): "workers", ord("3"): "workers",
54
+ ord("l"): "latency", ord("4"): "latency",
55
+ ord("s"): "slow", ord("5"): "slow",
56
+ }
57
+
58
+ SPARK_BLOCKS = " ▁▂▃▄▅▆▇█"
59
+ BAR_FULL = "█"
60
+ BAR_EMPTY = "░"
61
+
62
+ # Color pair IDs.
63
+ CP_HEADER = 1
64
+ CP_OK = 2
65
+ CP_WARN = 3
66
+ CP_CRIT = 4
67
+ CP_DIM = 5
68
+ CP_BAR = 6
69
+ CP_TAB = 7
70
+ CP_TAB_ACTIVE = 8
71
+
72
+
73
+ # ---------------------------------------------------------------------------
74
+ # State mirror
75
+ # ---------------------------------------------------------------------------
76
+
77
+
78
+ @dataclass
79
+ class ProcState:
80
+ """Client-side mirror of the ingester's per-pid rolling window."""
81
+ pid: int
82
+ hostname: str = ""
83
+ process_group: str = ""
84
+ mod_wsgi_version: str = ""
85
+ python_version: str = ""
86
+ apache_version: str = ""
87
+ mpm_name: str = ""
88
+ sample_period: float = 1.0
89
+ samples: deque = field(default_factory=lambda: deque(maxlen=SAMPLE_RETENTION))
90
+ last_seen: float = 0.0
91
+
92
+
93
+ @dataclass
94
+ class SlowEntry:
95
+ pid: int
96
+ thread_id: int
97
+ log_id: str
98
+ method: str
99
+ scheme: str
100
+ hostname: str
101
+ script_name: str
102
+ path_info: str
103
+ start_stamp: float
104
+ duration: float
105
+ state: int
106
+ input_bytes: int
107
+ input_reads: int
108
+ output_bytes: int
109
+ output_writes: int
110
+ cpu_user_time: float
111
+ cpu_system_time: float
112
+ status: int = 0
113
+ last_seen: float = 0.0
114
+
115
+ @classmethod
116
+ def from_dict(cls, d: dict) -> "SlowEntry":
117
+ return cls(
118
+ pid=int(d["pid"]),
119
+ thread_id=int(d["thread_id"]),
120
+ log_id=d["log_id"],
121
+ method=d["method"],
122
+ scheme=d["scheme"],
123
+ hostname=d["hostname"],
124
+ script_name=d["script_name"],
125
+ path_info=d["path_info"],
126
+ start_stamp=float(d["start_stamp"]),
127
+ duration=float(d["duration"]),
128
+ state=int(d["state"]),
129
+ input_bytes=int(d.get("input_bytes", 0)),
130
+ input_reads=int(d.get("input_reads", 0)),
131
+ output_bytes=int(d.get("output_bytes", 0)),
132
+ output_writes=int(d.get("output_writes", 0)),
133
+ cpu_user_time=float(d.get("cpu_user_time", 0.0)),
134
+ cpu_system_time=float(d.get("cpu_system_time", 0.0)),
135
+ status=int(d.get("status", 0)),
136
+ last_seen=time.monotonic(),
137
+ )
138
+
139
+ def url(self) -> str:
140
+ path = (self.script_name or "") + (self.path_info or "")
141
+ return path or "/"
142
+
143
+
144
+ class State:
145
+ """Snapshot + live-sample mirror of the ingester."""
146
+
147
+ def __init__(self) -> None:
148
+ self.processes: dict[int, ProcState] = {}
149
+ self.slow: dict[tuple, SlowEntry] = {}
150
+ self.connected: bool = False
151
+ self.url: str = ""
152
+ self.last_message: float = 0.0
153
+ self.last_error: str = ""
154
+ self.total_received: int = 0
155
+ self.decode_errors: int = 0
156
+
157
+ def apply(self, msg: dict) -> None:
158
+ t = msg.get("type")
159
+ if t == "snapshot":
160
+ self._apply_snapshot(msg)
161
+ elif t == "sample":
162
+ self._apply_sample(msg)
163
+ elif t == "slow_request":
164
+ self._apply_slow(msg)
165
+ elif t == "slow_clear":
166
+ self._apply_slow_clear(msg)
167
+ self.last_message = time.monotonic()
168
+
169
+ def _apply_snapshot(self, msg: dict) -> None:
170
+ self.total_received = int(msg.get("total_received", 0))
171
+ self.decode_errors = int(msg.get("decode_errors", 0))
172
+ self.processes.clear()
173
+ for p in msg.get("processes", []):
174
+ pid = int(p["pid"])
175
+ ps = ProcState(
176
+ pid=pid,
177
+ hostname=p.get("hostname", ""),
178
+ process_group=p.get("process_group", ""),
179
+ mod_wsgi_version=p.get("mod_wsgi_version", ""),
180
+ python_version=p.get("python_version", ""),
181
+ apache_version=p.get("apache_version", ""),
182
+ mpm_name=p.get("mpm_name", ""),
183
+ last_seen=time.monotonic(),
184
+ )
185
+ for s in p.get("samples", []):
186
+ ps.samples.append(s)
187
+ fields = s.get("fields", {})
188
+ sp = fields.get("sample_period")
189
+ if isinstance(sp, (int, float)) and sp > 0:
190
+ ps.sample_period = float(sp)
191
+ self.processes[pid] = ps
192
+ self.slow.clear()
193
+ for r in msg.get("slow_requests", []):
194
+ entry = SlowEntry.from_dict(r["entry"])
195
+ key = tuple(r["key"])
196
+ self.slow[key] = entry
197
+
198
+ def _apply_sample(self, msg: dict) -> None:
199
+ self.total_received += 1
200
+ pid = int(msg["pid"])
201
+ ps = self.processes.get(pid)
202
+ if ps is None:
203
+ ps = ProcState(pid=pid)
204
+ self.processes[pid] = ps
205
+ ps.samples.append(msg)
206
+ ps.last_seen = time.monotonic()
207
+ fields = msg.get("fields", {})
208
+
209
+ def _latch(name: str, attr: str) -> None:
210
+ v = fields.get(name)
211
+ if isinstance(v, str) and v:
212
+ setattr(ps, attr, v)
213
+
214
+ _latch("hostname", "hostname")
215
+ _latch("process_group", "process_group")
216
+ _latch("mod_wsgi_version", "mod_wsgi_version")
217
+ _latch("python_version", "python_version")
218
+ _latch("apache_version", "apache_version")
219
+ _latch("mpm_name", "mpm_name")
220
+
221
+ sp = fields.get("sample_period")
222
+ if isinstance(sp, (int, float)) and sp > 0:
223
+ ps.sample_period = float(sp)
224
+
225
+ def _apply_slow(self, msg: dict) -> None:
226
+ key = tuple(msg["key"])
227
+ self.slow[key] = SlowEntry.from_dict(msg["entry"])
228
+
229
+ def _apply_slow_clear(self, msg: dict) -> None:
230
+ kept = {tuple(k["key"]): SlowEntry.from_dict(k["entry"])
231
+ for k in msg.get("kept", [])}
232
+ self.slow = kept
233
+
234
+
235
+ # ---------------------------------------------------------------------------
236
+ # Aggregation helpers
237
+ # ---------------------------------------------------------------------------
238
+
239
+
240
+ def _samples_in_window(ps: ProcState, seconds: float) -> list[dict]:
241
+ """Newest-first samples within the given wall-clock window."""
242
+ if not ps.samples:
243
+ return []
244
+ newest = ps.samples[-1].get("stamp", 0.0)
245
+ cutoff = newest - seconds
246
+ out = []
247
+ for s in reversed(ps.samples):
248
+ if s.get("stamp", 0.0) < cutoff:
249
+ break
250
+ out.append(s)
251
+ return out
252
+
253
+
254
+ def _last_field(ps: ProcState, name: str, default=0):
255
+ if not ps.samples:
256
+ return default
257
+ return ps.samples[-1].get("fields", {}).get(name, default)
258
+
259
+
260
+ def _hdr_bucket_bounds(idx: int) -> tuple[float, float]:
261
+ """Lower/upper bound (seconds) of bucket `idx`."""
262
+ if idx >= HDR_OCTAVES * HDR_SUBS:
263
+ return (HDR_BASE_SECONDS * (1 << HDR_OCTAVES), float("inf"))
264
+ octave = idx // HDR_SUBS
265
+ sub = idx % HDR_SUBS
266
+ octave_lo = HDR_BASE_SECONDS * (1 << octave)
267
+ sub_width = octave_lo / HDR_SUBS
268
+ return (octave_lo + sub * sub_width, octave_lo + (sub + 1) * sub_width)
269
+
270
+
271
+ def _hdr_percentile(buckets: list[int], pct: float) -> float | None:
272
+ """Linear interpolation inside the bucket containing the percentile."""
273
+ total = sum(buckets)
274
+ if total <= 0:
275
+ return None
276
+ target = pct * total / 100.0
277
+ cumulative = 0
278
+ for i, c in enumerate(buckets):
279
+ if c <= 0:
280
+ continue
281
+ if cumulative + c >= target:
282
+ lo, hi = _hdr_bucket_bounds(i)
283
+ if math.isinf(hi):
284
+ return lo
285
+ frac = (target - cumulative) / c
286
+ return lo + frac * (hi - lo)
287
+ cumulative += c
288
+ return None
289
+
290
+
291
+ def _aggregate_buckets(
292
+ state: State, phase: str, seconds: float, group_filter: str | None
293
+ ) -> list[int]:
294
+ """Sum HDR buckets for `phase` across processes within the window."""
295
+ name = f"{phase}_time_buckets"
296
+ out = [0] * HDR_TOTAL
297
+ for ps in state.processes.values():
298
+ if group_filter and ps.process_group != group_filter:
299
+ continue
300
+ for s in _samples_in_window(ps, seconds):
301
+ buckets = s.get("fields", {}).get(name)
302
+ if isinstance(buckets, list) and len(buckets) == HDR_TOTAL:
303
+ for i, c in enumerate(buckets):
304
+ out[i] += c
305
+ return out
306
+
307
+
308
+ def _phase_min_max(
309
+ state: State, phase: str, seconds: float, group_filter: str | None
310
+ ) -> tuple[float | None, float | None]:
311
+ """Min-of-mins, max-of-maxes (seconds) for `phase` in window."""
312
+ min_name = f"{phase}_time_min"
313
+ max_name = f"{phase}_time_max"
314
+ lo: float | None = None
315
+ hi: float | None = None
316
+ for ps in state.processes.values():
317
+ if group_filter and ps.process_group != group_filter:
318
+ continue
319
+ for s in _samples_in_window(ps, seconds):
320
+ f = s.get("fields", {})
321
+ v = f.get(min_name)
322
+ if isinstance(v, (int, float)) and v > 0:
323
+ lo = v if lo is None else min(lo, v)
324
+ v = f.get(max_name)
325
+ if isinstance(v, (int, float)) and v > 0:
326
+ hi = v if hi is None else max(hi, v)
327
+ return lo, hi
328
+
329
+
330
+ def _process_groups(state: State) -> list[str]:
331
+ seen = sorted({ps.process_group for ps in state.processes.values()
332
+ if ps.process_group})
333
+ return seen
334
+
335
+
336
+ # ---------------------------------------------------------------------------
337
+ # Formatters
338
+ # ---------------------------------------------------------------------------
339
+
340
+
341
+ def fmt_bytes(n: float) -> str:
342
+ if n < 1024:
343
+ return f"{int(n)}B"
344
+ for unit in ("KB", "MB", "GB", "TB"):
345
+ n /= 1024
346
+ if n < 1024:
347
+ return f"{n:.1f}{unit}"
348
+ return f"{n:.1f}PB"
349
+
350
+
351
+ def fmt_seconds(s: float | int | None) -> str:
352
+ """Render a duration in seconds as a human-readable string."""
353
+ if s is None:
354
+ return "-"
355
+ if s < 0.001:
356
+ return f"{s*1_000_000:.0f}µs"
357
+ if s < 1.0:
358
+ return f"{s*1000:.1f}ms"
359
+ return f"{s:.2f}s"
360
+
361
+
362
+ def sparkline(values: list[float], width: int) -> str:
363
+ if width <= 0 or not values:
364
+ return ""
365
+ if len(values) > width:
366
+ # downsample by averaging windows
367
+ step = len(values) / width
368
+ out = []
369
+ for i in range(width):
370
+ lo = int(i * step)
371
+ hi = max(lo + 1, int((i + 1) * step))
372
+ out.append(sum(values[lo:hi]) / max(1, hi - lo))
373
+ values = out
374
+ elif len(values) < width:
375
+ values = [0.0] * (width - len(values)) + values
376
+ vmax = max(values) if values else 0.0
377
+ if vmax <= 0:
378
+ return SPARK_BLOCKS[0] * width
379
+ levels = len(SPARK_BLOCKS) - 1
380
+ return "".join(
381
+ SPARK_BLOCKS[min(levels, max(0, int(v / vmax * levels)))]
382
+ for v in values
383
+ )
384
+
385
+
386
+ # ---------------------------------------------------------------------------
387
+ # Curses-safe drawing
388
+ # ---------------------------------------------------------------------------
389
+
390
+
391
+ def safe_addstr(win, y: int, x: int, text: str, attr: int = 0) -> None:
392
+ """addstr that won't raise if it bumps the bottom-right cell."""
393
+ try:
394
+ h, w = win.getmaxyx()
395
+ if y < 0 or y >= h or x >= w:
396
+ return
397
+ if x < 0:
398
+ text = text[-x:]
399
+ x = 0
400
+ avail = w - x
401
+ if avail <= 0:
402
+ return
403
+ if len(text) > avail:
404
+ text = text[:avail]
405
+ win.addstr(y, x, text, attr)
406
+ except curses.error:
407
+ pass
408
+
409
+
410
+ # ---------------------------------------------------------------------------
411
+ # View renderers
412
+ # ---------------------------------------------------------------------------
413
+
414
+
415
+ @dataclass
416
+ class UIState:
417
+ view: str = "overview"
418
+ paused: bool = False
419
+ refresh_idx: int = 1 # 1.0 s
420
+ window_idx: int = 1 # 60 s
421
+ phase_idx: int = 4 # request
422
+ process_sort: str = "rps" # rps|cpu|rss|p95|slow|pid
423
+ slow_sort: str = "elapsed" # elapsed|pid|method|url
424
+ slow_state_filter: int = -1 # -1=any, 0=active, 1=completed
425
+ slow_search: str = ""
426
+ group_filter: str | None = None
427
+ show_help: bool = False
428
+
429
+ @property
430
+ def refresh(self) -> float:
431
+ return REFRESH_RATES[self.refresh_idx]
432
+
433
+ @property
434
+ def window(self) -> int:
435
+ return WINDOW_CHOICES[self.window_idx]
436
+
437
+ @property
438
+ def phase(self) -> str:
439
+ return PHASES[self.phase_idx]
440
+
441
+
442
+ def _aggregate_header(state: State, ui: UIState) -> dict:
443
+ """Compute the always-visible header values once per render."""
444
+ rps_now = 0.0
445
+ in_bytes_per_s = 0.0
446
+ out_bytes_per_s = 0.0
447
+ cpu_user = 0.0
448
+ cpu_sys = 0.0
449
+ rss_total = 0
450
+ rss_max = 0
451
+ busy = 0
452
+ threads_total = 0
453
+ queue_mean = 0.0
454
+ queue_count = 0
455
+
456
+ rps_window: dict[int, list[tuple[int, int]]] = {}
457
+ status_1xx = 0
458
+ status_2xx = 0
459
+ status_3xx = 0
460
+ status_4xx = 0
461
+ status_5xx = 0
462
+ interval_requests = 0
463
+ for ps in state.processes.values():
464
+ if ui.group_filter and ps.process_group != ui.group_filter:
465
+ continue
466
+ if not ps.samples:
467
+ continue
468
+ last = ps.samples[-1]
469
+ f = last.get("fields", {})
470
+ rps_now += float(f.get("request_throughput") or 0)
471
+ in_bytes_per_s += float(f.get("input_bytes_total") or 0) / max(1e-3, ps.sample_period)
472
+ out_bytes_per_s += float(f.get("output_bytes_total") or 0) / max(1e-3, ps.sample_period)
473
+ cpu_user += float(f.get("cpu_user_utilization") or 0)
474
+ cpu_sys += float(f.get("cpu_system_utilization") or 0)
475
+ rss = int(f.get("memory_rss") or 0)
476
+ rss_total += rss
477
+ rss_max = max(rss_max, int(f.get("memory_max_rss") or 0))
478
+ active = int(f.get("request_threads_active") or 0)
479
+ maximum = int(f.get("request_threads_maximum") or 0)
480
+ busy += active
481
+ threads_total += maximum
482
+ qt = f.get("queue_time")
483
+ if isinstance(qt, (int, float)) and qt > 0:
484
+ queue_mean += qt
485
+ queue_count += 1
486
+ status_1xx += int(f.get("status_1xx_total") or 0)
487
+ status_2xx += int(f.get("status_2xx_total") or 0)
488
+ status_3xx += int(f.get("status_3xx_total") or 0)
489
+ status_4xx += int(f.get("status_4xx_total") or 0)
490
+ status_5xx += int(f.get("status_5xx_total") or 0)
491
+ interval_requests += int(f.get("request_count") or 0)
492
+
493
+ # 1-min and 10-min RPS averages: total request_count over window / seconds.
494
+ rps_1m = _avg_rps(state, ui, 60)
495
+ rps_10m = _avg_rps(state, ui, 600)
496
+
497
+ buckets = _aggregate_buckets(state, "request", ui.window, ui.group_filter)
498
+ p50 = _hdr_percentile(buckets, 50)
499
+ p95 = _hdr_percentile(buckets, 95)
500
+ p99 = _hdr_percentile(buckets, 99)
501
+ lo, hi = _phase_min_max(state, "request", ui.window, ui.group_filter)
502
+
503
+ now = time.time()
504
+ active_slow = sum(1 for e in state.slow.values() if e.state == 0)
505
+ recent_slow = sum(
506
+ 1 for e in state.slow.values()
507
+ if (now - e.start_stamp) <= 60
508
+ )
509
+ total_slow = len(state.slow)
510
+
511
+ return {
512
+ "rps_now": rps_now,
513
+ "rps_1m": rps_1m,
514
+ "rps_10m": rps_10m,
515
+ "in_bps": in_bytes_per_s,
516
+ "out_bps": out_bytes_per_s,
517
+ "cpu_user": cpu_user,
518
+ "cpu_sys": cpu_sys,
519
+ "rss_total": rss_total,
520
+ "rss_max": rss_max,
521
+ "busy": busy,
522
+ "threads_total": threads_total,
523
+ "queue_mean": (queue_mean / queue_count) if queue_count else None,
524
+ "p50": p50, "p95": p95, "p99": p99,
525
+ "min": lo, "max": hi,
526
+ "active_slow": active_slow,
527
+ "recent_slow": recent_slow,
528
+ "total_slow": total_slow,
529
+ "status_1xx": status_1xx,
530
+ "status_2xx": status_2xx,
531
+ "status_3xx": status_3xx,
532
+ "status_4xx": status_4xx,
533
+ "status_5xx": status_5xx,
534
+ "interval_requests": interval_requests,
535
+ }
536
+
537
+
538
+ def _avg_rps(state: State, ui: UIState, seconds: float) -> float:
539
+ total = 0
540
+ spans: list[float] = []
541
+ for ps in state.processes.values():
542
+ if ui.group_filter and ps.process_group != ui.group_filter:
543
+ continue
544
+ win = _samples_in_window(ps, seconds)
545
+ if len(win) < 2:
546
+ continue
547
+ oldest = win[-1].get("stamp", 0.0)
548
+ newest = win[0].get("stamp", 0.0)
549
+ span = newest - oldest
550
+ if span <= 0:
551
+ continue
552
+ rc = sum(int(s.get("fields", {}).get("request_count") or 0) for s in win)
553
+ total += rc
554
+ spans.append(span)
555
+ if not spans:
556
+ return 0.0
557
+ return total / (sum(spans) / len(spans))
558
+
559
+
560
+ def render_header(win, state: State, ui: UIState, h: dict) -> int:
561
+ """Draw the persistent header. Returns the row count consumed."""
562
+ width = win.getmaxyx()[1]
563
+ hostname = ""
564
+ pgroup = ui.group_filter or ""
565
+ sample_period = 0.0
566
+ if state.processes:
567
+ ps = next(iter(state.processes.values()))
568
+ hostname = ps.hostname
569
+ if not pgroup:
570
+ groups = _process_groups(state)
571
+ pgroup = ",".join(groups) if groups else "(any)"
572
+ sample_period = ps.sample_period
573
+
574
+ # Line 0 — title bar + connection state.
575
+ title = f" mod_wsgi-telemetry top {hostname or '-'} · {pgroup} "
576
+ status = "[paused]" if ui.paused else ("[live]" if state.connected else "[disconnected]")
577
+ extras = f" tick={sample_period:.1f}s refresh={ui.refresh:g}s win={_win_label(ui.window)} {status} "
578
+ pad = " " * max(0, width - len(title) - len(extras))
579
+ safe_addstr(win, 0, 0, title + pad + extras, curses.color_pair(CP_HEADER) | curses.A_BOLD)
580
+
581
+ # Line 1 — throughput. No padding on rps_now so the value starts
582
+ # at column 13, lining up vertically with the labels on the rows
583
+ # below (Capacity:, CPU:, Status:, Latency:, Slow:).
584
+ safe_addstr(win, 1, 0,
585
+ f" Throughput: {h['rps_now']:.1f} req/s now "
586
+ f"{h['rps_1m']:.1f}/s 1m {h['rps_10m']:.1f}/s 10m "
587
+ f"in {fmt_bytes(h['in_bps'])}/s out {fmt_bytes(h['out_bps'])}/s")
588
+
589
+ # Line 2 — capacity bar.
590
+ cap_total = h["threads_total"]
591
+ cap_busy = h["busy"]
592
+ frac = (cap_busy / cap_total) if cap_total else 0.0
593
+ bar_w = max(10, min(30, width - 60))
594
+ filled = int(round(frac * bar_w))
595
+ bar = BAR_FULL * filled + BAR_EMPTY * (bar_w - filled)
596
+ cap_attr = _cap_color(frac)
597
+ qstr = (f"queue {fmt_seconds(h['queue_mean'])}"
598
+ if h["queue_mean"] is not None else "queue -")
599
+ safe_addstr(win, 2, 0, " Capacity: ")
600
+ safe_addstr(win, 2, 13, bar, cap_attr)
601
+ safe_addstr(win, 2, 13 + bar_w + 1,
602
+ f" {cap_busy:>4d}/{cap_total:<4d} threads busy "
603
+ f"({frac*100:5.1f}%) {qstr}")
604
+
605
+ # Line 3 — CPU + memory.
606
+ safe_addstr(win, 3, 0,
607
+ f" CPU: user {h['cpu_user']:.2f} sys {h['cpu_sys']:.2f} "
608
+ f"total {h['cpu_user']+h['cpu_sys']:.2f} cores "
609
+ f"Memory: RSS {fmt_bytes(h['rss_total'])} (max {fmt_bytes(h['rss_max'])})")
610
+
611
+ # Line 4 — per-class HTTP response distribution for the latest
612
+ # interval. 2xx / 3xx / 4xx / 5xx as percentages of request_count.
613
+ # 1xx is a PEP 3333 tripwire (a WSGI app should never return 1xx)
614
+ # so it carries its raw count and renders amber when > 0; dim when
615
+ # zero. 5xx renders red-bold when > 0; 4xx is dim (mostly noise).
616
+ requests = h["interval_requests"]
617
+ def _pct(n):
618
+ return (n / requests * 100.0) if requests > 0 else 0.0
619
+ safe_addstr(win, 4, 0, " Status: ")
620
+ x = len(" Status: ")
621
+ s1 = f"1xx {h['status_1xx']}"
622
+ s1_attr = (curses.color_pair(CP_WARN) | curses.A_BOLD) if h["status_1xx"] > 0 \
623
+ else curses.color_pair(CP_DIM)
624
+ safe_addstr(win, 4, x, s1, s1_attr)
625
+ x += len(s1) + 2
626
+ s2 = f"2xx {_pct(h['status_2xx']):5.1f}%"
627
+ safe_addstr(win, 4, x, s2)
628
+ x += len(s2) + 2
629
+ s3 = f"3xx {_pct(h['status_3xx']):5.1f}%"
630
+ safe_addstr(win, 4, x, s3)
631
+ x += len(s3) + 2
632
+ s4 = f"4xx {_pct(h['status_4xx']):5.1f}%"
633
+ safe_addstr(win, 4, x, s4, curses.color_pair(CP_DIM))
634
+ x += len(s4) + 2
635
+ s5 = f"5xx {_pct(h['status_5xx']):5.1f}%"
636
+ s5_attr = (curses.color_pair(CP_CRIT) | curses.A_BOLD) if h["status_5xx"] > 0 \
637
+ else curses.color_pair(CP_DIM)
638
+ safe_addstr(win, 4, x, s5, s5_attr)
639
+
640
+ # Line 5 — latency percentiles.
641
+ p50 = fmt_seconds(h["p50"]); p95 = fmt_seconds(h["p95"]); p99 = fmt_seconds(h["p99"])
642
+ mn = fmt_seconds(h["min"]); mx = fmt_seconds(h["max"])
643
+ safe_addstr(win, 5, 0,
644
+ f" Latency: p50 {p50:>8} p95 {p95:>8} p99 {p99:>8} "
645
+ f"min {mn:>8} max {mx:>8}")
646
+
647
+ # Line 6 — slow-request counters.
648
+ safe_addstr(win, 6, 0,
649
+ f" Slow: {h['active_slow']} active / "
650
+ f"{h['recent_slow']} 1m / {h['total_slow']} total")
651
+
652
+ # Line 7 — tab bar.
653
+ _render_tabs(win, 7, width, ui.view)
654
+ return 8
655
+
656
+
657
+ def _win_label(seconds: int) -> str:
658
+ if seconds < 60:
659
+ return f"{seconds}s"
660
+ if seconds < 3600:
661
+ return f"{seconds // 60}m"
662
+ return f"{seconds // 3600}h"
663
+
664
+
665
+ def _cap_color(frac: float) -> int:
666
+ if frac >= 0.9:
667
+ return curses.color_pair(CP_CRIT) | curses.A_BOLD
668
+ if frac >= 0.7:
669
+ return curses.color_pair(CP_WARN)
670
+ return curses.color_pair(CP_OK)
671
+
672
+
673
+ def _render_tabs(win, y: int, width: int, active: str) -> None:
674
+ safe_addstr(win, y, 0, " " * width, curses.color_pair(CP_TAB))
675
+ x = 1
676
+ for i, name in enumerate(VIEWS, start=1):
677
+ label = f" {i} {name} "
678
+ attr = curses.color_pair(CP_TAB_ACTIVE) | curses.A_BOLD if name == active \
679
+ else curses.color_pair(CP_TAB)
680
+ safe_addstr(win, y, x, label, attr)
681
+ x += len(label)
682
+ if x >= width:
683
+ break
684
+ hint = " ?=help q=quit "
685
+ safe_addstr(win, y, max(x, width - len(hint)), hint, curses.color_pair(CP_TAB))
686
+
687
+
688
+ # --- Overview view -----------------------------------------------------------
689
+
690
+
691
+ def render_overview(win, state: State, ui: UIState, h: dict, y0: int) -> None:
692
+ height, width = win.getmaxyx()
693
+ avail = height - y0 - 1
694
+ if avail <= 0:
695
+ return
696
+ # Build a per-second time series for the window using the most-active pid
697
+ # as the time reference. Aggregate across pids on each second.
698
+ seconds = ui.window
699
+ rps_series, cap_series, cpu_series, rss_series = _build_series(state, ui, seconds)
700
+
701
+ spark_w = max(20, width - 18)
702
+ rows = [
703
+ ("Throughput ", rps_series, lambda v: f"{v:.1f}/s now"),
704
+ ("Capacity % ", cap_series, lambda v: f"{v*100:.0f}% now"),
705
+ ("CPU cores ", cpu_series, lambda v: f"{v:.2f}/core"),
706
+ ("RSS MB ", rss_series, lambda v: f"{v/1_048_576:.0f} MB"),
707
+ ]
708
+ y = y0 + 1
709
+ safe_addstr(win, y0, 1, f"Overview — last {_win_label(seconds)} (use < > to change window)",
710
+ curses.A_BOLD)
711
+ for label, series, fmt in rows:
712
+ if y >= height - 1:
713
+ return
714
+ safe_addstr(win, y, 1, label)
715
+ line = sparkline(series, spark_w)
716
+ safe_addstr(win, y, 13, line)
717
+ if series:
718
+ safe_addstr(win, y, 14 + spark_w, fmt(series[-1]))
719
+ y += 1
720
+
721
+ y += 1
722
+ if y >= height - 1:
723
+ return
724
+ safe_addstr(win, y, 1, "Phase mean times (current tick):", curses.A_BOLD)
725
+ y += 1
726
+ if y >= height - 1:
727
+ return
728
+ parts = []
729
+ for phase in PHASES:
730
+ v = _aggregate_phase_mean(state, phase, ui.group_filter)
731
+ parts.append(f"{phase}={fmt_seconds(v) if v is not None else '-':>8}")
732
+ safe_addstr(win, y, 1, " ".join(parts))
733
+
734
+
735
+ def _build_series(state: State, ui: UIState, seconds: int):
736
+ """Per-second aggregated series across pids for the last `seconds`."""
737
+ # Bucket samples by their second-stamp.
738
+ buckets: dict[int, dict[str, float]] = {}
739
+ for ps in state.processes.values():
740
+ if ui.group_filter and ps.process_group != ui.group_filter:
741
+ continue
742
+ for s in _samples_in_window(ps, seconds):
743
+ t = int(s.get("stamp", 0.0))
744
+ f = s.get("fields", {})
745
+ b = buckets.setdefault(t, {"rps": 0.0, "cap_b": 0, "cap_t": 0,
746
+ "cpu": 0.0, "rss": 0})
747
+ b["rps"] += float(f.get("request_throughput") or 0)
748
+ b["cap_b"] += int(f.get("request_threads_active") or 0)
749
+ b["cap_t"] += int(f.get("request_threads_maximum") or 0)
750
+ b["cpu"] += float(f.get("cpu_utilization") or 0)
751
+ b["rss"] += int(f.get("memory_rss") or 0)
752
+ keys = sorted(buckets)
753
+ rps = [buckets[k]["rps"] for k in keys]
754
+ cap = [(buckets[k]["cap_b"] / buckets[k]["cap_t"]) if buckets[k]["cap_t"] else 0.0
755
+ for k in keys]
756
+ cpu = [buckets[k]["cpu"] for k in keys]
757
+ rss = [buckets[k]["rss"] for k in keys]
758
+ return rps, cap, cpu, rss
759
+
760
+
761
+ def _aggregate_phase_mean(state: State, phase: str, group_filter: str | None):
762
+ """Sample-weighted mean of last-tick `<phase>_time` across processes."""
763
+ name = f"{phase}_time"
764
+ total = 0.0
765
+ n = 0
766
+ for ps in state.processes.values():
767
+ if group_filter and ps.process_group != group_filter:
768
+ continue
769
+ if not ps.samples:
770
+ continue
771
+ v = ps.samples[-1].get("fields", {}).get(name)
772
+ if isinstance(v, (int, float)) and v > 0:
773
+ total += v
774
+ n += 1
775
+ return (total / n) if n else None
776
+
777
+
778
+ # --- Processes view ----------------------------------------------------------
779
+
780
+
781
+ def render_processes(win, state: State, ui: UIState, y0: int) -> None:
782
+ height, width = win.getmaxyx()
783
+ rows = []
784
+ for ps in state.processes.values():
785
+ if ui.group_filter and ps.process_group != ui.group_filter:
786
+ continue
787
+ if not ps.samples:
788
+ continue
789
+ f = ps.samples[-1].get("fields", {})
790
+ buckets = []
791
+ for s in _samples_in_window(ps, 60):
792
+ b = s.get("fields", {}).get("request_time_buckets")
793
+ if isinstance(b, list) and len(b) == HDR_TOTAL:
794
+ if not buckets:
795
+ buckets = list(b)
796
+ else:
797
+ for i, c in enumerate(b):
798
+ buckets[i] += c
799
+ p95 = _hdr_percentile(buckets, 95) if buckets else None
800
+ slow_count = sum(1 for k, e in state.slow.items() if e.pid == ps.pid)
801
+ rows.append({
802
+ "pid": ps.pid,
803
+ "group": ps.process_group or "-",
804
+ "active": int(f.get("request_threads_active") or 0),
805
+ "max": int(f.get("request_threads_maximum") or 0),
806
+ "rps": float(f.get("request_throughput") or 0),
807
+ "cpu": float(f.get("cpu_utilization") or 0),
808
+ "rss": int(f.get("memory_rss") or 0),
809
+ "p95": p95,
810
+ "slow": slow_count,
811
+ })
812
+
813
+ sort_key = ui.process_sort
814
+ keyfn = {
815
+ "rps": lambda r: -r["rps"],
816
+ "cpu": lambda r: -r["cpu"],
817
+ "rss": lambda r: -r["rss"],
818
+ "p95": lambda r: -(r["p95"] or 0),
819
+ "slow": lambda r: -r["slow"],
820
+ "pid": lambda r: r["pid"],
821
+ }.get(sort_key, lambda r: -r["rps"])
822
+ rows.sort(key=keyfn)
823
+
824
+ safe_addstr(win, y0, 1,
825
+ f"Processes — sort: {sort_key} (use < > to change) "
826
+ f"{len(rows)} pids", curses.A_BOLD)
827
+ header = f" {'PID':>7} {'GROUP':<16} {'THREADS':>9} {'RPS':>7} " \
828
+ f"{'CPU':>6} {'RSS':>9} {'P95':>9} {'SLOW':>5}"
829
+ safe_addstr(win, y0 + 1, 0, header, curses.A_REVERSE)
830
+ y = y0 + 2
831
+ for r in rows:
832
+ if y >= height - 1:
833
+ break
834
+ line = (f" {r['pid']:>7} {r['group'][:16]:<16} "
835
+ f"{r['active']:>4}/{r['max']:<4} "
836
+ f"{r['rps']:>7.1f} "
837
+ f"{r['cpu']:>6.2f} "
838
+ f"{fmt_bytes(r['rss']):>9} "
839
+ f"{fmt_seconds(r['p95']):>9} "
840
+ f"{r['slow']:>5}")
841
+ attr = 0
842
+ if r["max"] and r["active"] / r["max"] >= 0.9:
843
+ attr = curses.color_pair(CP_CRIT)
844
+ elif r["max"] and r["active"] / r["max"] >= 0.7:
845
+ attr = curses.color_pair(CP_WARN)
846
+ safe_addstr(win, y, 0, line, attr)
847
+ y += 1
848
+
849
+
850
+ # --- Workers / slots view ----------------------------------------------------
851
+
852
+
853
+ def render_workers(win, state: State, ui: UIState, y0: int) -> None:
854
+ height, width = win.getmaxyx()
855
+ safe_addstr(win, y0, 1,
856
+ "Workers — slot grid: . idle * <1s # 1-5s ! >=slow-threshold",
857
+ curses.A_BOLD)
858
+ y = y0 + 1
859
+ pids = sorted(state.processes.keys())
860
+ for pid in pids:
861
+ if y >= height - 1:
862
+ break
863
+ ps = state.processes[pid]
864
+ if ui.group_filter and ps.process_group != ui.group_filter:
865
+ continue
866
+ if not ps.samples:
867
+ continue
868
+ last = ps.samples[-1].get("fields", {})
869
+ elapsed = last.get("request_threads_current_elapsed")
870
+ slow_thresh = last.get("slow_requests_threshold")
871
+ slow_thresh = (slow_thresh
872
+ if isinstance(slow_thresh, (int, float)) and slow_thresh > 0
873
+ else None)
874
+ if not isinstance(elapsed, list):
875
+ continue
876
+ cells = []
877
+ longest_idx = -1
878
+ longest = 0.0
879
+ for i, e in enumerate(elapsed):
880
+ if e <= 0:
881
+ cells.append(".")
882
+ elif slow_thresh is not None and e >= slow_thresh:
883
+ cells.append("!")
884
+ elif e >= 5.0:
885
+ cells.append("#")
886
+ elif e >= 1.0:
887
+ cells.append("#")
888
+ else:
889
+ cells.append("*")
890
+ if e > longest:
891
+ longest = e
892
+ longest_idx = i
893
+
894
+ label = f" pid {pid:<6} {ps.process_group[:14]:<14}"
895
+ safe_addstr(win, y, 0, label)
896
+ # Truncate slot row to terminal width.
897
+ avail = width - len(label) - 2
898
+ row = "".join(cells[:avail]) if avail > 0 else ""
899
+ # Color cells: red for !, yellow for #, default for *.
900
+ cx = len(label) + 1
901
+ for ch in row:
902
+ if cx >= width:
903
+ break
904
+ attr = 0
905
+ if ch == "!":
906
+ attr = curses.color_pair(CP_CRIT) | curses.A_BOLD
907
+ elif ch == "#":
908
+ attr = curses.color_pair(CP_WARN)
909
+ elif ch == "*":
910
+ attr = curses.color_pair(CP_OK)
911
+ else:
912
+ attr = curses.color_pair(CP_DIM)
913
+ safe_addstr(win, y, cx, ch, attr)
914
+ cx += 1
915
+ y += 1
916
+ if y >= height - 1:
917
+ break
918
+ if longest_idx >= 0 and longest > 0:
919
+ note = (f" longest slot: #{longest_idx} "
920
+ f"running {fmt_seconds(longest)}")
921
+ # Look for an active slow request matching this pid.
922
+ active = [e for e in state.slow.values()
923
+ if e.pid == pid and e.state == 0]
924
+ active.sort(key=lambda e: -e.duration)
925
+ if active:
926
+ e = active[0]
927
+ note += f" {e.method} {e.url()[:60]}"
928
+ safe_addstr(win, y, 0, note, curses.color_pair(CP_DIM))
929
+ y += 1
930
+
931
+
932
+ # --- Latency view ------------------------------------------------------------
933
+
934
+
935
+ def render_latency(win, state: State, ui: UIState, y0: int) -> None:
936
+ height, width = win.getmaxyx()
937
+ phase = ui.phase
938
+ seconds = ui.window
939
+ buckets = _aggregate_buckets(state, phase, seconds, ui.group_filter)
940
+ total = sum(buckets)
941
+ p50 = _hdr_percentile(buckets, 50)
942
+ p95 = _hdr_percentile(buckets, 95)
943
+ p99 = _hdr_percentile(buckets, 99)
944
+ lo, hi = _phase_min_max(state, phase, seconds, ui.group_filter)
945
+
946
+ safe_addstr(win, y0, 1,
947
+ f"Latency — phase: {phase} window: {_win_label(seconds)} "
948
+ f"samples: {total} "
949
+ f"([ ] phase, < > window)", curses.A_BOLD)
950
+ safe_addstr(win, y0 + 1, 1,
951
+ f" p50 {fmt_seconds(p50):>8} p95 {fmt_seconds(p95):>8} "
952
+ f"p99 {fmt_seconds(p99):>8} "
953
+ f"min {fmt_seconds(lo):>8} max {fmt_seconds(hi):>8}")
954
+
955
+ # Histogram. 65 buckets: render one column per bucket where it fits;
956
+ # otherwise pair them up so the chart still spans the whole window.
957
+ chart_top = y0 + 3
958
+ chart_bottom = height - 2
959
+ chart_h = max(4, chart_bottom - chart_top - 1)
960
+ chart_left = 4
961
+ chart_w = width - chart_left - 1
962
+ if chart_w <= 0 or chart_h <= 0:
963
+ return
964
+
965
+ cols_per_bucket = max(1, chart_w // HDR_TOTAL)
966
+ n_cols = HDR_TOTAL * cols_per_bucket
967
+ if n_cols > chart_w:
968
+ # Fallback: collapse buckets into chart_w cells by binning.
969
+ per_col = math.ceil(HDR_TOTAL / chart_w)
970
+ binned = []
971
+ for i in range(0, HDR_TOTAL, per_col):
972
+ binned.append(sum(buckets[i:i + per_col]))
973
+ display = binned
974
+ cols_per_bucket = 1
975
+ else:
976
+ display = []
977
+ for c in buckets:
978
+ display.extend([c] * cols_per_bucket)
979
+
980
+ vmax = max(display) if display else 0
981
+ if vmax <= 0:
982
+ safe_addstr(win, chart_top, chart_left, "(no samples in window)",
983
+ curses.color_pair(CP_DIM))
984
+ return
985
+
986
+ levels = chart_h * 8
987
+ # 8 sub-pixels per row using upper partial blocks.
988
+ PARTIAL = " ▁▂▃▄▅▆▇█"
989
+ # Render columns top-to-bottom.
990
+ for col, count in enumerate(display):
991
+ x = chart_left + col
992
+ if x >= width - 1:
993
+ break
994
+ h_units = int(round(count / vmax * levels))
995
+ full_rows = h_units // 8
996
+ rem = h_units % 8
997
+ # Bottom row anchored at chart_bottom.
998
+ for r in range(full_rows):
999
+ y = chart_bottom - r
1000
+ if y < chart_top:
1001
+ break
1002
+ safe_addstr(win, y, x, BAR_FULL, curses.color_pair(CP_BAR))
1003
+ if rem and full_rows < chart_h:
1004
+ y = chart_bottom - full_rows
1005
+ if y >= chart_top:
1006
+ safe_addstr(win, y, x, PARTIAL[rem], curses.color_pair(CP_BAR))
1007
+
1008
+ # Octave labels along the bottom.
1009
+ label_y = chart_bottom + 1
1010
+ if label_y < height:
1011
+ # Print a label every 4 buckets (= every octave start).
1012
+ for octave in range(0, HDR_OCTAVES, 2):
1013
+ lo = HDR_BASE_SECONDS * (1 << octave)
1014
+ x = chart_left + octave * HDR_SUBS * cols_per_bucket
1015
+ if x >= width:
1016
+ break
1017
+ safe_addstr(win, label_y, x, _short_seconds(lo),
1018
+ curses.color_pair(CP_DIM))
1019
+
1020
+
1021
+ def _short_seconds(s: float) -> str:
1022
+ """Compact axis-tick label for a bucket boundary."""
1023
+ if s < 1.0:
1024
+ return f"{s*1000:.0f}ms"
1025
+ return f"{s:.0f}s"
1026
+
1027
+
1028
+ # --- Slow requests view ------------------------------------------------------
1029
+
1030
+
1031
+ def render_slow(win, state: State, ui: UIState, y0: int) -> None:
1032
+ height, width = win.getmaxyx()
1033
+ rows = []
1034
+ now = time.time()
1035
+ for key, e in state.slow.items():
1036
+ if ui.slow_state_filter != -1 and e.state != ui.slow_state_filter:
1037
+ continue
1038
+ if ui.group_filter:
1039
+ ps = state.processes.get(e.pid)
1040
+ if not ps or ps.process_group != ui.group_filter:
1041
+ continue
1042
+ if ui.slow_search and ui.slow_search.lower() not in e.url().lower():
1043
+ continue
1044
+ if e.state == 0:
1045
+ elapsed = max(e.duration, now - e.start_stamp)
1046
+ else:
1047
+ elapsed = e.duration
1048
+ rows.append((elapsed, key, e))
1049
+
1050
+ sort_key = ui.slow_sort
1051
+ if sort_key == "elapsed":
1052
+ rows.sort(key=lambda r: -r[0])
1053
+ elif sort_key == "pid":
1054
+ rows.sort(key=lambda r: (r[2].pid, -r[0]))
1055
+ elif sort_key == "method":
1056
+ rows.sort(key=lambda r: (r[2].method, -r[0]))
1057
+ elif sort_key == "url":
1058
+ rows.sort(key=lambda r: (r[2].url(), -r[0]))
1059
+
1060
+ state_filter = {-1: "any", 0: "active", 1: "completed"}[ui.slow_state_filter]
1061
+ title = (f"Slow requests — sort: {sort_key} state: {state_filter} "
1062
+ f"({len(rows)} shown / {len(state.slow)} total) "
1063
+ f"(< > sort, f filter, / search)")
1064
+ safe_addstr(win, y0, 1, title, curses.A_BOLD)
1065
+ if ui.slow_search:
1066
+ safe_addstr(win, y0, len(title) + 2, f"search: {ui.slow_search!r}",
1067
+ curses.color_pair(CP_WARN))
1068
+
1069
+ header = f" {'STATE':<6} {'ELAPSED':>9} {'PID':>7} " \
1070
+ f"{'METHOD':<7} {'STATUS':>6} {'LOG ID':<24} URL"
1071
+ safe_addstr(win, y0 + 1, 0, header, curses.A_REVERSE)
1072
+ y = y0 + 2
1073
+ for elapsed, key, e in rows:
1074
+ if y >= height - 1:
1075
+ break
1076
+ st = "active" if e.state == 0 else "done"
1077
+ attr = curses.color_pair(CP_CRIT) | curses.A_BOLD if e.state == 0 \
1078
+ else curses.color_pair(CP_DIM)
1079
+ # status==0 means start_response wasn't called yet: show a dash
1080
+ # so it's visibly distinct from a real status code.
1081
+ status_str = "-" if not e.status else str(e.status)
1082
+ line = (f" {st:<6} {fmt_seconds(elapsed):>9} {e.pid:>7} "
1083
+ f"{e.method:<7} {status_str:>6} "
1084
+ f"{(e.log_id or '-')[:24]:<24} {e.url()}")
1085
+ safe_addstr(win, y, 0, line, attr)
1086
+ y += 1
1087
+
1088
+ if y < height - 1:
1089
+ # Show a brief I/O footer for the top entry.
1090
+ if rows:
1091
+ _, _, top = rows[0]
1092
+ footer = (f" top: cpu user={fmt_seconds(top.cpu_user_time)} "
1093
+ f"sys={fmt_seconds(top.cpu_system_time)} "
1094
+ f"i/o in={fmt_bytes(top.input_bytes)}/{top.input_reads}r "
1095
+ f"out={fmt_bytes(top.output_bytes)}/{top.output_writes}w")
1096
+ safe_addstr(win, height - 2, 0, footer, curses.color_pair(CP_DIM))
1097
+
1098
+
1099
+ # --- Help overlay ------------------------------------------------------------
1100
+
1101
+
1102
+ HELP_LINES = [
1103
+ "Keys:",
1104
+ " o / 1 overview p / 2 processes",
1105
+ " w / 3 workers l / 4 latency",
1106
+ " s / 5 slow requests",
1107
+ "",
1108
+ " space pause/resume rendering r reset/clear (slow view)",
1109
+ " + / - change refresh rate ? toggle this help",
1110
+ " < / > change sort or window [ / ] cycle phase (latency)",
1111
+ " f cycle filters (group, slow state)",
1112
+ " / slow URL search (slow view; type then Enter; Esc clears)",
1113
+ " q quit",
1114
+ ]
1115
+
1116
+
1117
+ def render_help(win) -> None:
1118
+ h, w = win.getmaxyx()
1119
+ box_w = max(60, max(len(line) for line in HELP_LINES) + 4)
1120
+ box_h = len(HELP_LINES) + 4
1121
+ y0 = max(2, (h - box_h) // 2)
1122
+ x0 = max(2, (w - box_w) // 2)
1123
+ for r in range(box_h):
1124
+ safe_addstr(win, y0 + r, x0, " " * box_w, curses.color_pair(CP_TAB))
1125
+ safe_addstr(win, y0, x0, " " * box_w, curses.color_pair(CP_HEADER) | curses.A_BOLD)
1126
+ safe_addstr(win, y0, x0 + 2, " mod_wsgi-telemetry top — help ",
1127
+ curses.color_pair(CP_HEADER) | curses.A_BOLD)
1128
+ for i, line in enumerate(HELP_LINES):
1129
+ safe_addstr(win, y0 + 2 + i, x0 + 2, line, curses.color_pair(CP_TAB))
1130
+
1131
+
1132
+ # ---------------------------------------------------------------------------
1133
+ # WebSocket client
1134
+ # ---------------------------------------------------------------------------
1135
+
1136
+
1137
+ async def ws_client(url: str, state: State, stop: asyncio.Event) -> None:
1138
+ """Connect, drain, reconnect on failure with backoff."""
1139
+ backoff = 1.0
1140
+ state.url = url
1141
+ while not stop.is_set():
1142
+ try:
1143
+ timeout = aiohttp.ClientTimeout(total=None, sock_connect=5, sock_read=None)
1144
+ async with aiohttp.ClientSession(timeout=timeout) as session:
1145
+ async with session.ws_connect(url, heartbeat=30) as ws:
1146
+ state.connected = True
1147
+ state.last_error = ""
1148
+ backoff = 1.0
1149
+ async for msg in ws:
1150
+ if msg.type == aiohttp.WSMsgType.TEXT:
1151
+ try:
1152
+ payload = json.loads(msg.data)
1153
+ except json.JSONDecodeError:
1154
+ continue
1155
+ state.apply(payload)
1156
+ elif msg.type in (aiohttp.WSMsgType.CLOSED,
1157
+ aiohttp.WSMsgType.ERROR):
1158
+ break
1159
+ if stop.is_set():
1160
+ break
1161
+ except asyncio.CancelledError:
1162
+ break
1163
+ except Exception as e:
1164
+ state.last_error = f"{type(e).__name__}: {e}"
1165
+ state.connected = False
1166
+ if stop.is_set():
1167
+ break
1168
+ try:
1169
+ await asyncio.wait_for(stop.wait(), timeout=backoff)
1170
+ break
1171
+ except asyncio.TimeoutError:
1172
+ pass
1173
+ backoff = min(backoff * 2, 10.0)
1174
+
1175
+
1176
+ # ---------------------------------------------------------------------------
1177
+ # Curses setup + dispatch
1178
+ # ---------------------------------------------------------------------------
1179
+
1180
+
1181
+ def init_colors() -> None:
1182
+ if not curses.has_colors():
1183
+ return
1184
+ curses.start_color()
1185
+ try:
1186
+ curses.use_default_colors()
1187
+ bg = -1
1188
+ except curses.error:
1189
+ bg = curses.COLOR_BLACK
1190
+ curses.init_pair(CP_HEADER, curses.COLOR_BLACK, curses.COLOR_CYAN)
1191
+ curses.init_pair(CP_OK, curses.COLOR_GREEN, bg)
1192
+ curses.init_pair(CP_WARN, curses.COLOR_YELLOW, bg)
1193
+ curses.init_pair(CP_CRIT, curses.COLOR_RED, bg)
1194
+ curses.init_pair(CP_DIM, curses.COLOR_WHITE, bg)
1195
+ curses.init_pair(CP_BAR, curses.COLOR_CYAN, bg)
1196
+ curses.init_pair(CP_TAB, curses.COLOR_WHITE, curses.COLOR_BLUE)
1197
+ curses.init_pair(CP_TAB_ACTIVE, curses.COLOR_BLACK, curses.COLOR_WHITE)
1198
+
1199
+
1200
+ class App:
1201
+ def __init__(self, stdscr, state: State, ui: UIState, stop: asyncio.Event,
1202
+ *, no_color: bool) -> None:
1203
+ self.stdscr = stdscr
1204
+ self.state = state
1205
+ self.ui = ui
1206
+ self.stop = stop
1207
+ self.no_color = no_color
1208
+ self.search_mode = False
1209
+ self.search_buf = ""
1210
+
1211
+ def handle_key(self, ch: int) -> None:
1212
+ ui = self.ui
1213
+ if self.search_mode:
1214
+ self._handle_search_key(ch)
1215
+ return
1216
+ if ch in (ord("q"), 27): # q or ESC
1217
+ self.stop.set()
1218
+ return
1219
+ if ch == ord("?") or ch == ord("h"):
1220
+ ui.show_help = not ui.show_help
1221
+ return
1222
+ if ch == ord(" "):
1223
+ ui.paused = not ui.paused
1224
+ return
1225
+ if ch == ord("+"):
1226
+ ui.refresh_idx = max(0, ui.refresh_idx - 1)
1227
+ return
1228
+ if ch == ord("-"):
1229
+ ui.refresh_idx = min(len(REFRESH_RATES) - 1, ui.refresh_idx + 1)
1230
+ return
1231
+ if ch in VIEW_KEYS:
1232
+ ui.view = VIEW_KEYS[ch]
1233
+ return
1234
+ if ch == ord("<") or ch == ord(","):
1235
+ self._cycle_left()
1236
+ return
1237
+ if ch == ord(">") or ch == ord("."):
1238
+ self._cycle_right()
1239
+ return
1240
+ if ch == ord("[") and ui.view == "latency":
1241
+ ui.phase_idx = (ui.phase_idx - 1) % len(PHASES)
1242
+ return
1243
+ if ch == ord("]") and ui.view == "latency":
1244
+ ui.phase_idx = (ui.phase_idx + 1) % len(PHASES)
1245
+ return
1246
+ if ch == ord("f"):
1247
+ self._cycle_filter()
1248
+ return
1249
+ if ch == ord("r") and ui.view == "slow":
1250
+ ui.slow_search = ""
1251
+ ui.slow_state_filter = -1
1252
+ return
1253
+ if ch == ord("/") and ui.view == "slow":
1254
+ self.search_mode = True
1255
+ self.search_buf = ui.slow_search
1256
+ return
1257
+
1258
+ def _handle_search_key(self, ch: int) -> None:
1259
+ if ch in (10, 13):
1260
+ self.ui.slow_search = self.search_buf
1261
+ self.search_mode = False
1262
+ elif ch == 27:
1263
+ self.search_mode = False
1264
+ self.search_buf = ""
1265
+ self.ui.slow_search = ""
1266
+ elif ch in (curses.KEY_BACKSPACE, 127, 8):
1267
+ self.search_buf = self.search_buf[:-1]
1268
+ elif 32 <= ch < 127:
1269
+ self.search_buf += chr(ch)
1270
+
1271
+ def _cycle_left(self) -> None:
1272
+ ui = self.ui
1273
+ if ui.view in ("overview", "latency"):
1274
+ ui.window_idx = (ui.window_idx - 1) % len(WINDOW_CHOICES)
1275
+ elif ui.view == "processes":
1276
+ keys = ["rps", "cpu", "rss", "p95", "slow", "pid"]
1277
+ i = keys.index(ui.process_sort) if ui.process_sort in keys else 0
1278
+ ui.process_sort = keys[(i - 1) % len(keys)]
1279
+ elif ui.view == "slow":
1280
+ keys = ["elapsed", "pid", "method", "url"]
1281
+ i = keys.index(ui.slow_sort) if ui.slow_sort in keys else 0
1282
+ ui.slow_sort = keys[(i - 1) % len(keys)]
1283
+
1284
+ def _cycle_right(self) -> None:
1285
+ ui = self.ui
1286
+ if ui.view in ("overview", "latency"):
1287
+ ui.window_idx = (ui.window_idx + 1) % len(WINDOW_CHOICES)
1288
+ elif ui.view == "processes":
1289
+ keys = ["rps", "cpu", "rss", "p95", "slow", "pid"]
1290
+ i = keys.index(ui.process_sort) if ui.process_sort in keys else 0
1291
+ ui.process_sort = keys[(i + 1) % len(keys)]
1292
+ elif ui.view == "slow":
1293
+ keys = ["elapsed", "pid", "method", "url"]
1294
+ i = keys.index(ui.slow_sort) if ui.slow_sort in keys else 0
1295
+ ui.slow_sort = keys[(i + 1) % len(keys)]
1296
+
1297
+ def _cycle_filter(self) -> None:
1298
+ ui = self.ui
1299
+ if ui.view == "slow":
1300
+ seq = [-1, 0, 1]
1301
+ i = seq.index(ui.slow_state_filter) if ui.slow_state_filter in seq else 0
1302
+ ui.slow_state_filter = seq[(i + 1) % len(seq)]
1303
+ return
1304
+ groups = _process_groups(self.state)
1305
+ if not groups:
1306
+ ui.group_filter = None
1307
+ return
1308
+ cycle = [None] + groups
1309
+ try:
1310
+ i = cycle.index(ui.group_filter)
1311
+ except ValueError:
1312
+ i = 0
1313
+ ui.group_filter = cycle[(i + 1) % len(cycle)]
1314
+
1315
+ def render(self) -> None:
1316
+ stdscr = self.stdscr
1317
+ stdscr.erase()
1318
+ h = _aggregate_header(self.state, self.ui)
1319
+ y0 = render_header(stdscr, self.state, self.ui, h)
1320
+ view = self.ui.view
1321
+ if view == "overview":
1322
+ render_overview(stdscr, self.state, self.ui, h, y0)
1323
+ elif view == "processes":
1324
+ render_processes(stdscr, self.state, self.ui, y0)
1325
+ elif view == "workers":
1326
+ render_workers(stdscr, self.state, self.ui, y0)
1327
+ elif view == "latency":
1328
+ render_latency(stdscr, self.state, self.ui, y0)
1329
+ elif view == "slow":
1330
+ render_slow(stdscr, self.state, self.ui, y0)
1331
+
1332
+ # Footer.
1333
+ height, width = stdscr.getmaxyx()
1334
+ footer = self._footer_text()
1335
+ safe_addstr(stdscr, height - 1, 0, footer.ljust(width)[:width],
1336
+ curses.color_pair(CP_TAB))
1337
+ if self.ui.show_help:
1338
+ render_help(stdscr)
1339
+ stdscr.noutrefresh()
1340
+ curses.doupdate()
1341
+
1342
+ def _footer_text(self) -> str:
1343
+ if self.search_mode:
1344
+ return f" / search: {self.search_buf}_ (Enter=apply Esc=cancel) "
1345
+ st = self.state
1346
+ if not st.connected:
1347
+ return f" disconnected ({st.last_error or 'connecting...'}) url={st.url} "
1348
+ age = time.monotonic() - st.last_message if st.last_message else 0
1349
+ return (f" connected to {st.url} pids={len(st.processes)} "
1350
+ f"slow={len(st.slow)} rcvd={st.total_received} "
1351
+ f"last_msg={age:.1f}s ago "
1352
+ f"{'PAUSED' if self.ui.paused else ''}")
1353
+
1354
+
1355
+ async def run_app(stdscr, args, state: State, ui: UIState) -> int:
1356
+ init_colors() if not args.no_color else None
1357
+ curses.curs_set(0)
1358
+ stdscr.nodelay(True)
1359
+ stdscr.keypad(True)
1360
+
1361
+ stop = asyncio.Event()
1362
+ loop = asyncio.get_running_loop()
1363
+ for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
1364
+ try:
1365
+ loop.add_signal_handler(sig, stop.set)
1366
+ except (NotImplementedError, ValueError):
1367
+ pass
1368
+
1369
+ ws_task = asyncio.create_task(ws_client(args.url, state, stop))
1370
+ app = App(stdscr, state, ui, stop, no_color=args.no_color)
1371
+
1372
+ try:
1373
+ last_render = 0.0
1374
+ while not stop.is_set():
1375
+ # Drain all pending keys.
1376
+ while True:
1377
+ ch = stdscr.getch()
1378
+ if ch == -1:
1379
+ break
1380
+ app.handle_key(ch)
1381
+ now = time.monotonic()
1382
+ if now - last_render >= ui.refresh and not ui.paused:
1383
+ app.render()
1384
+ last_render = now
1385
+ elif now - last_render >= ui.refresh:
1386
+ app.render()
1387
+ last_render = now
1388
+ await asyncio.sleep(0.05)
1389
+ finally:
1390
+ ws_task.cancel()
1391
+ try:
1392
+ await ws_task
1393
+ except (asyncio.CancelledError, Exception):
1394
+ pass
1395
+
1396
+ return 0
1397
+
1398
+
1399
+ def render_once_text(state: State, ui: UIState) -> str:
1400
+ """Plain-text snapshot for --once. No curses, no colors."""
1401
+ h = _aggregate_header(state, ui)
1402
+ lines = []
1403
+ pgroup = ui.group_filter or ",".join(_process_groups(state)) or "(any)"
1404
+ hostname = next(iter(state.processes.values())).hostname if state.processes else "-"
1405
+ lines.append(f"mod_wsgi-telemetry top host={hostname} group={pgroup}")
1406
+ lines.append(f" rps: now {h['rps_now']:.1f} 1m {h['rps_1m']:.1f} "
1407
+ f"10m {h['rps_10m']:.1f} "
1408
+ f"in {fmt_bytes(h['in_bps'])}/s out {fmt_bytes(h['out_bps'])}/s")
1409
+ cap_total = h["threads_total"]
1410
+ cap_busy = h["busy"]
1411
+ frac = (cap_busy / cap_total) if cap_total else 0.0
1412
+ lines.append(f" cap: {cap_busy}/{cap_total} threads ({frac*100:.1f}%) "
1413
+ f"queue {fmt_seconds(h['queue_mean']) if h['queue_mean'] is not None else '-'}")
1414
+ lines.append(f" cpu: user {h['cpu_user']:.2f} sys {h['cpu_sys']:.2f} "
1415
+ f"rss {fmt_bytes(h['rss_total'])} (max {fmt_bytes(h['rss_max'])})")
1416
+ # status==0 (no start_response, exception path) is folded into 5xx
1417
+ # on the C side; the percentages here are share of completed
1418
+ # requests in the latest interval. 1xx is a PEP 3333 tripwire and
1419
+ # only included in the line when it has fired.
1420
+ requests = h["interval_requests"]
1421
+ pct = lambda n: (n / requests * 100.0) if requests > 0 else 0.0
1422
+ s_parts = []
1423
+ if h["status_1xx"] > 0:
1424
+ s_parts.append(f"1xx {h['status_1xx']} (PEP 3333 violation)")
1425
+ s_parts.append(f"2xx {pct(h['status_2xx']):.1f}%")
1426
+ s_parts.append(f"3xx {pct(h['status_3xx']):.1f}%")
1427
+ s_parts.append(f"4xx {pct(h['status_4xx']):.1f}%")
1428
+ s_parts.append(f"5xx {pct(h['status_5xx']):.1f}%")
1429
+ lines.append(" stat: " + " ".join(s_parts))
1430
+ lines.append(f" lat: p50 {fmt_seconds(h['p50'])} "
1431
+ f"p95 {fmt_seconds(h['p95'])} p99 {fmt_seconds(h['p99'])} "
1432
+ f"min {fmt_seconds(h['min'])} max {fmt_seconds(h['max'])}")
1433
+ lines.append(f" slow: {h['active_slow']} active / {h['recent_slow']} 1m / "
1434
+ f"{h['total_slow']} total")
1435
+ lines.append("")
1436
+ lines.append("processes:")
1437
+ rows = []
1438
+ for ps in state.processes.values():
1439
+ if ui.group_filter and ps.process_group != ui.group_filter:
1440
+ continue
1441
+ if not ps.samples:
1442
+ continue
1443
+ f = ps.samples[-1].get("fields", {})
1444
+ rows.append((
1445
+ ps.pid, ps.process_group or "-",
1446
+ int(f.get("request_threads_active") or 0),
1447
+ int(f.get("request_threads_maximum") or 0),
1448
+ float(f.get("request_throughput") or 0),
1449
+ float(f.get("cpu_utilization") or 0),
1450
+ int(f.get("memory_rss") or 0),
1451
+ ))
1452
+ rows.sort(key=lambda r: -r[4])
1453
+ lines.append(f" {'PID':>7} {'GROUP':<16} {'THR':>9} {'RPS':>7} "
1454
+ f"{'CPU':>6} {'RSS':>9}")
1455
+ for pid, grp, act, mx, rps, cpu, rss in rows:
1456
+ lines.append(f" {pid:>7} {grp[:16]:<16} {act:>4}/{mx:<4} "
1457
+ f"{rps:>7.1f} {cpu:>6.2f} {fmt_bytes(rss):>9}")
1458
+ return "\n".join(lines)
1459
+
1460
+
1461
+ # ---------------------------------------------------------------------------
1462
+ # Entry point
1463
+ # ---------------------------------------------------------------------------
1464
+
1465
+
1466
+ def main(argv: list[str] | None = None) -> int:
1467
+ ap = argparse.ArgumentParser(
1468
+ description="Terminal monitor for mod_wsgi telemetry."
1469
+ )
1470
+ ap.add_argument("--url", default=os.environ.get("MOD_WSGI_TELEMETRY_URL", DEFAULT_URL),
1471
+ help=f"WebSocket URL of the telemetry server (default: {DEFAULT_URL})")
1472
+ ap.add_argument("--refresh", type=float, default=1.0,
1473
+ help="Render refresh interval in seconds (default: 1.0)")
1474
+ ap.add_argument("--view", choices=VIEWS, default="overview",
1475
+ help="Initial view (default: overview)")
1476
+ ap.add_argument("--group", default=None,
1477
+ help="Filter to a single process group")
1478
+ ap.add_argument("--no-color", action="store_true",
1479
+ help="Disable ANSI colour")
1480
+ ap.add_argument("--once", action="store_true",
1481
+ help="Render one frame to stdout and exit (scriptable)")
1482
+ ap.add_argument("--once-timeout", type=float, default=3.0,
1483
+ help="Max seconds to wait for the snapshot in --once mode")
1484
+ ap.add_argument("--log-file", default=None,
1485
+ help="Write debug log to this file (curses hides stderr)")
1486
+ args = ap.parse_args(argv)
1487
+
1488
+ if args.log_file:
1489
+ logging.basicConfig(filename=args.log_file, level=logging.DEBUG,
1490
+ format="%(asctime)s %(levelname)s %(name)s %(message)s")
1491
+
1492
+ state = State()
1493
+ ui = UIState(view=args.view, group_filter=args.group)
1494
+ # Snap refresh to the closest preset so the cycler stays consistent.
1495
+ ui.refresh_idx = min(
1496
+ range(len(REFRESH_RATES)),
1497
+ key=lambda i: abs(REFRESH_RATES[i] - args.refresh),
1498
+ )
1499
+
1500
+ if args.once:
1501
+ return _run_once(args, state, ui)
1502
+
1503
+ return curses.wrapper(_curses_main, args, state, ui)
1504
+
1505
+
1506
+ def _curses_main(stdscr, args, state, ui) -> int:
1507
+ return asyncio.run(run_app(stdscr, args, state, ui))
1508
+
1509
+
1510
+ def _run_once(args, state: State, ui: UIState) -> int:
1511
+ async def grab() -> int:
1512
+ stop = asyncio.Event()
1513
+ ws_task = asyncio.create_task(ws_client(args.url, state, stop))
1514
+ deadline = time.monotonic() + args.once_timeout
1515
+ try:
1516
+ while time.monotonic() < deadline:
1517
+ if state.last_message:
1518
+ # Wait one extra tick to catch any in-flight live update.
1519
+ await asyncio.sleep(0.1)
1520
+ break
1521
+ await asyncio.sleep(0.05)
1522
+ sys.stdout.write(render_once_text(state, ui) + "\n")
1523
+ return 0 if state.last_message else 2
1524
+ finally:
1525
+ stop.set()
1526
+ ws_task.cancel()
1527
+ try:
1528
+ await ws_task
1529
+ except (asyncio.CancelledError, Exception):
1530
+ pass
1531
+
1532
+ return asyncio.run(grab())
1533
+
1534
+
1535
+ if __name__ == "__main__":
1536
+ sys.exit(main())